ace-assign 0.63.1 → 0.66.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ace-defaults/assign/catalog/steps/auto-merge.step.yml +11 -0
- data/.ace-defaults/assign/catalog/steps/create-pr.step.yml +2 -2
- data/.ace-defaults/assign/catalog/steps/mark-pr-ready.step.yml +1 -1
- data/.ace-defaults/assign/catalog/steps/release-minor.step.yml +4 -2
- data/.ace-defaults/assign/catalog/steps/release.step.yml +4 -2
- data/.ace-defaults/assign/catalog/steps/update-pr-desc.step.yml +1 -1
- data/.ace-defaults/assign/presets/fix-bug.yml +15 -3
- data/.ace-defaults/assign/presets/work-on-task-ace-development.yml +16 -4
- data/.ace-defaults/assign/presets/work-on-task-auto-merge.yml +15 -3
- data/.ace-defaults/assign/presets/work-on-task.yml +16 -4
- data/CHANGELOG.md +62 -0
- data/docs/demo/canonical-skill-source.tape.yml +4 -4
- data/docs/protected-authority.md +282 -0
- data/docs/usage.md +205 -3
- data/exe/ace-scope-ready +55 -0
- data/handbook/workflow-instructions/assign/drive.wf.md +28 -3
- data/handbook/workflow-instructions/assign/prepare.wf.md +2 -2
- data/lib/ace/assign/atoms/attempt_state_machine.rb +8 -0
- data/lib/ace/assign/atoms/delivery_parameters.rb +19 -0
- data/lib/ace/assign/authority/assignment_inventory.rb +290 -0
- data/lib/ace/assign/authority/campaign_consumer_policy.rb +100 -0
- data/lib/ace/assign/authority/campaign_execution.rb +119 -0
- data/lib/ace/assign/authority/campaign_round_transfer.rb +82 -0
- data/lib/ace/assign/authority/candidate_transfer.rb +327 -0
- data/lib/ace/assign/authority/cleanup_result_protection.rb +53 -0
- data/lib/ace/assign/authority/client.rb +257 -0
- data/lib/ace/assign/authority/deployment.rb +627 -0
- data/lib/ace/assign/authority/deployment_history.rb +146 -0
- data/lib/ace/assign/authority/endcap.rb +455 -0
- data/lib/ace/assign/authority/endcap_attempts.rb +274 -0
- data/lib/ace/assign/authority/endcap_campaign_results.rb +232 -0
- data/lib/ace/assign/authority/endcap_campaign_rounds.rb +266 -0
- data/lib/ace/assign/authority/endcap_contexts.rb +23 -0
- data/lib/ace/assign/authority/endcap_inboxes.rb +37 -0
- data/lib/ace/assign/authority/endcap_prepared_work.rb +80 -0
- data/lib/ace/assign/authority/endcap_publication.rb +207 -0
- data/lib/ace/assign/authority/endcap_results.rb +398 -0
- data/lib/ace/assign/authority/endcap_review_requests.rb +183 -0
- data/lib/ace/assign/authority/endcap_reviews.rb +130 -0
- data/lib/ace/assign/authority/endcap_service_settlement.rb +152 -0
- data/lib/ace/assign/authority/endcap_services.rb +598 -0
- data/lib/ace/assign/authority/evidence_transfer.rb +67 -0
- data/lib/ace/assign/authority/execution_scope_observation.rb +903 -0
- data/lib/ace/assign/authority/inbox_context_original.rb +113 -0
- data/lib/ace/assign/authority/launch_attempt_consumers.rb +51 -0
- data/lib/ace/assign/authority/launch_campaign_registration.rb +214 -0
- data/lib/ace/assign/authority/launch_control_channel.rb +313 -0
- data/lib/ace/assign/authority/launch_control_exclusions.rb +141 -0
- data/lib/ace/assign/authority/launch_driver.rb +393 -0
- data/lib/ace/assign/authority/launch_input_inhibition.rb +132 -0
- data/lib/ace/assign/authority/launch_lifecycle.rb +997 -0
- data/lib/ace/assign/authority/launch_prepared_work.rb +122 -0
- data/lib/ace/assign/authority/launch_review.rb +91 -0
- data/lib/ace/assign/authority/launch_scope_abort.rb +81 -0
- data/lib/ace/assign/authority/launch_scope_admission.rb +215 -0
- data/lib/ace/assign/authority/launch_scope_close.rb +146 -0
- data/lib/ace/assign/authority/launch_scope_parent.rb +44 -0
- data/lib/ace/assign/authority/launch_scope_release.rb +859 -0
- data/lib/ace/assign/authority/launch_steering.rb +353 -0
- data/lib/ace/assign/authority/launch_stop.rb +254 -0
- data/lib/ace/assign/authority/maintenance_workspace_retirement.rb +229 -0
- data/lib/ace/assign/authority/original_inbox_selection.rb +41 -0
- data/lib/ace/assign/authority/posix_acl.rb +59 -0
- data/lib/ace/assign/authority/prepared_input.rb +130 -0
- data/lib/ace/assign/authority/prepared_queue.rb +185 -0
- data/lib/ace/assign/authority/prepared_task_context.rb +116 -0
- data/lib/ace/assign/authority/prepared_work.rb +304 -0
- data/lib/ace/assign/authority/prepared_worker.rb +69 -0
- data/lib/ace/assign/authority/private_directory.rb +33 -0
- data/lib/ace/assign/authority/protected_assignment_context.rb +226 -0
- data/lib/ace/assign/authority/readiness_hook.rb +120 -0
- data/lib/ace/assign/authority/receipt_transfer.rb +48 -0
- data/lib/ace/assign/authority/router.rb +76 -0
- data/lib/ace/assign/authority/server.rb +300 -0
- data/lib/ace/assign/authority/service_cleanup_dispatch.rb +88 -0
- data/lib/ace/assign/authority/service_cleanup_evidence.rb +180 -0
- data/lib/ace/assign/authority/service_delivery_evidence.rb +101 -0
- data/lib/ace/assign/authority/service_evidence.rb +286 -0
- data/lib/ace/assign/authority/service_publication_evidence.rb +110 -0
- data/lib/ace/assign/authority/task_context_entry.rb +32 -0
- data/lib/ace/assign/authority/transfer_codec.rb +187 -0
- data/lib/ace/assign/cli/commands/assignment_target.rb +15 -2
- data/lib/ace/assign/cli/commands/attempt/base.rb +49 -0
- data/lib/ace/assign/cli/commands/attempt/finish.rb +20 -4
- data/lib/ace/assign/cli/commands/attempt/reconcile.rb +11 -0
- data/lib/ace/assign/cli/commands/authority/inbox_context_principal.rb +25 -0
- data/lib/ace/assign/cli/commands/authority/inbox_context_selection.rb +28 -0
- data/lib/ace/assign/cli/commands/authority/launch.rb +97 -0
- data/lib/ace/assign/cli/commands/authority/serve.rb +31 -0
- data/lib/ace/assign/cli/commands/authority/status.rb +26 -0
- data/lib/ace/assign/cli/commands/authority/task_context.rb +31 -0
- data/lib/ace/assign/cli/commands/authority/task_context_principal.rb +25 -0
- data/lib/ace/assign/cli/commands/authority/task_context_selection.rb +30 -0
- data/lib/ace/assign/cli/commands/authority/terminate.rb +29 -0
- data/lib/ace/assign/cli/commands/authority/worker.rb +23 -0
- data/lib/ace/assign/cli/commands/campaign_export_result.rb +55 -0
- data/lib/ace/assign/cli/commands/campaign_record_round.rb +57 -0
- data/lib/ace/assign/cli/commands/create.rb +11 -1
- data/lib/ace/assign/cli/commands/delivery.rb +107 -0
- data/lib/ace/assign/cli/commands/fail.rb +6 -0
- data/lib/ace/assign/cli/commands/finish.rb +23 -2
- data/lib/ace/assign/cli/commands/fork_session.rb +6 -2
- data/lib/ace/assign/cli/commands/inbox_bind.rb +34 -0
- data/lib/ace/assign/cli/commands/protected_submission.rb +61 -0
- data/lib/ace/assign/cli/commands/resume.rb +38 -0
- data/lib/ace/assign/cli/commands/select.rb +2 -0
- data/lib/ace/assign/cli/commands/start.rb +2 -0
- data/lib/ace/assign/cli/commands/status.rb +4 -2
- data/lib/ace/assign/cli/commands/step.rb +3 -1
- data/lib/ace/assign/cli/commands/submit_candidate.rb +34 -0
- data/lib/ace/assign/cli/commands/submit_result.rb +61 -0
- data/lib/ace/assign/cli.rb +56 -4
- data/lib/ace/assign/errors.rb +139 -0
- data/lib/ace/assign/models/assignment.rb +26 -4
- data/lib/ace/assign/models/evidence_event.rb +4 -2
- data/lib/ace/assign/molecules/assignment_manager.rb +12 -2
- data/lib/ace/assign/molecules/attempt_reconciler.rb +51 -12
- data/lib/ace/assign/molecules/canonical_attempt_state.rb +151 -0
- data/lib/ace/assign/molecules/canonical_evidence.rb +153 -0
- data/lib/ace/assign/molecules/canonical_read_snapshot.rb +374 -0
- data/lib/ace/assign/molecules/evidence_journal.rb +922 -91
- data/lib/ace/assign/molecules/execution_identity_resolver.rb +49 -9
- data/lib/ace/assign/molecules/execution_scope_lineage.rb +345 -0
- data/lib/ace/assign/molecules/fork_session_launcher.rb +16 -6
- data/lib/ace/assign/molecules/journal_input_inhibition.rb +78 -0
- data/lib/ace/assign/molecules/journal_mutation.rb +271 -0
- data/lib/ace/assign/molecules/journal_prompt_mutation.rb +241 -0
- data/lib/ace/assign/molecules/lifecycle_exclusion.rb +182 -26
- data/lib/ace/assign/molecules/original_launch_binding.rb +37 -0
- data/lib/ace/assign/molecules/proposal_journal.rb +151 -0
- data/lib/ace/assign/molecules/protected_control_exclusion.rb +303 -0
- data/lib/ace/assign/molecules/protected_workspace_exclusion.rb +330 -0
- data/lib/ace/assign/molecules/protected_workspace_provisioner.rb +164 -0
- data/lib/ace/assign/molecules/queue_scanner.rb +26 -11
- data/lib/ace/assign/molecules/receipt_verifier.rb +68 -32
- data/lib/ace/assign/molecules/step_writer.rb +27 -5
- data/lib/ace/assign/molecules/terminal_scope_receipt.rb +88 -0
- data/lib/ace/assign/organisms/assignment_executor.rb +10 -4
- data/lib/ace/assign/organisms/attempt_coordinator.rb +302 -13
- data/lib/ace/assign/organisms/delivery_coordinator.rb +342 -0
- data/lib/ace/assign/organisms/prepared_work_builder.rb +194 -0
- data/lib/ace/assign/organisms/protected_delivery_coordinator.rb +100 -0
- data/lib/ace/assign/organisms/task_assignment_creator.rb +11 -4
- data/lib/ace/assign/version.rb +1 -1
- data/lib/ace/assign.rb +4 -102
- metadata +186 -8
|
@@ -0,0 +1,997 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require "digest"
|
|
3
|
+
require "json"
|
|
4
|
+
require "securerandom"
|
|
5
|
+
require "time"
|
|
6
|
+
require "fileutils"
|
|
7
|
+
require "open3"
|
|
8
|
+
require "ace/herdr/molecules/guarded_native_origin"
|
|
9
|
+
require_relative "launch_control_channel"
|
|
10
|
+
require_relative "deployment"
|
|
11
|
+
require_relative "deployment_history"
|
|
12
|
+
require_relative "../molecules/execution_scope_lineage"
|
|
13
|
+
require_relative "../molecules/original_launch_binding"
|
|
14
|
+
require_relative "../molecules/canonical_attempt_state"
|
|
15
|
+
require_relative "execution_scope_observation"
|
|
16
|
+
require_relative "prepared_work"
|
|
17
|
+
|
|
18
|
+
module Ace
|
|
19
|
+
module Assign
|
|
20
|
+
module Authority
|
|
21
|
+
# Launch facts and release permission live in qjl authority_mutation data.
|
|
22
|
+
# Process handles/streams are observations, never a replacement ledger.
|
|
23
|
+
class LaunchLifecycle
|
|
24
|
+
MUTATIONS = {
|
|
25
|
+
"register_assignment" => %w[mapping_id assignment_id definition_digest prepared_head prepared_tree manifest_sha256 expected_generation transfer],
|
|
26
|
+
"reserve_attempt" => %w[mapping_id assignment_id scope worker_uid runtime base_head launcher_process_binding expected_generation],
|
|
27
|
+
"record_launch" => %w[mapping_id assignment_id attempt_id launch_ticket process_binding guarded_origin expected_generation],
|
|
28
|
+
"bind_process" => %w[mapping_id assignment_id attempt_id launch_ticket process_binding expected_generation],
|
|
29
|
+
"release_launch" => %w[mapping_id assignment_id attempt_id launch_ticket process_binding expected_generation],
|
|
30
|
+
"abort_launch" => %w[mapping_id assignment_id attempt_id launch_ticket failure_evidence failure_digest expected_generation]
|
|
31
|
+
}.freeze
|
|
32
|
+
TRANSFER_OPERATIONS = {"register_assignment" => {direction: :upload, purpose: :candidate, roles: [:launcher]}, "prompt_attempt" => {direction: :upload, purpose: :prompt_text, roles: %i[launcher supervisor]}}.freeze
|
|
33
|
+
OPERATIONS = (MUTATIONS.keys + %w[assignment_inventory stop_attempt prompt_attempt prompt_status launch_review_intent launch_input_inhibit_selection launch_input_inhibit_completion launch_prompt_intent launch_prompt_completion launch_preflight registration_status attempt_status inspect_launch observe_execution_scope close_execution_scope]).freeze
|
|
34
|
+
TERMINAL = %w[succeeded failed stopped].freeze
|
|
35
|
+
|
|
36
|
+
attr_reader :mutex, :journals, :deployment, :deployment_history
|
|
37
|
+
def initialize(deployment:, kernel: Ace::Runtime::Molecules::ProtectedLinux.new, journals: nil, mutex: Mutex.new, scope_observer_factory: nil, deployment_history: nil, control_exclusion_factory: nil)
|
|
38
|
+
@deployment, @kernel = deployment, kernel
|
|
39
|
+
if deployment_history && (!deployment_history.is_a?(DeploymentHistory) ||
|
|
40
|
+
!deployment_history.selects?(deployment))
|
|
41
|
+
raise ArgumentError, "protected history transaction does not select installed descriptor"
|
|
42
|
+
end
|
|
43
|
+
@deployment_history = deployment_history
|
|
44
|
+
@journals = journals || {}
|
|
45
|
+
@control_exclusion_factory = control_exclusion_factory || ->(**options) {
|
|
46
|
+
Molecules::LifecycleExclusion.control_exclusion(**options)
|
|
47
|
+
}
|
|
48
|
+
@mutex = mutex
|
|
49
|
+
@observations = {}
|
|
50
|
+
@streams = {}
|
|
51
|
+
@native_issuers = {}
|
|
52
|
+
@control_channels = {}
|
|
53
|
+
@slot_exclusions = {}
|
|
54
|
+
@scope_observers = {}
|
|
55
|
+
@scope_observer_factory = scope_observer_factory || ->(mapping_id) {
|
|
56
|
+
ExecutionScopeObservation.new(mapping_id: mapping_id, deployment: @deployment, kernel: @kernel)
|
|
57
|
+
}
|
|
58
|
+
@changed = ConditionVariable.new
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def attach_result_owner(owner)
|
|
62
|
+
raise ArgumentError, "result status owner already attached" if @result_owner
|
|
63
|
+
@result_owner = owner
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Existing service owners call this while holding slot-before-assignment
|
|
67
|
+
# exclusion. It is a canonical seal gate, never a new effect journal.
|
|
68
|
+
def scope_open_for_effect!(events:, params:, map:)
|
|
69
|
+
chain = events.select { |event| event["attempt_id"] == params.fetch("attempt_id") }
|
|
70
|
+
Molecules::ExecutionScopeLineage.new(events: chain, project_id: map.fetch("project_id"),
|
|
71
|
+
assignment_id: params.fetch("assignment_id"), attempt_id: params.fetch("attempt_id"),
|
|
72
|
+
mapping_id: params.fetch("mapping_id")).require_launch_bound!
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def dispatch(request:, peer:, role:, transfer: nil)
|
|
76
|
+
operation, params = request.values_at("operation", "params")
|
|
77
|
+
return assignment_inventory!(request: request, peer: peer, role: role) if operation == "assignment_inventory"
|
|
78
|
+
return stop_attempt!(request: request, peer: peer, role: role) if operation == "stop_attempt"
|
|
79
|
+
return prompt_attempt!(request: request, peer: peer, role: role, transfer: transfer) if operation == "prompt_attempt"
|
|
80
|
+
return prompt_status!(request: request, peer: peer, role: role) if operation == "prompt_status"
|
|
81
|
+
return launch_input_inhibit_selection!(request: request, peer: peer, role: role) if operation == "launch_input_inhibit_selection"
|
|
82
|
+
return launch_input_inhibit_completion!(request: request, peer: peer, role: role) if operation == "launch_input_inhibit_completion"
|
|
83
|
+
return launch_review_intent!(request: request, peer: peer, role: role) if operation == "launch_review_intent"
|
|
84
|
+
return launch_prompt_intent!(request: request, peer: peer, role: role) if operation == "launch_prompt_intent"
|
|
85
|
+
return launch_prompt_completion!(request: request, peer: peer, role: role) if operation == "launch_prompt_completion"
|
|
86
|
+
if operation == "observe_execution_scope"
|
|
87
|
+
raise ArgumentError, "scope observation mutation ID must be null" unless request.fetch("mutation_id").nil?
|
|
88
|
+
return {data: observe_execution_scope!(params: params, peer: peer, role: role), replayed: false}
|
|
89
|
+
end
|
|
90
|
+
if operation == "close_execution_scope"
|
|
91
|
+
strict!(params, %w[mapping_id assignment_id attempt_id expected_generation])
|
|
92
|
+
return close_execution_scope!(params: params.merge("mutation_id" => request.fetch("mutation_id")), peer: peer, role: role)
|
|
93
|
+
end
|
|
94
|
+
if operation == "launch_preflight"
|
|
95
|
+
strict!(params, %w[mapping_id])
|
|
96
|
+
raise AttemptErrors::UnauthorizedIdentity, "preflight requires mapped launcher" unless role == :launcher
|
|
97
|
+
map = @deployment.mapping(params.fetch("mapping_id"))
|
|
98
|
+
return {data: {"project_id" => map.fetch("project_id"), "runtime" => "herdr", "supported" => true}, replayed: false}
|
|
99
|
+
end
|
|
100
|
+
if operation == "registration_status"
|
|
101
|
+
strict!(params, %w[mapping_id assignment_id])
|
|
102
|
+
raise AttemptErrors::UnauthorizedIdentity, "registration status requires mapped launcher" unless role == :launcher
|
|
103
|
+
map = @deployment.mapping(params.fetch("mapping_id"))
|
|
104
|
+
token!(params.fetch("assignment_id"))
|
|
105
|
+
journal = journal_for(map)
|
|
106
|
+
@mutex.synchronize do
|
|
107
|
+
registration = definition(journal, params.fetch("assignment_id"))
|
|
108
|
+
return {data: registration || {"assignment_id" => params.fetch("assignment_id"), "generation" => 0}, replayed: false}
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
if operation == "inspect_launch"
|
|
112
|
+
strict!(params, %w[mapping_id assignment_id attempt_id])
|
|
113
|
+
raise AttemptErrors::UnauthorizedIdentity, "inspection requires launcher or supervisor" unless %i[launcher supervisor].include?(role)
|
|
114
|
+
return {data: inspect_launch(params, peer: peer, role: role), replayed: false}
|
|
115
|
+
end
|
|
116
|
+
if operation == "attempt_status"
|
|
117
|
+
keys = %w[mapping_id assignment_id attempt_id]
|
|
118
|
+
keys += ["result_candidate_generation"] if @result_owner
|
|
119
|
+
strict!(params, keys)
|
|
120
|
+
if @result_owner
|
|
121
|
+
raise ArgumentError, "status mutation ID must be null" unless request.fetch("mutation_id").nil?
|
|
122
|
+
selector = params.fetch("result_candidate_generation")
|
|
123
|
+
generation!(selector, allow_nil: true)
|
|
124
|
+
end
|
|
125
|
+
return {data: status(params, peer: peer, role: role), replayed: false}
|
|
126
|
+
end
|
|
127
|
+
unless role == :launcher || (role == :supervisor && operation == "abort_launch")
|
|
128
|
+
raise AttemptErrors::UnauthorizedIdentity, "launch mutation requires mapped launcher or authorized recovery supervisor"
|
|
129
|
+
end
|
|
130
|
+
keys = MUTATIONS.fetch(operation) { raise ArgumentError, "unknown authority operation" }
|
|
131
|
+
strict!(params, keys)
|
|
132
|
+
token!(request.fetch("mutation_id"))
|
|
133
|
+
token!(params.fetch("assignment_id"))
|
|
134
|
+
generation!(params.fetch("expected_generation"), allow_nil: operation == "register_assignment")
|
|
135
|
+
map = @deployment.mapping(params.fetch("mapping_id"))
|
|
136
|
+
bounded_scope!(params.fetch("scope")) if operation == "reserve_attempt"
|
|
137
|
+
@kernel.live!(peer)
|
|
138
|
+
journal = journal_for(map)
|
|
139
|
+
digest = Digest::SHA256.hexdigest(JSON.generate(canonical(params)))
|
|
140
|
+
prepared = if operation == "register_assignment"
|
|
141
|
+
authorize_transfer!(request: request, peer: peer, role: role)
|
|
142
|
+
admitted = admit_prepared_registration!(params, map, transfer)
|
|
143
|
+
digest = Digest::SHA256.hexdigest(JSON.generate(canonical(params.merge("task_context_entry" => admitted.fetch(:task_context_entry)))))
|
|
144
|
+
params = params.merge("definition_bytes" => admitted.fetch(:definition_bytes))
|
|
145
|
+
admitted
|
|
146
|
+
end
|
|
147
|
+
native_start = nil
|
|
148
|
+
outcome = with_exclusion(params, map, journal) do |control_context|
|
|
149
|
+
if operation == "reserve_attempt"
|
|
150
|
+
existing = @mutex.synchronize { journal.mutation_result(request.fetch("mutation_id")) }
|
|
151
|
+
unless existing
|
|
152
|
+
# Retire the exact released predecessor before reserving a fresh
|
|
153
|
+
# generation. No native action occurs inside the journal CAS.
|
|
154
|
+
commit = journal.ref_value
|
|
155
|
+
retire_released_parent!(mapping_id: params.fetch("mapping_id"), journal: journal, commit: commit)
|
|
156
|
+
end
|
|
157
|
+
end
|
|
158
|
+
response = @mutex.synchronize do
|
|
159
|
+
replay = journal.mutation_result(request.fetch("mutation_id"))
|
|
160
|
+
if replay && operation == "abort_launch"
|
|
161
|
+
replay_events = journal.read_events(params.fetch("assignment_id")).select { |event| event["attempt_id"] == params.fetch("attempt_id") }
|
|
162
|
+
scope_close_owner!(params, map, replay_events, peer, role)
|
|
163
|
+
end
|
|
164
|
+
attempt_id = replay&.fetch("attempt_id") || mutation_attempt_id(operation, params)
|
|
165
|
+
if operation == "release_launch" && !replay
|
|
166
|
+
deadline = wire.deadline(5)
|
|
167
|
+
until @streams.key?(attempt_id)
|
|
168
|
+
remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
169
|
+
raise AttemptErrors::EvidenceUnavailable, "exact gate readiness is unavailable" if remaining <= 0
|
|
170
|
+
@changed.wait(@mutex, [remaining, 0.1].min)
|
|
171
|
+
end
|
|
172
|
+
end
|
|
173
|
+
reservation_handle = nil
|
|
174
|
+
begin
|
|
175
|
+
fresh = false
|
|
176
|
+
released = false
|
|
177
|
+
result = with_parent_campaign_registration(params, control_context, replay: !replay.nil?) do |campaign_guard|
|
|
178
|
+
journal.mutate(assignment_id: params.fetch("assignment_id"), attempt_id: attempt_id,
|
|
179
|
+
mutation_id: request.fetch("mutation_id"), operation: operation, parameters_digest: digest,
|
|
180
|
+
expected_generation: operation == "reserve_attempt" ? 0 : (params.fetch("expected_generation") || 0), with_replay: true) do |events, commit, generation|
|
|
181
|
+
fresh = true
|
|
182
|
+
recheck_control_registration!(control_context, params, map, journal, commit: commit)
|
|
183
|
+
campaign_guard&.call(journal, commit)
|
|
184
|
+
plan = case operation
|
|
185
|
+
when "register_assignment" then register(params, map, journal, commit, generation, prepared: prepared,
|
|
186
|
+
lifecycle_control: control_context.fetch(:selection))
|
|
187
|
+
when "reserve_attempt" then reserve(params, map, journal, commit, generation, peer, attempt_id)
|
|
188
|
+
when "record_launch" then record(params, map, events, peer)
|
|
189
|
+
when "bind_process" then bind(params, map, events, peer)
|
|
190
|
+
when "release_launch"
|
|
191
|
+
released = true
|
|
192
|
+
release(params, map, events, peer, journal: journal, commit: commit)
|
|
193
|
+
when "abort_launch" then abort(params, map, events, peer, supervisor: role == :supervisor, journal: journal, commit: commit)
|
|
194
|
+
end
|
|
195
|
+
bounded_reply!(plan.fetch(:data), generation + 1)
|
|
196
|
+
# Validate the complete plan before opening a handle, but acquire
|
|
197
|
+
# it before returning any events for staging/CAS. Reuse across
|
|
198
|
+
# retries; a competing accepted replay never owns this handle.
|
|
199
|
+
reservation_handle ||= @kernel.pin(peer) if operation == "reserve_attempt"
|
|
200
|
+
plan
|
|
201
|
+
end
|
|
202
|
+
end
|
|
203
|
+
fresh = !result.fetch(:replayed)
|
|
204
|
+
result = result.fetch(:data)
|
|
205
|
+
if fresh && operation == "reserve_attempt"
|
|
206
|
+
@observations[result.fetch("attempt_id")] = {launcher: peer, launcher_handle: reservation_handle, deadline: wire.deadline(30), ticket: result.fetch("launch_ticket"), assignment_id: result.fetch("assignment_id")}
|
|
207
|
+
reservation_handle = nil
|
|
208
|
+
end
|
|
209
|
+
# No release in mutation replay. A crash here retains issued uncertainty.
|
|
210
|
+
if fresh && released
|
|
211
|
+
stream = @streams.fetch(result.fetch("attempt_id"))
|
|
212
|
+
permission = {"operation" => "release", "launch_ticket" => result.fetch("launch_ticket"),
|
|
213
|
+
"attempt_id" => result.fetch("attempt_id"), "assignment_id" => result.fetch("assignment_id"),
|
|
214
|
+
"generation" => result.fetch("generation"), "journal_commit" => result.fetch("journal_commit"),
|
|
215
|
+
"prepared_input" => result.fetch("prepared_input")}
|
|
216
|
+
begin
|
|
217
|
+
wire.write(stream.fetch(:socket), permission, deadline: wire.deadline(1))
|
|
218
|
+
rescue StandardError
|
|
219
|
+
# Durable permission cannot be undone or retransmitted after write loss.
|
|
220
|
+
nil
|
|
221
|
+
ensure
|
|
222
|
+
stream[:issued] = true
|
|
223
|
+
stream.fetch(:condition).broadcast
|
|
224
|
+
end
|
|
225
|
+
end
|
|
226
|
+
reap_terminal_observations(journal, params.fetch("assignment_id"))
|
|
227
|
+
materialize_definition(result, map, journal) if operation == "register_assignment"
|
|
228
|
+
{data: result, replayed: !fresh}
|
|
229
|
+
ensure
|
|
230
|
+
reservation_handle&.close
|
|
231
|
+
end
|
|
232
|
+
end
|
|
233
|
+
if operation == "reserve_attempt" && !response.fetch(:replayed)
|
|
234
|
+
begin
|
|
235
|
+
parent = provision_reserved_parent_held!(response.fetch(:data), map, journal)
|
|
236
|
+
admission = response.fetch(:data).slice("mapping_id", "assignment_id", "attempt_id", "launch_ticket").merge(
|
|
237
|
+
"mutation_id" => "scope-admission-#{Digest::SHA256.hexdigest(response.dig(:data, 'attempt_id'))[0, 48]}",
|
|
238
|
+
"expected_generation" => parent.dig(:data, "generation"))
|
|
239
|
+
digest = Digest::SHA256.hexdigest(JSON.generate(canonical(admission.reject { |key, _| key == "mutation_id" })))
|
|
240
|
+
admitted = admit_native_service_held!(admission, map, journal, peer, role, digest)
|
|
241
|
+
native_start = [admission, admitted] unless admitted.fetch(:replayed)
|
|
242
|
+
rescue Ace::Runtime::RuntimeUnavailableError, AttemptErrors::EvidenceUnavailable
|
|
243
|
+
# Reservation is already canonical. A failed/lost parent job is
|
|
244
|
+
# held for exact inspection; replay must not activate it again.
|
|
245
|
+
# The immutable reservation reply remains its original outcome.
|
|
246
|
+
nil
|
|
247
|
+
end
|
|
248
|
+
end
|
|
249
|
+
if operation == "abort_launch" && response.dig(:data, "abort_observation", "kind") == "protected_scope_before_release"
|
|
250
|
+
resume_scope_release_held!(response, map, journal, peer, role)
|
|
251
|
+
end
|
|
252
|
+
response
|
|
253
|
+
end
|
|
254
|
+
if native_start
|
|
255
|
+
begin
|
|
256
|
+
complete_native_start!(native_start.first, map, journal, peer, role, native_start.last)
|
|
257
|
+
rescue Ace::Runtime::RuntimeUnavailableError, AttemptErrors::EvidenceUnavailable
|
|
258
|
+
# Canonical admission remains uncertain; original reserve reply is immutable.
|
|
259
|
+
nil
|
|
260
|
+
end
|
|
261
|
+
end
|
|
262
|
+
outcome
|
|
263
|
+
ensure
|
|
264
|
+
settle_native_issuer!(native_start.first, map) if native_start
|
|
265
|
+
end
|
|
266
|
+
|
|
267
|
+
def gate_ready(request:, peer:, socket:, deadline:)
|
|
268
|
+
params = request.fetch("params")
|
|
269
|
+
strict!(params, %w[mapping_id launch_ticket])
|
|
270
|
+
token!(params.fetch("launch_ticket"))
|
|
271
|
+
map = @deployment.mapping(params.fetch("mapping_id"))
|
|
272
|
+
admitted = nil
|
|
273
|
+
until admitted || Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
|
|
274
|
+
@mutex.synchronize do
|
|
275
|
+
journal = journal_for(map)
|
|
276
|
+
@observations.each do |attempt, observation|
|
|
277
|
+
next unless observation[:ticket] == params["launch_ticket"]
|
|
278
|
+
events = journal.read_events(observation.fetch(:assignment_id))
|
|
279
|
+
state = states(events)[attempt]
|
|
280
|
+
next unless state && state["mapping_id"] == params["mapping_id"] && state["launch_ticket"] == params["launch_ticket"]
|
|
281
|
+
next unless %w[recorded bound].include?(state["phase"])
|
|
282
|
+
identity = state.fetch("process_binding").fetch("process_identity")
|
|
283
|
+
next unless @kernel.same?(peer, identity)
|
|
284
|
+
@kernel.live!(peer)
|
|
285
|
+
next unless observation[:child] && @kernel.same?(observation.fetch(:child), peer) && launcher_live?(observation)
|
|
286
|
+
raise AttemptErrors::Conflict, "gate stream already established" if observation[:gate_established]
|
|
287
|
+
condition = ConditionVariable.new
|
|
288
|
+
admitted = {socket: socket, condition: condition, issued: false, state: state}
|
|
289
|
+
observation[:gate_established] = true
|
|
290
|
+
@streams[attempt] = admitted
|
|
291
|
+
@changed.broadcast
|
|
292
|
+
wire.write(socket, {"status" => "ok", "data" => {"phase" => "ready"}}, deadline: deadline)
|
|
293
|
+
break
|
|
294
|
+
end
|
|
295
|
+
end
|
|
296
|
+
sleep 0.02 unless admitted
|
|
297
|
+
end
|
|
298
|
+
raise AttemptErrors::EvidenceUnavailable, "gate was not positively admitted" unless admitted
|
|
299
|
+
@mutex.synchronize do
|
|
300
|
+
until admitted[:issued]
|
|
301
|
+
remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
302
|
+
break if remaining <= 0 || !launcher_live?(@observations[admitted.fetch(:state).fetch("attempt_id")])
|
|
303
|
+
admitted.fetch(:condition).wait(@mutex, [remaining, 0.1].min)
|
|
304
|
+
end
|
|
305
|
+
end
|
|
306
|
+
ensure
|
|
307
|
+
if admitted
|
|
308
|
+
@mutex.synchronize { @streams.delete(admitted.fetch(:state).fetch("attempt_id")) }
|
|
309
|
+
end
|
|
310
|
+
end
|
|
311
|
+
|
|
312
|
+
# Endcap calls this against its already locked canonical attempt chain.
|
|
313
|
+
# Only launch operations may establish origin; later business mutations
|
|
314
|
+
# cannot replace immutable native identity by echoing launch fields.
|
|
315
|
+
def origin(events, assignment_id:, attempt_id:, mapping_id:)
|
|
316
|
+
state = states(events)[attempt_id]
|
|
317
|
+
unless state && state["assignment_id"] == assignment_id && state["mapping_id"] == mapping_id
|
|
318
|
+
raise AttemptErrors::NotFound, "canonical protected launch origin is missing"
|
|
319
|
+
end
|
|
320
|
+
JSON.parse(JSON.generate(state))
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
# Source-owned consumers enter before taking the shared authority mutex.
|
|
324
|
+
# Registration is reread after exclusion; a changed task cannot borrow
|
|
325
|
+
# the lock acquired for a previous registration.
|
|
326
|
+
def with_assignment(params:, map:, exclusive: false)
|
|
327
|
+
id = params.fetch("assignment_id")
|
|
328
|
+
token!(id)
|
|
329
|
+
journal = journal_for(map)
|
|
330
|
+
context = control_registration_context!(params, map, journal)
|
|
331
|
+
exclusion = context.fetch(:exclusion)
|
|
332
|
+
enter = proc do
|
|
333
|
+
@mutex.synchronize do
|
|
334
|
+
recheck_control_registration!(context, params, map, journal)
|
|
335
|
+
yield journal, JSON.parse(JSON.generate(context.fetch(:registration)))
|
|
336
|
+
end
|
|
337
|
+
end
|
|
338
|
+
with_slot(context.fetch(:map), deployment: context.fetch(:descriptor)) do
|
|
339
|
+
if exclusive
|
|
340
|
+
enter_exclusive = lambda do |offset|
|
|
341
|
+
if offset == context.fetch(:keys).size
|
|
342
|
+
enter.call
|
|
343
|
+
else
|
|
344
|
+
exclusion.with_exclusive(context.fetch(:keys).fetch(offset)) { enter_exclusive.call(offset + 1) }
|
|
345
|
+
end
|
|
346
|
+
end
|
|
347
|
+
enter_exclusive.call(0)
|
|
348
|
+
else
|
|
349
|
+
exclusion.with_shared_multi(context.fetch(:keys)) { enter.call }
|
|
350
|
+
end
|
|
351
|
+
end
|
|
352
|
+
end
|
|
353
|
+
|
|
354
|
+
# Shutdown calls this only after all admitted handlers have ended.
|
|
355
|
+
def close
|
|
356
|
+
@mutex.synchronize do
|
|
357
|
+
@observations.each_value { |observation| close_observation(observation) }
|
|
358
|
+
@observations.clear
|
|
359
|
+
@control_channels.each_value(&:close)
|
|
360
|
+
@control_channels.clear
|
|
361
|
+
@scope_observers.each_value { |observer| observer.close if observer.is_a?(ExecutionScopeObservation) }
|
|
362
|
+
@scope_observers.clear
|
|
363
|
+
end
|
|
364
|
+
end
|
|
365
|
+
|
|
366
|
+
private
|
|
367
|
+
|
|
368
|
+
def protected_descriptor_sha256!
|
|
369
|
+
reference = @deployment.artifact_reference
|
|
370
|
+
unless reference.is_a?(Hash) && reference["sha256"].is_a?(String) &&
|
|
371
|
+
Molecules::ExecutionScopeLineage::DIGEST.match?(reference["sha256"])
|
|
372
|
+
raise AttemptErrors::EvidenceUnavailable, "selected deployment has no protected whole-descriptor provenance"
|
|
373
|
+
end
|
|
374
|
+
reference.fetch("sha256")
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
def scope_observer_for(mapping_id)
|
|
378
|
+
@scope_observers[mapping_id] ||= @scope_observer_factory.call(mapping_id)
|
|
379
|
+
end
|
|
380
|
+
|
|
381
|
+
def bounded_scope!(scope)
|
|
382
|
+
unless scope.is_a?(String) && scope.bytesize.between?(1, 128) && scope.split(".", -1).length <= 16
|
|
383
|
+
raise ArgumentError, "assignment scope exceeds protected launch bounds"
|
|
384
|
+
end
|
|
385
|
+
Atoms::AssignmentScope.canonicalize(scope)
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
def bounded_reply!(data, generation)
|
|
389
|
+
envelope = {"status" => "ok", "data" => data.merge("generation" => generation, "journal_commit" => "0" * 64),
|
|
390
|
+
"transport" => {"replayed" => false}}
|
|
391
|
+
raise ArgumentError, "accepted launch reply exceeds transport bounds" if JSON.generate(envelope).bytesize + 1 > 16_384
|
|
392
|
+
if data.key?("prepared_input")
|
|
393
|
+
permission = data.slice("launch_ticket", "attempt_id", "assignment_id", "prepared_input").merge(
|
|
394
|
+
"operation" => "release", "generation" => generation, "journal_commit" => "0" * 64)
|
|
395
|
+
raise ArgumentError, "accepted release exceeds transport bounds" if JSON.generate(permission).bytesize + 1 > 16_384
|
|
396
|
+
end
|
|
397
|
+
end
|
|
398
|
+
|
|
399
|
+
def close_observation(observation)
|
|
400
|
+
%i[launcher_handle child_handle].each do |key|
|
|
401
|
+
observation[key]&.close
|
|
402
|
+
end
|
|
403
|
+
end
|
|
404
|
+
|
|
405
|
+
def reap_terminal_observations(journal, assignment)
|
|
406
|
+
events = journal.read_events(assignment)
|
|
407
|
+
@observations.keys.each do |attempt|
|
|
408
|
+
chain = events.select { |event| event["attempt_id"] == attempt }
|
|
409
|
+
next if chain.empty? || !terminal_events?(chain)
|
|
410
|
+
close_observation(@observations.delete(attempt))
|
|
411
|
+
@streams[attempt]&.fetch(:condition)&.broadcast
|
|
412
|
+
end
|
|
413
|
+
end
|
|
414
|
+
|
|
415
|
+
def exclusion_for(map, _journal, deployment:, selection:)
|
|
416
|
+
@control_exclusion_factory.call(authority: deployment.authority(map.fetch("authority_id")),
|
|
417
|
+
project_id: map.fetch("project_id"), descriptor_sha256: deployment.artifact_reference.fetch("sha256"),
|
|
418
|
+
root_identity: selection&.fetch("root_identity"))
|
|
419
|
+
end
|
|
420
|
+
|
|
421
|
+
def with_exclusion(params, map, journal)
|
|
422
|
+
context = control_registration_context!(params, map, journal)
|
|
423
|
+
context = campaign_child_context!(params, context, map, journal)
|
|
424
|
+
enter = proc { with_selected_control_exclusion(context, params, map, journal) { yield context } }
|
|
425
|
+
enter.call
|
|
426
|
+
end
|
|
427
|
+
|
|
428
|
+
# Containment/recovery never settles Inbox work or grants fresh input.
|
|
429
|
+
# It retains the same canonical task/slot/assignment exclusions while
|
|
430
|
+
# permitting the proof prerequisites needed to recover unknown effects.
|
|
431
|
+
def with_containment_exclusion(params, map, journal)
|
|
432
|
+
context = control_registration_context!(params, map, journal)
|
|
433
|
+
with_selected_control_exclusion(context, params, map, journal) { yield context }
|
|
434
|
+
end
|
|
435
|
+
|
|
436
|
+
def with_selected_control_exclusion(context, params, map, journal)
|
|
437
|
+
exclusion = context.fetch(:exclusion)
|
|
438
|
+
owners = [context, context[:campaign]&.fetch(:context)].compact
|
|
439
|
+
keys = owners.flat_map { |owner| owner.fetch(:keys) }.uniq
|
|
440
|
+
exclusion.provision_keys!(keys: keys) if params.key?("definition_bytes")
|
|
441
|
+
slots = owners.sort_by { |owner| [owner.fetch(:descriptor).authority(owner.fetch(:map).fetch("authority_id")).fetch("state_root"), owner.fetch(:map).fetch("execution_scope").fetch("slot_id")] }
|
|
442
|
+
.uniq { |owner| [owner.fetch(:descriptor).authority(owner.fetch(:map).fetch("authority_id")).fetch("state_root"), owner.fetch(:map).fetch("execution_scope").fetch("slot_id")] }
|
|
443
|
+
enter_slot = lambda do |offset, &body|
|
|
444
|
+
return body.call if offset == slots.size
|
|
445
|
+
owner = slots.fetch(offset)
|
|
446
|
+
with_slot(owner.fetch(:map), deployment: owner.fetch(:descriptor)) { enter_slot.call(offset + 1, &body) }
|
|
447
|
+
end
|
|
448
|
+
enter_slot.call(0) do
|
|
449
|
+
exclusion.with_shared_multi(keys) do
|
|
450
|
+
@mutex.synchronize { recheck_control_registration!(context, params, map, journal) }
|
|
451
|
+
yield context
|
|
452
|
+
end
|
|
453
|
+
end
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
# Slot ownership spans assignment chains and survives authority restart
|
|
457
|
+
# in the existing canonical journal. The file lock only serializes a
|
|
458
|
+
# fresh read/CAS or bounded manager action; its contents convey no facts.
|
|
459
|
+
def with_slot(map, deployment: @deployment, deadline: nil)
|
|
460
|
+
service = deployment.authority(map.fetch("authority_id"))
|
|
461
|
+
scope = map.fetch("execution_scope")
|
|
462
|
+
exclusion = @slot_exclusions[service.fetch("state_root")] ||= Molecules::LifecycleExclusion.new(
|
|
463
|
+
root: File.join(service.fetch("state_root"), "execution-slot-exclusion"))
|
|
464
|
+
key = [object_id, service.fetch("state_root"), exclusion.slot_key(scope.fetch("slot_id"))]
|
|
465
|
+
held = Thread.current[:ace_assign_scope_exclusions] ||= {}
|
|
466
|
+
raise AttemptErrors::Conflict, "execution slot exclusion cannot be entered recursively" if held[key]
|
|
467
|
+
exclusion.with_exclusive(exclusion.slot_key(scope.fetch("slot_id")), deadline: deadline) do
|
|
468
|
+
held[key] = true
|
|
469
|
+
snapshots = Thread.current[:ace_assign_history_operations] ||= {}
|
|
470
|
+
owns_snapshot = !snapshots.key?(object_id)
|
|
471
|
+
snapshots[object_id] = {} if owns_snapshot
|
|
472
|
+
begin
|
|
473
|
+
yield
|
|
474
|
+
ensure
|
|
475
|
+
snapshots.delete(object_id) if owns_snapshot
|
|
476
|
+
held.delete(key)
|
|
477
|
+
end
|
|
478
|
+
end
|
|
479
|
+
end
|
|
480
|
+
|
|
481
|
+
def materialize_definition(result, map, journal)
|
|
482
|
+
project = @deployment.project(map.fetch("project_id"))
|
|
483
|
+
directory = File.join(project.fetch("assignment_root"), result.fetch("assignment_id"))
|
|
484
|
+
raise AttemptErrors::EvidenceUnavailable, "assignment cache is substituted" if File.symlink?(directory)
|
|
485
|
+
FileUtils.mkdir_p(directory, mode: 0o700)
|
|
486
|
+
stat = File.stat(directory)
|
|
487
|
+
unless stat.directory? && stat.uid == Process.uid && (stat.mode & 0o077).zero?
|
|
488
|
+
raise AttemptErrors::EvidenceUnavailable, "assignment cache is not owner-private"
|
|
489
|
+
end
|
|
490
|
+
bytes = journal.blob(result.fetch("definition_ref"), commit: result.fetch("journal_commit"))
|
|
491
|
+
unless Digest::SHA256.hexdigest(bytes) == result.fetch("definition_digest")
|
|
492
|
+
raise AttemptErrors::EvidenceUnavailable, "canonical assignment definition is corrupt"
|
|
493
|
+
end
|
|
494
|
+
temporary = File.join(directory, ".definition-#{SecureRandom.hex(16)}")
|
|
495
|
+
File.open(temporary, File::WRONLY | File::CREAT | File::EXCL | File::NOFOLLOW, 0o600) do |file|
|
|
496
|
+
file.write(bytes)
|
|
497
|
+
file.flush
|
|
498
|
+
file.fsync
|
|
499
|
+
end
|
|
500
|
+
File.rename(temporary, File.join(directory, "assignment.yaml"))
|
|
501
|
+
ensure
|
|
502
|
+
File.unlink(temporary) if temporary && File.exist?(temporary)
|
|
503
|
+
end
|
|
504
|
+
|
|
505
|
+
def journal_for(map)
|
|
506
|
+
@journals[map.fetch("project_id")] ||= begin
|
|
507
|
+
project = @deployment.project(map.fetch("project_id"))
|
|
508
|
+
Molecules::EvidenceJournal.new(repo_root: project.fetch("journal_repository"),
|
|
509
|
+
checkout_root: project.fetch("evidence_checkout_root"), ref: project.fetch("evidence_git_ref"))
|
|
510
|
+
end
|
|
511
|
+
end
|
|
512
|
+
|
|
513
|
+
def mutation_attempt_id(operation, params)
|
|
514
|
+
return "definition-#{params.fetch('assignment_id')}" if operation == "register_assignment"
|
|
515
|
+
return "launch-#{SecureRandom.hex(12)}" if operation == "reserve_attempt"
|
|
516
|
+
token!(params.fetch("attempt_id"))
|
|
517
|
+
end
|
|
518
|
+
|
|
519
|
+
def admit_prepared_registration!(params, map, transfer)
|
|
520
|
+
entry = TaskContextEntry.validate!(map.fetch("task_context_entry"))
|
|
521
|
+
raise AttemptErrors::MalformedTransfer, "Prepared registration transfer unavailable" unless transfer && transfer.count == 1
|
|
522
|
+
descriptor = params.fetch("transfer")
|
|
523
|
+
bytes = transfer.bytes
|
|
524
|
+
work = PreparedWork.admit(bytes: bytes, head: params.fetch("prepared_head"), tree: params.fetch("prepared_tree"),
|
|
525
|
+
sha256: descriptor.fetch("sha256"), size: descriptor.fetch("bytes"), root: @deployment.project(map.fetch("project_id")).fetch("candidate_root"))
|
|
526
|
+
unless work.manifest.values_at("assignment_id", "project_id") == [params.fetch("assignment_id"), map.fetch("project_id")] &&
|
|
527
|
+
work.manifest_sha256 == params.fetch("manifest_sha256")
|
|
528
|
+
raise ArgumentError, "prepared_input_mismatch: registration association"
|
|
529
|
+
end
|
|
530
|
+
definition_bytes = work.definition_bytes(head: params.fetch("prepared_head"), tree: params.fetch("prepared_tree"))
|
|
531
|
+
unless Digest::SHA256.hexdigest(definition_bytes) == params.fetch("definition_digest")
|
|
532
|
+
raise ArgumentError, "prepared_input_mismatch: derived definition digest"
|
|
533
|
+
end
|
|
534
|
+
{work: work, bytes: bytes.freeze, definition_bytes: definition_bytes,
|
|
535
|
+
task_context_entry: immutable_maintenance_projection(entry)}.freeze
|
|
536
|
+
end
|
|
537
|
+
|
|
538
|
+
def register(params, map, journal, commit, generation, prepared:, lifecycle_control:)
|
|
539
|
+
assignment = validated_registration_definition!(params, map)
|
|
540
|
+
bytes = params.fetch("definition_bytes")
|
|
541
|
+
value = JSON.parse(bytes)
|
|
542
|
+
previous = definition(journal, params.fetch("assignment_id"), commit: commit)
|
|
543
|
+
if previous
|
|
544
|
+
original = inventory_definition!(journal, commit,
|
|
545
|
+
{selector: {"assignment_id" => params.fetch("assignment_id"), "attempt_id" => nil}, registration: previous})
|
|
546
|
+
if original.campaign_execution &&
|
|
547
|
+
[assignment.campaign_execution, assignment.parent, assignment.task_id, assignment.project_id] !=
|
|
548
|
+
[original.campaign_execution, original.parent, original.task_id, original.project_id]
|
|
549
|
+
raise AttemptErrors::Conflict, "registered campaign child linkage is immutable"
|
|
550
|
+
end
|
|
551
|
+
end
|
|
552
|
+
if previous && previous.fetch("lifecycle_control") != lifecycle_control
|
|
553
|
+
raise AttemptErrors::Conflict, "same assignment cannot replace original lifecycle control"
|
|
554
|
+
end
|
|
555
|
+
if previous && previous["definition_digest"] == params["definition_digest"] &&
|
|
556
|
+
previous["task_context_entry"] != prepared.fetch(:task_context_entry)
|
|
557
|
+
raise AttemptErrors::Conflict, "same prepared definition cannot replace original task context entry"
|
|
558
|
+
end
|
|
559
|
+
if previous && previous["definition_digest"] != params["definition_digest"] && active_events?(journal.read_events(params.fetch("assignment_id")))
|
|
560
|
+
raise AttemptErrors::Conflict, "active assignment definition cannot change"
|
|
561
|
+
end
|
|
562
|
+
if previous && previous["definition_digest"] == params["definition_digest"] &&
|
|
563
|
+
previous.values_at("prepared_bundle_bytes", "prepared_bundle_sha256") != [prepared.fetch(:bytes).bytesize, Digest::SHA256.hexdigest(prepared.fetch(:bytes))]
|
|
564
|
+
raise AttemptErrors::Conflict, "same prepared definition cannot replace original bundle bytes"
|
|
565
|
+
end
|
|
566
|
+
path = "execution/definitions/#{params.fetch('assignment_id')}-#{params.fetch('definition_digest')}.json"
|
|
567
|
+
bundle = prepared.fetch(:bytes)
|
|
568
|
+
bundle_sha = Digest::SHA256.hexdigest(bundle)
|
|
569
|
+
bundle_path = "execution/prepared/#{params.fetch('assignment_id')}-#{bundle_sha}.bundle"
|
|
570
|
+
{events: [], blobs: {path => bytes, bundle_path => bundle}, data: {"prepared_work" => value.fetch("prepared_work"),
|
|
571
|
+
"lifecycle_control" => lifecycle_control,
|
|
572
|
+
"task_context_entry" => prepared.fetch(:task_context_entry),
|
|
573
|
+
"prepared_bundle_ref" => bundle_path, "prepared_bundle_bytes" => bundle.bytesize, "prepared_bundle_sha256" => bundle_sha,
|
|
574
|
+
"selection_sha256" => prepared.fetch(:work).selection_sha256, "assignment_id" => params.fetch("assignment_id"),
|
|
575
|
+
"project_id" => map.fetch("project_id"), "mapping_id" => params.fetch("mapping_id"),
|
|
576
|
+
"phase" => "registered", "definition_ref" => path, "definition_digest" => params.fetch("definition_digest"),
|
|
577
|
+
"definition_generation" => previous && previous["definition_digest"] == params["definition_digest"] ? previous.fetch("definition_generation") : generation + 1, "task_id" => assignment.task_id}}
|
|
578
|
+
end
|
|
579
|
+
|
|
580
|
+
def reserve(params, map, journal, commit, generation, peer, attempt_id)
|
|
581
|
+
registration = definition(journal, params.fetch("assignment_id"), commit: commit)
|
|
582
|
+
raise AttemptErrors::NotFound, "assignment is unregistered" unless registration && registration["mapping_id"] == params["mapping_id"]
|
|
583
|
+
unless registration["definition_generation"] == params["expected_generation"] && params["worker_uid"] == map["worker_uid"] &&
|
|
584
|
+
params["runtime"] == "herdr" && @kernel.same?(params["launcher_process_binding"], peer)
|
|
585
|
+
raise AttemptErrors::UnauthorizedIdentity, "reservation identity differs"
|
|
586
|
+
end
|
|
587
|
+
scope = Atoms::AssignmentScope.canonicalize(params.fetch("scope"))
|
|
588
|
+
unless registration.dig("prepared_work", "scope") == scope && registration.dig("prepared_work", "task_id") == registration["task_id"]
|
|
589
|
+
raise AttemptErrors::Conflict, "reservation differs from prepared selection"
|
|
590
|
+
end
|
|
591
|
+
ensure_slot_available!(map, journal)
|
|
592
|
+
events = journal.read_events(params.fetch("assignment_id"))
|
|
593
|
+
events.group_by { |event| event["attempt_id"] }.each_value do |chain|
|
|
594
|
+
intent = chain.find { |event| event["type"] == "intent" }
|
|
595
|
+
next unless intent && !terminal_events?(chain)
|
|
596
|
+
old_scope = intent.fetch("payload").fetch("scope")
|
|
597
|
+
if old_scope == scope || old_scope.start_with?("#{scope}.") || scope.start_with?("#{old_scope}.")
|
|
598
|
+
raise AttemptErrors::Conflict, "scope is already owned"
|
|
599
|
+
end
|
|
600
|
+
end
|
|
601
|
+
head = params.fetch("base_head")
|
|
602
|
+
raise ArgumentError, "invalid base revision" unless head.is_a?(String) && head.match?(/\A[0-9a-f]{40}\z/)
|
|
603
|
+
registered_definition = inventory_definition!(journal, commit,
|
|
604
|
+
{selector: {"assignment_id" => params.fetch("assignment_id"), "attempt_id" => nil}, registration: registration})
|
|
605
|
+
if registered_definition.campaign_execution && head != registered_definition.campaign_execution.fetch("base")
|
|
606
|
+
raise AttemptErrors::Conflict, "campaign child base differs from pinned parent candidate"
|
|
607
|
+
end
|
|
608
|
+
ticket = SecureRandom.hex(24)
|
|
609
|
+
payload = {"assignment_id" => params.fetch("assignment_id"), "scope" => scope,
|
|
610
|
+
"project_id" => map.fetch("project_id"), "task_id" => registration.fetch("task_id"),
|
|
611
|
+
"base_head" => head, "actor" => map.fetch("worker_actor"), "role" => "worker", "runtime" => "herdr",
|
|
612
|
+
"launcher_identity" => peer, "launch_ticket" => ticket}
|
|
613
|
+
provisioning = {"slot_id" => map.fetch("execution_scope").fetch("slot_id"),
|
|
614
|
+
"deployment_digest" => Digest::SHA256.hexdigest(JSON.generate(canonical(map))),
|
|
615
|
+
"descriptor_sha256" => protected_descriptor_sha256!,
|
|
616
|
+
"reservation_generation" => generation + 1}
|
|
617
|
+
{events: [{type: "intent", payload: payload}, {type: "scope_provisioning", payload: provisioning}], blobs: {}, data: payload.merge(
|
|
618
|
+
"attempt_id" => attempt_id, "phase" => "reserved", "mapping_id" => params.fetch("mapping_id"),
|
|
619
|
+
"reservation_generation" => generation + 1, "handshake_deadline" => (Time.now.utc + 30).iso8601(6))}
|
|
620
|
+
end
|
|
621
|
+
|
|
622
|
+
def ensure_slot_available!(map, journal, commit: journal.ref_value)
|
|
623
|
+
if @deployment_history
|
|
624
|
+
protected = maintenance_journal_for(@deployment, map)
|
|
625
|
+
unless protected.ref_value == commit
|
|
626
|
+
raise AttemptErrors::Conflict, "historical admission canonical ref changed"
|
|
627
|
+
end
|
|
628
|
+
return with_maintenance_inbox_inventory(@deployment_history.candidate) do
|
|
629
|
+
maintenance_slot_lineages!(nil, protected, commit, selected_context: [@deployment, map])
|
|
630
|
+
raise AttemptErrors::Conflict, "historical admission canonical ref changed" unless protected.ref_value == commit
|
|
631
|
+
true
|
|
632
|
+
end
|
|
633
|
+
end
|
|
634
|
+
slot = map.fetch("execution_scope").fetch("slot_id")
|
|
635
|
+
journal.assignment_ids(commit: commit).each do |assignment_id|
|
|
636
|
+
journal.read_events(assignment_id, commit: commit).group_by { |event| event.fetch("attempt_id") }.each do |attempt_id, chain|
|
|
637
|
+
unless Models::EvidenceEvent.chain_valid?(chain)
|
|
638
|
+
raise AttemptErrors::EvidenceUnavailable, "slot reservation chain is corrupt"
|
|
639
|
+
end
|
|
640
|
+
reservations = chain.select do |event|
|
|
641
|
+
event["type"] == "authority_mutation" && event.dig("payload", "operation") == "reserve_attempt"
|
|
642
|
+
end
|
|
643
|
+
next if reservations.empty?
|
|
644
|
+
raise AttemptErrors::EvidenceUnavailable, "attempt has conflicting reservations" unless reservations.size == 1
|
|
645
|
+
reservation = reservations.first.fetch("payload").fetch("data")
|
|
646
|
+
provisioning = chain.select { |event| event["type"] == "scope_provisioning" }
|
|
647
|
+
unless provisioning.size == 1
|
|
648
|
+
raise AttemptErrors::EvidenceUnavailable, "canonical slot provisioning identity is incomplete"
|
|
649
|
+
end
|
|
650
|
+
owner = provisioning.first.fetch("payload")
|
|
651
|
+
unless owner.is_a?(Hash) && owner.keys.sort == %w[deployment_digest descriptor_sha256 reservation_generation slot_id] &&
|
|
652
|
+
owner["reservation_generation"] == reservation.fetch("reservation_generation") &&
|
|
653
|
+
owner["descriptor_sha256"].is_a?(String) && Molecules::ExecutionScopeLineage::DIGEST.match?(owner["descriptor_sha256"]) &&
|
|
654
|
+
owner["deployment_digest"].is_a?(String) && Molecules::ExecutionScopeLineage::DIGEST.match?(owner["deployment_digest"])
|
|
655
|
+
raise AttemptErrors::EvidenceUnavailable, "canonical slot provisioning identity differs"
|
|
656
|
+
end
|
|
657
|
+
unless owner.fetch("descriptor_sha256") == protected_descriptor_sha256!
|
|
658
|
+
raise AttemptErrors::EvidenceUnavailable, "historical descriptor requires authenticated maintenance before normal slot reuse"
|
|
659
|
+
end
|
|
660
|
+
prior_map = @deployment.mapping(reservation.fetch("mapping_id"))
|
|
661
|
+
unless prior_map.fetch("execution_scope").fetch("slot_id") == owner.fetch("slot_id") &&
|
|
662
|
+
prior_map.fetch("project_id") == reservation.fetch("project_id")
|
|
663
|
+
raise AttemptErrors::EvidenceUnavailable, "historical slot owner was removed or moved"
|
|
664
|
+
end
|
|
665
|
+
next unless owner.fetch("slot_id") == slot
|
|
666
|
+
lineage = Molecules::ExecutionScopeLineage.new(events: chain, project_id: map.fetch("project_id"),
|
|
667
|
+
assignment_id: assignment_id, attempt_id: attempt_id, mapping_id: reservation.fetch("mapping_id"))
|
|
668
|
+
unless lineage.binding.nil? || lineage.binding.fetch("deployment_digest") == owner.fetch("deployment_digest")
|
|
669
|
+
raise AttemptErrors::EvidenceUnavailable, "parent binding differs from original slot provisioning"
|
|
670
|
+
end
|
|
671
|
+
# Terminality alone never clears this slot: canonical closure is
|
|
672
|
+
# separately mandatory, including after the authority restarts.
|
|
673
|
+
unless terminal_events?(chain) && lineage.proof_id && scope_reservation_released?(chain, lineage, journal, commit)
|
|
674
|
+
raise AttemptErrors::Conflict, "execution slot has an unreleased canonical reservation"
|
|
675
|
+
end
|
|
676
|
+
scope_settlement_complete!(journal: journal, events: chain, params: {"mapping_id" => reservation.fetch("mapping_id"),
|
|
677
|
+
"assignment_id" => assignment_id, "attempt_id" => attempt_id}, map: prior_map, commit: commit)
|
|
678
|
+
end
|
|
679
|
+
end
|
|
680
|
+
true
|
|
681
|
+
rescue KeyError
|
|
682
|
+
raise AttemptErrors::EvidenceUnavailable, "canonical slot ownership is incomplete"
|
|
683
|
+
end
|
|
684
|
+
|
|
685
|
+
def record(params, map, events, peer)
|
|
686
|
+
state, observation = owned(params, events, peer)
|
|
687
|
+
raise AttemptErrors::Conflict, "launch cannot record another child" unless state["phase"] == "reserved"
|
|
688
|
+
binding = validate_binding!(params.fetch("process_binding"), map, mapping_id: params.fetch("mapping_id"), launch_ticket: params.fetch("launch_ticket"), assignment_id: params.fetch("assignment_id"), attempt_id: params.fetch("attempt_id"), events: events)
|
|
689
|
+
child = binding.fetch("process_identity")
|
|
690
|
+
guard = Ace::Herdr::Molecules::GuardedNativeOrigin.verify!(params.fetch("guarded_origin"),
|
|
691
|
+
terminal_id: binding.fetch("terminal_id"), child: child)
|
|
692
|
+
@kernel.live!(child)
|
|
693
|
+
if observation[:child_handle]
|
|
694
|
+
unless @kernel.same?(observation.fetch(:child), child)
|
|
695
|
+
raise AttemptErrors::Conflict, "record retry differs from observed child"
|
|
696
|
+
end
|
|
697
|
+
else
|
|
698
|
+
observation[:child_handle] = @kernel.pin(child)
|
|
699
|
+
observation[:child] = child
|
|
700
|
+
end
|
|
701
|
+
{events: [], blobs: {}, data: state.merge("phase" => "recorded", "process_binding" => binding, "guarded_origin" => guard)}
|
|
702
|
+
end
|
|
703
|
+
|
|
704
|
+
def bind(params, map, events, peer)
|
|
705
|
+
state, = owned(params, events, peer)
|
|
706
|
+
unless state["phase"] == "recorded" && state["process_binding"] == params["process_binding"]
|
|
707
|
+
raise AttemptErrors::Conflict, "binding differs from recorded original child"
|
|
708
|
+
end
|
|
709
|
+
validate_binding!(params.fetch("process_binding"), map, mapping_id: params.fetch("mapping_id"), launch_ticket: params.fetch("launch_ticket"), assignment_id: params.fetch("assignment_id"), attempt_id: params.fetch("attempt_id"), events: events)
|
|
710
|
+
@kernel.live!(params.fetch("process_binding").fetch("process_identity"))
|
|
711
|
+
payload = {"actor" => map.fetch("worker_actor"), "role" => "worker", "runtime" => "herdr",
|
|
712
|
+
"pid" => params.dig("process_binding", "process_identity", "pid"),
|
|
713
|
+
"process_identity" => params.dig("process_binding", "process_identity"), "runtime_binding" => params.fetch("process_binding")}
|
|
714
|
+
lineage = Molecules::ExecutionScopeLineage.new(events: events, project_id: map.fetch("project_id"),
|
|
715
|
+
assignment_id: params.fetch("assignment_id"), attempt_id: params.fetch("attempt_id"), mapping_id: params.fetch("mapping_id"))
|
|
716
|
+
child = {"scope_generation" => lineage.binding.fetch("scope_generation"),
|
|
717
|
+
"scope_binding_event_id" => lineage.binding_event.fetch("digest"), "native_binding_event_id" => lineage.native_event.fetch("digest"),
|
|
718
|
+
"original_process_binding" => params.fetch("process_binding")}
|
|
719
|
+
{events: [{type: "scope_child_bound", payload: child}, {type: "process_start", payload: payload}], blobs: {}, data: state.merge("phase" => "bound")}
|
|
720
|
+
end
|
|
721
|
+
|
|
722
|
+
def release(params, map, events, peer, journal:, commit:)
|
|
723
|
+
state, observation = owned(params, events, peer)
|
|
724
|
+
unless state["phase"] == "bound" && state["process_binding"] == params["process_binding"] && @streams.key?(state.fetch("attempt_id"))
|
|
725
|
+
raise AttemptErrors::Conflict, "exact live bound gate is unavailable"
|
|
726
|
+
end
|
|
727
|
+
validate_binding!(params.fetch("process_binding"), map, mapping_id: params.fetch("mapping_id"), launch_ticket: params.fetch("launch_ticket"), assignment_id: params.fetch("assignment_id"), attempt_id: params.fetch("attempt_id"), events: events)
|
|
728
|
+
original = scope_open_for_effect!(events: events, params: params, map: map)
|
|
729
|
+
raise AttemptErrors::Conflict, "canonical original child differs" unless original == params.fetch("process_binding")
|
|
730
|
+
@kernel.live!(observation.fetch(:child))
|
|
731
|
+
raise AttemptErrors::Conflict, "launcher has exited" unless launcher_live?(observation)
|
|
732
|
+
prepared = original_bound_prepared_registration!(journal: journal, params: params, commit: commit)
|
|
733
|
+
{events: [], blobs: {}, data: state.merge("phase" => "issued", "execution" => "potentially_executed",
|
|
734
|
+
"prepared_input" => compact_prepared_input(prepared))}
|
|
735
|
+
end
|
|
736
|
+
|
|
737
|
+
def abort(params, map, events, peer, supervisor: false, journal:, commit:)
|
|
738
|
+
evidence = params.fetch("failure_evidence")
|
|
739
|
+
unless evidence.is_a?(String) && evidence.bytesize <= 16_384 && Digest::SHA256.hexdigest(evidence) == params["failure_digest"]
|
|
740
|
+
raise ArgumentError, "invalid bounded failure evidence"
|
|
741
|
+
end
|
|
742
|
+
scope_failure = begin
|
|
743
|
+
JSON.parse(evidence)
|
|
744
|
+
rescue JSON::ParserError
|
|
745
|
+
nil
|
|
746
|
+
end
|
|
747
|
+
if scope_failure.is_a?(Hash) && scope_failure["kind"] == "protected_scope_before_release"
|
|
748
|
+
return scope_abort_plan(params, map, events, peer, journal: journal, commit: commit, supervisor: supervisor)
|
|
749
|
+
end
|
|
750
|
+
state, observation = owned(params, events, peer, require_launcher_live: false, supervisor: supervisor)
|
|
751
|
+
issued = events.any? do |event|
|
|
752
|
+
event["type"] == "authority_mutation" && event.dig("payload", "operation") == "release_launch"
|
|
753
|
+
end
|
|
754
|
+
positive = %w[recorded bound uncertain].include?(state["phase"]) && !issued &&
|
|
755
|
+
state["execution"] != "potentially_executed" && observation[:child_handle] && observation[:child] &&
|
|
756
|
+
state.dig("process_binding", "process_identity") == observation[:child] &&
|
|
757
|
+
@kernel.exited?(observation.fetch(:child_handle))
|
|
758
|
+
target = positive ? "failed" : "uncertain"
|
|
759
|
+
proof = if positive
|
|
760
|
+
{"process_identity" => observation.fetch(:child), "exit_observed_at" => Time.now.utc.iso8601(6),
|
|
761
|
+
"bootstrap_sha256" => map.fetch("bootstrap_sha256"), "server_identity" => map.dig("native", "server_identity"),
|
|
762
|
+
"release" => "not_issued", "yama" => 2, "capability_sets" => "empty", "no_new_privs" => 1}
|
|
763
|
+
end
|
|
764
|
+
evidence_ref = "evidence/imports/launch-failure-#{state.fetch('attempt_id')}-#{params.fetch('failure_digest')}"
|
|
765
|
+
reason = positive ? "protected_gate_exited_before_release" : "protected_launch_requires_positive_termination_proof"
|
|
766
|
+
lifecycle_event = if positive && state["phase"] == "uncertain"
|
|
767
|
+
{type: "reconciliation", payload: {"resolution" => "failed", "reason" => reason,
|
|
768
|
+
"failure_digest" => params.fetch("failure_digest"), "abort_observation" => proof}}
|
|
769
|
+
else
|
|
770
|
+
{type: "transition", payload: {"from" => state["phase"] == "uncertain" ? "uncertain" :
|
|
771
|
+
(%w[bound issued].include?(state["phase"]) ? "running" : "reserved"), "to" => target, "reason" => reason}}
|
|
772
|
+
end
|
|
773
|
+
{events: [lifecycle_event],
|
|
774
|
+
blobs: {evidence_ref => evidence}, data: state.merge("phase" => target, "failure_digest" => params.fetch("failure_digest"),
|
|
775
|
+
"failure_ref" => evidence_ref, "abort_observation" => proof,
|
|
776
|
+
"required_action" => positive ? nil : "supervisor_inspect_exact_child_and_release_uncertainty")}
|
|
777
|
+
end
|
|
778
|
+
|
|
779
|
+
def owned(params, events, peer, require_launcher_live: true, supervisor: false)
|
|
780
|
+
state = states(events)[params.fetch("attempt_id")]
|
|
781
|
+
unless state && state["mapping_id"] == params["mapping_id"] && state["launch_ticket"] == params["launch_ticket"] &&
|
|
782
|
+
(supervisor || @kernel.same?(state["launcher_identity"], peer))
|
|
783
|
+
raise AttemptErrors::UnauthorizedIdentity, "launch is not owned by this exact launcher"
|
|
784
|
+
end
|
|
785
|
+
observation = @observations[params.fetch("attempt_id")]
|
|
786
|
+
raise AttemptErrors::EvidenceUnavailable, "exact launch handles require supervisor recovery" unless observation
|
|
787
|
+
if require_launcher_live && observation.fetch(:deadline) <= Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
788
|
+
raise AttemptErrors::EvidenceUnavailable, "protected launch handshake expired; supervisor inspection required"
|
|
789
|
+
end
|
|
790
|
+
if require_launcher_live && !launcher_live?(observation)
|
|
791
|
+
raise AttemptErrors::Conflict, "launcher has exited"
|
|
792
|
+
end
|
|
793
|
+
[state, observation]
|
|
794
|
+
end
|
|
795
|
+
|
|
796
|
+
def validate_binding!(binding, map, mapping_id:, launch_ticket:, assignment_id:, attempt_id:, events:)
|
|
797
|
+
strict!(binding, %w[runtime session pane terminal_id process_identity shell_identity native_origin])
|
|
798
|
+
raise ArgumentError, "native runtime differs" unless binding["runtime"] == "herdr" && binding["shell_identity"] == binding["process_identity"]
|
|
799
|
+
identity = binding.fetch("process_identity")
|
|
800
|
+
strict!(identity, %w[pid uid gid groups started_at host parent_pid])
|
|
801
|
+
origin = binding.fetch("native_origin")
|
|
802
|
+
strict!(origin, %w[workspace tab pane server_identity socket_identity command cwd])
|
|
803
|
+
unless origin["command"] == [map.fetch("bootstrap"), mapping_id, launch_ticket] && origin["cwd"] == map.fetch("worker_cwd") &&
|
|
804
|
+
binding["terminal_id"].is_a?(String) && !binding["terminal_id"].empty? && binding["session"].is_a?(String) && binding["pane"].is_a?(String)
|
|
805
|
+
raise AttemptErrors::UnauthorizedIdentity, "native origin command or terminal differs"
|
|
806
|
+
end
|
|
807
|
+
lineage = Molecules::ExecutionScopeLineage.new(events: events, project_id: map.fetch("project_id"),
|
|
808
|
+
assignment_id: assignment_id, attempt_id: attempt_id, mapping_id: mapping_id)
|
|
809
|
+
lineage.require_open!
|
|
810
|
+
native = lineage.native_event&.fetch("payload")
|
|
811
|
+
raise AttemptErrors::EvidenceUnavailable, "original canonical native stage is missing" unless native
|
|
812
|
+
server = native.fetch("server_identity")
|
|
813
|
+
unless identity.is_a?(Hash) && identity["uid"] == map["worker_uid"] && identity["gid"] == map["worker_gid"] &&
|
|
814
|
+
identity["groups"] == map["worker_groups"] && identity["parent_pid"] == server["pid"] &&
|
|
815
|
+
binding.dig("native_origin", "server_identity") == server &&
|
|
816
|
+
binding.dig("native_origin", "socket_identity") == native.fetch("socket_identity") &&
|
|
817
|
+
binding.dig("native_origin", "workspace") == map.dig("native", "workspace_id") &&
|
|
818
|
+
binding.dig("native_origin", "workspace") == binding["session"] && binding.dig("native_origin", "pane") == binding["pane"]
|
|
819
|
+
raise AttemptErrors::UnauthorizedIdentity, "original native child lineage differs"
|
|
820
|
+
end
|
|
821
|
+
@kernel.live!(server)
|
|
822
|
+
binding
|
|
823
|
+
end
|
|
824
|
+
|
|
825
|
+
def launcher_live?(observation)
|
|
826
|
+
return false unless observation && observation[:launcher_handle] && !@kernel.exited?(observation.fetch(:launcher_handle))
|
|
827
|
+
@kernel.live!(observation.fetch(:launcher))
|
|
828
|
+
true
|
|
829
|
+
rescue StandardError
|
|
830
|
+
false
|
|
831
|
+
end
|
|
832
|
+
|
|
833
|
+
def definition(journal, id, commit: journal.ref_value)
|
|
834
|
+
journal.read_events(id, commit: commit).reverse.filter_map do |event|
|
|
835
|
+
next unless event["type"] == "authority_mutation" && event.dig("payload", "operation") == "register_assignment"
|
|
836
|
+
event.dig("payload", "data")
|
|
837
|
+
end.first
|
|
838
|
+
end
|
|
839
|
+
|
|
840
|
+
def states(events)
|
|
841
|
+
events.each_with_object({}) do |event, result|
|
|
842
|
+
next unless event["type"] == "authority_mutation" && MUTATIONS.key?(event.dig("payload", "operation"))
|
|
843
|
+
data = event.dig("payload", "data")
|
|
844
|
+
next unless data.is_a?(Hash) && data["launch_ticket"]
|
|
845
|
+
result[event.fetch("attempt_id")] = data
|
|
846
|
+
end
|
|
847
|
+
end
|
|
848
|
+
|
|
849
|
+
def inspect_launch(params, peer:, role:)
|
|
850
|
+
token!(params.fetch("assignment_id"))
|
|
851
|
+
token!(params.fetch("attempt_id"))
|
|
852
|
+
map = @deployment.mapping(params.fetch("mapping_id"))
|
|
853
|
+
journal = journal_for(map)
|
|
854
|
+
@kernel.live!(peer)
|
|
855
|
+
with_exclusion(params, map, journal) do
|
|
856
|
+
@mutex.synchronize do
|
|
857
|
+
commit = journal.ref_value
|
|
858
|
+
events = journal.read_events(params.fetch("assignment_id"), commit: commit).select { |event| event["attempt_id"] == params.fetch("attempt_id") }
|
|
859
|
+
state = origin(events, assignment_id: params.fetch("assignment_id"),
|
|
860
|
+
attempt_id: params.fetch("attempt_id"), mapping_id: params.fetch("mapping_id"))
|
|
861
|
+
if role == :launcher && !@kernel.same?(state.fetch("launcher_identity"), peer)
|
|
862
|
+
raise AttemptErrors::UnauthorizedIdentity, "launch belongs to another launcher incarnation"
|
|
863
|
+
end
|
|
864
|
+
lineage = Molecules::ExecutionScopeLineage.new(events: events, project_id: map.fetch("project_id"),
|
|
865
|
+
assignment_id: params.fetch("assignment_id"), attempt_id: params.fetch("attempt_id"), mapping_id: params.fetch("mapping_id"))
|
|
866
|
+
state = state.merge("generation" => journal.authority_generation(events), "journal_commit" => commit)
|
|
867
|
+
unless state["process_binding"]
|
|
868
|
+
return state.merge("native_binding" => lineage.native_event&.fetch("payload"),
|
|
869
|
+
"scope_binding_event_id" => lineage.binding_event&.fetch("digest"),
|
|
870
|
+
"scope_failure_evidence" => lineage.proof_event && {"kind" => "protected_scope_before_release",
|
|
871
|
+
"scope_generation" => lineage.binding.fetch("scope_generation"),
|
|
872
|
+
"scope_binding_event_id" => lineage.binding_event.fetch("digest"),
|
|
873
|
+
"seal_event_id" => lineage.seal_event.fetch("digest"), "proof_id" => lineage.proof_id},
|
|
874
|
+
"required_action" => lineage.binding ? "close_scope_before_release" : "inspect_exact_scope")
|
|
875
|
+
end
|
|
876
|
+
identity = state.fetch("process_binding") { raise AttemptErrors::EvidenceUnavailable, "no recorded original child; inspection cannot infer absence" }.fetch("process_identity")
|
|
877
|
+
observation = @observations[params.fetch("attempt_id")]
|
|
878
|
+
if observation && observation[:child_handle] && observation[:child] == identity && @kernel.exited?(observation.fetch(:child_handle))
|
|
879
|
+
return state.merge("journal_commit" => journal.ref_value,
|
|
880
|
+
"termination_observation" => {"status" => "exited", "process_identity" => identity},
|
|
881
|
+
"required_action" => "request_abort_using_retained_exact_exit_proof")
|
|
882
|
+
end
|
|
883
|
+
native = lineage.native_event&.fetch("payload")
|
|
884
|
+
raise AttemptErrors::EvidenceUnavailable, "original canonical native stage is missing" unless native
|
|
885
|
+
@kernel.live!(native.fetch("server_identity"))
|
|
886
|
+
@kernel.live!(identity)
|
|
887
|
+
unless observation
|
|
888
|
+
child_handle = @kernel.pin(identity)
|
|
889
|
+
@observations[params.fetch("attempt_id")] = {child: identity, child_handle: child_handle,
|
|
890
|
+
launcher: state.fetch("launcher_identity"), launcher_handle: nil, deadline: 0}
|
|
891
|
+
end
|
|
892
|
+
state.merge("journal_commit" => journal.ref_value,
|
|
893
|
+
"termination_observation" => {"status" => "alive", "process_identity" => identity},
|
|
894
|
+
"required_action" => "close_original_native_child_then_request_abort")
|
|
895
|
+
end
|
|
896
|
+
end
|
|
897
|
+
end
|
|
898
|
+
|
|
899
|
+
def status(params, peer:, role:)
|
|
900
|
+
map = @deployment.mapping(params.fetch("mapping_id"))
|
|
901
|
+
token!(params.fetch("assignment_id")); token!(params.fetch("attempt_id"))
|
|
902
|
+
journal = journal_for(map)
|
|
903
|
+
with_containment_exclusion(params, map, journal) do
|
|
904
|
+
@mutex.synchronize do
|
|
905
|
+
commit = journal.ref_value
|
|
906
|
+
events = journal.read_events(params.fetch("assignment_id"), commit: commit)
|
|
907
|
+
state = states(events)[params.fetch("attempt_id")]
|
|
908
|
+
raise AttemptErrors::NotFound, "launch not found" unless state && state["mapping_id"] == params["mapping_id"]
|
|
909
|
+
if role == :launcher && !@kernel.same?(state.fetch("launcher_identity"), peer)
|
|
910
|
+
raise AttemptErrors::UnauthorizedIdentity, "launch belongs to another launcher incarnation"
|
|
911
|
+
end
|
|
912
|
+
chain = events.select { |event| event["attempt_id"] == params.fetch("attempt_id") }
|
|
913
|
+
projection = state.slice("assignment_id", "attempt_id", "phase", "scope", "launch_ticket", "reservation_generation", "generation", "execution", "required_action", "handshake_deadline")
|
|
914
|
+
projection.merge!("state" => journal.canonical_attempt_state(chain), "generation" => journal.authority_generation(chain))
|
|
915
|
+
if %i[launcher supervisor].include?(role)
|
|
916
|
+
projection.merge!(state.slice("process_binding", "launcher_identity", "mapping_id"))
|
|
917
|
+
original = original_prompt_record!(chain, state, params: params) if chain.any? { |event| event["type"] == "authority_mutation" && event.dig("payload", "operation") == "record_launch" }
|
|
918
|
+
projection.merge!("original_binding_digest" => original&.fetch("binding_digest"), "terminal_event_id" => nil, "reservation_release_event_id" => nil)
|
|
919
|
+
if TERMINAL.include?(projection.fetch("state"))
|
|
920
|
+
lineage = Molecules::ExecutionScopeLineage.new(events: chain, project_id: map.fetch("project_id"),
|
|
921
|
+
assignment_id: params.fetch("assignment_id"), attempt_id: params.fetch("attempt_id"), mapping_id: params.fetch("mapping_id"))
|
|
922
|
+
released = scope_reservation_released?(chain, lineage, journal, commit)
|
|
923
|
+
if released
|
|
924
|
+
release = chain.find { |event| event["type"] == "authority_mutation" && event.dig("payload", "operation") == "scope_reservation_release" }
|
|
925
|
+
projection.merge!("terminal_event_id" => release.dig("payload", "data", "terminal_event_id"), "reservation_release_event_id" => release.fetch("digest"))
|
|
926
|
+
else
|
|
927
|
+
projection["terminal_event_id"] = terminal_scope_receipt!(chain, lineage, journal, commit).fetch("digest")
|
|
928
|
+
end
|
|
929
|
+
end
|
|
930
|
+
end
|
|
931
|
+
observation = @observations[params.fetch("attempt_id")]
|
|
932
|
+
expired = !%w[issued failed].include?(state["phase"]) && state["handshake_deadline"] && Time.iso8601(state.fetch("handshake_deadline")) <= Time.now.utc
|
|
933
|
+
child_exited = observation && observation[:child_handle] &&
|
|
934
|
+
observation[:child] == state.dig("process_binding", "process_identity") && @kernel.exited?(observation.fetch(:child_handle))
|
|
935
|
+
if !TERMINAL.include?(state["phase"]) && (!observation || !launcher_live?(observation) || expired || child_exited)
|
|
936
|
+
projection.merge!("phase" => "uncertain", "required_action" => "supervisor_inspect_exact_child_and_release_uncertainty")
|
|
937
|
+
end
|
|
938
|
+
if @result_owner
|
|
939
|
+
projection.merge!(@result_owner.result_status(journal: journal, events: chain, params: params,
|
|
940
|
+
map: map, peer: peer, role: role, commit: commit))
|
|
941
|
+
end
|
|
942
|
+
projection.merge("journal_commit" => commit)
|
|
943
|
+
end
|
|
944
|
+
end
|
|
945
|
+
end
|
|
946
|
+
|
|
947
|
+
def active_events?(events)
|
|
948
|
+
events.group_by { |event| event["attempt_id"] }.values.any? { |chain| chain.any? { |event| event["type"] == "intent" } && !terminal_events?(chain) }
|
|
949
|
+
end
|
|
950
|
+
def terminal_events?(events)
|
|
951
|
+
TERMINAL.include?(Molecules::CanonicalAttemptState.derive(events))
|
|
952
|
+
end
|
|
953
|
+
|
|
954
|
+
def strict!(value, keys)
|
|
955
|
+
raise ArgumentError, "authority fields differ" unless value.is_a?(Hash) && value.keys.sort == keys.sort
|
|
956
|
+
end
|
|
957
|
+
def token!(value)
|
|
958
|
+
raise ArgumentError, "invalid public identifier" unless value.is_a?(String) && Deployment::TOKEN.match?(value)
|
|
959
|
+
value
|
|
960
|
+
end
|
|
961
|
+
def generation!(value, allow_nil: false)
|
|
962
|
+
return if allow_nil && value.nil?
|
|
963
|
+
raise ArgumentError, "invalid journal generation" unless value.is_a?(Integer) && value >= 0
|
|
964
|
+
end
|
|
965
|
+
def canonical(value)
|
|
966
|
+
case value
|
|
967
|
+
when Hash then value.sort.to_h.transform_values { |item| canonical(item) }
|
|
968
|
+
when Array then value.map { |item| canonical(item) }
|
|
969
|
+
else value
|
|
970
|
+
end
|
|
971
|
+
end
|
|
972
|
+
def wire
|
|
973
|
+
Ace::Runtime::Molecules::ProtectedSocket
|
|
974
|
+
end
|
|
975
|
+
end
|
|
976
|
+
end
|
|
977
|
+
end
|
|
978
|
+
end
|
|
979
|
+
|
|
980
|
+
require_relative "launch_scope_abort"
|
|
981
|
+
require_relative "launch_scope_admission"
|
|
982
|
+
require_relative "launch_scope_close"
|
|
983
|
+
require_relative "launch_scope_release"
|
|
984
|
+
require_relative "launch_scope_parent"
|
|
985
|
+
|
|
986
|
+
require_relative "launch_steering"
|
|
987
|
+
require_relative "launch_input_inhibition"
|
|
988
|
+
|
|
989
|
+
require_relative "launch_stop"
|
|
990
|
+
require_relative "assignment_inventory"
|
|
991
|
+
require_relative "launch_prepared_work"
|
|
992
|
+
|
|
993
|
+
require_relative "launch_review"
|
|
994
|
+
require_relative "launch_attempt_consumers"
|
|
995
|
+
require_relative "launch_control_exclusions"
|
|
996
|
+
|
|
997
|
+
require_relative "launch_campaign_registration"
|