ace-hitl 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +19 -0
- data/docs/usage.md +27 -34
- data/lib/ace/hitl/cli/commands/proposal.rb +4 -2
- data/lib/ace/hitl/lifecycle/binding.rb +1 -1
- data/lib/ace/hitl/lifecycle/client.rb +63 -25
- data/lib/ace/hitl/lifecycle/kinds.rb +4 -1
- data/lib/ace/hitl/lifecycle/proposals.rb +48 -42
- data/lib/ace/hitl/lifecycle/service.rb +36 -13
- data/lib/ace/hitl/lifecycle/service_publication_binding.rb +97 -0
- data/lib/ace/hitl/lifecycle/store.rb +104 -36
- data/lib/ace/hitl/live_client.rb +9 -27
- data/lib/ace/hitl/providers/lab/assignment_binding.rb +10 -3
- data/lib/ace/hitl/providers/lab/protected_assignment_binding.rb +75 -0
- data/lib/ace/hitl/providers/lab.rb +6 -2
- data/lib/ace/hitl/version.rb +1 -1
- metadata +8 -6
|
@@ -48,6 +48,8 @@ module Ace
|
|
|
48
48
|
@proposal_clock = proposal_clock
|
|
49
49
|
@max_connections = MAX_CONNECTIONS
|
|
50
50
|
@connections = 0
|
|
51
|
+
@handler_threads = []
|
|
52
|
+
@handler_sockets = []
|
|
51
53
|
@connections_mutex = Mutex.new
|
|
52
54
|
@stopping = false
|
|
53
55
|
@server = nil
|
|
@@ -57,8 +59,9 @@ module Ace
|
|
|
57
59
|
|
|
58
60
|
# Serve until #stop. Installs TERM/INT handlers for a clean
|
|
59
61
|
# shutdown that unlinks the socket file.
|
|
60
|
-
def run
|
|
62
|
+
def run(on_ready: nil)
|
|
61
63
|
prepare!
|
|
64
|
+
on_ready.call if on_ready
|
|
62
65
|
if Thread.current == Thread.main
|
|
63
66
|
%i[TERM INT].each do |signal|
|
|
64
67
|
trap(signal) { stop }
|
|
@@ -86,12 +89,21 @@ module Ace
|
|
|
86
89
|
end
|
|
87
90
|
next
|
|
88
91
|
end
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
92
|
+
@connections_mutex.synchronize do
|
|
93
|
+
@handler_sockets << connection
|
|
94
|
+
@handler_threads.reject! { |thread| !thread.alive? }
|
|
95
|
+
thread = Thread.new(connection) do |socket|
|
|
96
|
+
begin
|
|
97
|
+
serve_connection(socket)
|
|
98
|
+
ensure
|
|
99
|
+
@connections_mutex.synchronize do
|
|
100
|
+
@connections -= 1
|
|
101
|
+
@handler_sockets.delete(socket)
|
|
102
|
+
end
|
|
103
|
+
end
|
|
94
104
|
end
|
|
105
|
+
thread.report_on_exception = false
|
|
106
|
+
@handler_threads << thread
|
|
95
107
|
end
|
|
96
108
|
end
|
|
97
109
|
ensure
|
|
@@ -152,6 +164,16 @@ module Ace
|
|
|
152
164
|
end
|
|
153
165
|
|
|
154
166
|
def shutdown!
|
|
167
|
+
stop
|
|
168
|
+
@connections_mutex.synchronize { @handler_sockets.dup }.each { |socket| socket.close rescue nil }
|
|
169
|
+
failures = []
|
|
170
|
+
@connections_mutex.synchronize { @handler_threads.dup }.each do |thread|
|
|
171
|
+
begin
|
|
172
|
+
thread.join
|
|
173
|
+
rescue Exception => error
|
|
174
|
+
failures << error
|
|
175
|
+
end
|
|
176
|
+
end
|
|
155
177
|
begin
|
|
156
178
|
@server&.close
|
|
157
179
|
rescue SystemCallError, IOError
|
|
@@ -167,6 +189,7 @@ module Ace
|
|
|
167
189
|
@endpoint_lock = nil
|
|
168
190
|
end
|
|
169
191
|
log("stopped")
|
|
192
|
+
raise failures.first unless failures.empty?
|
|
170
193
|
end
|
|
171
194
|
|
|
172
195
|
# The socket's parent directory is part of the trust boundary: it
|
|
@@ -213,7 +236,7 @@ module Ace
|
|
|
213
236
|
policy: @policy,
|
|
214
237
|
identity: peer,
|
|
215
238
|
vault: @vault,
|
|
216
|
-
proposal_clock: @proposal_clock
|
|
239
|
+
proposal_clock: @proposal_clock, stopping: -> { @stopping }
|
|
217
240
|
)
|
|
218
241
|
end
|
|
219
242
|
|
|
@@ -232,18 +255,18 @@ module Ace
|
|
|
232
255
|
case op
|
|
233
256
|
when "ping" then {"pong" => true}
|
|
234
257
|
when "proposal-create" then store.proposal_create(**symbolize(params))
|
|
235
|
-
when "proposal-show" then store.proposal_show(required(params, "id"), history_after: params.fetch("history_after", 0))
|
|
236
|
-
when "proposal-revise" then store.proposal_revise(required(params, "id"), expected_revision: required(params, "expected_revision"),
|
|
258
|
+
when "proposal-show" then store.proposal_show(required(params, "id"), project: required(params, "project"), history_after: params.fetch("history_after", 0))
|
|
259
|
+
when "proposal-revise" then store.proposal_revise(required(params, "id"), project: required(params, "project"), expected_revision: required(params, "expected_revision"),
|
|
237
260
|
operation_id: required(params, "operation_id"), document: required(params, "document"))
|
|
238
261
|
when "proposal-ack"
|
|
239
|
-
store.proposal_acknowledge(required(params, "id"), submitted_at: required(params, "submitted_at"))
|
|
262
|
+
store.proposal_acknowledge(required(params, "id"), project: required(params, "project"), submitted_at: required(params, "submitted_at"))
|
|
240
263
|
when "proposal-reply"
|
|
241
|
-
store.proposal_reply(required(params, "id"), answer: required(params, "answer"),
|
|
264
|
+
store.proposal_reply(required(params, "id"), project: required(params, "project"), answer: required(params, "answer"),
|
|
242
265
|
received_at: required(params, "received_at"), sequence: required(params, "sequence"))
|
|
243
266
|
when "proposal-reconcile"
|
|
244
|
-
store.proposal_reconcile(required(params, "id"), checkpoint: required(params, "checkpoint"))
|
|
267
|
+
store.proposal_reconcile(required(params, "id"), project: required(params, "project"), checkpoint: required(params, "checkpoint"))
|
|
245
268
|
when "proposal-wake" then store.proposal_wake(project: required(params, "project"), after: params["after"])
|
|
246
|
-
when "proposal-due" then store.proposal_due(after: params["after"])
|
|
269
|
+
when "proposal-due" then store.proposal_due(project: required(params, "project"), after: params["after"])
|
|
247
270
|
when "proposal-history" then store.proposal_history(project: required(params, "project"),
|
|
248
271
|
query: params.fetch("query", ""), after: params["after"])
|
|
249
272
|
when "create" then store.create(**symbolize(params))
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
require_relative "binding"
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Ace
|
|
6
|
+
module Hitl
|
|
7
|
+
module Lifecycle
|
|
8
|
+
# Constructed beside the same-process authority, never from wire data.
|
|
9
|
+
# The selector is routing data; only the authority's held callback can
|
|
10
|
+
# admit a store transition for the original receiver process.
|
|
11
|
+
class ServicePublicationBinding < Binding
|
|
12
|
+
FIELDS = %w[schema mapping_id assignment_id attempt_id request_id claim_binding input_digest
|
|
13
|
+
candidate_generation head challenge_digest].freeze
|
|
14
|
+
|
|
15
|
+
def initialize(ordinary:, authority:, kernel:)
|
|
16
|
+
@ordinary, @authority, @kernel = ordinary, authority, kernel
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def validate_request(**params)
|
|
20
|
+
deny_executor_ordinary!(params.fetch(:project), params[:caller_pid])
|
|
21
|
+
@ordinary.validate_request(**params)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def reverse_address(**params)
|
|
25
|
+
@ordinary.reverse_address(**params)
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def with_active(**params, &block)
|
|
29
|
+
@ordinary.with_active(**params, &block)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def proposal_journal(project:)
|
|
33
|
+
@ordinary.proposal_journal(project: project)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def proposal_projects
|
|
37
|
+
@ordinary.proposal_projects
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
def with_publication!(binding:, identity:, assignment:, attempt:, project:, otp:, original_peer: nil, requester_required: true)
|
|
41
|
+
unless binding.is_a?(Hash) && binding.keys.sort == FIELDS.sort &&
|
|
42
|
+
binding["schema"] == "ace.hitl-publication-binding/v1" &&
|
|
43
|
+
binding["assignment_id"] == assignment && binding["attempt_id"] == attempt &&
|
|
44
|
+
%w[mapping_id assignment_id attempt_id request_id].all? { |key| binding[key].is_a?(String) && binding[key].match?(/\A[a-zA-Z0-9][a-zA-Z0-9._:-]{0,127}\z/) } &&
|
|
45
|
+
%w[claim_binding input_digest challenge_digest].all? { |key| binding[key].is_a?(String) && binding[key].match?(/\A[0-9a-f]{64}\z/) } &&
|
|
46
|
+
binding["head"].is_a?(String) && binding["head"].match?(/\A[0-9a-f]{40}\z/) &&
|
|
47
|
+
binding["candidate_generation"].is_a?(Integer) && binding["candidate_generation"].positive? &&
|
|
48
|
+
otp.is_a?(Hash) && otp["operation"] == "publish" && otp["input_digest"] == binding["input_digest"]
|
|
49
|
+
raise BindingError, "publication HITL selector differs"
|
|
50
|
+
end
|
|
51
|
+
caller = requester_required ? actual_peer!(identity) : original_peer
|
|
52
|
+
raise BindingError, "publication original receiver differs" if original_peer && caller != original_peer
|
|
53
|
+
@authority.with_publication_hitl!(binding: binding, project: project, peer: caller) do |selection|
|
|
54
|
+
unless selection.fetch("challenge_ref") == otp["result_ref"] && selection.fetch("executor_process_binding") == caller
|
|
55
|
+
raise BindingError, "publication challenge or receiver differs"
|
|
56
|
+
end
|
|
57
|
+
yield selection
|
|
58
|
+
end
|
|
59
|
+
rescue Ace::Assign::Error, Ace::Runtime::RuntimeUnavailableError => e
|
|
60
|
+
raise BindingError, "publication authority unavailable (#{e.class.name})"
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def verify_publication_requester!(identity:, original_peer:)
|
|
64
|
+
raise BindingError, "publication original receiver differs" unless actual_peer!(identity) == original_peer
|
|
65
|
+
true
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
private
|
|
69
|
+
|
|
70
|
+
def actual_peer!(identity)
|
|
71
|
+
unless identity.respond_to?(:pid) && identity.pid.is_a?(Integer) && identity.pid.positive?
|
|
72
|
+
raise BindingError, "publication kernel peer is unavailable"
|
|
73
|
+
end
|
|
74
|
+
peer = @kernel.capture(identity.pid)
|
|
75
|
+
unless peer["uid"] == identity.uid && peer["gid"] == identity.gid
|
|
76
|
+
raise BindingError, "publication kernel credentials differ"
|
|
77
|
+
end
|
|
78
|
+
@kernel.live!(peer)
|
|
79
|
+
peer
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def deny_executor_ordinary!(project, pid)
|
|
83
|
+
unless pid.is_a?(Integer) && pid.positive?
|
|
84
|
+
raise BindingError, "HITL kernel requester is unavailable"
|
|
85
|
+
end
|
|
86
|
+
peer = @kernel.capture(pid)
|
|
87
|
+
@kernel.live!(peer)
|
|
88
|
+
if @authority.publication_executor_identity?(project: project, uid: peer.fetch("uid"))
|
|
89
|
+
raise BindingError, "executor cannot select an ordinary worker HITL request"
|
|
90
|
+
end
|
|
91
|
+
rescue Ace::Assign::Error, Ace::Runtime::RuntimeUnavailableError => error
|
|
92
|
+
raise BindingError, "HITL kernel requester unavailable (#{error.class.name})"
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
@@ -79,7 +79,7 @@ module Ace
|
|
|
79
79
|
|
|
80
80
|
def initialize(root:, binding:, policy: AccessPolicy.new,
|
|
81
81
|
identity: Identity, ownership: AtomicJson::DEFAULT_OWNERSHIP,
|
|
82
|
-
poll_seconds: CONSUME_POLL_SECONDS, vault: :file, proposal_clock: -> { Time.now.utc })
|
|
82
|
+
poll_seconds: CONSUME_POLL_SECONDS, vault: :file, proposal_clock: -> { Time.now.utc }, stopping: -> { false })
|
|
83
83
|
raise ArgumentError, "a binding policy is required (fail closed without one)" unless binding
|
|
84
84
|
raise ArgumentError, "an access policy is required (fail closed without one)" unless policy
|
|
85
85
|
|
|
@@ -91,6 +91,7 @@ module Ace
|
|
|
91
91
|
@poll_seconds = poll_seconds
|
|
92
92
|
@vault = vault == :file ? OtpVault::FileVault : vault
|
|
93
93
|
@proposal_clock = proposal_clock
|
|
94
|
+
@stopping = stopping
|
|
94
95
|
end
|
|
95
96
|
|
|
96
97
|
# ---- requester side ------------------------------------------------
|
|
@@ -101,16 +102,39 @@ module Ace
|
|
|
101
102
|
# validates its binding — unknown identity/authority is an
|
|
102
103
|
# error, never permission.
|
|
103
104
|
def create(id:, attempt:, plan:, question:, ace_hitl_id:, project: "ace", harness: "lab-admin",
|
|
104
|
-
assignment:, kind: "text", options: [], effect: nil, otp: nil)
|
|
105
|
+
assignment:, kind: "text", options: [], effect: nil, otp: nil, publication_binding: nil)
|
|
105
106
|
raise StateError, "proposal kind must use proposal_create" if kind == "proposal"
|
|
106
107
|
value = prepare_request(id: id, attempt: attempt, plan: plan, question: question,
|
|
107
108
|
ace_hitl_id: ace_hitl_id, project: project, harness: harness, assignment: assignment,
|
|
108
|
-
kind: kind, options: options, effect: effect, otp: otp)
|
|
109
|
-
persist_request(value)
|
|
109
|
+
kind: kind, options: options, effect: effect, otp: otp, publication_binding: publication_binding)
|
|
110
|
+
return persist_request(value) unless value["publication_binding"]
|
|
111
|
+
with_request_lock(value.fetch("id")) do
|
|
112
|
+
with_live_authority!(value, requester: @identity.username) do
|
|
113
|
+
if value["publication_binding"]
|
|
114
|
+
if load_terminal(value.fetch("id"))
|
|
115
|
+
raise StateError, "publication HITL id is already terminal"
|
|
116
|
+
end
|
|
117
|
+
duplicate = request_path(value.fetch("id")).exist? && load_request(value.fetch("id"))
|
|
118
|
+
if duplicate
|
|
119
|
+
unless duplicate.slice("publication_binding", "publication_requester_peer", "otp") == value.slice("publication_binding", "publication_requester_peer", "otp")
|
|
120
|
+
raise StateError, "publication HITL id has different binding"
|
|
121
|
+
end
|
|
122
|
+
return {"id" => duplicate.fetch("id"), "requested" => true, "replay" => true, "envelope" => duplicate.fetch("envelope")}
|
|
123
|
+
end
|
|
124
|
+
requests_dir.glob("*.json").each do |path|
|
|
125
|
+
existing = AtomicJson.read(path)
|
|
126
|
+
if existing["publication_binding"] == value["publication_binding"]
|
|
127
|
+
raise StateError, "publication challenge already has an active HITL request"
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
persist_request_locked!(value)
|
|
132
|
+
end
|
|
133
|
+
end
|
|
110
134
|
end
|
|
111
135
|
|
|
112
136
|
def prepare_request(id:, attempt:, plan:, question:, ace_hitl_id:, project: "ace", harness: "lab-admin",
|
|
113
|
-
assignment:, kind: "text", options: [], effect: nil, otp: nil)
|
|
137
|
+
assignment:, kind: "text", options: [], effect: nil, otp: nil, publication_binding: nil)
|
|
114
138
|
requester = @identity.username
|
|
115
139
|
request_id = safe_id(id)
|
|
116
140
|
validate_binding_ids!(assignment, attempt)
|
|
@@ -143,7 +167,13 @@ module Ace
|
|
|
143
167
|
challenge = normalize_otp_challenge!(otp)
|
|
144
168
|
end
|
|
145
169
|
|
|
146
|
-
|
|
170
|
+
if publication_binding
|
|
171
|
+
raise StateError, "publication HITL must be OTP" unless kind == "otp"
|
|
172
|
+
publication_binding = JSON.parse(JSON.generate(publication_binding))
|
|
173
|
+
reverse = nil
|
|
174
|
+
else
|
|
175
|
+
reverse = validate_request_binding(assignment, attempt, project, requester)
|
|
176
|
+
end
|
|
147
177
|
|
|
148
178
|
value = {
|
|
149
179
|
"id" => request_id,
|
|
@@ -161,6 +191,7 @@ module Ace
|
|
|
161
191
|
"created_at" => Time.now.to_i
|
|
162
192
|
}
|
|
163
193
|
value["otp"] = challenge if challenge
|
|
194
|
+
value["publication_binding"] = publication_binding if publication_binding
|
|
164
195
|
Effects.validate_declaration!(effect) if effect
|
|
165
196
|
value["effect"] = Effects.normalized_declaration(effect) if effect
|
|
166
197
|
value["incarnation"] = SecureRandom.hex(8)
|
|
@@ -182,6 +213,10 @@ module Ace
|
|
|
182
213
|
private :prepare_request
|
|
183
214
|
|
|
184
215
|
def persist_request(value)
|
|
216
|
+
with_request_lock(value.fetch("id")) { persist_request_locked!(value) }
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
def persist_request_locked!(value)
|
|
185
220
|
request_id = value.fetch("id")
|
|
186
221
|
attempt = value.fetch("attempt")
|
|
187
222
|
request_path = requests_dir.join("#{request_id}.json")
|
|
@@ -197,7 +232,7 @@ module Ace
|
|
|
197
232
|
rescue Errno::EEXIST
|
|
198
233
|
raise StateError, "HITL request already exists"
|
|
199
234
|
end
|
|
200
|
-
|
|
235
|
+
initialize_projection_locked!(value)
|
|
201
236
|
{
|
|
202
237
|
"id" => request_id,
|
|
203
238
|
"assignment" => value["assignment"],
|
|
@@ -207,7 +242,7 @@ module Ace
|
|
|
207
242
|
}
|
|
208
243
|
end
|
|
209
244
|
|
|
210
|
-
private :persist_request
|
|
245
|
+
private :persist_request, :persist_request_locked!
|
|
211
246
|
|
|
212
247
|
# Non-secret request facts for the requester of record (or the
|
|
213
248
|
# transport). Never carries answer content.
|
|
@@ -217,7 +252,7 @@ module Ace
|
|
|
217
252
|
terminal = load_terminal(request_id)
|
|
218
253
|
if terminal
|
|
219
254
|
gate_read_access!(terminal)
|
|
220
|
-
return terminal.slice("id", "assignment", "attempt", "project", "requester", "sensitive", "envelope", "state", "native_delivery", "effect_receipt_ref")
|
|
255
|
+
return terminal.slice("id", "assignment", "attempt", "project", "requester", "sensitive", "envelope", "state", "native_delivery", "effect_receipt_ref", "publication_binding", "publication_requester_peer", "otp")
|
|
221
256
|
.merge("kind" => terminal.dig("envelope", "kind"))
|
|
222
257
|
end
|
|
223
258
|
end
|
|
@@ -235,13 +270,15 @@ module Ace
|
|
|
235
270
|
"options" => value["options"],
|
|
236
271
|
"plan" => value["plan"],
|
|
237
272
|
"otp" => value["otp"],
|
|
273
|
+
"publication_binding" => value["publication_binding"],
|
|
274
|
+
"publication_requester_peer" => value["publication_requester_peer"],
|
|
238
275
|
"requester" => value["requester"],
|
|
239
276
|
"envelope" => value["envelope"],
|
|
240
277
|
"state" => public_state(request_id, value),
|
|
241
278
|
"effect_receipt_ref" => effect_receipt_ref(value),
|
|
242
279
|
"effect" => value["effect_state"]
|
|
243
280
|
}
|
|
244
|
-
facts["proposal"] = proposal_for_request!(request_id) if value["kind"] == "proposal"
|
|
281
|
+
facts["proposal"] = proposal_for_request!(request_id, project: value.fetch("project")) if value["kind"] == "proposal"
|
|
245
282
|
facts
|
|
246
283
|
end
|
|
247
284
|
|
|
@@ -273,6 +310,7 @@ module Ace
|
|
|
273
310
|
verify_operation!(value, operation)
|
|
274
311
|
deadline = timeout.positive? ? Time.now.to_i + timeout : nil
|
|
275
312
|
loop do
|
|
313
|
+
raise TransportError, "HITL boundary is stopping; request remains pending" if @stopping.call
|
|
276
314
|
answer = nil
|
|
277
315
|
with_request_lock(request_id) do
|
|
278
316
|
begin
|
|
@@ -294,6 +332,7 @@ module Ace
|
|
|
294
332
|
verify_native_consumer!(value) if native_delivery
|
|
295
333
|
verify_operation!(value, operation)
|
|
296
334
|
verify_otp_deadline!(value)
|
|
335
|
+
raise TransportError, "HITL boundary is stopping; request remains pending" if @stopping.call
|
|
297
336
|
answer = read_answer(value)
|
|
298
337
|
unless answer
|
|
299
338
|
next
|
|
@@ -354,11 +393,19 @@ module Ace
|
|
|
354
393
|
unless @identity.username == value["requester"].to_s
|
|
355
394
|
raise PermissionError, "only the requesting role can cancel this request"
|
|
356
395
|
end
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
396
|
+
cancel = lambda do
|
|
397
|
+
requester_gate!(value)
|
|
398
|
+
commit_terminal!(value, "cancelled", audit)
|
|
399
|
+
update_public(value, "cancelled", audit: audit)
|
|
400
|
+
@vault.discard(self, value)
|
|
401
|
+
remove_request(value, keep_public: true)
|
|
402
|
+
locked_value = value
|
|
403
|
+
end
|
|
404
|
+
if value["publication_binding"]
|
|
405
|
+
with_live_authority!(value, requester: @identity.username, &cancel)
|
|
406
|
+
else
|
|
407
|
+
cancel.call
|
|
408
|
+
end
|
|
362
409
|
end
|
|
363
410
|
{
|
|
364
411
|
"id" => request_id,
|
|
@@ -625,19 +672,18 @@ module Ace
|
|
|
625
672
|
# lock first and already delivered can never be regressed to
|
|
626
673
|
# "created" — a current-incarnation projection is left untouched.
|
|
627
674
|
def initialize_projection!(value)
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
end
|
|
675
|
+
with_request_lock(safe_id(value.fetch("id"))) { initialize_projection_locked!(value) }
|
|
676
|
+
end
|
|
677
|
+
|
|
678
|
+
def initialize_projection_locked!(value)
|
|
679
|
+
request_id = safe_id(value.fetch("id"))
|
|
680
|
+
current = AtomicJson.read(public_path(request_id))
|
|
681
|
+
predecessor = current.nil? || current["incarnation"] != value["incarnation"]
|
|
682
|
+
if predecessor
|
|
683
|
+
public_path(request_id).unlink if public_path(request_id).exist?
|
|
684
|
+
effects_dir.join("#{request_id}.json").unlink if effects_dir.join("#{request_id}.json").exist?
|
|
685
|
+
terminals_dir.join("#{request_id}.json").unlink if terminals_dir.join("#{request_id}.json").exist?
|
|
686
|
+
update_public(value, "created")
|
|
641
687
|
end
|
|
642
688
|
nil
|
|
643
689
|
end
|
|
@@ -722,7 +768,7 @@ module Ace
|
|
|
722
768
|
|
|
723
769
|
def verify_native_consumer!(value)
|
|
724
770
|
raise StateError, "OTP requires protected local consume" if value["sensitive"] == true
|
|
725
|
-
reverse = @binding.reverse_address(attempt: value["attempt"],
|
|
771
|
+
reverse = @binding.reverse_address(assignment: value["assignment"], project: value["project"], attempt: value["attempt"],
|
|
726
772
|
caller_pid: @identity.respond_to?(:pid) ? @identity.pid : nil)
|
|
727
773
|
unless reverse && reverse == value.dig("envelope", "reverse")
|
|
728
774
|
raise BindingError, "native delivery requires the exact original requester process"
|
|
@@ -730,7 +776,8 @@ module Ace
|
|
|
730
776
|
end
|
|
731
777
|
|
|
732
778
|
def validate_binding_ids!(assignment, attempt)
|
|
733
|
-
unless Kinds::COMPACT_ID.match?(assignment.to_s) &&
|
|
779
|
+
unless Kinds::COMPACT_ID.match?(assignment.to_s) &&
|
|
780
|
+
(Kinds::COMPACT_ID.match?(attempt.to_s) || Kinds::PROTECTED_ATTEMPT_ID.match?(attempt.to_s))
|
|
734
781
|
raise StateError, "HITL request requires the exact managed assignment and attempt ids"
|
|
735
782
|
end
|
|
736
783
|
end
|
|
@@ -783,10 +830,16 @@ module Ace
|
|
|
783
830
|
unless @identity.username == value["requester"].to_s
|
|
784
831
|
raise PermissionError, "only the requesting role can consume this answer"
|
|
785
832
|
end
|
|
833
|
+
if value["publication_binding"]
|
|
834
|
+
@binding.verify_publication_requester!(identity: @identity, original_peer: value.fetch("publication_requester_peer"))
|
|
835
|
+
end
|
|
786
836
|
end
|
|
787
837
|
|
|
788
838
|
def gate_read_access!(value)
|
|
789
|
-
|
|
839
|
+
if @identity.username == value["requester"].to_s
|
|
840
|
+
requester_gate!(value)
|
|
841
|
+
return
|
|
842
|
+
end
|
|
790
843
|
|
|
791
844
|
require_transport!("read", value)
|
|
792
845
|
end
|
|
@@ -829,6 +882,7 @@ module Ace
|
|
|
829
882
|
raise PermissionError, "only the requesting role can #{operation == "cancelled" ? "cancel" : "consume"} this request"
|
|
830
883
|
end
|
|
831
884
|
|
|
885
|
+
requester_gate!(terminal) if terminal["publication_binding"]
|
|
832
886
|
case terminal["state"]
|
|
833
887
|
when "cancelled"
|
|
834
888
|
return replayed_cancel(request_id, terminal) if operation == "cancelled"
|
|
@@ -889,6 +943,9 @@ module Ace
|
|
|
889
943
|
"effect_receipt_ref" => effect_receipt_ref(value),
|
|
890
944
|
"at" => Time.now.to_i
|
|
891
945
|
}
|
|
946
|
+
%w[publication_binding publication_requester_peer otp].each do |key|
|
|
947
|
+
record[key] = value[key] if value.key?(key)
|
|
948
|
+
end
|
|
892
949
|
record["answer"] = extra[:answer] if extra[:answer]
|
|
893
950
|
record["native_delivery"] = true if extra[:native_delivery]
|
|
894
951
|
record["audit"] = extra["audit"] if extra["audit"]
|
|
@@ -907,11 +964,22 @@ module Ace
|
|
|
907
964
|
# the request stays pending and the classified error propagates
|
|
908
965
|
# for retry (review 8x32r9b0).
|
|
909
966
|
def with_live_authority!(value, requester:)
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
967
|
+
if value["publication_binding"]
|
|
968
|
+
unless @binding.respond_to?(:with_publication!)
|
|
969
|
+
raise BindingError, "publication HITL source binding is unavailable"
|
|
970
|
+
end
|
|
971
|
+
@binding.with_publication!(binding: value.fetch("publication_binding"), identity: @identity,
|
|
972
|
+
assignment: value.fetch("assignment"), attempt: value.fetch("attempt"), project: value.fetch("project"),
|
|
973
|
+
otp: value.fetch("otp"), original_peer: value["publication_requester_peer"],
|
|
974
|
+
requester_required: @identity.username == value.fetch("requester")) do |selection|
|
|
975
|
+
value["publication_requester_peer"] ||= JSON.parse(JSON.generate(selection.fetch("executor_process_binding")))
|
|
976
|
+
yield
|
|
977
|
+
end
|
|
978
|
+
else
|
|
979
|
+
@binding.with_active(
|
|
980
|
+
assignment: value["assignment"], attempt: value["attempt"],
|
|
981
|
+
project: value["project"], requester: requester
|
|
982
|
+
) { yield }
|
|
915
983
|
end
|
|
916
984
|
rescue EndedAttemptError
|
|
917
985
|
update_public(value, "cancelled")
|
data/lib/ace/hitl/live_client.rb
CHANGED
|
@@ -8,8 +8,9 @@ require_relative "providers/lab"
|
|
|
8
8
|
module Ace
|
|
9
9
|
module Hitl
|
|
10
10
|
# Explicit in-process client. The scoped service owns answer availability
|
|
11
|
-
# and business callbacks; Herdr owns
|
|
12
|
-
#
|
|
11
|
+
# and business callbacks; Herdr owns delivery to the selected live terminal.
|
|
12
|
+
# A watcher lives only as long as its requesting agent. Delivery status is
|
|
13
|
+
# read from that original Inbox owner, never a duplicate recovery projection.
|
|
13
14
|
class LiveClient
|
|
14
15
|
def initialize(boundary: nil, coordinator: nil, inbox: nil, root: Dir.pwd)
|
|
15
16
|
@boundary = boundary
|
|
@@ -63,9 +64,6 @@ module Ace
|
|
|
63
64
|
event = event_id(envelope)
|
|
64
65
|
queued = inbox.enqueue(event: event, attempt: envelope["attempt_id"], ref: reverse, payload: answer,
|
|
65
66
|
managed_envelope: delivery, expected_target: target)
|
|
66
|
-
# A signed supersession authorizes an explicit reconciliation retry,
|
|
67
|
-
# not another ordinary delivery/watch call.
|
|
68
|
-
return queued if queued["state"] == "queued" && queued.dig("reconciliation", "outcome") == "superseded"
|
|
69
67
|
coordinator.bind_inbox(attempt_id: envelope["attempt_id"], event_id: event, inbox: inbox)
|
|
70
68
|
inbox.deliver(event: event)
|
|
71
69
|
ensure
|
|
@@ -84,36 +82,20 @@ module Ace
|
|
|
84
82
|
next facts unless facts["native_delivery"] == true
|
|
85
83
|
envelope = envelope_for(facts, facts["id"])
|
|
86
84
|
delivery = delivery_status(envelope)
|
|
87
|
-
next if delivery["state"] == "
|
|
85
|
+
next if delivery["state"] == "delivered"
|
|
88
86
|
facts.merge("delivery" => delivery)
|
|
89
87
|
end
|
|
90
88
|
end
|
|
91
89
|
|
|
92
|
-
# Only the existing signed verifier + accepted attempt registration may
|
|
93
|
-
# settle uncertainty. Explicit retry is admitted solely by verified
|
|
94
|
-
# supersession, including its verified replacement native target.
|
|
95
|
-
def reconcile(request:, receipt_path:, retry_delivery: false)
|
|
96
|
-
facts = boundary.read(request)
|
|
97
|
-
envelope = envelope_for(facts, request)
|
|
98
|
-
event = event_id(envelope)
|
|
99
|
-
result = coordinator.reconcile_inbox(attempt_id: envelope["attempt_id"], event_id: event,
|
|
100
|
-
receipt_path: receipt_path, inbox: inbox)
|
|
101
|
-
if retry_delivery && result["state"] == "queued" && !result["reconciliation_refusal"] &&
|
|
102
|
-
result.dig("reconciliation", "outcome") == "superseded"
|
|
103
|
-
return inbox.deliver(event: event)
|
|
104
|
-
end
|
|
105
|
-
result
|
|
106
|
-
end
|
|
107
|
-
|
|
108
90
|
private
|
|
109
91
|
|
|
110
92
|
def delivery_status(envelope)
|
|
111
93
|
event = event_id(envelope)
|
|
112
|
-
#
|
|
113
|
-
#
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
94
|
+
# This is transport history from its original owner, not proof that
|
|
95
|
+
# an agent read an answer or a cached view of the current terminal.
|
|
96
|
+
inbox.status(event: event)
|
|
97
|
+
rescue Ace::Herdr::Organisms::Inbox::MissingEventError
|
|
98
|
+
{"event_id" => event, "state" => "not-submitted"}
|
|
117
99
|
end
|
|
118
100
|
|
|
119
101
|
def envelope_for(result, request)
|
|
@@ -23,13 +23,13 @@ module Ace
|
|
|
23
23
|
|
|
24
24
|
def validate_request(assignment:, attempt:, project:, requester:, caller_pid: nil)
|
|
25
25
|
with_verified(assignment, attempt, project, requester) do
|
|
26
|
-
reverse_address(attempt: attempt, caller_pid: caller_pid)
|
|
26
|
+
reverse_address(assignment: assignment, attempt: attempt, project: project, caller_pid: caller_pid)
|
|
27
27
|
end
|
|
28
28
|
end
|
|
29
29
|
|
|
30
30
|
# Read-only owner verification can run inside an already-held
|
|
31
31
|
# assignment exclusion without reacquiring that exclusion.
|
|
32
|
-
def reverse_address(attempt:, caller_pid:)
|
|
32
|
+
def reverse_address(assignment:, attempt:, project:, caller_pid:)
|
|
33
33
|
unless caller_pid.is_a?(Integer) && caller_pid.positive?
|
|
34
34
|
raise Lifecycle::BindingError, "kernel requester process identity is unavailable"
|
|
35
35
|
end
|
|
@@ -45,10 +45,17 @@ module Ace
|
|
|
45
45
|
with_verified(assignment, attempt, project, requester) { yield }
|
|
46
46
|
end
|
|
47
47
|
|
|
48
|
-
def proposal_journal
|
|
48
|
+
def proposal_journal(project:)
|
|
49
|
+
unless project.is_a?(String) && !project.empty?
|
|
50
|
+
raise Lifecycle::BindingError, "proposal requires an explicit project"
|
|
51
|
+
end
|
|
49
52
|
coordinator.send(:journal_for)
|
|
50
53
|
end
|
|
51
54
|
|
|
55
|
+
def proposal_projects
|
|
56
|
+
coordinator.send(:journal_for).proposals.map { |record| record.fetch("project_id") }.uniq.sort
|
|
57
|
+
end
|
|
58
|
+
|
|
52
59
|
private
|
|
53
60
|
|
|
54
61
|
def with_verified(assignment_id, attempt_id, project_id, requester)
|