ace-hitl 0.9.0 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +17 -0
- data/README.md +33 -1
- data/docs/usage.md +84 -13
- data/lib/ace/hitl/cli/commands/ask.rb +60 -54
- data/lib/ace/hitl/cli/commands/cancel.rb +34 -0
- data/lib/ace/hitl/cli/commands/consume.rb +34 -0
- data/lib/ace/hitl/cli/commands/deliver.rb +32 -0
- data/lib/ace/hitl/cli/commands/duty.rb +29 -0
- data/lib/ace/hitl/cli/commands/lifecycle_command.rb +36 -0
- data/lib/ace/hitl/cli/commands/overseer_ack.rb +30 -0
- data/lib/ace/hitl/cli/commands/overseer_pending.rb +28 -0
- data/lib/ace/hitl/cli/commands/overseer_send.rb +35 -0
- data/lib/ace/hitl/cli/commands/pending.rb +29 -0
- data/lib/ace/hitl/cli/commands/states.rb +28 -0
- data/lib/ace/hitl/cli/commands/wait.rb +2 -2
- data/lib/ace/hitl/cli.rb +28 -1
- data/lib/ace/hitl/lifecycle/atomic_json.rb +79 -0
- data/lib/ace/hitl/lifecycle/binding.rb +27 -0
- data/lib/ace/hitl/lifecycle/duty.rb +43 -0
- data/lib/ace/hitl/lifecycle/effects.rb +254 -0
- data/lib/ace/hitl/lifecycle/errors.rb +27 -0
- data/lib/ace/hitl/lifecycle/identity.rb +66 -0
- data/lib/ace/hitl/lifecycle/kinds.rb +57 -0
- data/lib/ace/hitl/lifecycle/overseer.rb +111 -0
- data/lib/ace/hitl/lifecycle/store.rb +544 -0
- data/lib/ace/hitl/lifecycle.rb +15 -0
- data/lib/ace/hitl/molecules/lab_projection_observer.rb +9 -1
- data/lib/ace/hitl/providers/errors.rb +24 -0
- data/lib/ace/hitl/providers/lab/daemon_binding.rb +206 -0
- data/lib/ace/hitl/providers/lab.rb +124 -0
- data/lib/ace/hitl/providers/providers.rb +41 -0
- data/lib/ace/hitl/providers/ref.rb +62 -0
- data/lib/ace/hitl/version.rb +1 -1
- data/lib/ace/hitl.rb +3 -0
- metadata +27 -3
- data/lib/ace/hitl/molecules/lab_request_submitter.rb +0 -82
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ace/support/cli"
|
|
4
|
+
require_relative "lifecycle_command"
|
|
5
|
+
|
|
6
|
+
module Ace
|
|
7
|
+
module Hitl
|
|
8
|
+
module CLI
|
|
9
|
+
module Commands
|
|
10
|
+
# Host-broker listing of answerable relay requests; never purges
|
|
11
|
+
# or cancels anything (W651).
|
|
12
|
+
class Pending < Ace::Support::Cli::Command
|
|
13
|
+
include Ace::Support::Cli::Base
|
|
14
|
+
include LifecycleCommand
|
|
15
|
+
|
|
16
|
+
desc "List answerable HITL relay requests (host-broker operation)"
|
|
17
|
+
|
|
18
|
+
option :quiet, type: :boolean, aliases: %w[-q], desc: "Suppress non-essential output"
|
|
19
|
+
|
|
20
|
+
def call(**options)
|
|
21
|
+
emit(lifecycle_store.pending)
|
|
22
|
+
rescue Lifecycle::Error => e
|
|
23
|
+
raise_lifecycle_error(e.message)
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "ace/support/cli"
|
|
4
|
+
require_relative "lifecycle_command"
|
|
5
|
+
|
|
6
|
+
module Ace
|
|
7
|
+
module Hitl
|
|
8
|
+
module CLI
|
|
9
|
+
module Commands
|
|
10
|
+
# Host-broker listing of all public lifecycle projections.
|
|
11
|
+
class States < Ace::Support::Cli::Command
|
|
12
|
+
include Ace::Support::Cli::Base
|
|
13
|
+
include LifecycleCommand
|
|
14
|
+
|
|
15
|
+
desc "List public HITL lifecycle projections (host-broker operation)"
|
|
16
|
+
|
|
17
|
+
option :quiet, type: :boolean, aliases: %w[-q], desc: "Suppress non-essential output"
|
|
18
|
+
|
|
19
|
+
def call(**options)
|
|
20
|
+
emit(lifecycle_store.states)
|
|
21
|
+
rescue Lifecycle::Error => e
|
|
22
|
+
raise_lifecycle_error(e.message)
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
@@ -55,11 +55,11 @@ module Ace
|
|
|
55
55
|
lab_request_id = event.metadata["lab_request_id"]
|
|
56
56
|
puts "Lab request delivered: #{event.id} (#{result[:lab_state]})"
|
|
57
57
|
puts "Lab request: #{lab_request_id}"
|
|
58
|
-
puts "Consume the answer when ready
|
|
58
|
+
puts "Consume the answer when ready from the lab relay."
|
|
59
59
|
if result[:lab_state] == "callback-ok"
|
|
60
60
|
puts "Effect callback: ok"
|
|
61
61
|
elsif result[:lab_state] == "callback-escalated"
|
|
62
|
-
puts "Effect callback: escalated; inspect
|
|
62
|
+
puts "Effect callback: escalated; inspect the lab duty projection"
|
|
63
63
|
end
|
|
64
64
|
when :timeout
|
|
65
65
|
raise Ace::Support::Cli::Error.new("Timed out waiting for HITL event '#{ref}'")
|
data/lib/ace/hitl/cli.rb
CHANGED
|
@@ -8,6 +8,15 @@ require_relative "cli/commands/show"
|
|
|
8
8
|
require_relative "cli/commands/list"
|
|
9
9
|
require_relative "cli/commands/update"
|
|
10
10
|
require_relative "cli/commands/wait"
|
|
11
|
+
require_relative "cli/commands/deliver"
|
|
12
|
+
require_relative "cli/commands/consume"
|
|
13
|
+
require_relative "cli/commands/cancel"
|
|
14
|
+
require_relative "cli/commands/pending"
|
|
15
|
+
require_relative "cli/commands/states"
|
|
16
|
+
require_relative "cli/commands/duty"
|
|
17
|
+
require_relative "cli/commands/overseer_send"
|
|
18
|
+
require_relative "cli/commands/overseer_pending"
|
|
19
|
+
require_relative "cli/commands/overseer_ack"
|
|
11
20
|
|
|
12
21
|
module Ace
|
|
13
22
|
module Hitl
|
|
@@ -22,7 +31,16 @@ module Ace
|
|
|
22
31
|
["show", "Show HITL event details"],
|
|
23
32
|
["list", "List HITL events"],
|
|
24
33
|
["update", "Update HITL event metadata or answer"],
|
|
25
|
-
["wait", "Wait for an answer on a specific HITL event"]
|
|
34
|
+
["wait", "Wait for an answer on a specific HITL event"],
|
|
35
|
+
["deliver", "Deliver an answer (stdin) to a pending HITL relay request"],
|
|
36
|
+
["consume", "Wait for and consume the answer of one own HITL relay request"],
|
|
37
|
+
["cancel", "Cancel one HITL relay request with an audited reason"],
|
|
38
|
+
["pending", "List answerable HITL relay requests (host-broker)"],
|
|
39
|
+
["states", "List public HITL lifecycle projections (host-broker)"],
|
|
40
|
+
["duty", "Project pending and escalated HITL requests (host-broker)"],
|
|
41
|
+
["overseer-send", "Queue a bounded, type-tagged Overseer response (stdin)"],
|
|
42
|
+
["overseer-pending", "List queued Overseer responses (host-broker)"],
|
|
43
|
+
["overseer-ack", "Acknowledge one relayed Overseer response"]
|
|
26
44
|
].freeze
|
|
27
45
|
|
|
28
46
|
HELP_EXAMPLES = [
|
|
@@ -40,6 +58,15 @@ module Ace
|
|
|
40
58
|
register "list", CLI::Commands::List
|
|
41
59
|
register "update", CLI::Commands::Update
|
|
42
60
|
register "wait", CLI::Commands::Wait
|
|
61
|
+
register "deliver", CLI::Commands::Deliver
|
|
62
|
+
register "consume", CLI::Commands::Consume
|
|
63
|
+
register "cancel", CLI::Commands::Cancel
|
|
64
|
+
register "pending", CLI::Commands::Pending
|
|
65
|
+
register "states", CLI::Commands::States
|
|
66
|
+
register "duty", CLI::Commands::Duty
|
|
67
|
+
register "overseer-send", CLI::Commands::OverseerSend
|
|
68
|
+
register "overseer-pending", CLI::Commands::OverseerPending
|
|
69
|
+
register "overseer-ack", CLI::Commands::OverseerAck
|
|
43
70
|
|
|
44
71
|
version_cmd = Ace::Support::Cli::VersionCommand.build(
|
|
45
72
|
gem_name: "ace-hitl",
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "securerandom"
|
|
6
|
+
|
|
7
|
+
module Ace
|
|
8
|
+
module Hitl
|
|
9
|
+
module Lifecycle
|
|
10
|
+
# Durable, permission-preserving JSON record writes (port of
|
|
11
|
+
# atomic_json): O_EXCL|O_NOFOLLOW tempfile in the target directory,
|
|
12
|
+
# fsync, chmod, chown, rename. Ownership transitions go through the
|
|
13
|
+
# injectable ownership strategy; production performs real chown(2),
|
|
14
|
+
# tests map names onto the current identity (spec 8wm.t.y21 §2).
|
|
15
|
+
module AtomicJson
|
|
16
|
+
Ownership = Struct.new(:uid, :gid)
|
|
17
|
+
|
|
18
|
+
DEFAULT_OWNERSHIP = Object.new.tap do |strategy|
|
|
19
|
+
strategy.define_singleton_method(:chown) do |path, ownership|
|
|
20
|
+
File.chown(ownership.uid, ownership.gid, path) if ownership
|
|
21
|
+
end
|
|
22
|
+
end.freeze
|
|
23
|
+
|
|
24
|
+
module_function
|
|
25
|
+
|
|
26
|
+
# mode is an Integer permission bits value; ownership nil keeps
|
|
27
|
+
# the current identity. exclusive commits with link(2) instead of
|
|
28
|
+
# rename(2), so an existing destination raises Errno::EEXIST
|
|
29
|
+
# instead of being silently replaced (create-once writes).
|
|
30
|
+
def call(path, value, mode:, ownership: nil, ownership_strategy: DEFAULT_OWNERSHIP,
|
|
31
|
+
exclusive: false)
|
|
32
|
+
path = Pathname.new(path)
|
|
33
|
+
path.parent.mkpath
|
|
34
|
+
# The random suffix makes the temporary unique per invocation:
|
|
35
|
+
# a fixed pid-based name lets a competing writer's cleanup
|
|
36
|
+
# delete our in-flight temporary and fail both writes (review
|
|
37
|
+
# 8wq2zttt on PR#336). Cleanup only ever touches this call's
|
|
38
|
+
# own file.
|
|
39
|
+
temporary = path.parent.join(".#{path.basename}.tmp.#{$PROCESS_ID}.#{SecureRandom.hex(4)}")
|
|
40
|
+
flags = File::WRONLY | File::CREAT | File::EXCL | File::NOFOLLOW
|
|
41
|
+
fd = IO.sysopen(temporary, flags, mode)
|
|
42
|
+
begin
|
|
43
|
+
io = IO.for_fd(fd, mode: "w")
|
|
44
|
+
begin
|
|
45
|
+
io.write(JSON.generate(value))
|
|
46
|
+
io.write("\n")
|
|
47
|
+
io.flush
|
|
48
|
+
io.fsync
|
|
49
|
+
ensure
|
|
50
|
+
io.close
|
|
51
|
+
end
|
|
52
|
+
rescue
|
|
53
|
+
begin
|
|
54
|
+
temporary.unlink
|
|
55
|
+
rescue
|
|
56
|
+
nil
|
|
57
|
+
end
|
|
58
|
+
raise
|
|
59
|
+
end
|
|
60
|
+
File.chmod(mode, temporary)
|
|
61
|
+
ownership_strategy.chown(temporary, ownership)
|
|
62
|
+
if exclusive
|
|
63
|
+
File.link(temporary, path)
|
|
64
|
+
else
|
|
65
|
+
File.rename(temporary, path)
|
|
66
|
+
end
|
|
67
|
+
ensure
|
|
68
|
+
temporary.unlink if temporary&.exist?
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def read(path)
|
|
72
|
+
JSON.parse(File.read(path))
|
|
73
|
+
rescue SystemCallError, JSON::ParserError
|
|
74
|
+
nil
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ace
|
|
4
|
+
module Hitl
|
|
5
|
+
module Lifecycle
|
|
6
|
+
# The binding seam between the generic request lifecycle and the
|
|
7
|
+
# domain that owns Work/Attempt liveness and attribution (spec
|
|
8
|
+
# 8wm.t.y21 §1). The store REQUIRES a binding; the generic core
|
|
9
|
+
# never carries Work/Attempt semantics itself.
|
|
10
|
+
#
|
|
11
|
+
# Implementations must fail closed: any doubt raises BindingError.
|
|
12
|
+
class Binding
|
|
13
|
+
# Validate a request creation against the requester's exact live
|
|
14
|
+
# attempt. Raises BindingError on any violation or unavailability.
|
|
15
|
+
def validate_request(work:, attempt:, project:, requester:)
|
|
16
|
+
raise NotImplementedError
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Re-verify that the attempt is still live before an answer is
|
|
20
|
+
# delivered or consumed. Raises BindingError when terminal.
|
|
21
|
+
def require_active(work:, attempt:)
|
|
22
|
+
raise NotImplementedError
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require_relative "errors"
|
|
4
|
+
|
|
5
|
+
module Ace
|
|
6
|
+
module Hitl
|
|
7
|
+
module Lifecycle
|
|
8
|
+
# The standing-duty projection (spec 8wm.t.y21 §7; contract of the
|
|
9
|
+
# deployed hitl_status/duty surface): pending answerable requests
|
|
10
|
+
# with their effect declaration visibility, plus everything the
|
|
11
|
+
# effect layer has escalated. Root-only, like the store reads it
|
|
12
|
+
# projects from.
|
|
13
|
+
module Duty
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
def project(store)
|
|
17
|
+
store.require_root!("duty")
|
|
18
|
+
{
|
|
19
|
+
"pending" => store.pending.map { |value| pending_entry(value) },
|
|
20
|
+
"escalated" => store.states.select do |record|
|
|
21
|
+
record["effect_state"] == Effects::OUTCOME_ESCALATED
|
|
22
|
+
end
|
|
23
|
+
}
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def pending_entry(value)
|
|
27
|
+
{
|
|
28
|
+
"id" => value["id"],
|
|
29
|
+
"work" => value["work"],
|
|
30
|
+
"attempt" => value["attempt"],
|
|
31
|
+
"project" => value["project"],
|
|
32
|
+
"harness" => value["harness"],
|
|
33
|
+
"kind" => value["kind"],
|
|
34
|
+
"state" => "created",
|
|
35
|
+
"requester" => value["requester"],
|
|
36
|
+
"created_at" => value["created_at"],
|
|
37
|
+
"has_effect" => value["effect"].is_a?(Hash) && Array(value["effect"]["argv"]).any?
|
|
38
|
+
}
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -0,0 +1,254 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
require_relative "errors"
|
|
5
|
+
require_relative "../atoms/hitl_effect_validator"
|
|
6
|
+
|
|
7
|
+
module Ace
|
|
8
|
+
module Hitl
|
|
9
|
+
module Lifecycle
|
|
10
|
+
# The requester-declared answer-effect layer (spec 8wm.t.y21 §5;
|
|
11
|
+
# recovered from the deployed 8wl.t.ga9 contract). A request may
|
|
12
|
+
# declare a callback that executes exactly once when the answer
|
|
13
|
+
# arrives — always AFTER the answer is relayed — AS THE REQUESTER,
|
|
14
|
+
# through exec-style argv (a shell never sees the answer). The
|
|
15
|
+
# answer content and the substituted argv never appear in any log;
|
|
16
|
+
# full attempts live only in the root-only effects log.
|
|
17
|
+
module Effects
|
|
18
|
+
DEFAULT_TIMEOUT_S = 120
|
|
19
|
+
TERMINATE_GRACE_SECONDS = 0.2
|
|
20
|
+
OUTCOME_OK = "callback-ok"
|
|
21
|
+
OUTCOME_ESCALATED = "callback-escalated"
|
|
22
|
+
|
|
23
|
+
# A Lifecycle::Error so every lifecycle caller's typed rescue
|
|
24
|
+
# (notably the provider seam's orphan-event ask wrapper) catches
|
|
25
|
+
# declaration violations instead of a raw backtrace escaping to
|
|
26
|
+
# the CLI (review F-R1 on W696).
|
|
27
|
+
DeclarationError = Class.new(Lifecycle::Error)
|
|
28
|
+
|
|
29
|
+
module_function
|
|
30
|
+
|
|
31
|
+
# Validated by Atoms::HitlEffectValidator at the CLI boundary;
|
|
32
|
+
# the store re-applies the declaration checks so direct API use
|
|
33
|
+
# cannot bypass the bounds. Declarations are never rewritten.
|
|
34
|
+
def validate_declaration!(effect)
|
|
35
|
+
match = effect[:match]&.to_s
|
|
36
|
+
cwd = (effect[:cwd] || effect[:effect_cwd])&.to_s
|
|
37
|
+
# cwd is a required declaration field (spec §5: {argv:, cwd:,
|
|
38
|
+
# match:, timeout_s:}). A cwd-less declaration would otherwise
|
|
39
|
+
# pass validation and fail at answer time as a misleading
|
|
40
|
+
# Errno::ENOENT spawn escalation (review F-A on W696).
|
|
41
|
+
if cwd.nil? || cwd.strip.empty?
|
|
42
|
+
raise DeclarationError,
|
|
43
|
+
"--effect-cwd is required for an effect callback (absolute path to an existing directory)"
|
|
44
|
+
end
|
|
45
|
+
Atoms::HitlEffectValidator.validate!(
|
|
46
|
+
match: match,
|
|
47
|
+
effect_args: Array(effect[:effect_args] || effect[:argv]).map(&:to_s),
|
|
48
|
+
effect_cwd: cwd,
|
|
49
|
+
effect_timeout: (effect[:timeout_s] || effect[:effect_timeout]).to_s
|
|
50
|
+
)
|
|
51
|
+
rescue Atoms::HitlEffectValidator::ValidationError => e
|
|
52
|
+
raise DeclarationError, e.message
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The persisted record shape, byte-compatible with the deployed
|
|
56
|
+
# contract: {argv:, cwd:, match:, timeout_s:}.
|
|
57
|
+
def normalized_declaration(effect)
|
|
58
|
+
timeout = effect[:timeout_s] || effect[:effect_timeout] || DEFAULT_TIMEOUT_S
|
|
59
|
+
{
|
|
60
|
+
"argv" => Array(effect[:effect_args] || effect[:argv]).map(&:to_s),
|
|
61
|
+
"cwd" => (effect[:cwd] || effect[:effect_cwd]).to_s,
|
|
62
|
+
"match" => effect[:match]&.to_s,
|
|
63
|
+
"timeout_s" => Integer(timeout)
|
|
64
|
+
}
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Executed inside deliver's locked critical section, after the
|
|
68
|
+
# answer is relayed. Exactly one attempt; failure escalates once.
|
|
69
|
+
def run(store, value, answer, requester_uid:, requester_gid:, identity: Identity,
|
|
70
|
+
spawner: Process, group_dropper: nil)
|
|
71
|
+
declaration = value["effect"]
|
|
72
|
+
return nil unless declaration.is_a?(Hash) && Array(declaration["argv"]).any?
|
|
73
|
+
|
|
74
|
+
if !identity.root? && requester_uid != identity.euid
|
|
75
|
+
raise Lifecycle::PermissionError,
|
|
76
|
+
"effect callback requires the requester's identity and root authority to drop to it"
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
match_ok = declaration["match"].nil? || fullmatch?(declaration["match"], answer)
|
|
80
|
+
attempt = {
|
|
81
|
+
"at" => Time.now.to_i,
|
|
82
|
+
"match_ok" => match_ok,
|
|
83
|
+
"timed_out" => false,
|
|
84
|
+
"exit_status" => nil,
|
|
85
|
+
"duration_s" => nil
|
|
86
|
+
}
|
|
87
|
+
if match_ok
|
|
88
|
+
start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
89
|
+
begin
|
|
90
|
+
status, timed_out = execute(
|
|
91
|
+
declaration, answer, requester_uid, requester_gid, spawner,
|
|
92
|
+
group_dropper: group_dropper
|
|
93
|
+
)
|
|
94
|
+
rescue SystemCallError => e
|
|
95
|
+
# A spawn/wait failure (missing binary, non-executable
|
|
96
|
+
# argv, EPERM) must never escape deliver AFTER the answer
|
|
97
|
+
# was relayed: the operator signal would be silently lost.
|
|
98
|
+
# It is an escalation outcome exactly like a timeout or a
|
|
99
|
+
# nonzero exit (spec §5; review F2 on W696).
|
|
100
|
+
attempt["error"] = e.class.name
|
|
101
|
+
status = nil
|
|
102
|
+
timed_out = false
|
|
103
|
+
end
|
|
104
|
+
attempt["timed_out"] = timed_out
|
|
105
|
+
attempt["exit_status"] = status
|
|
106
|
+
attempt["duration_s"] = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - start).round(3)
|
|
107
|
+
end
|
|
108
|
+
attempt["outcome"] = outcome_of(attempt)
|
|
109
|
+
|
|
110
|
+
record_effect_attempt(store, value, attempt)
|
|
111
|
+
effect_state = (attempt["outcome"] == "ok") ? OUTCOME_OK : OUTCOME_ESCALATED
|
|
112
|
+
value["effect_state"] = effect_state
|
|
113
|
+
store.update_public(value, "answer-delivered")
|
|
114
|
+
if effect_state == OUTCOME_ESCALATED
|
|
115
|
+
escalate_once(store, value, attempt)
|
|
116
|
+
end
|
|
117
|
+
effect_state
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
def outcome_of(attempt)
|
|
121
|
+
(attempt["match_ok"] && !attempt["timed_out"] && attempt["exit_status"] == 0) ? "ok" : "escalated"
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# The root-only effects log: redacted attempts (no answer, no
|
|
125
|
+
# argv) plus the deduped escalation marker.
|
|
126
|
+
def record_effect_attempt(store, value, attempt)
|
|
127
|
+
store.effects_dir.mkpath
|
|
128
|
+
path = store.effects_dir.join("#{value["id"]}.json")
|
|
129
|
+
record = AtomicJson.read(path)
|
|
130
|
+
record = {"id" => value["id"], "attempts" => [], "escalated" => nil} unless record.is_a?(Hash)
|
|
131
|
+
record["attempts"] << attempt
|
|
132
|
+
if attempt["outcome"] != "ok" && record["escalated"].nil?
|
|
133
|
+
record["escalated"] = {"at" => Time.now.to_i, "outcome" => attempt["outcome"]}
|
|
134
|
+
end
|
|
135
|
+
AtomicJson.call(path, record, mode: 0o600)
|
|
136
|
+
nil
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# The deduped escalation: recorded once per request in the
|
|
140
|
+
# effects log; the wake/spool glue stays behind the sink seam.
|
|
141
|
+
def escalate_once(store, value, attempt)
|
|
142
|
+
return if value["escalation_spooled"]
|
|
143
|
+
|
|
144
|
+
sink = store.escalation_sink
|
|
145
|
+
value["escalation_spooled"] = true
|
|
146
|
+
return unless sink
|
|
147
|
+
|
|
148
|
+
sink.call(
|
|
149
|
+
request_id: value["id"],
|
|
150
|
+
work: value["work"],
|
|
151
|
+
attempt: value["attempt"],
|
|
152
|
+
project: value["project"],
|
|
153
|
+
outcome: attempt["outcome"]
|
|
154
|
+
)
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Python re.fullmatch equivalent: the whole answer must match.
|
|
158
|
+
def fullmatch?(source, answer)
|
|
159
|
+
Regexp.new("\\A(?:#{source})\\z").match?(answer)
|
|
160
|
+
rescue RegexpError
|
|
161
|
+
false
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
# Exec-style spawn, never a shell. {answer} substitutes once per
|
|
165
|
+
# element as a plain string replace. Output is discarded
|
|
166
|
+
# (redaction). Process.spawn applies only setgid/setuid, so the
|
|
167
|
+
# supplementary-group drop happens around the spawn window (see
|
|
168
|
+
# drop_child_groups) and the forked child inherits the dropped
|
|
169
|
+
# list. Returns [exit_status, timed_out].
|
|
170
|
+
def execute(declaration, answer, requester_uid, requester_gid, spawner = Process,
|
|
171
|
+
group_dropper: nil)
|
|
172
|
+
group_dropper ||= method(:drop_child_groups)
|
|
173
|
+
# Block form: the answer is inserted literally. The
|
|
174
|
+
# replacement-string form would interpret backslash sequences
|
|
175
|
+
# in the answer as backreferences (review 8wq2ztty on PR#336).
|
|
176
|
+
argv = declaration["argv"].map { |element| element.gsub("{answer}") { answer } }
|
|
177
|
+
timeout = declaration["timeout_s"] || DEFAULT_TIMEOUT_S
|
|
178
|
+
deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
|
|
179
|
+
status = nil
|
|
180
|
+
timed_out = false
|
|
181
|
+
group_dropper.call(requester_gid) do
|
|
182
|
+
# [cmd, cmd] forces exec/argv semantics: a single-element
|
|
183
|
+
# argv would otherwise be handed to a shell as a command
|
|
184
|
+
# string (review 8wq2ztu2 on PR#336), and a substituted
|
|
185
|
+
# answer must never become shell syntax.
|
|
186
|
+
#
|
|
187
|
+
# pgroup: true makes the child a process group leader so the
|
|
188
|
+
# timeout can signal its whole group (review 8wq2zttw on
|
|
189
|
+
# PR#336).
|
|
190
|
+
pid = spawner.spawn(
|
|
191
|
+
[argv.first, argv.first],
|
|
192
|
+
*argv.drop(1),
|
|
193
|
+
chdir: declaration["cwd"],
|
|
194
|
+
gid: requester_gid,
|
|
195
|
+
uid: requester_uid,
|
|
196
|
+
pgroup: true,
|
|
197
|
+
out: File::NULL,
|
|
198
|
+
err: File::NULL
|
|
199
|
+
)
|
|
200
|
+
loop do
|
|
201
|
+
_, status = spawner.waitpid2(pid, Process::WNOHANG)
|
|
202
|
+
break if status
|
|
203
|
+
|
|
204
|
+
if Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
|
|
205
|
+
timed_out = true
|
|
206
|
+
# -pid addresses the process group: a callback that
|
|
207
|
+
# forked leaves no descendants running after the
|
|
208
|
+
# timeout. ESRCH means the group is already gone; macOS
|
|
209
|
+
# additionally raises EPERM when signaling a group whose
|
|
210
|
+
# only remaining members are zombies — both are fine,
|
|
211
|
+
# the direct child is reaped by the wait below.
|
|
212
|
+
begin
|
|
213
|
+
spawner.kill("TERM", -pid)
|
|
214
|
+
rescue Errno::ESRCH, Errno::EPERM
|
|
215
|
+
end
|
|
216
|
+
sleep_after_terminate
|
|
217
|
+
begin
|
|
218
|
+
spawner.kill("KILL", -pid)
|
|
219
|
+
rescue Errno::ESRCH, Errno::EPERM
|
|
220
|
+
end
|
|
221
|
+
_, status = spawner.waitpid2(pid)
|
|
222
|
+
break
|
|
223
|
+
end
|
|
224
|
+
sleep 0.05
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
[status&.exitstatus, timed_out]
|
|
228
|
+
end
|
|
229
|
+
|
|
230
|
+
# Supplementary-group isolation for the effect child (spec §5;
|
|
231
|
+
# review F7 on W696): spawn has no groups hook, so root swaps the
|
|
232
|
+
# process supplementary list for exactly the requester's group
|
|
233
|
+
# for the duration of the block and restores it afterwards; the
|
|
234
|
+
# child inherits the dropped list. A non-root process has nothing
|
|
235
|
+
# to drop and would hit EPERM, so it yields unchanged.
|
|
236
|
+
def drop_child_groups(gid)
|
|
237
|
+
return yield unless Process.euid.zero?
|
|
238
|
+
|
|
239
|
+
previous = Process.groups
|
|
240
|
+
Process::Sys.setgroups([gid])
|
|
241
|
+
begin
|
|
242
|
+
yield
|
|
243
|
+
ensure
|
|
244
|
+
Process::Sys.setgroups(previous)
|
|
245
|
+
end
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
def sleep_after_terminate
|
|
249
|
+
sleep(TERMINATE_GRACE_SECONDS)
|
|
250
|
+
end
|
|
251
|
+
end
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
end
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ace
|
|
4
|
+
module Hitl
|
|
5
|
+
module Lifecycle
|
|
6
|
+
# Error model for the generic HITL request lifecycle
|
|
7
|
+
# (spec 8wm.t.y21 §9; ports of the migrated transport CLI error surfaces).
|
|
8
|
+
class Error < StandardError; end
|
|
9
|
+
|
|
10
|
+
# The Work/Attempt binding policy rejected the operation, or the
|
|
11
|
+
# binding authority is unreachable. Never a silent pass.
|
|
12
|
+
class BindingError < Error; end
|
|
13
|
+
|
|
14
|
+
# The operation requires an identity the caller does not have
|
|
15
|
+
# (root-only host-broker operations, foreign-requester answers).
|
|
16
|
+
class PermissionError < Error; end
|
|
17
|
+
|
|
18
|
+
# The request record is unknown, already transitioned, or the
|
|
19
|
+
# requested lifecycle transition does not hold.
|
|
20
|
+
class StateError < Error; end
|
|
21
|
+
|
|
22
|
+
# The answer violates its kind's shape or carries secret-shaped
|
|
23
|
+
# content.
|
|
24
|
+
class AnswerError < Error; end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
end
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "etc"
|
|
4
|
+
|
|
5
|
+
module Ace
|
|
6
|
+
module Hitl
|
|
7
|
+
module Lifecycle
|
|
8
|
+
# The single identity gateway of the generic lifecycle: euid, user
|
|
9
|
+
# and group lookups, and the privilege drop for effect execution.
|
|
10
|
+
# Every identity decision goes through here so tests can pin a
|
|
11
|
+
# faithful unprivileged fixture without patching call sites
|
|
12
|
+
# (spec 8wm.t.y21 §2).
|
|
13
|
+
module Identity
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
def euid
|
|
17
|
+
Process.euid
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def uid
|
|
21
|
+
Process.uid
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def gid
|
|
25
|
+
Process.gid
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def root?
|
|
29
|
+
euid.zero?
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def username
|
|
33
|
+
Etc.getpwuid(euid)&.name || Process.uid.to_s
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# The requester's uid/gid; unknown names fail closed.
|
|
37
|
+
def user_ids(name)
|
|
38
|
+
entry = Etc.getpwnam(name.to_s)
|
|
39
|
+
[entry.uid, entry.gid]
|
|
40
|
+
rescue ArgumentError
|
|
41
|
+
raise Lifecycle::Error, "unknown requester identity: #{name}"
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def group_id(name)
|
|
45
|
+
Etc.getgrnam(name.to_s).gid
|
|
46
|
+
rescue ArgumentError
|
|
47
|
+
raise Lifecycle::Error, "unknown group identity: #{name}"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Drop to the exact requester identity for effect execution.
|
|
51
|
+
# Root drops fully (setgroups/setgid/setuid); a non-root process
|
|
52
|
+
# may only "drop" to itself, so a foreign requester fails closed.
|
|
53
|
+
def drop_to!(uid, gid)
|
|
54
|
+
if root?
|
|
55
|
+
Process::Sys.setgroups([gid])
|
|
56
|
+
Process::Sys.setgid(gid)
|
|
57
|
+
Process::Sys.setuid(uid)
|
|
58
|
+
elsif uid != euid
|
|
59
|
+
raise Lifecycle::PermissionError,
|
|
60
|
+
"effect callback requires the requester's identity and root authority to drop to it"
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Ace
|
|
4
|
+
module Hitl
|
|
5
|
+
module Lifecycle
|
|
6
|
+
# Request kinds and the secret/OTP answer gates (port of the
|
|
7
|
+
# migrated kind model; spec 8wm.t.y21 §4).
|
|
8
|
+
module Kinds
|
|
9
|
+
ALL = %w[
|
|
10
|
+
text choice confirm secret review
|
|
11
|
+
question decision verification otp
|
|
12
|
+
].freeze
|
|
13
|
+
SECRET = %w[otp].freeze
|
|
14
|
+
|
|
15
|
+
REQUEST_ID = /\A[A-Za-z0-9_-]{6,64}\z/
|
|
16
|
+
WORK_ID = /\AW[0-9]+\z/
|
|
17
|
+
ATTEMPT_ID = /\AA-[0-9a-f]{24}\z/
|
|
18
|
+
SAFE_LABEL = /\A[A-Za-z0-9_. -]{1,48}\z/
|
|
19
|
+
OTP_ANSWER = /\A[0-9]{6}\z/
|
|
20
|
+
|
|
21
|
+
# Secret-shape scrubbing: tokens that must never enter a
|
|
22
|
+
# non-OTP answer or a channel message.
|
|
23
|
+
SECRET_SHAPED = Regexp.new(
|
|
24
|
+
"(?:github_pat_[A-Za-z0-9_]+|gh[pousr]_[A-Za-z0-9]+|" \
|
|
25
|
+
"sk-[A-Za-z0-9_-]{20,}|-----BEGIN [A-Z ]+PRIVATE KEY-----|" \
|
|
26
|
+
"\\b(?:otp|token|secret|password)\\s*[:=]\\s*\\S+|" \
|
|
27
|
+
"\\b[0-9]{6}\\b)",
|
|
28
|
+
Regexp::IGNORECASE
|
|
29
|
+
).freeze
|
|
30
|
+
|
|
31
|
+
class << self
|
|
32
|
+
def secret?(kind)
|
|
33
|
+
SECRET.include?(kind.to_s)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def valid?(kind)
|
|
37
|
+
ALL.include?(kind.to_s)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# The deliver/consume answer gate (port of check_answer):
|
|
41
|
+
# OTP answers must be exactly six ASCII digits; every other
|
|
42
|
+
# kind rejects secret-shaped content before persistence.
|
|
43
|
+
def check_answer!(kind, answer)
|
|
44
|
+
if secret?(kind)
|
|
45
|
+
unless OTP_ANSWER.match?(answer)
|
|
46
|
+
raise AnswerError, "OTP answer must be exactly six ASCII digits"
|
|
47
|
+
end
|
|
48
|
+
elsif SECRET_SHAPED.match?(answer)
|
|
49
|
+
raise AnswerError, "secret-shaped content is forbidden in HITL answers"
|
|
50
|
+
end
|
|
51
|
+
nil
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|