ace-test-runner-e2e 0.29.6 → 0.38.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ace-defaults/e2e-runner/config.yml +14 -2
- data/CHANGELOG.md +187 -0
- data/README.md +2 -2
- data/exe/ace-test-e2e-sh +9 -4
- data/handbook/guides/e2e-testing.g.md +43 -9
- data/handbook/guides/scenario-yml-reference.g.md +16 -8
- data/handbook/guides/tc-authoring.g.md +12 -5
- data/handbook/skills/as-e2e-fix/SKILL.md +2 -2
- data/handbook/skills/as-e2e-review/SKILL.md +2 -2
- data/handbook/templates/ace-taskflow-fixture.template.md +17 -17
- data/handbook/templates/agent-experience-report.template.md +3 -2
- data/handbook/templates/scenario.yml.template.yml +13 -2
- data/handbook/templates/tc-file.template.md +14 -4
- data/handbook/workflow-instructions/e2e/analyze-failures.wf.md +53 -6
- data/handbook/workflow-instructions/e2e/create.wf.md +139 -23
- data/handbook/workflow-instructions/e2e/execute.wf.md +11 -7
- data/handbook/workflow-instructions/e2e/fix.wf.md +65 -15
- data/handbook/workflow-instructions/e2e/plan-changes.wf.md +17 -1
- data/handbook/workflow-instructions/e2e/review.wf.md +44 -28
- data/handbook/workflow-instructions/e2e/rewrite.wf.md +17 -3
- data/handbook/workflow-instructions/e2e/run.wf.md +50 -26
- data/handbook/workflow-instructions/e2e/setup-sandbox.wf.md +4 -4
- data/lib/ace/test/end_to_end_runner/atoms/skill_prompt_builder.rb +7 -5
- data/lib/ace/test/end_to_end_runner/atoms/skill_result_parser.rb +73 -7
- data/lib/ace/test/end_to_end_runner/cli/commands/run_test.rb +21 -8
- data/lib/ace/test/end_to_end_runner/models/test_case.rb +8 -2
- data/lib/ace/test/end_to_end_runner/models/test_result.rb +9 -3
- data/lib/ace/test/end_to_end_runner/models/test_scenario.rb +4 -2
- data/lib/ace/test/end_to_end_runner/molecules/affected_detector.rb +7 -2
- data/lib/ace/test/end_to_end_runner/molecules/bwrap_sandbox_backend.rb +271 -0
- data/lib/ace/test/end_to_end_runner/molecules/config_loader.rb +28 -1
- data/lib/ace/test/end_to_end_runner/molecules/integration_runner.rb +122 -0
- data/lib/ace/test/end_to_end_runner/molecules/pipeline_executor.rb +165 -25
- data/lib/ace/test/end_to_end_runner/molecules/pipeline_prompt_bundler.rb +121 -8
- data/lib/ace/test/end_to_end_runner/molecules/pipeline_report_generator.rb +91 -19
- data/lib/ace/test/end_to_end_runner/molecules/pipeline_sandbox_builder.rb +119 -18
- data/lib/ace/test/end_to_end_runner/molecules/report_writer.rb +13 -12
- data/lib/ace/test/end_to_end_runner/molecules/sandbox_runtime_builder.rb +282 -0
- data/lib/ace/test/end_to_end_runner/molecules/scenario_loader.rb +85 -5
- data/lib/ace/test/end_to_end_runner/molecules/setup_executor.rb +98 -16
- data/lib/ace/test/end_to_end_runner/molecules/suite_report_writer.rb +241 -97
- data/lib/ace/test/end_to_end_runner/molecules/test_discoverer.rb +38 -13
- data/lib/ace/test/end_to_end_runner/molecules/test_executor.rb +27 -5
- data/lib/ace/test/end_to_end_runner/organisms/suite_orchestrator.rb +73 -15
- data/lib/ace/test/end_to_end_runner/organisms/test_orchestrator.rb +120 -19
- data/lib/ace/test/end_to_end_runner/version.rb +1 -1
- data/lib/ace/test/end_to_end_runner.rb +2 -0
- metadata +19 -2
|
@@ -13,6 +13,7 @@ module Ace
|
|
|
13
13
|
# - **Failed**: 0
|
|
14
14
|
# - **Total**: 8
|
|
15
15
|
# - **Report Paths**: 8p5jo2-lint-ts001-reports/*
|
|
16
|
+
# - **Observations**: None
|
|
16
17
|
# - **Issues**: None
|
|
17
18
|
#
|
|
18
19
|
# Falls back to ResultParser.parse() for JSON responses.
|
|
@@ -45,6 +46,7 @@ module Ace
|
|
|
45
46
|
fields[:failed] = extract_field(text, "Failed")
|
|
46
47
|
fields[:total] = extract_field(text, "Total")
|
|
47
48
|
fields[:report_paths] = extract_field(text, "Report Paths")
|
|
49
|
+
fields[:observations] = extract_field(text, "Observations")
|
|
48
50
|
fields[:issues] = extract_field(text, "Issues")
|
|
49
51
|
|
|
50
52
|
# Need at least test_id and status for a valid parse
|
|
@@ -69,8 +71,7 @@ module Ace
|
|
|
69
71
|
passed.times { |i| test_cases << {id: "TC-#{format("%03d", i + 1)}", description: "", status: "pass", actual: "", notes: ""} }
|
|
70
72
|
failed.times { |i| test_cases << {id: "TC-#{format("%03d", passed + i + 1)}", description: "", status: "fail", actual: "", notes: ""} }
|
|
71
73
|
|
|
72
|
-
|
|
73
|
-
observations = (issues && issues.downcase != "none") ? issues : ""
|
|
74
|
+
observations = normalize_observations(parsed[:observations], parsed[:issues])
|
|
74
75
|
|
|
75
76
|
{
|
|
76
77
|
test_id: parsed[:test_id],
|
|
@@ -131,8 +132,8 @@ module Ace
|
|
|
131
132
|
fields[:failed_tcs] = extract_field(text, "Failed TCs")
|
|
132
133
|
fields[:issues] = extract_field(text, "Issues")
|
|
133
134
|
|
|
134
|
-
return
|
|
135
|
-
|
|
135
|
+
return parse_minimal_verifier(text) unless fields[:test_id] && fields[:status]
|
|
136
|
+
return parse(text) unless fields[:tcs_passed] && fields[:tcs_failed] && fields[:tcs_total]
|
|
136
137
|
|
|
137
138
|
passed = fields[:tcs_passed].to_i
|
|
138
139
|
failed = fields[:tcs_failed].to_i
|
|
@@ -180,6 +181,58 @@ module Ace
|
|
|
180
181
|
}
|
|
181
182
|
end
|
|
182
183
|
|
|
184
|
+
def self.parse_minimal_verifier(text)
|
|
185
|
+
compact = text.to_s.strip
|
|
186
|
+
results_match = compact.match(/Results:\s*(\d+)\s*\/\s*(\d+)\s*passed/i)
|
|
187
|
+
if results_match
|
|
188
|
+
passed = results_match[1].to_i
|
|
189
|
+
total = results_match[2].to_i
|
|
190
|
+
status = if total.zero?
|
|
191
|
+
"fail"
|
|
192
|
+
elsif passed == total
|
|
193
|
+
"pass"
|
|
194
|
+
elsif passed.zero?
|
|
195
|
+
"fail"
|
|
196
|
+
else
|
|
197
|
+
"partial"
|
|
198
|
+
end
|
|
199
|
+
failed = [total - passed, 0].max
|
|
200
|
+
test_cases = []
|
|
201
|
+
passed.times { |i| test_cases << {id: "TC-#{format("%03d", i + 1)}", description: "", status: "pass", actual: "", notes: ""} }
|
|
202
|
+
failed.times { |i| test_cases << {id: "TC-#{format("%03d", passed + i + 1)}", description: "", status: "fail", actual: "", notes: "", category: "unknown"} }
|
|
203
|
+
|
|
204
|
+
return {
|
|
205
|
+
test_id: "",
|
|
206
|
+
status: status,
|
|
207
|
+
test_cases: test_cases,
|
|
208
|
+
summary: "#{passed}/#{total} passed",
|
|
209
|
+
observations: compact
|
|
210
|
+
}
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
status_match = compact.match(/\b(PASS|FAIL|PARTIAL|ERROR)\b/i)
|
|
214
|
+
return parse(text) unless status_match
|
|
215
|
+
|
|
216
|
+
status = normalize_status(status_match[1])
|
|
217
|
+
evidence = compact.sub(/^.*?\b#{Regexp.escape(status_match[1])}\b[:\-\s]*/i, "").strip
|
|
218
|
+
tc_status = (status == "pass") ? "pass" : "fail"
|
|
219
|
+
|
|
220
|
+
{
|
|
221
|
+
test_id: "",
|
|
222
|
+
status: status,
|
|
223
|
+
test_cases: [{
|
|
224
|
+
id: "TC-001",
|
|
225
|
+
description: "",
|
|
226
|
+
status: tc_status,
|
|
227
|
+
actual: "",
|
|
228
|
+
notes: evidence,
|
|
229
|
+
category: ((tc_status == "fail") ? "unknown" : nil)
|
|
230
|
+
}],
|
|
231
|
+
summary: evidence.empty? ? status : evidence,
|
|
232
|
+
observations: evidence
|
|
233
|
+
}
|
|
234
|
+
end
|
|
235
|
+
|
|
183
236
|
# Parse TC-level markdown return contract
|
|
184
237
|
def self.parse_tc_markdown(text)
|
|
185
238
|
fields = {}
|
|
@@ -188,6 +241,7 @@ module Ace
|
|
|
188
241
|
fields[:tc_id] = extract_field(text, "TC ID")
|
|
189
242
|
fields[:status] = extract_field(text, "Status")
|
|
190
243
|
fields[:report_paths] = extract_field(text, "Report Paths")
|
|
244
|
+
fields[:observations] = extract_field(text, "Observations")
|
|
191
245
|
fields[:issues] = extract_field(text, "Issues")
|
|
192
246
|
|
|
193
247
|
# Need test_id, tc_id, and status for a valid TC parse
|
|
@@ -200,8 +254,7 @@ module Ace
|
|
|
200
254
|
def self.to_tc_normalized(parsed)
|
|
201
255
|
parsed[:status] = normalize_status(parsed[:status])
|
|
202
256
|
|
|
203
|
-
|
|
204
|
-
observations = (issues && issues.downcase != "none") ? issues : ""
|
|
257
|
+
observations = normalize_observations(parsed[:observations], parsed[:issues])
|
|
205
258
|
|
|
206
259
|
{
|
|
207
260
|
test_id: parsed[:test_id],
|
|
@@ -234,9 +287,22 @@ module Ace
|
|
|
234
287
|
end
|
|
235
288
|
end
|
|
236
289
|
|
|
290
|
+
def self.normalize_observations(primary, fallback = nil)
|
|
291
|
+
[primary, fallback].each do |value|
|
|
292
|
+
next if value.nil?
|
|
293
|
+
|
|
294
|
+
normalized = value.to_s.strip
|
|
295
|
+
next if normalized.empty? || normalized.casecmp("none").zero?
|
|
296
|
+
|
|
297
|
+
return normalized
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
""
|
|
301
|
+
end
|
|
302
|
+
|
|
237
303
|
private_class_method :parse_markdown, :to_normalized, :extract_field,
|
|
238
304
|
:parse_tc_markdown, :to_tc_normalized, :normalize_status,
|
|
239
|
-
:parse_failed_tcs
|
|
305
|
+
:parse_failed_tcs, :parse_minimal_verifier, :normalize_observations
|
|
240
306
|
end
|
|
241
307
|
end
|
|
242
308
|
end
|
|
@@ -20,9 +20,9 @@ module Ace
|
|
|
20
20
|
desc <<~DESC.strip
|
|
21
21
|
Run E2E tests via LLM execution
|
|
22
22
|
|
|
23
|
-
Discovers and executes
|
|
24
|
-
Tests are sent to an LLM
|
|
25
|
-
structured results.
|
|
23
|
+
Discovers and executes deterministic preflight tests from test/feat
|
|
24
|
+
before TS-* agent scenarios from test/e2e. Tests are sent to an LLM
|
|
25
|
+
provider which executes the scenario steps and returns structured results.
|
|
26
26
|
|
|
27
27
|
Output:
|
|
28
28
|
Exit codes: 0 (all pass), 1 (any fail/error)
|
|
@@ -35,7 +35,7 @@ module Ace
|
|
|
35
35
|
"ace-lint --provider gemini:flash # Use specific provider",
|
|
36
36
|
"ace-lint --provider glite # Use API provider (predict mode)",
|
|
37
37
|
"ace-lint --tags smoke # Run only smoke-tagged scenarios",
|
|
38
|
-
"ace-lint TS-LINT-003 --dry-run # Preview
|
|
38
|
+
"ace-lint TS-LINT-003 --dry-run # Preview preflight and scenario phases"
|
|
39
39
|
]
|
|
40
40
|
|
|
41
41
|
argument :package, required: true, desc: "Package name (e.g., ace-lint)"
|
|
@@ -55,7 +55,7 @@ module Ace
|
|
|
55
55
|
option :report_dir, type: :string,
|
|
56
56
|
desc: "Explicit report directory path (overrides computed path)"
|
|
57
57
|
option :dry_run, type: :boolean,
|
|
58
|
-
desc: "Preview which scenarios would run without executing"
|
|
58
|
+
desc: "Preview which preflight tests and scenarios would run without executing"
|
|
59
59
|
option :tags, type: :string,
|
|
60
60
|
desc: "Comma-separated scenario tags to include"
|
|
61
61
|
option :verify, type: :boolean,
|
|
@@ -110,7 +110,7 @@ module Ace
|
|
|
110
110
|
|
|
111
111
|
private
|
|
112
112
|
|
|
113
|
-
# Handle dry-run mode: preview which scenarios would run
|
|
113
|
+
# Handle dry-run mode: preview which preflight tests and scenarios would run
|
|
114
114
|
#
|
|
115
115
|
# @param package [String] Package name
|
|
116
116
|
# @param test_id [String, nil] Test ID
|
|
@@ -125,15 +125,28 @@ module Ace
|
|
|
125
125
|
tags: tags,
|
|
126
126
|
base_dir: Dir.pwd
|
|
127
127
|
)
|
|
128
|
-
|
|
128
|
+
preflight_files = discoverer.find_integration_tests(package: package, base_dir: Dir.pwd)
|
|
129
|
+
if files.empty? && preflight_files.empty?
|
|
129
130
|
raise Ace::Support::Cli::Error.new(
|
|
130
131
|
"No tests found for package '#{package}'" +
|
|
131
132
|
(test_id ? " with ID '#{test_id}'" : "")
|
|
132
133
|
)
|
|
133
134
|
end
|
|
134
135
|
|
|
135
|
-
output.puts "Dry run: preview of
|
|
136
|
+
output.puts "Dry run: preview of execution phases"
|
|
136
137
|
output.puts ""
|
|
138
|
+
output.puts "Phase 1: deterministic preflight"
|
|
139
|
+
if preflight_files.empty?
|
|
140
|
+
output.puts " (none)"
|
|
141
|
+
else
|
|
142
|
+
preflight_files.each do |file|
|
|
143
|
+
output.puts " [preflight] #{file}"
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
output.puts ""
|
|
147
|
+
output.puts "Phase 2: scenarios"
|
|
148
|
+
output.puts " (none)" if files.empty?
|
|
149
|
+
output.puts "" unless files.empty?
|
|
137
150
|
|
|
138
151
|
files.each do |file|
|
|
139
152
|
scenario = loader.load(File.dirname(file))
|
|
@@ -9,7 +9,8 @@ module Ace
|
|
|
9
9
|
# Contains parsed frontmatter metadata and the full markdown body
|
|
10
10
|
# from an independent test case file within a scenario directory.
|
|
11
11
|
class TestCase
|
|
12
|
-
attr_reader :tc_id, :title, :content, :file_path, :pending, :goal_format
|
|
12
|
+
attr_reader :tc_id, :title, :content, :file_path, :pending, :goal_format,
|
|
13
|
+
:declared_artifacts, :optional_artifacts
|
|
13
14
|
|
|
14
15
|
# @param tc_id [String] Test case identifier (e.g., "TC-001")
|
|
15
16
|
# @param title [String] Test case title from frontmatter
|
|
@@ -17,13 +18,18 @@ module Ace
|
|
|
17
18
|
# @param file_path [String] Absolute path to the source test file
|
|
18
19
|
# @param pending [String, nil] Pending reason (presence = pending, value = reason)
|
|
19
20
|
# @param goal_format [String, nil] Test case source format ("standalone")
|
|
20
|
-
|
|
21
|
+
# @param declared_artifacts [Array<String>] Required artifact paths under results/tc/*
|
|
22
|
+
# @param optional_artifacts [Array<String>] Optional artifact paths under results/tc/*
|
|
23
|
+
def initialize(tc_id:, title:, content:, file_path:, pending: nil, goal_format: nil,
|
|
24
|
+
declared_artifacts: [], optional_artifacts: [])
|
|
21
25
|
@tc_id = tc_id
|
|
22
26
|
@title = title
|
|
23
27
|
@content = content
|
|
24
28
|
@file_path = file_path
|
|
25
29
|
@pending = pending
|
|
26
30
|
@goal_format = goal_format
|
|
31
|
+
@declared_artifacts = declared_artifacts
|
|
32
|
+
@optional_artifacts = optional_artifacts
|
|
27
33
|
end
|
|
28
34
|
|
|
29
35
|
# Whether this test case is pending (should be skipped)
|
|
@@ -10,7 +10,7 @@ module Ace
|
|
|
10
10
|
# from executing a test scenario via LLM.
|
|
11
11
|
class TestResult
|
|
12
12
|
attr_reader :test_id, :status, :test_cases, :summary,
|
|
13
|
-
:started_at, :completed_at, :report_dir, :error
|
|
13
|
+
:started_at, :completed_at, :report_dir, :error, :metadata, :observations
|
|
14
14
|
|
|
15
15
|
# @param test_id [String] Test identifier
|
|
16
16
|
# @param status [String] Overall status: "pass", "fail", "partial", "error"
|
|
@@ -20,8 +20,10 @@ module Ace
|
|
|
20
20
|
# @param completed_at [Time] When execution completed
|
|
21
21
|
# @param report_dir [String, nil] Path to the reports directory
|
|
22
22
|
# @param error [String, nil] Error message if execution failed
|
|
23
|
+
# @param observations [String] Runner/verifier observations for report context
|
|
24
|
+
# @param metadata [Hash] Additional structured phase/report metadata
|
|
23
25
|
def initialize(test_id:, status:, test_cases: [], summary: "",
|
|
24
|
-
started_at: nil, completed_at: nil, report_dir: nil, error: nil)
|
|
26
|
+
started_at: nil, completed_at: nil, report_dir: nil, error: nil, observations: "", metadata: {})
|
|
25
27
|
@test_id = test_id
|
|
26
28
|
@status = status
|
|
27
29
|
@test_cases = test_cases
|
|
@@ -30,6 +32,8 @@ module Ace
|
|
|
30
32
|
@completed_at = completed_at || Time.now
|
|
31
33
|
@report_dir = report_dir
|
|
32
34
|
@error = error
|
|
35
|
+
@observations = observations.to_s
|
|
36
|
+
@metadata = metadata
|
|
33
37
|
end
|
|
34
38
|
|
|
35
39
|
# Check if the test passed
|
|
@@ -94,7 +98,9 @@ module Ace
|
|
|
94
98
|
started_at: started_at,
|
|
95
99
|
completed_at: completed_at,
|
|
96
100
|
report_dir: dir,
|
|
97
|
-
error: error
|
|
101
|
+
error: error,
|
|
102
|
+
observations: observations,
|
|
103
|
+
metadata: metadata
|
|
98
104
|
)
|
|
99
105
|
end
|
|
100
106
|
|
|
@@ -12,7 +12,7 @@ module Ace
|
|
|
12
12
|
attr_reader :test_id, :title, :area, :package, :priority, :duration,
|
|
13
13
|
:requires, :file_path, :content, :timeout,
|
|
14
14
|
:setup_steps, :dir_path, :fixture_path, :test_cases,
|
|
15
|
-
:tags, :tool_under_test, :sandbox_layout
|
|
15
|
+
:tags, :tool_under_test, :sandbox_layout, :sandbox_profile
|
|
16
16
|
|
|
17
17
|
# @param test_id [String] Test identifier (e.g., "TS-LINT-001")
|
|
18
18
|
# @param title [String] Test title
|
|
@@ -31,11 +31,12 @@ module Ace
|
|
|
31
31
|
# @param tags [Array<String>] Scenario-level tags for discovery-time filtering
|
|
32
32
|
# @param tool_under_test [String, nil] Primary tool under test
|
|
33
33
|
# @param sandbox_layout [Hash] Declared sandbox artifact layout
|
|
34
|
+
# @param sandbox_profile [String] Sandbox bootstrap profile
|
|
34
35
|
def initialize(test_id:, title:, area:, package:, file_path:, content:,
|
|
35
36
|
priority: "medium", duration: "~5min", requires: {},
|
|
36
37
|
setup_steps: [], dir_path: nil, fixture_path: nil, test_cases: [],
|
|
37
38
|
timeout: nil, tags: [], tool_under_test: nil,
|
|
38
|
-
sandbox_layout: {})
|
|
39
|
+
sandbox_layout: {}, sandbox_profile: "ace-default")
|
|
39
40
|
@test_id = test_id
|
|
40
41
|
@title = title
|
|
41
42
|
@area = area
|
|
@@ -53,6 +54,7 @@ module Ace
|
|
|
53
54
|
@tags = tags
|
|
54
55
|
@tool_under_test = tool_under_test
|
|
55
56
|
@sandbox_layout = sandbox_layout
|
|
57
|
+
@sandbox_profile = sandbox_profile
|
|
56
58
|
end
|
|
57
59
|
|
|
58
60
|
# Generate short package name (without ace- prefix)
|
|
@@ -39,10 +39,15 @@ module Ace
|
|
|
39
39
|
# @return [Array<String>] Changed file paths
|
|
40
40
|
def get_changed_files(base_dir, ref)
|
|
41
41
|
# Run git diff to get changed files using array-based command for security
|
|
42
|
-
output, status = Open3.
|
|
42
|
+
output, stderr, status = Open3.capture3("git", "diff", "--name-only", ref, "--",
|
|
43
43
|
chdir: base_dir)
|
|
44
44
|
|
|
45
|
-
|
|
45
|
+
unless status.success?
|
|
46
|
+
message = stderr.to_s.strip
|
|
47
|
+
message = "git diff exited with status #{status.exitstatus}" if message.empty?
|
|
48
|
+
warn "Warning: git detection failed: #{message}"
|
|
49
|
+
return []
|
|
50
|
+
end
|
|
46
51
|
|
|
47
52
|
output.lines.map(&:strip).reject(&:empty?)
|
|
48
53
|
rescue => e
|
|
@@ -0,0 +1,271 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "etc"
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "open3"
|
|
6
|
+
|
|
7
|
+
module Ace
|
|
8
|
+
module Test
|
|
9
|
+
module EndToEndRunner
|
|
10
|
+
module Molecules
|
|
11
|
+
# Wraps subprocesses in a lightweight bubblewrap sandbox on Linux.
|
|
12
|
+
class BwrapSandboxBackend
|
|
13
|
+
BUNDLER_ENV_PREFIXES = %w[BUNDLE BUNDLER].freeze
|
|
14
|
+
STRIPPED_ENV_KEYS = %w[RUBYOPT RUBYLIB].freeze
|
|
15
|
+
PROVIDER_HOME_MOUNTS = [
|
|
16
|
+
[".claude", ".claude"],
|
|
17
|
+
[".codex", ".codex"],
|
|
18
|
+
[".gemini", ".gemini"],
|
|
19
|
+
[".pi", ".pi"],
|
|
20
|
+
[".local/share/opencode", ".local/share/opencode"]
|
|
21
|
+
].freeze
|
|
22
|
+
DEFAULT_SYSTEM_MOUNTS = %w[/usr /bin /sbin /lib /lib64 /etc /opt /var/lib/flatpak/exports].freeze
|
|
23
|
+
|
|
24
|
+
attr_reader :sandbox_root
|
|
25
|
+
|
|
26
|
+
def self.supported?
|
|
27
|
+
Gem.win_platform? == false && RUBY_PLATFORM.include?("linux")
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def self.available?(bwrap_path: "bwrap")
|
|
31
|
+
return false unless supported?
|
|
32
|
+
|
|
33
|
+
system("which", bwrap_path, out: File::NULL, err: File::NULL)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def initialize(sandbox_root:, source_root: nil, bwrap_path: "bwrap", outer_env: nil)
|
|
37
|
+
@sandbox_root = File.expand_path(sandbox_root)
|
|
38
|
+
@source_root = source_root ? File.expand_path(source_root) : infer_source_root
|
|
39
|
+
@bwrap_path = bwrap_path
|
|
40
|
+
@outer_env = sanitized_outer_env(outer_env)
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def ensure_available!
|
|
44
|
+
return if self.class.available?(bwrap_path: @bwrap_path)
|
|
45
|
+
|
|
46
|
+
raise "bubblewrap is required for Linux E2E sandboxing but '#{@bwrap_path}' is not available"
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
def prepared_env(base_env = {})
|
|
50
|
+
env = stringify_keys(base_env)
|
|
51
|
+
STRIPPED_ENV_KEYS.each { |key| env.delete(key) }
|
|
52
|
+
env["PROJECT_ROOT_PATH"] = @sandbox_root
|
|
53
|
+
env["ACE_E2E_SOURCE_ROOT"] ||= @source_root if @source_root
|
|
54
|
+
env["HOME"] = sandbox_home
|
|
55
|
+
env["TMPDIR"] = sandbox_tmp
|
|
56
|
+
env["XDG_RUNTIME_DIR"] = sandbox_runtime_dir
|
|
57
|
+
env["TMUX_TMPDIR"] ||= sandbox_runtime_dir
|
|
58
|
+
env["BUNDLE_GEMFILE"] ||= File.join(@sandbox_root, ".ace-local", "e2e-runtime", "Gemfile")
|
|
59
|
+
env["ACE_CONFIG_PATH"] ||= File.join(@sandbox_root, ".ace")
|
|
60
|
+
env["BUNDLE_APP_CONFIG"] ||= bundler_app_config
|
|
61
|
+
env["BUNDLE_USER_HOME"] ||= bundler_home
|
|
62
|
+
env["BUNDLE_USER_CACHE"] ||= bundler_cache
|
|
63
|
+
env["BUNDLE_USER_CONFIG"] ||= bundler_user_config
|
|
64
|
+
env["PATH"] ||= ENV["PATH"].to_s
|
|
65
|
+
env
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def wrap_command(cmd, chdir:, env: {})
|
|
69
|
+
command_prefix(chdir: chdir, env: env) + Array(cmd)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def command_prefix(chdir:, env: {})
|
|
73
|
+
ensure_available!
|
|
74
|
+
merged_env = prepared_env(env)
|
|
75
|
+
ensure_runtime_dirs(merged_env)
|
|
76
|
+
|
|
77
|
+
args = [
|
|
78
|
+
@bwrap_path,
|
|
79
|
+
"--clearenv",
|
|
80
|
+
"--die-with-parent",
|
|
81
|
+
"--new-session",
|
|
82
|
+
"--proc", "/proc",
|
|
83
|
+
"--dev-bind", "/dev", "/dev",
|
|
84
|
+
"--tmpfs", "/tmp",
|
|
85
|
+
"--tmpfs", home_root,
|
|
86
|
+
"--dir", merged_env.fetch("HOME"),
|
|
87
|
+
"--bind", merged_env.fetch("HOME"), merged_env.fetch("HOME")
|
|
88
|
+
]
|
|
89
|
+
|
|
90
|
+
host_mounts(merged_env.fetch("PATH")).each do |mount|
|
|
91
|
+
append_bind(args, mount, mount, read_only: true)
|
|
92
|
+
end
|
|
93
|
+
append_bind(args, @source_root, @source_root, read_only: true) if @source_root
|
|
94
|
+
append_bind(args, @sandbox_root, @sandbox_root, read_only: false)
|
|
95
|
+
append_bind(args, support_root, support_root, read_only: false)
|
|
96
|
+
ruby_root = merged_env["ACE_E2E_SANDBOX_RUBY_ROOT"].to_s
|
|
97
|
+
append_bind(args, ruby_root, ruby_root, read_only: true) if !ruby_root.empty? && File.exist?(ruby_root)
|
|
98
|
+
bind_provider_homes(args, merged_env.fetch("HOME"))
|
|
99
|
+
|
|
100
|
+
setenvs = {
|
|
101
|
+
"HOME" => merged_env.fetch("HOME"),
|
|
102
|
+
"TMPDIR" => merged_env.fetch("TMPDIR"),
|
|
103
|
+
"XDG_RUNTIME_DIR" => merged_env.fetch("XDG_RUNTIME_DIR"),
|
|
104
|
+
"TMUX_TMPDIR" => merged_env.fetch("TMUX_TMPDIR"),
|
|
105
|
+
"PATH" => merged_env.fetch("PATH"),
|
|
106
|
+
"PROJECT_ROOT_PATH" => merged_env.fetch("PROJECT_ROOT_PATH")
|
|
107
|
+
}
|
|
108
|
+
setenvs["ACE_E2E_SOURCE_ROOT"] = merged_env["ACE_E2E_SOURCE_ROOT"] if merged_env["ACE_E2E_SOURCE_ROOT"]
|
|
109
|
+
setenvs["ACE_E2E_SANDBOX_RUBY_ROOT"] = ruby_root unless ruby_root.empty?
|
|
110
|
+
setenvs["ACE_TMUX_SESSION"] = merged_env["ACE_TMUX_SESSION"] if merged_env["ACE_TMUX_SESSION"]
|
|
111
|
+
|
|
112
|
+
merged_env.each do |key, value|
|
|
113
|
+
next if setenvs.key?(key)
|
|
114
|
+
next if value.nil?
|
|
115
|
+
|
|
116
|
+
setenvs[key] = value.to_s
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
setenvs.each do |key, value|
|
|
120
|
+
args.concat(["--setenv", key, value])
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
args.concat(["--chdir", File.expand_path(chdir), "--"])
|
|
124
|
+
args
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def capture3(cmd, chdir:, env: {}, stdin_data: nil)
|
|
128
|
+
Open3.capture3(
|
|
129
|
+
@outer_env,
|
|
130
|
+
*wrap_command(cmd, chdir: chdir, env: env),
|
|
131
|
+
chdir: "/",
|
|
132
|
+
stdin_data: stdin_data
|
|
133
|
+
)
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def exec(cmd, chdir:, env: {})
|
|
137
|
+
Kernel.exec(@outer_env, *wrap_command(cmd, chdir: chdir, env: env))
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
private
|
|
141
|
+
|
|
142
|
+
def sandbox_home
|
|
143
|
+
File.join(support_root, "home")
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def sandbox_tmp
|
|
147
|
+
File.join(support_root, "tmp")
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def sandbox_runtime_dir
|
|
151
|
+
File.join(support_root, "runtime")
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
def bundler_root
|
|
155
|
+
File.join(support_root, "bundler")
|
|
156
|
+
end
|
|
157
|
+
|
|
158
|
+
def bundler_home
|
|
159
|
+
File.join(bundler_root, "home")
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def bundler_cache
|
|
163
|
+
File.join(bundler_root, "cache")
|
|
164
|
+
end
|
|
165
|
+
|
|
166
|
+
def bundler_user_config
|
|
167
|
+
File.join(bundler_root, "config")
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
def bundler_app_config
|
|
171
|
+
File.join(bundler_root, "app-config")
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def home_root
|
|
175
|
+
File.dirname(actual_home)
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def actual_home
|
|
179
|
+
File.expand_path(ENV.fetch("HOME"))
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def infer_source_root
|
|
183
|
+
env_root = ENV["ACE_E2E_SOURCE_ROOT"].to_s.strip
|
|
184
|
+
return File.expand_path(env_root) unless env_root.empty?
|
|
185
|
+
|
|
186
|
+
File.expand_path(Dir.pwd)
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
def support_root
|
|
190
|
+
"#{@sandbox_root}.support"
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def host_mounts(path_value)
|
|
194
|
+
mounts = DEFAULT_SYSTEM_MOUNTS.select { |path| File.exist?(path) }
|
|
195
|
+
path_value.to_s.split(File::PATH_SEPARATOR).each do |entry|
|
|
196
|
+
next if entry.to_s.strip.empty?
|
|
197
|
+
|
|
198
|
+
expanded = File.expand_path(entry)
|
|
199
|
+
next unless File.exist?(expanded)
|
|
200
|
+
|
|
201
|
+
mounts << expanded
|
|
202
|
+
mounts << mise_install_root(expanded) if expanded.include?("/.local/share/mise/installs/")
|
|
203
|
+
end
|
|
204
|
+
|
|
205
|
+
mounts.compact.uniq.sort
|
|
206
|
+
end
|
|
207
|
+
|
|
208
|
+
def mise_install_root(path)
|
|
209
|
+
match = path.match(%r{\A(.*/\.local/share/mise/installs/[^/]+/[^/]+)})
|
|
210
|
+
match && match[1]
|
|
211
|
+
end
|
|
212
|
+
|
|
213
|
+
def ensure_runtime_dirs(env)
|
|
214
|
+
[
|
|
215
|
+
@sandbox_root,
|
|
216
|
+
support_root,
|
|
217
|
+
env.fetch("HOME"),
|
|
218
|
+
env.fetch("TMPDIR"),
|
|
219
|
+
env.fetch("XDG_RUNTIME_DIR"),
|
|
220
|
+
env.fetch("BUNDLE_APP_CONFIG"),
|
|
221
|
+
env.fetch("BUNDLE_USER_HOME"),
|
|
222
|
+
env.fetch("BUNDLE_USER_CACHE")
|
|
223
|
+
].each { |path| FileUtils.mkdir_p(path) }
|
|
224
|
+
FileUtils.chmod(0o700, env.fetch("XDG_RUNTIME_DIR")) if File.exist?(env.fetch("XDG_RUNTIME_DIR"))
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
def bind_provider_homes(args, sandbox_home_dir)
|
|
228
|
+
PROVIDER_HOME_MOUNTS.each do |source_suffix, target_suffix|
|
|
229
|
+
source = File.join(actual_home, source_suffix)
|
|
230
|
+
next unless File.exist?(source)
|
|
231
|
+
|
|
232
|
+
target = File.join(sandbox_home_dir, target_suffix)
|
|
233
|
+
FileUtils.mkdir_p(File.dirname(target))
|
|
234
|
+
FileUtils.mkdir_p(target) if File.directory?(source)
|
|
235
|
+
append_bind(args, source, target, read_only: false)
|
|
236
|
+
end
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
def append_bind(args, source, target, read_only:)
|
|
240
|
+
source = File.expand_path(source)
|
|
241
|
+
target = File.expand_path(target)
|
|
242
|
+
FileUtils.mkdir_p(File.dirname(target))
|
|
243
|
+
FileUtils.mkdir_p(target) if File.directory?(source)
|
|
244
|
+
args.concat([read_only ? "--ro-bind" : "--bind", source, target])
|
|
245
|
+
end
|
|
246
|
+
|
|
247
|
+
def stringify_keys(hash)
|
|
248
|
+
hash.each_with_object({}) { |(key, value), acc| acc[key.to_s] = value }
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
def sanitized_process_env(base_env)
|
|
252
|
+
stringify_keys(base_env).each_with_object({}) do |(key, value), env|
|
|
253
|
+
next if strip_env_key?(key)
|
|
254
|
+
|
|
255
|
+
env[key] = value
|
|
256
|
+
end
|
|
257
|
+
end
|
|
258
|
+
|
|
259
|
+
def sanitized_outer_env(outer_env)
|
|
260
|
+
path = stringify_keys(outer_env || {"PATH" => ENV["PATH"].to_s})["PATH"]
|
|
261
|
+
{"PATH" => path.to_s.empty? ? ENV["PATH"].to_s : path.to_s}
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
def strip_env_key?(key)
|
|
265
|
+
STRIPPED_ENV_KEYS.include?(key) || BUNDLER_ENV_PREFIXES.any? { |prefix| key.start_with?(prefix) }
|
|
266
|
+
end
|
|
267
|
+
end
|
|
268
|
+
end
|
|
269
|
+
end
|
|
270
|
+
end
|
|
271
|
+
end
|
|
@@ -23,7 +23,22 @@ module Ace
|
|
|
23
23
|
# @return [String] Default provider from config
|
|
24
24
|
def self.default_provider
|
|
25
25
|
config = load
|
|
26
|
-
config.dig("execution", "
|
|
26
|
+
config.dig("execution", "runner_provider") ||
|
|
27
|
+
config.dig("execution", "provider") || "claude:sonnet"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# @return [String] Default runner provider from config
|
|
31
|
+
def self.default_runner_provider
|
|
32
|
+
config = load
|
|
33
|
+
config.dig("execution", "runner_provider") ||
|
|
34
|
+
config.dig("execution", "provider") || "claude:sonnet"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
# @return [String] Default verifier provider from config
|
|
38
|
+
def self.default_verifier_provider
|
|
39
|
+
config = load
|
|
40
|
+
config.dig("execution", "verifier_provider") ||
|
|
41
|
+
config.dig("execution", "provider") || "claude:sonnet"
|
|
27
42
|
end
|
|
28
43
|
|
|
29
44
|
# @return [Integer] Default timeout from config
|
|
@@ -38,6 +53,18 @@ module Ace
|
|
|
38
53
|
config.dig("execution", "parallel") || 3
|
|
39
54
|
end
|
|
40
55
|
|
|
56
|
+
# @return [String] Default sandbox bootstrap profile
|
|
57
|
+
def self.default_sandbox_profile
|
|
58
|
+
config = load
|
|
59
|
+
config.dig("sandbox", "profile") || "ace-default"
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# @return [String] Dedicated Ruby version for sandbox runtime
|
|
63
|
+
def self.default_sandbox_ruby_version
|
|
64
|
+
config = load
|
|
65
|
+
config.dig("sandbox", "ruby_version") || "3.4.9"
|
|
66
|
+
end
|
|
67
|
+
|
|
41
68
|
# @return [Array<String>] CLI provider names
|
|
42
69
|
def self.cli_providers
|
|
43
70
|
config = load
|