ace-test-runner-e2e 0.38.11 → 0.40.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +55 -0
- data/handbook/guides/e2e-testing.g.md +35 -3
- data/handbook/guides/scenario-yml-reference.g.md +8 -3
- data/handbook/guides/tc-authoring.g.md +15 -4
- data/handbook/templates/tc-file.template.md +4 -2
- data/handbook/workflow-instructions/e2e/create.wf.md +13 -3
- data/handbook/workflow-instructions/e2e/fix.wf.md +19 -0
- data/handbook/workflow-instructions/e2e/plan-changes.wf.md +16 -0
- data/handbook/workflow-instructions/e2e/review.wf.md +14 -10
- data/handbook/workflow-instructions/e2e/rewrite.wf.md +10 -3
- data/lib/ace/test/end_to_end_runner/atoms/artifact_contract_validator.rb +138 -0
- data/lib/ace/test/end_to_end_runner/cli/commands/run_suite.rb +195 -5
- data/lib/ace/test/end_to_end_runner/cli/commands/run_test.rb +37 -1
- data/lib/ace/test/end_to_end_runner/molecules/artifact_pruner.rb +61 -0
- data/lib/ace/test/end_to_end_runner/molecules/pipeline_executor.rb +90 -14
- data/lib/ace/test/end_to_end_runner/molecules/pipeline_prompt_bundler.rb +43 -5
- data/lib/ace/test/end_to_end_runner/molecules/pipeline_sandbox_builder.rb +7 -5
- data/lib/ace/test/end_to_end_runner/molecules/report_writer.rb +2 -0
- data/lib/ace/test/end_to_end_runner/molecules/sandbox_runtime_builder.rb +101 -9
- data/lib/ace/test/end_to_end_runner/molecules/scenario_loader.rb +28 -30
- data/lib/ace/test/end_to_end_runner/molecules/setup_executor.rb +24 -1
- data/lib/ace/test/end_to_end_runner/molecules/suite_report_writer.rb +182 -1
- data/lib/ace/test/end_to_end_runner/organisms/suite_orchestrator.rb +25 -3
- data/lib/ace/test/end_to_end_runner/organisms/test_orchestrator.rb +44 -5
- data/lib/ace/test/end_to_end_runner/version.rb +1 -1
- data/lib/ace/test/end_to_end_runner.rb +2 -0
- metadata +4 -2
|
@@ -133,6 +133,7 @@ module Ace
|
|
|
133
133
|
return [nil, nil, nil] unless cli_provider? && scenario.setup_steps.any?
|
|
134
134
|
|
|
135
135
|
sandbox_dir = File.join(@base_dir, ".ace-local", "test-e2e", scenario.dir_name(timestamp))
|
|
136
|
+
setup_steps = effective_setup_steps_for(scenario)
|
|
136
137
|
package_copy = Ace::TestSupport::SandboxPackageCopy.new(source_root: @base_dir)
|
|
137
138
|
package_source = File.join(@base_dir, scenario.package.to_s)
|
|
138
139
|
package_copy_result = if File.directory?(package_source)
|
|
@@ -148,7 +149,8 @@ module Ace
|
|
|
148
149
|
}
|
|
149
150
|
}
|
|
150
151
|
end
|
|
151
|
-
Molecules::PipelineSandboxBuilder.new(config_root: @base_dir)
|
|
152
|
+
sandbox_builder = Molecules::PipelineSandboxBuilder.new(config_root: @base_dir)
|
|
153
|
+
protocol_packages = sandbox_builder.sync_protocol_sources_into(sandbox_dir)
|
|
152
154
|
runtime_result = @runtime_builder.prepare(
|
|
153
155
|
sandbox_root: sandbox_dir,
|
|
154
156
|
env: package_copy_result[:env],
|
|
@@ -164,12 +166,17 @@ module Ace
|
|
|
164
166
|
@setup_executor_factory.call(sandbox_backend: sandbox_backend)
|
|
165
167
|
end
|
|
166
168
|
result = setup_executor.execute(
|
|
167
|
-
setup_steps:
|
|
169
|
+
setup_steps: setup_steps,
|
|
168
170
|
sandbox_dir: sandbox_dir,
|
|
169
171
|
fixture_source: scenario.fixture_path,
|
|
170
172
|
scenario_name: scenario.test_id,
|
|
171
173
|
run_id: timestamp,
|
|
172
|
-
initial_env: runtime_result[:env]
|
|
174
|
+
initial_env: runtime_result[:env],
|
|
175
|
+
git_excludes: sandbox_support_git_excludes(
|
|
176
|
+
scenario,
|
|
177
|
+
protocol_packages,
|
|
178
|
+
setup_steps: setup_steps
|
|
179
|
+
)
|
|
173
180
|
)
|
|
174
181
|
|
|
175
182
|
unless result[:success]
|
|
@@ -190,10 +197,10 @@ module Ace
|
|
|
190
197
|
steps = Array(scenario.setup_steps)
|
|
191
198
|
return steps unless scenario.sandbox_profile == "ace-default"
|
|
192
199
|
|
|
193
|
-
|
|
200
|
+
has_config_sync = setup_contains_command?(steps, "ace-config sync")
|
|
194
201
|
has_handbook_sync = setup_contains_command?(steps, "ace-handbook sync")
|
|
195
202
|
bootstrap = []
|
|
196
|
-
bootstrap << {"run" => "ace-config
|
|
203
|
+
bootstrap << {"run" => "ace-config sync ace-llm-providers-cli"} unless has_config_sync
|
|
197
204
|
bootstrap << {"run" => "ace-handbook sync"} unless has_handbook_sync
|
|
198
205
|
return steps if bootstrap.empty?
|
|
199
206
|
|
|
@@ -209,6 +216,38 @@ module Ace
|
|
|
209
216
|
end
|
|
210
217
|
end
|
|
211
218
|
|
|
219
|
+
def sandbox_support_git_excludes(scenario, protocol_packages, setup_steps:)
|
|
220
|
+
package_paths = ([scenario.package] + Array(protocol_packages))
|
|
221
|
+
.map(&:to_s)
|
|
222
|
+
.map(&:strip)
|
|
223
|
+
.reject(&:empty?)
|
|
224
|
+
.uniq
|
|
225
|
+
.sort
|
|
226
|
+
.map { |name| "#{name}/" }
|
|
227
|
+
|
|
228
|
+
# Most scenarios need the copied package tree to stay visible to tools
|
|
229
|
+
# like rg/fd. Only fixture-commit setup flows should hide the copied
|
|
230
|
+
# support trees from git staging.
|
|
231
|
+
return [] unless setup_contains_initial_commit?(setup_steps)
|
|
232
|
+
|
|
233
|
+
support_paths = [".ace-handbook/", ".ace/git/", ".ace/llm/providers/"]
|
|
234
|
+
(package_paths + support_paths).uniq
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def setup_contains_initial_commit?(steps)
|
|
238
|
+
staged_changes = false
|
|
239
|
+
|
|
240
|
+
Array(steps).any? do |step|
|
|
241
|
+
next false unless step.is_a?(Hash)
|
|
242
|
+
|
|
243
|
+
command = step["run"].to_s
|
|
244
|
+
staged_changes ||= command.match?(/\bgit\s+add\b/m)
|
|
245
|
+
|
|
246
|
+
command.match?(/\bgit\s+commit\b/m) &&
|
|
247
|
+
(staged_changes || command.match?(/\bgit\s+commit\b(?=.*(?:\s--all\b|\s-[A-Za-z]*a[A-Za-z]*\b))/m))
|
|
248
|
+
end
|
|
249
|
+
end
|
|
250
|
+
|
|
212
251
|
# Run a single test
|
|
213
252
|
# @param test_cases [Array<String>, nil] Optional test case IDs to filter
|
|
214
253
|
# @param report_dir [String, nil] Explicit report directory path (overrides computed path)
|
|
@@ -13,6 +13,7 @@ require_relative "end_to_end_runner/atoms/result_parser"
|
|
|
13
13
|
require_relative "end_to_end_runner/atoms/skill_prompt_builder"
|
|
14
14
|
require_relative "end_to_end_runner/atoms/skill_result_parser"
|
|
15
15
|
require_relative "end_to_end_runner/atoms/suite_report_prompt_builder"
|
|
16
|
+
require_relative "end_to_end_runner/atoms/artifact_contract_validator"
|
|
16
17
|
require_relative "end_to_end_runner/atoms/test_case_parser"
|
|
17
18
|
require_relative "end_to_end_runner/atoms/tc_fidelity_validator"
|
|
18
19
|
require_relative "end_to_end_runner/atoms/display_helpers"
|
|
@@ -32,6 +33,7 @@ require_relative "end_to_end_runner/molecules/pipeline_report_generator"
|
|
|
32
33
|
require_relative "end_to_end_runner/molecules/pipeline_executor"
|
|
33
34
|
require_relative "end_to_end_runner/molecules/report_writer"
|
|
34
35
|
require_relative "end_to_end_runner/molecules/suite_report_writer"
|
|
36
|
+
require_relative "end_to_end_runner/molecules/artifact_pruner"
|
|
35
37
|
require_relative "end_to_end_runner/molecules/simple_display_manager"
|
|
36
38
|
require_relative "end_to_end_runner/molecules/progress_display_manager"
|
|
37
39
|
require_relative "end_to_end_runner/molecules/suite_simple_display_manager"
|
metadata
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ace-test-runner-e2e
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.40.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Michal Czyz
|
|
8
8
|
bindir: exe
|
|
9
9
|
cert_chain: []
|
|
10
|
-
date: 2026-04-
|
|
10
|
+
date: 2026-04-27 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
12
|
- !ruby/object:Gem::Dependency
|
|
13
13
|
name: ace-support-cli
|
|
@@ -172,6 +172,7 @@ files:
|
|
|
172
172
|
- handbook/workflow-instructions/e2e/run.wf.md
|
|
173
173
|
- handbook/workflow-instructions/e2e/setup-sandbox.wf.md
|
|
174
174
|
- lib/ace/test/end_to_end_runner.rb
|
|
175
|
+
- lib/ace/test/end_to_end_runner/atoms/artifact_contract_validator.rb
|
|
175
176
|
- lib/ace/test/end_to_end_runner/atoms/display_helpers.rb
|
|
176
177
|
- lib/ace/test/end_to_end_runner/atoms/prompt_builder.rb
|
|
177
178
|
- lib/ace/test/end_to_end_runner/atoms/result_parser.rb
|
|
@@ -186,6 +187,7 @@ files:
|
|
|
186
187
|
- lib/ace/test/end_to_end_runner/models/test_result.rb
|
|
187
188
|
- lib/ace/test/end_to_end_runner/models/test_scenario.rb
|
|
188
189
|
- lib/ace/test/end_to_end_runner/molecules/affected_detector.rb
|
|
190
|
+
- lib/ace/test/end_to_end_runner/molecules/artifact_pruner.rb
|
|
189
191
|
- lib/ace/test/end_to_end_runner/molecules/bwrap_sandbox_backend.rb
|
|
190
192
|
- lib/ace/test/end_to_end_runner/molecules/config_loader.rb
|
|
191
193
|
- lib/ace/test/end_to_end_runner/molecules/failure_finder.rb
|