ace-test-runner 0.27.1 → 0.28.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.ace-defaults/test-runner/config.yml +2 -0
- data/CHANGELOG.md +27 -0
- data/exe/ace-test-suite +13 -4
- data/lib/ace/test_runner/atoms/command_builder.rb +60 -86
- data/lib/ace/test_runner/atoms/line_number_resolver.rb +70 -63
- data/lib/ace/test_runner/atoms/result_parser.rb +1 -1
- data/lib/ace/test_runner/cli/commands/test.rb +2 -3
- data/lib/ace/test_runner/formatters/json_formatter.rb +3 -1
- data/lib/ace/test_runner/formatters/progress_file_formatter.rb +10 -7
- data/lib/ace/test_runner/formatters/progress_formatter.rb +10 -7
- data/lib/ace/test_runner/models/test_report.rb +1 -0
- data/lib/ace/test_runner/models/test_result.rb +12 -3
- data/lib/ace/test_runner/models/test_selection_plan.rb +44 -0
- data/lib/ace/test_runner/molecules/cli_argument_parser.rb +9 -1
- data/lib/ace/test_runner/molecules/execution_evidence.rb +102 -0
- data/lib/ace/test_runner/molecules/failed_package_reporter.rb +4 -65
- data/lib/ace/test_runner/molecules/in_process_runner.rb +48 -37
- data/lib/ace/test_runner/molecules/pattern_resolver.rb +8 -5
- data/lib/ace/test_runner/molecules/report_storage.rb +24 -20
- data/lib/ace/test_runner/molecules/selection_resolver.rb +45 -0
- data/lib/ace/test_runner/molecules/selection_verifier.rb +78 -0
- data/lib/ace/test_runner/molecules/smart_test_executor.rb +4 -0
- data/lib/ace/test_runner/molecules/test_executor.rb +11 -4
- data/lib/ace/test_runner/organisms/sequential_target_executor.rb +6 -7
- data/lib/ace/test_runner/organisms/test_orchestrator.rb +44 -21
- data/lib/ace/test_runner/suite/display_manager.rb +7 -6
- data/lib/ace/test_runner/suite/orchestrator.rb +5 -3
- data/lib/ace/test_runner/suite/process_monitor.rb +37 -119
- data/lib/ace/test_runner/suite/result_aggregator.rb +17 -89
- data/lib/ace/test_runner/suite/simple_display_manager.rb +2 -1
- data/lib/ace/test_runner/version.rb +1 -1
- data/lib/ace/test_runner.rb +1 -0
- metadata +34 -2
|
@@ -1,42 +1,26 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
require "json"
|
|
4
|
-
|
|
5
3
|
module Ace
|
|
6
4
|
module TestRunner
|
|
7
5
|
module Suite
|
|
8
6
|
class ResultAggregator
|
|
9
7
|
attr_reader :packages
|
|
10
8
|
|
|
11
|
-
def initialize(packages,
|
|
9
|
+
def initialize(packages, runtime_results: {})
|
|
12
10
|
@packages = packages
|
|
13
|
-
@report_root = report_root
|
|
14
11
|
@runtime_results = runtime_results || {}
|
|
15
12
|
end
|
|
16
13
|
|
|
17
14
|
def aggregate
|
|
18
15
|
results = collect_results
|
|
19
16
|
|
|
20
|
-
# Calculate assertion totals
|
|
21
|
-
total_assertions = 0
|
|
22
|
-
assertions_failed = 0
|
|
23
|
-
|
|
24
|
-
results.each do |r|
|
|
25
|
-
# Get assertions from either summary or report data
|
|
26
|
-
if r[:assertions]
|
|
27
|
-
total_assertions += r[:assertions]
|
|
28
|
-
elsif r[:report_data]
|
|
29
|
-
total_assertions += r[:report_data].dig(:result, :assertions) || 0
|
|
30
|
-
end
|
|
31
|
-
end
|
|
32
|
-
|
|
33
17
|
{
|
|
34
18
|
total_tests: results.sum { |r| r[:total] || 0 },
|
|
35
19
|
total_passed: results.sum { |r| r[:passed] || 0 },
|
|
36
20
|
total_failed: results.sum { |r| (r[:failed] || 0) + (r[:errors] || 0) },
|
|
37
21
|
total_skipped: results.sum { |r| r[:skipped] || 0 },
|
|
38
|
-
total_assertions:
|
|
39
|
-
assertions_failed:
|
|
22
|
+
total_assertions: results.sum { |result| result[:assertions] || 0 },
|
|
23
|
+
assertions_failed: 0,
|
|
40
24
|
total_duration: results.map { |r| r[:duration] || 0 }.max,
|
|
41
25
|
packages_passed: results.count { |r| r[:success] },
|
|
42
26
|
packages_failed: results.count { |r| !r[:success] },
|
|
@@ -47,81 +31,22 @@ module Ace
|
|
|
47
31
|
|
|
48
32
|
def collect_results
|
|
49
33
|
@packages.map do |package|
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
report_root: @report_root,
|
|
56
|
-
package_name: package["name"]
|
|
57
|
-
)
|
|
58
|
-
summary_path = reports_dir ? File.join(reports_dir, "summary.json") : nil
|
|
59
|
-
|
|
60
|
-
if summary_path && File.exist?(summary_path)
|
|
61
|
-
begin
|
|
62
|
-
data = JSON.parse(File.read(summary_path), symbolize_names: true)
|
|
63
|
-
data[:package] = package["name"]
|
|
64
|
-
data[:path] = package["path"]
|
|
65
|
-
data[:report_root] = @report_root
|
|
66
|
-
|
|
67
|
-
# Try to get assertions from report.json if not in summary
|
|
68
|
-
if !data[:assertions] || data[:assertions] == 0
|
|
69
|
-
report_path = File.join(reports_dir, "report.json")
|
|
70
|
-
if File.exist?(report_path)
|
|
71
|
-
begin
|
|
72
|
-
report_data = JSON.parse(File.read(report_path), symbolize_names: true)
|
|
73
|
-
data[:assertions] = report_data.dig(:result, :assertions) || 0
|
|
74
|
-
data[:report_data] = report_data
|
|
75
|
-
rescue JSON::ParserError
|
|
76
|
-
# Ignore if report.json can't be parsed
|
|
77
|
-
end
|
|
78
|
-
end
|
|
79
|
-
end
|
|
80
|
-
|
|
81
|
-
data
|
|
82
|
-
rescue JSON::ParserError => e
|
|
83
|
-
# If we can't parse the summary, create a failure result
|
|
84
|
-
{
|
|
85
|
-
package: package["name"],
|
|
86
|
-
path: package["path"],
|
|
87
|
-
report_root: @report_root,
|
|
88
|
-
success: false,
|
|
89
|
-
error: "Failed to parse summary.json: #{e.message}",
|
|
90
|
-
total: 0,
|
|
91
|
-
passed: 0,
|
|
92
|
-
failed: 0,
|
|
93
|
-
errors: 1
|
|
94
|
-
}
|
|
95
|
-
end
|
|
96
|
-
else
|
|
97
|
-
runtime || {
|
|
98
|
-
package: package["name"],
|
|
99
|
-
path: package["path"],
|
|
100
|
-
report_root: @report_root,
|
|
101
|
-
success: false,
|
|
102
|
-
error: "No test results found (summary.json missing)",
|
|
103
|
-
total: 0,
|
|
104
|
-
passed: 0,
|
|
105
|
-
failed: 0,
|
|
106
|
-
errors: 1
|
|
107
|
-
}
|
|
108
|
-
end
|
|
34
|
+
runtime_result(package) || {
|
|
35
|
+
package: package["name"], path: package["path"], entry_id: package["entry_id"],
|
|
36
|
+
report_dir: nil, success: false, error: "No verified execution completion evidence",
|
|
37
|
+
total: 0, passed: 0, failed: 0, errors: 1, assertions: 0
|
|
38
|
+
}
|
|
109
39
|
end
|
|
110
40
|
end
|
|
111
41
|
|
|
112
|
-
def runtime_result_overrides_summary?(package, runtime)
|
|
113
|
-
return false unless runtime
|
|
114
|
-
|
|
115
|
-
status = @runtime_results[package["name"]] || {}
|
|
116
|
-
status[:timed_out] || status[:interrupted] || runtime[:success] == false
|
|
117
|
-
end
|
|
118
|
-
|
|
119
42
|
def collect_failed_packages(results)
|
|
120
43
|
results.select { |r| !r[:success] }.map do |result|
|
|
121
44
|
{
|
|
122
45
|
name: result[:package],
|
|
123
46
|
path: result[:path],
|
|
124
|
-
|
|
47
|
+
report_dir: result[:report_dir],
|
|
48
|
+
execution_id: result[:execution_id],
|
|
49
|
+
entry_id: result[:entry_id],
|
|
125
50
|
failures: result[:failed] || 0,
|
|
126
51
|
errors: result[:errors] || 0,
|
|
127
52
|
error_message: result[:error]
|
|
@@ -130,7 +55,7 @@ module Ace
|
|
|
130
55
|
end
|
|
131
56
|
|
|
132
57
|
def runtime_result(package)
|
|
133
|
-
status = @runtime_results[package["
|
|
58
|
+
status = @runtime_results[package["entry_id"]]
|
|
134
59
|
return nil unless status && status[:completed]
|
|
135
60
|
|
|
136
61
|
results = status[:results] || {}
|
|
@@ -142,11 +67,14 @@ module Ace
|
|
|
142
67
|
{
|
|
143
68
|
package: package["name"],
|
|
144
69
|
path: package["path"],
|
|
145
|
-
|
|
70
|
+
report_dir: results[:report_dir],
|
|
71
|
+
completion_path: results[:completion_path],
|
|
72
|
+
execution_id: results[:execution_id],
|
|
73
|
+
entry_id: package["entry_id"],
|
|
146
74
|
success: status[:success],
|
|
147
75
|
error: results[:error],
|
|
148
76
|
total: total,
|
|
149
|
-
passed:
|
|
77
|
+
passed: results[:passed] || 0,
|
|
150
78
|
failed: failures,
|
|
151
79
|
errors: errors,
|
|
152
80
|
skipped: skipped,
|
|
@@ -31,7 +31,7 @@ module Ace
|
|
|
31
31
|
|
|
32
32
|
# Called when a package status changes. Prints a line when package completes.
|
|
33
33
|
def update_package(package, status, _output = nil)
|
|
34
|
-
@package_status[package["
|
|
34
|
+
@package_status[package["entry_id"]] = status
|
|
35
35
|
|
|
36
36
|
# Only print when package completes
|
|
37
37
|
return unless status[:completed]
|
|
@@ -88,6 +88,7 @@ module Ace
|
|
|
88
88
|
line += " #{skipped} skip" if skipped > 0
|
|
89
89
|
line += " timeout" if status[:timed_out]
|
|
90
90
|
|
|
91
|
+
line += results[:report_dir] ? " report: #{results[:report_dir]}" : " no saved report"
|
|
91
92
|
puts line
|
|
92
93
|
end
|
|
93
94
|
|
data/lib/ace/test_runner.rb
CHANGED
|
@@ -45,6 +45,7 @@ require_relative "test_runner/atoms/environment_sanitizer"
|
|
|
45
45
|
require_relative "test_runner/molecules/fixture_environment"
|
|
46
46
|
require_relative "test_runner/molecules/test_executor"
|
|
47
47
|
require_relative "test_runner/molecules/failure_analyzer"
|
|
48
|
+
require_relative "test_runner/molecules/execution_evidence"
|
|
48
49
|
require_relative "test_runner/molecules/report_storage"
|
|
49
50
|
require_relative "test_runner/molecules/config_loader"
|
|
50
51
|
require_relative "test_runner/molecules/pattern_resolver"
|
metadata
CHANGED
|
@@ -1,14 +1,28 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: ace-test-runner
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.28.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Michal Czyz
|
|
8
8
|
bindir: exe
|
|
9
9
|
cert_chain: []
|
|
10
|
-
date: 2026-
|
|
10
|
+
date: 2026-10-08 00:00:00.000000000 Z
|
|
11
11
|
dependencies:
|
|
12
|
+
- !ruby/object:Gem::Dependency
|
|
13
|
+
name: ace-support-fs
|
|
14
|
+
requirement: !ruby/object:Gem::Requirement
|
|
15
|
+
requirements:
|
|
16
|
+
- - "~>"
|
|
17
|
+
- !ruby/object:Gem::Version
|
|
18
|
+
version: '0.3'
|
|
19
|
+
type: :runtime
|
|
20
|
+
prerelease: false
|
|
21
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
22
|
+
requirements:
|
|
23
|
+
- - "~>"
|
|
24
|
+
- !ruby/object:Gem::Version
|
|
25
|
+
version: '0.3'
|
|
12
26
|
- !ruby/object:Gem::Dependency
|
|
13
27
|
name: ace-support-cli
|
|
14
28
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -107,6 +121,20 @@ dependencies:
|
|
|
107
121
|
- - "~>"
|
|
108
122
|
- !ruby/object:Gem::Version
|
|
109
123
|
version: '1.8'
|
|
124
|
+
- !ruby/object:Gem::Dependency
|
|
125
|
+
name: prism
|
|
126
|
+
requirement: !ruby/object:Gem::Requirement
|
|
127
|
+
requirements:
|
|
128
|
+
- - "~>"
|
|
129
|
+
- !ruby/object:Gem::Version
|
|
130
|
+
version: '1.9'
|
|
131
|
+
type: :runtime
|
|
132
|
+
prerelease: false
|
|
133
|
+
version_requirements: !ruby/object:Gem::Requirement
|
|
134
|
+
requirements:
|
|
135
|
+
- - "~>"
|
|
136
|
+
- !ruby/object:Gem::Version
|
|
137
|
+
version: '1.9'
|
|
110
138
|
- !ruby/object:Gem::Dependency
|
|
111
139
|
name: ostruct
|
|
112
140
|
requirement: !ruby/object:Gem::Requirement
|
|
@@ -193,10 +221,12 @@ files:
|
|
|
193
221
|
- lib/ace/test_runner/models/test_failure.rb
|
|
194
222
|
- lib/ace/test_runner/models/test_report.rb
|
|
195
223
|
- lib/ace/test_runner/models/test_result.rb
|
|
224
|
+
- lib/ace/test_runner/models/test_selection_plan.rb
|
|
196
225
|
- lib/ace/test_runner/models/test_target.rb
|
|
197
226
|
- lib/ace/test_runner/molecules/cli_argument_parser.rb
|
|
198
227
|
- lib/ace/test_runner/molecules/config_loader.rb
|
|
199
228
|
- lib/ace/test_runner/molecules/deprecation_fixer.rb
|
|
229
|
+
- lib/ace/test_runner/molecules/execution_evidence.rb
|
|
200
230
|
- lib/ace/test_runner/molecules/failed_package_reporter.rb
|
|
201
231
|
- lib/ace/test_runner/molecules/failure_analyzer.rb
|
|
202
232
|
- lib/ace/test_runner/molecules/fixture_environment.rb
|
|
@@ -205,6 +235,8 @@ files:
|
|
|
205
235
|
- lib/ace/test_runner/molecules/pattern_resolver.rb
|
|
206
236
|
- lib/ace/test_runner/molecules/rake_integration.rb
|
|
207
237
|
- lib/ace/test_runner/molecules/report_storage.rb
|
|
238
|
+
- lib/ace/test_runner/molecules/selection_resolver.rb
|
|
239
|
+
- lib/ace/test_runner/molecules/selection_verifier.rb
|
|
208
240
|
- lib/ace/test_runner/molecules/smart_test_executor.rb
|
|
209
241
|
- lib/ace/test_runner/molecules/test_executor.rb
|
|
210
242
|
- lib/ace/test_runner/organisms/agent_reporter.rb
|