ceedling 1.1.2 → 1.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/GIT_COMMIT_SHA +1 -1
- data/README.md +1 -1
- data/assets/fake_crash_test_fixture.rb +14 -0
- data/assets/partial_all_module.h +11 -0
- data/assets/run_with_ubsan.rb +15 -0
- data/assets/same_dir_pairing_module.c +4 -0
- data/assets/same_dir_pairing_module.h +8 -0
- data/assets/same_dir_pairing_module_func.h +6 -0
- data/assets/same_dir_pairing_other.c +4 -0
- data/assets/same_dir_pairing_other.h +8 -0
- data/assets/test_example_file_crash_sigsegv_with_param.c +32 -0
- data/assets/test_example_file_ub_shift_overflow.c +22 -0
- data/assets/test_partial_all_module.c +16 -0
- data/assets/test_same_dir_pairing.c +23 -0
- data/bin/actions_wrapper.rb +7 -6
- data/docs/Changelog.md +29 -1
- data/docs/mkdocs/configuration/reference/index.md +8 -0
- data/docs/mkdocs/configuration/reference/partials.md +38 -0
- data/examples/wondrous_forest/src/LightSensor.c +21 -4
- data/examples/wondrous_forest/src/LightSensor.h +12 -4
- data/examples/wondrous_forest/test/TestLightSensor.c +34 -4
- data/lib/ceedling/c_extractor/c_extractor.rb +37 -11
- data/lib/ceedling/c_extractor/c_extractor_constants.rb +0 -3
- data/lib/ceedling/c_extractor/c_extractor_declarations.rb +26 -25
- data/lib/ceedling/c_extractor/c_extractor_functions.rb +5 -12
- data/lib/ceedling/config/configurator.rb +8 -1
- data/lib/ceedling/config/configurator_setup.rb +26 -0
- data/lib/ceedling/defaults.rb +16 -2
- data/lib/ceedling/file_path_utils.rb +7 -0
- data/lib/ceedling/file_wrapper.rb +8 -0
- data/lib/ceedling/generators/generator.rb +15 -0
- data/lib/ceedling/generators/generator_helper.rb +25 -6
- data/lib/ceedling/generators/generator_partials.rb +84 -3
- data/lib/ceedling/generators/generator_test_results_backtrace.rb +268 -100
- data/lib/ceedling/generators/generator_test_runner.rb +20 -2
- data/lib/ceedling/includes/includes.rb +66 -11
- data/lib/ceedling/objects.yml +5 -0
- data/lib/ceedling/partials/partializer.rb +18 -3
- data/lib/ceedling/partials/partializer_utils.rb +6 -1
- data/lib/ceedling/preprocess/preprocessinator_includes_handler.rb +43 -16
- data/lib/ceedling/test_invoker/test_build_executor.rb +19 -8
- data/lib/version.rb +1 -1
- data/mkdocs.yml +1 -0
- data/plugins/gcov/lib/gcov.rb +2 -2
- data/site-local/404.html +30 -0
- data/site-local/configuration/environment-vars.html +30 -0
- data/site-local/configuration/global-collections.html +30 -0
- data/site-local/configuration/index.html +30 -0
- data/site-local/configuration/loading.html +30 -0
- data/site-local/configuration/mixins.html +30 -0
- data/site-local/configuration/parallel-builds.html +30 -0
- data/site-local/configuration/project-file.html +30 -0
- data/site-local/configuration/reference/cexception.html +30 -0
- data/site-local/configuration/reference/cmock.html +30 -0
- data/site-local/configuration/reference/defines.html +30 -0
- data/site-local/configuration/reference/environment.html +30 -0
- data/site-local/configuration/reference/extension.html +30 -0
- data/site-local/configuration/reference/files.html +30 -0
- data/site-local/configuration/reference/flags.html +30 -0
- data/site-local/configuration/reference/index.html +36 -0
- data/site-local/configuration/reference/libraries.html +30 -0
- data/site-local/configuration/reference/mixins.html +30 -0
- data/site-local/configuration/reference/partials.html +4257 -0
- data/site-local/configuration/reference/paths.html +30 -0
- data/site-local/configuration/reference/plugins.html +30 -0
- data/site-local/configuration/reference/project.html +30 -0
- data/site-local/configuration/reference/release-build.html +31 -1
- data/site-local/configuration/reference/test-build.html +31 -1
- data/site-local/configuration/reference/test-runner.html +30 -0
- data/site-local/configuration/reference/tools.html +30 -0
- data/site-local/configuration/reference/unity.html +30 -0
- data/site-local/configuration/which-ceedling.html +30 -0
- data/site-local/development/index.html +30 -0
- data/site-local/development/plugins/configuration.html +30 -0
- data/site-local/development/plugins/index.html +30 -0
- data/site-local/development/plugins/plugin-subclass.html +30 -0
- data/site-local/development/plugins/rake-tasks.html +30 -0
- data/site-local/development/workflow.html +30 -0
- data/site-local/getting-started/command-line.html +30 -0
- data/site-local/getting-started/example-projects/cipher-quest.html +30 -0
- data/site-local/getting-started/example-projects/index.html +30 -0
- data/site-local/getting-started/example-projects/temp-sensor.html +30 -0
- data/site-local/getting-started/example-projects/wondrous-forest.html +30 -0
- data/site-local/getting-started/index.html +30 -0
- data/site-local/getting-started/installation.html +30 -0
- data/site-local/getting-started/quick-start.html +30 -0
- data/site-local/help.html +30 -0
- data/site-local/index.html +30 -0
- data/site-local/overview/build-system.html +30 -0
- data/site-local/overview/index.html +30 -0
- data/site-local/overview/test-environments.html +30 -0
- data/site-local/overview/tools-and-frameworks.html +30 -0
- data/site-local/plugins/beep.html +30 -0
- data/site-local/plugins/bullseye.html +30 -0
- data/site-local/plugins/command-hooks.html +30 -0
- data/site-local/plugins/compile-commands-json-db.html +30 -0
- data/site-local/plugins/cppcheck.html +30 -0
- data/site-local/plugins/dependencies.html +30 -0
- data/site-local/plugins/fff.html +30 -0
- data/site-local/plugins/gcov/examples.html +30 -0
- data/site-local/plugins/gcov/gcovr.html +30 -0
- data/site-local/plugins/gcov/index.html +30 -0
- data/site-local/plugins/gcov/overview.html +30 -0
- data/site-local/plugins/gcov/reportgenerator.html +30 -0
- data/site-local/plugins/gcov/reporting.html +30 -0
- data/site-local/plugins/gcov/setup.html +30 -0
- data/site-local/plugins/gcov/tool-versions.html +30 -0
- data/site-local/plugins/gcov/troubleshooting.html +30 -0
- data/site-local/plugins/index.html +30 -0
- data/site-local/plugins/module-generator.html +30 -0
- data/site-local/plugins/report-build-warnings-log.html +30 -0
- data/site-local/plugins/report-tests-gtestlike-stdout.html +30 -0
- data/site-local/plugins/report-tests-ide-stdout.html +30 -0
- data/site-local/plugins/report-tests-log-factory.html +30 -0
- data/site-local/plugins/report-tests-pretty-stdout.html +30 -0
- data/site-local/plugins/report-tests-raw-output-log.html +30 -0
- data/site-local/plugins/report-tests-teamcity-stdout.html +30 -0
- data/site-local/plugins/valgrind.html +30 -0
- data/site-local/project/release-history.html +30 -0
- data/site-local/project/upgrade.html +30 -0
- data/site-local/reference/build-directives.html +30 -0
- data/site-local/reference/command-line.html +30 -0
- data/site-local/reference/environment-vars.html +30 -0
- data/site-local/reference/gcov-plugin.html +30 -0
- data/site-local/reference/global-collections.html +30 -0
- data/site-local/reference/index.html +30 -0
- data/site-local/reference/partials-macros.html +30 -0
- data/site-local/reference/project-configuration.html +30 -0
- data/site-local/sitemap.xml.gz +0 -0
- data/site-local/snapshot/examples/cipher_quest/index.html +30 -0
- data/site-local/snapshot/examples/temp_sensor/index.html +30 -0
- data/site-local/snapshot/examples/wondrous_forest/index.html +30 -0
- data/site-local/testing-guide/build-directives.html +30 -0
- data/site-local/testing-guide/conventions.html +30 -0
- data/site-local/testing-guide/frameworks.html +30 -0
- data/site-local/testing-guide/index.html +30 -0
- data/site-local/testing-guide/partials/configuration.html +30 -0
- data/site-local/testing-guide/partials/conventions.html +30 -0
- data/site-local/testing-guide/partials/directives.html +30 -0
- data/site-local/testing-guide/partials/example.html +30 -0
- data/site-local/testing-guide/partials/index.html +30 -0
- data/site-local/testing-guide/partials/overview.html +30 -0
- data/site-local/testing-guide/partials/variables.html +30 -0
- data/site-local/testing-guide/test-cases.html +30 -0
- data/site-local/testing-guide/test-sample.html +30 -0
- data/site-local/testing-guide/test-suite-anatomy.html +30 -0
- data/spec/support/system/spec_system_helper.rb +25 -0
- data/spec/support/system/system_context.rb +15 -0
- data/spec/system/deployment_as_gem_spec.rb +14 -0
- data/spec/system/deployment_as_vendor_spec.rb +19 -0
- data/spec/system/encoding_stress_spec.rb +4 -4
- data/spec/system/example_wondrous_forest_spec.rb +2 -2
- data/spec/system/support/common_test_cases.rb +227 -0
- data/spec/units/c_extractor/c_extractor_declarations_spec.rb +27 -4
- data/spec/units/c_extractor/c_extractor_functions_spec.rb +1 -11
- data/spec/units/c_extractor/c_extractor_integration_spec.rb +18 -11
- data/spec/units/c_extractor/c_extractor_spec.rb +57 -1
- data/spec/units/config/configurator_setup_spec.rb +61 -0
- data/spec/units/configurator_spec.rb +45 -0
- data/spec/units/file_path_utils_spec.rb +17 -0
- data/spec/units/generators/generator_helper_spec.rb +34 -7
- data/spec/units/generators/generator_partials_spec.rb +120 -12
- data/spec/units/generators/generator_test_results_backtrace_spec.rb +248 -41
- data/spec/units/generators/generator_test_runner_spec.rb +26 -2
- data/spec/units/includes/include_spec.rb +30 -18
- data/spec/units/includes/includes_spec.rb +100 -10
- data/spec/units/partials/partializer_spec.rb +113 -1
- data/spec/units/partials/partializer_utils_spec.rb +11 -0
- data/spec/units/preprocess/preprocessinator_includes_handler_spec.rb +96 -7
- data/spec/units/system_wrapper_spec.rb +72 -0
- data/spec/units/test_context_extractor_spec.rb +3 -3
- metadata +20 -2
|
@@ -7,13 +7,17 @@
|
|
|
7
7
|
|
|
8
8
|
class GeneratorTestResultsBacktrace
|
|
9
9
|
|
|
10
|
-
constructor :configurator, :tool_executor, :generator_test_results, :
|
|
10
|
+
constructor :configurator, :tool_executor, :generator_test_results, :generator_helper,
|
|
11
|
+
:file_path_utils, :file_wrapper, :loginator
|
|
11
12
|
|
|
12
13
|
def setup()
|
|
13
14
|
@RESULTS_COLLECTOR = Struct.new( :passed, :failed, :ignored, :output, keyword_init:true )
|
|
15
|
+
# Alias, matching Generator's own convention for the same dependency.
|
|
16
|
+
@helper = @generator_helper
|
|
14
17
|
end
|
|
15
18
|
|
|
16
|
-
# Re-runs each test case
|
|
19
|
+
# Re-runs each test case (or, for a parameterized test, each group of parameterized
|
|
20
|
+
# cases -- see `group_test_cases`) under gdb to identify which one(s) crashed and why.
|
|
17
21
|
# Writes the full gdb transcript to a per-test-case log file and assembles a
|
|
18
22
|
# terse crash label (signal + description, optional source line in backticks)
|
|
19
23
|
# for each failing test case. Returns a modified shell_result with regenerated output.
|
|
@@ -28,98 +32,174 @@ class GeneratorTestResultsBacktrace
|
|
|
28
32
|
|
|
29
33
|
test_name = File.basename( filename, '.*' )
|
|
30
34
|
|
|
31
|
-
#
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
35
|
+
# True once some retry group has actually shown crash evidence of its own -- an
|
|
36
|
+
# unresolved member, or (below) a group whose real status contradicts a fully clean
|
|
37
|
+
# set of matches. If this stays false across every group, the whole diagnostic never
|
|
38
|
+
# reproduced or attributed the crash the main run already detected, and none of it
|
|
39
|
+
# can be trusted -- see the fallback after the loop.
|
|
40
|
+
any_group_crashed = false
|
|
36
41
|
|
|
37
|
-
|
|
42
|
+
# Iterate on test cases, one sub-process run per group (see `group_test_cases`)
|
|
43
|
+
group_test_cases( test_cases ).each do |group|
|
|
44
|
+
# Build the test fixture to run with our test case (or parameterized group) of interest
|
|
38
45
|
command = @tool_executor.build_command_line(
|
|
39
46
|
@configurator.tools_test_backtrace_gdb, [],
|
|
40
47
|
gdb_script_filepath,
|
|
41
48
|
executable,
|
|
42
|
-
|
|
49
|
+
unity_filter_arg( group )
|
|
43
50
|
)
|
|
44
51
|
# Things are gonna go boom, so ignore booms to get output
|
|
45
52
|
command[:options][:boom] = false
|
|
46
53
|
|
|
47
54
|
crash_result = @tool_executor.exec( command )
|
|
48
55
|
|
|
49
|
-
# Sum execution time for each
|
|
56
|
+
# Sum execution time for each sub-process run
|
|
50
57
|
# Note: Running tests separately increases total execution time
|
|
51
58
|
shell_result[:time] += crash_result[:time].to_f()
|
|
52
59
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
#
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
#
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
# Line number
|
|
83
|
-
line_number = matched[1]
|
|
84
|
-
|
|
85
|
-
# Build terse signal label: "[SIGNAL] Description"
|
|
86
|
-
signal_label = format_signal_label( crash_result[:output] )
|
|
60
|
+
# Buffered separately from test_case_results and only merged in afterward, since
|
|
61
|
+
# the status check below can still discard every match here in favor of a crash
|
|
62
|
+
# attribution, once the whole group has been seen.
|
|
63
|
+
group_results = @RESULTS_COLLECTOR.new( passed:0, failed:0, ignored:0, output:[] )
|
|
64
|
+
unresolved = []
|
|
65
|
+
|
|
66
|
+
# Attribute each group member its own real result line, if Unity printed one
|
|
67
|
+
group.each do |test_case|
|
|
68
|
+
case crash_result[:output]
|
|
69
|
+
# Success test case
|
|
70
|
+
when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:PASS\s*$)/
|
|
71
|
+
group_results[:passed] += 1
|
|
72
|
+
group_results[:output] << $1
|
|
73
|
+
|
|
74
|
+
# Ignored test case
|
|
75
|
+
when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:IGNORE\s*$)/
|
|
76
|
+
group_results[:ignored] += 1
|
|
77
|
+
group_results[:output] << $1
|
|
78
|
+
|
|
79
|
+
when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:FAIL(:.+)?\s*$)/
|
|
80
|
+
group_results[:failed] += 1
|
|
81
|
+
group_results[:output] << $1
|
|
82
|
+
|
|
83
|
+
# No result line for this member -- either it crashed, or it never got to run
|
|
84
|
+
# because an earlier member in this same group crashed. Resolved below.
|
|
85
|
+
else
|
|
86
|
+
unresolved << test_case
|
|
87
|
+
end
|
|
88
|
+
end
|
|
87
89
|
|
|
88
|
-
|
|
89
|
-
|
|
90
|
+
if !unresolved.empty?
|
|
91
|
+
# Prefer whichever unresolved member's own C symbol is actually named in the gdb
|
|
92
|
+
# backtrace (works regardless of position in the group); fall back to the first
|
|
93
|
+
# unresolved member if no member's symbol can be found in the transcript (e.g. a
|
|
94
|
+
# brief crash report with no frame information at all).
|
|
95
|
+
crashed_case = unresolved.find do |tc|
|
|
96
|
+
crash_result[:output].match?( /#{Regexp.escape(tc[:symbol])}\s*\(\)\sat/ )
|
|
97
|
+
end
|
|
98
|
+
crashed_case ||= unresolved.first
|
|
90
99
|
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
100
|
+
# Per-test-case log file: <log_path>/<context>/<test_name>/<test_case>.gdb.log
|
|
101
|
+
log_path = @file_path_utils.form_test_gdb_log( test_name, context: context, name: crashed_case[:test] )
|
|
102
|
+
@file_wrapper.mkdir( File.dirname( log_path ) )
|
|
103
|
+
@file_wrapper.write( log_path, "=== #{crashed_case[:test]} ===\n#{crash_result[:output]}\n", 'a' )
|
|
95
104
|
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
"#{filename}:#{line_number}:#{test_case[:test]}:FAIL: Test case crashed" \
|
|
99
|
-
" >> #{signal_label}" \
|
|
100
|
-
"#{crash_detail}" \
|
|
101
|
-
"#{NEWLINE_TOKEN}(#{log_path})"
|
|
105
|
+
unresolved.each do |test_case|
|
|
106
|
+
group_results[:failed] += 1
|
|
102
107
|
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
108
|
+
if !test_case.equal?( crashed_case )
|
|
109
|
+
# An earlier case in this same parameterized group already crashed the
|
|
110
|
+
# process -- this member never got a chance to run.
|
|
111
|
+
group_results[:output] <<
|
|
112
|
+
"#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: " \
|
|
113
|
+
"Test case not run -- an earlier case in this parameterized test group crashed"
|
|
114
|
+
next
|
|
115
|
+
end
|
|
107
116
|
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
117
|
+
# Collect file_name and line in which crash occurred.
|
|
118
|
+
# Match against the actual C symbol (`:symbol`), not the human-facing test name
|
|
119
|
+
# (`:test`): a parameterized test case crashes inside a generated wrapper function
|
|
120
|
+
# (`runner_args<N>_<test>`), not a function literally named `<test>(<args>)`.
|
|
121
|
+
matched = crash_result[:output].match( /#{Regexp.escape(test_case[:symbol])}\s*\(\)\sat.+#{filename}:(\d+)\n/ )
|
|
122
|
+
|
|
123
|
+
# If we found an error report line containing `test_case() at filename.c:###` in `gdb` output
|
|
124
|
+
if matched
|
|
125
|
+
# Line number
|
|
126
|
+
line_number = matched[1]
|
|
127
|
+
|
|
128
|
+
# Build terse signal label: "[SIGNAL] Description"
|
|
129
|
+
signal_label = format_signal_label( crash_result[:output] )
|
|
130
|
+
|
|
131
|
+
# Extract the offending source line (nil for assertion crashes or when unavailable)
|
|
132
|
+
source_line = extract_source_line( crash_result[:output], test_case[:symbol], filename )
|
|
133
|
+
|
|
134
|
+
# Unity's test executable output is line oriented.
|
|
135
|
+
# Multi-line output is not possible (it looks like random `printf()` statements to the results parser).
|
|
136
|
+
# "Encode" newlines in multiline string to be handled by the test results parser.
|
|
137
|
+
crash_detail = source_line ? "#{NEWLINE_TOKEN}`#{source_line}`" : ''
|
|
138
|
+
|
|
139
|
+
# Log path appears on its own encoded line so the results parser treats it separately
|
|
140
|
+
group_results[:output] <<
|
|
141
|
+
"#{filename}:#{line_number}:#{test_case[:test]}:FAIL: Test case crashed" \
|
|
142
|
+
" >> #{signal_label}" \
|
|
143
|
+
"#{crash_detail}" \
|
|
112
144
|
"#{NEWLINE_TOKEN}(#{log_path})"
|
|
145
|
+
|
|
146
|
+
# Try to extract a useful label even when no crash location frame was found.
|
|
147
|
+
# A brief Windows assertion failure may report only the assertion text without frames.
|
|
113
148
|
else
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
149
|
+
label = format_signal_label( crash_result[:output] )
|
|
150
|
+
|
|
151
|
+
if !label.empty?
|
|
152
|
+
group_results[:output] <<
|
|
153
|
+
"#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed" \
|
|
154
|
+
" >> #{label}" \
|
|
155
|
+
"#{NEWLINE_TOKEN}(#{log_path})"
|
|
156
|
+
else
|
|
157
|
+
group_results[:output] <<
|
|
158
|
+
"#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: " \
|
|
159
|
+
"Test case crashed (failed to extract `gdb` report)" \
|
|
160
|
+
"#{NEWLINE_TOKEN}(#{log_path})"
|
|
161
|
+
end
|
|
118
162
|
end
|
|
119
163
|
end
|
|
120
164
|
end
|
|
121
165
|
|
|
122
|
-
|
|
166
|
+
# Every member in this group resolved via a matched result line -- but gdb
|
|
167
|
+
# normally exits successfully after handling a crash regardless of what happened
|
|
168
|
+
# to the debuggee, so a clean-looking group here is a weaker signal than it is for
|
|
169
|
+
# `do_simple`. Still checked for the same reason: a diagnostic retry's own real
|
|
170
|
+
# status contradicting a fully clean set of matches is never trustworthy, whatever
|
|
171
|
+
# tool produced it.
|
|
172
|
+
if unresolved.empty? && @helper.test_crash?( filename, executable, crash_result )
|
|
173
|
+
group_results = @RESULTS_COLLECTOR.new(
|
|
174
|
+
passed: 0, ignored: 0, failed: group.size,
|
|
175
|
+
output: group.map { |test_case|
|
|
176
|
+
"#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed" \
|
|
177
|
+
" >> diagnostic retry's own exit status contradicts its reported clean result"
|
|
178
|
+
}
|
|
179
|
+
)
|
|
180
|
+
unresolved = group # non-empty now, marks this group as having shown crash evidence
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
any_group_crashed ||= !unresolved.empty?
|
|
184
|
+
|
|
185
|
+
test_case_results[:passed] += group_results[:passed]
|
|
186
|
+
test_case_results[:ignored] += group_results[:ignored]
|
|
187
|
+
test_case_results[:failed] += group_results[:failed]
|
|
188
|
+
test_case_results[:output].concat( group_results[:output] )
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# No retry group, across the entire file, ever showed real crash evidence, yet this
|
|
192
|
+
# method only runs because the main run was already established as a crash. An
|
|
193
|
+
# all-clean diagnostic must never be allowed to overrule that -- fall back to the
|
|
194
|
+
# same whole-file crash-as-failure construction the :none backtrace setting uses.
|
|
195
|
+
unless any_group_crashed
|
|
196
|
+
@loginator.log(
|
|
197
|
+
"Diagnostic retry for `#{File.basename(executable)}` could not reproduce the crash " \
|
|
198
|
+
"already detected on the main run -- reporting the original crash rather than trusting " \
|
|
199
|
+
"an all-clear diagnostic that contradicts it.",
|
|
200
|
+
Verbosity::ERRORS, LogLabels::CRASH
|
|
201
|
+
)
|
|
202
|
+
return @generator_test_results.create_crash_failure( filename, shell_result, test_cases )
|
|
123
203
|
end
|
|
124
204
|
|
|
125
205
|
# Reset shell result exit code and output
|
|
@@ -135,7 +215,8 @@ class GeneratorTestResultsBacktrace
|
|
|
135
215
|
return shell_result
|
|
136
216
|
end
|
|
137
217
|
|
|
138
|
-
# Re-runs each test case
|
|
218
|
+
# Re-runs each test case (or, for a parameterized test, each group of parameterized
|
|
219
|
+
# cases -- see `group_test_cases`) individually to determine which one(s) crashed.
|
|
139
220
|
# For crash cases, captures any extra output from the test binary (e.g.
|
|
140
221
|
# assertion messages on stderr) and includes it in the failure report.
|
|
141
222
|
# Returns a modified shell_result with regenerated output.
|
|
@@ -146,50 +227,109 @@ class GeneratorTestResultsBacktrace
|
|
|
146
227
|
# Reset time
|
|
147
228
|
shell_result[:time] = 0
|
|
148
229
|
|
|
149
|
-
#
|
|
150
|
-
|
|
151
|
-
|
|
230
|
+
# True once some retry group has actually shown crash evidence of its own -- a
|
|
231
|
+
# member with no result line, or (below) a group whose real status contradicts a
|
|
232
|
+
# fully clean set of matches. If this stays false across every group, the whole
|
|
233
|
+
# diagnostic never reproduced or attributed the crash the main run already
|
|
234
|
+
# detected, and none of it can be trusted -- see the fallback after the loop.
|
|
235
|
+
any_group_crashed = false
|
|
236
|
+
|
|
237
|
+
# Iterate on test cases, one sub-process run per group (see `group_test_cases`)
|
|
238
|
+
group_test_cases( test_cases ).each do |group|
|
|
239
|
+
# Build the test fixture to run with our test case (or parameterized group) of interest
|
|
152
240
|
command = @tool_executor.build_command_line(
|
|
153
241
|
@configurator.tools_test_fixture_simple_backtrace, [],
|
|
154
242
|
executable,
|
|
155
|
-
|
|
243
|
+
unity_filter_arg( group )
|
|
156
244
|
)
|
|
157
245
|
# Things are gonna go boom, so ignore booms to get output
|
|
158
246
|
command[:options][:boom] = false
|
|
159
247
|
|
|
160
248
|
crash_result = @tool_executor.exec( command )
|
|
161
249
|
|
|
162
|
-
# Sum execution time for each
|
|
250
|
+
# Sum execution time for each sub-process run
|
|
163
251
|
# Note: Running tests separately increases total execution time
|
|
164
252
|
shell_result[:time] += crash_result[:time].to_f()
|
|
165
253
|
|
|
166
|
-
#
|
|
167
|
-
|
|
168
|
-
#
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
254
|
+
# Buffered separately from test_case_results and only merged in afterward, since
|
|
255
|
+
# the status check below can still discard every match here in favor of a crash
|
|
256
|
+
# attribution, once the whole group has been seen.
|
|
257
|
+
group_results = @RESULTS_COLLECTOR.new( passed:0, failed:0, ignored:0, output:[] )
|
|
258
|
+
crashed = false # Has the actual crash in this group already been attributed?
|
|
259
|
+
|
|
260
|
+
# Attribute each group member its own real result line, if Unity printed one
|
|
261
|
+
group.each do |test_case|
|
|
262
|
+
case crash_result[:output]
|
|
263
|
+
# Success test case
|
|
264
|
+
when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:PASS\s*$)/
|
|
265
|
+
group_results[:passed] += 1
|
|
266
|
+
group_results[:output] << $1
|
|
267
|
+
|
|
268
|
+
# Ignored test case
|
|
269
|
+
when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:IGNORE\s*$)/
|
|
270
|
+
group_results[:ignored] += 1
|
|
271
|
+
group_results[:output] << $1
|
|
272
|
+
|
|
273
|
+
when /(^#{Regexp.escape(filename)}:\d+:#{Regexp.escape(test_case[:test])}:FAIL(:.+)?\s*$)/
|
|
274
|
+
group_results[:failed] += 1
|
|
275
|
+
group_results[:output] << $1
|
|
276
|
+
|
|
277
|
+
# No result line for this member -- either it crashed, or it never got to run
|
|
278
|
+
# because an earlier member in this same group crashed.
|
|
279
|
+
else
|
|
280
|
+
group_results[:failed] += 1
|
|
281
|
+
|
|
282
|
+
if crashed
|
|
283
|
+
group_results[:output] <<
|
|
284
|
+
"#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: " \
|
|
285
|
+
"Test case not run -- an earlier case in this parameterized test group crashed"
|
|
286
|
+
else
|
|
287
|
+
crashed = true
|
|
288
|
+
# Collect any non-result, non-blank lines (e.g. assertion messages on stderr)
|
|
289
|
+
extra = extract_simple_crash_output( crash_result[:output], filename )
|
|
290
|
+
test_output = "#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed"
|
|
291
|
+
test_output += " >> #{extra.join(NEWLINE_TOKEN)}" unless extra.empty?
|
|
292
|
+
group_results[:output] << test_output
|
|
293
|
+
end
|
|
294
|
+
end
|
|
189
295
|
end
|
|
190
296
|
|
|
191
|
-
#
|
|
192
|
-
|
|
297
|
+
# Every member in this group resolved via a matched result line -- but a diagnostic
|
|
298
|
+
# retry running through its own, separately-configured tool (e.g. without the main
|
|
299
|
+
# :test_fixture's sanitizer options) can complete cleanly and print a legitimate-
|
|
300
|
+
# looking PASS even when it isn't a trustworthy stand-in for what actually happened.
|
|
301
|
+
# The same real-status check the main run itself relies on applies here too.
|
|
302
|
+
if !crashed && @helper.test_crash?( filename, executable, crash_result )
|
|
303
|
+
crashed = true
|
|
304
|
+
group_results = @RESULTS_COLLECTOR.new(
|
|
305
|
+
passed: 0, ignored: 0, failed: group.size,
|
|
306
|
+
output: group.map { |test_case|
|
|
307
|
+
"#{filename}:#{test_case[:line_number]}:#{test_case[:test]}:FAIL: Test case crashed" \
|
|
308
|
+
" >> diagnostic retry's own exit status contradicts its reported clean result"
|
|
309
|
+
}
|
|
310
|
+
)
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
any_group_crashed ||= crashed
|
|
314
|
+
|
|
315
|
+
test_case_results[:passed] += group_results[:passed]
|
|
316
|
+
test_case_results[:ignored] += group_results[:ignored]
|
|
317
|
+
test_case_results[:failed] += group_results[:failed]
|
|
318
|
+
test_case_results[:output].concat( group_results[:output] )
|
|
319
|
+
end
|
|
320
|
+
|
|
321
|
+
# No retry group, across the entire file, ever showed real crash evidence, yet this
|
|
322
|
+
# method only runs because the main run was already established as a crash. An
|
|
323
|
+
# all-clean diagnostic must never be allowed to overrule that -- fall back to the
|
|
324
|
+
# same whole-file crash-as-failure construction the :none backtrace setting uses.
|
|
325
|
+
unless any_group_crashed
|
|
326
|
+
@loginator.log(
|
|
327
|
+
"Diagnostic retry for `#{File.basename(executable)}` could not reproduce the crash " \
|
|
328
|
+
"already detected on the main run -- reporting the original crash rather than trusting " \
|
|
329
|
+
"an all-clear diagnostic that contradicts it.",
|
|
330
|
+
Verbosity::ERRORS, LogLabels::CRASH
|
|
331
|
+
)
|
|
332
|
+
return @generator_test_results.create_crash_failure( filename, shell_result, test_cases )
|
|
193
333
|
end
|
|
194
334
|
|
|
195
335
|
# Reset shell result exit code and output
|
|
@@ -208,6 +348,34 @@ class GeneratorTestResultsBacktrace
|
|
|
208
348
|
### Private ###
|
|
209
349
|
private
|
|
210
350
|
|
|
351
|
+
# Groups test cases so each parameterized test's cases are isolated together as one
|
|
352
|
+
# sub-process run instead of one run per case. Unity's `-n`/`-f` command-line filter
|
|
353
|
+
# parser treats a comma as a separator between multiple OR'd filter clauses, so an
|
|
354
|
+
# exact filter can never match a parameterized case's runtime name once it has more
|
|
355
|
+
# than one argument (e.g. `name(5, 3, 2)`). Grouping by base name and matching each
|
|
356
|
+
# member's own result line out of the group's single shared sub-run output sidesteps
|
|
357
|
+
# that limitation entirely. Non-parameterized test cases are unaffected: each forms
|
|
358
|
+
# its own single-member group, isolated exactly as before.
|
|
359
|
+
def group_test_cases(test_cases)
|
|
360
|
+
test_cases.group_by { |test_case| test_case[:test].sub(/\(.*\)\z/, '') }.values
|
|
361
|
+
end
|
|
362
|
+
|
|
363
|
+
# Builds the Unity command-line filter argument for isolating one group (see
|
|
364
|
+
# `group_test_cases`). A non-parameterized (single-member, no-args) group is isolated
|
|
365
|
+
# with an exact-match filter as before. A parameterized group is isolated with a
|
|
366
|
+
# non-strict prefix filter on its shared base name -- the only reliable way to select
|
|
367
|
+
# such a group, since its members' runtime names contain commas.
|
|
368
|
+
def unity_filter_arg(group)
|
|
369
|
+
test_name = group.first[:test]
|
|
370
|
+
|
|
371
|
+
if test_name.include?('(')
|
|
372
|
+
base_name = test_name.sub(/\(.*\)\z/, '')
|
|
373
|
+
%(-f "#{base_name}")
|
|
374
|
+
else
|
|
375
|
+
%(-n "#{test_name}")
|
|
376
|
+
end
|
|
377
|
+
end
|
|
378
|
+
|
|
211
379
|
# Builds a terse crash label from gdb output.
|
|
212
380
|
# Rules:
|
|
213
381
|
# - Named signal explicitly in output → "[SIGNAL] Description"
|
|
@@ -62,8 +62,26 @@ class GeneratorTestRunner
|
|
|
62
62
|
# Unity's runner generator `find_tests()` produces an array of hashes with the following keys...
|
|
63
63
|
# { test:, args:, call:, params:, line_number: }
|
|
64
64
|
|
|
65
|
-
# For external use
|
|
66
|
-
|
|
65
|
+
# For external use, reduce down to test name, runtime C symbol, and line number.
|
|
66
|
+
# A parameterized test (`:args` populated) is expanded into one entry per TEST_CASE /
|
|
67
|
+
# TEST_RANGE / TEST_MATRIX row, since Unity's generated runner registers each invocation
|
|
68
|
+
# under its own name (matching `generate_test_runner.rb`'s `"#{test}(#{args})"` naming) and
|
|
69
|
+
# runs it through its own wrapper function (`runner_args<N>_<test>`). Collapsing these to a
|
|
70
|
+
# single bare-name entry breaks exact-match test case isolation (crash handling) for
|
|
71
|
+
# parameterized tests.
|
|
72
|
+
@test_cases = @test_cases_internal.flat_map do |hash|
|
|
73
|
+
if hash[:args].nil? || hash[:args].empty?
|
|
74
|
+
[ { test: hash[:test], symbol: hash[:test], line_number: hash[:line_number] } ]
|
|
75
|
+
else
|
|
76
|
+
hash[:args].each_with_index.map do |args, idx|
|
|
77
|
+
{
|
|
78
|
+
test: "#{hash[:test]}(#{args})",
|
|
79
|
+
symbol: "runner_args#{idx + 1}_#{hash[:test]}",
|
|
80
|
+
line_number: hash[:line_number]
|
|
81
|
+
}
|
|
82
|
+
end
|
|
83
|
+
end
|
|
84
|
+
end
|
|
67
85
|
end
|
|
68
86
|
|
|
69
87
|
def extract_test_cases(source_contents)
|
|
@@ -32,10 +32,15 @@ class Includes
|
|
|
32
32
|
else raise ArgumentError, "Unknown Include type: #{include.class}"
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
-
{
|
|
35
|
+
hash = {
|
|
36
36
|
'type' => type,
|
|
37
37
|
'filepath' => include.filepath,
|
|
38
38
|
}
|
|
39
|
+
# `include_path` is only ever meaningful on a reconciled SystemInclude, so it's
|
|
40
|
+
# written only when actually set -- a cached build's YAML stays uncluttered for
|
|
41
|
+
# the common, plain case.
|
|
42
|
+
hash['include_path'] = include.include_path if include.include_path
|
|
43
|
+
hash
|
|
39
44
|
end
|
|
40
45
|
end
|
|
41
46
|
|
|
@@ -63,11 +68,11 @@ class Includes
|
|
|
63
68
|
|
|
64
69
|
case hash['type']
|
|
65
70
|
when 'user'
|
|
66
|
-
UserInclude.new(hash['filepath'])
|
|
71
|
+
UserInclude.new(hash['filepath'], include_path: hash['include_path'])
|
|
67
72
|
when 'mock'
|
|
68
73
|
MockInclude.new(hash['filepath'])
|
|
69
74
|
when 'system'
|
|
70
|
-
SystemInclude.new(hash['filepath'])
|
|
75
|
+
SystemInclude.new(hash['filepath'], include_path: hash['include_path'])
|
|
71
76
|
else
|
|
72
77
|
raise ArgumentError, "Invalid include type: #{hash['type']}. Must be 'user' or 'system'"
|
|
73
78
|
end
|
|
@@ -153,6 +158,16 @@ class Includes
|
|
|
153
158
|
# Compares bare includes against user and system includes and applies the following rules:
|
|
154
159
|
# 1. Intersection of bare includes and system includes.
|
|
155
160
|
# 2. Intersection of bare includes and user includes.
|
|
161
|
+
#
|
|
162
|
+
# A matched system include's `filepath` is its real, resolved location (possibly
|
|
163
|
+
# absolute), which is what identity and deduplication need but is wrong to render
|
|
164
|
+
# verbatim into a generated file. Each matched system include is therefore rebuilt
|
|
165
|
+
# carrying `include_path:` recovered from `bare` -- the original, as-written directive
|
|
166
|
+
# text -- via `best_bare_match`, so `<sys/stat.h>` renders as written instead of
|
|
167
|
+
# collapsing to `<stat.h>`. User includes are left untouched; unlike a system header's
|
|
168
|
+
# `<...>` text, a quoted include's own bare-includes pass can be resolved by GCC's
|
|
169
|
+
# directory-relative search before reconciliation ever sees it, so `bare` isn't a
|
|
170
|
+
# reliable source of "the literal, as-written spelling" for a user include.
|
|
156
171
|
def self.reconcile(bare:, user:, system:)
|
|
157
172
|
# Validate input types
|
|
158
173
|
|
|
@@ -179,10 +194,15 @@ class Includes
|
|
|
179
194
|
# Create set of bare include filenames for O(1) lookup
|
|
180
195
|
bare_filenames = Set.new(bare.map(&:filename))
|
|
181
196
|
|
|
182
|
-
# Intersect system includes with bare includes based on filename
|
|
183
|
-
#
|
|
197
|
+
# Intersect system includes with bare includes based on filename, then rebuild each
|
|
198
|
+
# matched system include carrying the original, as-written directive spelling
|
|
199
|
+
# recovered from bare via best_bare_match, since filepath alone (the real, resolved
|
|
200
|
+
# location) is wrong to render verbatim.
|
|
184
201
|
system_includes = system.select do |include|
|
|
185
202
|
bare_filenames.include?(include.filename)
|
|
203
|
+
end.map do |include|
|
|
204
|
+
original = best_bare_match(bare, include.filepath)
|
|
205
|
+
SystemInclude.new(include.filepath, include_path: original.filepath)
|
|
186
206
|
end
|
|
187
207
|
|
|
188
208
|
# Intersect user includes with bare includes based on filename.
|
|
@@ -196,6 +216,36 @@ class Includes
|
|
|
196
216
|
return (system_includes + user_includes)
|
|
197
217
|
end
|
|
198
218
|
|
|
219
|
+
# Two filepaths correspond if the shorter one's path segments equal the longer one's
|
|
220
|
+
# own trailing segments, exactly, in order -- checked without regard for which side is
|
|
221
|
+
# shorter, since either a bare entry or its candidate may be the one carrying less path.
|
|
222
|
+
def self.paths_correspond?(a, b)
|
|
223
|
+
segments_a = a.split(/[\\\/]/).reject(&:empty?)
|
|
224
|
+
segments_b = b.split(/[\\\/]/).reject(&:empty?)
|
|
225
|
+
shorter, longer = segments_a.length <= segments_b.length ? [segments_a, segments_b] : [segments_b, segments_a]
|
|
226
|
+
return longer.last(shorter.length) == shorter
|
|
227
|
+
end
|
|
228
|
+
private_class_method :paths_correspond?
|
|
229
|
+
|
|
230
|
+
# Finds the bare entry that best identifies a resolved system include's original,
|
|
231
|
+
# as-written spelling. A bare entry carries no information about whether its own
|
|
232
|
+
# #include was quoted or bracketed, so a project's own same-named quoted include
|
|
233
|
+
# (`#include "stat.h"`) can sit in `bare` right alongside a system one
|
|
234
|
+
# (`#include <sys/stat.h>`). Preferring the *longest* path correspondence, rather than
|
|
235
|
+
# merely the first one found, keeps a short, unrelated bare entry -- which trivially
|
|
236
|
+
# "corresponds" to any longer resolved path ending in the same filename -- from
|
|
237
|
+
# shadowing the correct, more specific entry. Falls back to the longest same-filename
|
|
238
|
+
# candidate, even without path correspondence, only when the compiler's real
|
|
239
|
+
# search-path structure doesn't literally match the directive's own spelling (a
|
|
240
|
+
# symlinked or vendored include root, for instance).
|
|
241
|
+
def self.best_bare_match(bare, filepath)
|
|
242
|
+
candidates = bare.select { |bare_include| bare_include.filename == File.basename(filepath) }
|
|
243
|
+
corresponding = candidates.select { |bare_include| paths_correspond?(bare_include.filepath, filepath) }
|
|
244
|
+
pool = corresponding.empty? ? candidates : corresponding
|
|
245
|
+
return pool.max_by { |bare_include| bare_include.filepath.split(/[\\\/]/).reject(&:empty?).length }
|
|
246
|
+
end
|
|
247
|
+
private_class_method :best_bare_match
|
|
248
|
+
|
|
199
249
|
# Sort list so system includes are at the beginning
|
|
200
250
|
# (Best practice)
|
|
201
251
|
def self.sort(includes)
|
|
@@ -216,6 +266,7 @@ class Include
|
|
|
216
266
|
attr_reader :filepath
|
|
217
267
|
attr_reader :filename
|
|
218
268
|
attr_reader :path
|
|
269
|
+
attr_reader :include_path
|
|
219
270
|
|
|
220
271
|
# Initialize an Include object from a C include statement or simple filepath.
|
|
221
272
|
#
|
|
@@ -224,18 +275,21 @@ class Include
|
|
|
224
275
|
# - #include <stdio.h>
|
|
225
276
|
# - A quoted/bracketed filepath (e.g., '"header.h"' or <stdio.h>')
|
|
226
277
|
# - A plain filepath (e.g., 'path/to/header.h')
|
|
227
|
-
# @param
|
|
228
|
-
# -
|
|
229
|
-
#
|
|
278
|
+
# @param include_path [String, nil] (default: nil)
|
|
279
|
+
# - When set, this exact spelling is used in the include directive, taking
|
|
280
|
+
# precedence over `filepath`/`filename`. This exists for a reconciled
|
|
281
|
+
# SystemInclude, whose `filepath` is deliberately the header's real, resolved
|
|
282
|
+
# location (needed elsewhere for identity -- see `Includes.sanitize!`) rather than
|
|
283
|
+
# the original, as-written directive text a reconciled render needs.
|
|
230
284
|
# @raise [ArgumentError] If the statement is empty or becomes empty after cleaning
|
|
231
|
-
def initialize(statement,
|
|
285
|
+
def initialize(statement, include_path: nil)
|
|
232
286
|
@filepath = clean(statement)
|
|
233
287
|
|
|
234
288
|
raise ArgumentError, "Empty include statement" if @filepath.empty?
|
|
235
289
|
|
|
236
290
|
@filename = File.basename(@filepath)
|
|
237
291
|
@path = File.dirname(@filepath)
|
|
238
|
-
@
|
|
292
|
+
@include_path = clean(include_path) if include_path
|
|
239
293
|
end
|
|
240
294
|
|
|
241
295
|
# Method specialized by subclasses
|
|
@@ -294,7 +348,8 @@ class Include
|
|
|
294
348
|
|
|
295
349
|
# Returns the configured entry to use in the include directive
|
|
296
350
|
def include()
|
|
297
|
-
|
|
351
|
+
return @include_path if @include_path
|
|
352
|
+
@filename
|
|
298
353
|
end
|
|
299
354
|
|
|
300
355
|
private
|
data/lib/ceedling/objects.yml
CHANGED
|
@@ -249,8 +249,10 @@ generator_test_results_backtrace:
|
|
|
249
249
|
- configurator
|
|
250
250
|
- tool_executor
|
|
251
251
|
- generator_test_results
|
|
252
|
+
- generator_helper
|
|
252
253
|
- file_path_utils
|
|
253
254
|
- file_wrapper
|
|
255
|
+
- loginator
|
|
254
256
|
|
|
255
257
|
generator_test_results_sanity_checker:
|
|
256
258
|
compose:
|
|
@@ -312,6 +314,7 @@ preprocessinator_includes_handler:
|
|
|
312
314
|
- tool_executor
|
|
313
315
|
- parsing_parcels
|
|
314
316
|
- file_wrapper
|
|
317
|
+
- file_path_utils
|
|
315
318
|
- yaml_wrapper
|
|
316
319
|
- loginator
|
|
317
320
|
- reportinator
|
|
@@ -354,6 +357,8 @@ c_extractor:
|
|
|
354
357
|
- c_extractor_functions
|
|
355
358
|
- c_extractor_preprocessing
|
|
356
359
|
- c_extractor_definitions
|
|
360
|
+
- configurator
|
|
361
|
+
- loginator
|
|
357
362
|
|
|
358
363
|
partializer_config:
|
|
359
364
|
compose:
|