branchproof 0.11.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +89 -0
- data/README.md +440 -29
- data/doc/Branchproof/CLI.md +1 -1
- data/doc/Branchproof/ChangedCoverage.md +19 -0
- data/doc/Branchproof/ChangedCoveragePolicy.md +24 -0
- data/doc/Branchproof/ChangedDecisionMap.md +20 -0
- data/doc/Branchproof/ChangedScope.md +18 -0
- data/doc/Branchproof/Collation.md +23 -0
- data/doc/Branchproof/CollationManifest.md +29 -0
- data/doc/Branchproof/CoverageSummary.md +20 -0
- data/doc/Branchproof/Doctor/TerminalRenderer.md +15 -0
- data/doc/Branchproof/Doctor.md +34 -0
- data/doc/Branchproof/FallbackInstrumentation.md +8 -0
- data/doc/Branchproof/FallbackRuntime.md +12 -0
- data/doc/Branchproof/FallbackSyntax.md +13 -0
- data/doc/Branchproof/GitChanges.md +19 -0
- data/doc/Branchproof/GithubReport.md +25 -0
- data/doc/Branchproof/HtmlReport.md +22 -0
- data/doc/Branchproof/Instrumenter.md +1 -1
- data/doc/Branchproof/MinitestAdapter.md +4 -0
- data/doc/Branchproof/RakeTask.md +121 -0
- data/doc/Branchproof/Report.md +41 -3
- data/doc/Branchproof/ReportSelection.md +10 -1
- data/doc/Branchproof/Runtime.md +4 -1
- data/doc/Branchproof/Source.md +6 -1
- data/doc/Branchproof/SummaryRanking/Decision.md +86 -0
- data/doc/Branchproof/SummaryRanking/SourceFile.md +43 -0
- data/doc/Branchproof/SummaryRanking.md +36 -0
- data/doc/Branchproof/SummaryReport.md +42 -0
- data/doc/Branchproof/Worker.md +1 -1
- data/doc/Branchproof.md +21 -1
- data/doc/CHANGELOG.md +89 -0
- data/doc/README.md +440 -29
- data/lib/branchproof/analyzer.rb +2 -52
- data/lib/branchproof/changed_coverage.rb +177 -0
- data/lib/branchproof/changed_decision_map.rb +359 -0
- data/lib/branchproof/changed_scope.rb +93 -0
- data/lib/branchproof/cli.rb +242 -21
- data/lib/branchproof/collation.rb +525 -0
- data/lib/branchproof/collation_manifest.rb +129 -0
- data/lib/branchproof/comparison.rb +11 -3
- data/lib/branchproof/comparison_report.rb +2 -0
- data/lib/branchproof/configuration.rb +12 -2
- data/lib/branchproof/coverage_policy.rb +122 -0
- data/lib/branchproof/coverage_summary.rb +94 -0
- data/lib/branchproof/doctor.rb +241 -0
- data/lib/branchproof/evidence.rb +5 -1
- data/lib/branchproof/exception_instrumentation.rb +4 -3
- data/lib/branchproof/exception_syntax.rb +15 -1
- data/lib/branchproof/fallback_instrumentation.rb +25 -0
- data/lib/branchproof/fallback_runtime.rb +14 -0
- data/lib/branchproof/fallback_syntax.rb +90 -0
- data/lib/branchproof/focused_report.rb +81 -22
- data/lib/branchproof/git_changes.rb +188 -0
- data/lib/branchproof/github_report.rb +292 -0
- data/lib/branchproof/html_report.rb +412 -0
- data/lib/branchproof/instrumenter.rb +2 -0
- data/lib/branchproof/minimizer.rb +4 -2
- data/lib/branchproof/minitest_adapter.rb +45 -5
- data/lib/branchproof/rails_support.rb +2 -2
- data/lib/branchproof/rake_task.rb +186 -0
- data/lib/branchproof/report.rb +383 -71
- data/lib/branchproof/report_selection.rb +26 -5
- data/lib/branchproof/runtime.rb +2 -0
- data/lib/branchproof/saved_report.rb +323 -5
- data/lib/branchproof/source.rb +25 -4
- data/lib/branchproof/summary_ranking.rb +165 -0
- data/lib/branchproof/summary_report.rb +241 -0
- data/lib/branchproof/version.rb +1 -1
- data/lib/branchproof/worker.rb +14 -9
- data/lib/branchproof.rb +12 -0
- data/llms.txt +21 -1
- data/sig/branchproof.rbs +206 -5
- metadata +38 -22
|
@@ -0,0 +1,525 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# rubocop:disable Metrics/AbcSize, Metrics/BlockLength, Metrics/ClassLength, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/ParameterLists, Metrics/PerceivedComplexity
|
|
4
|
+
|
|
5
|
+
require "digest"
|
|
6
|
+
require "json"
|
|
7
|
+
require "pathname"
|
|
8
|
+
require "stringio"
|
|
9
|
+
require_relative "evidence"
|
|
10
|
+
require_relative "saved_report"
|
|
11
|
+
require_relative "analyzer"
|
|
12
|
+
require_relative "minimizer"
|
|
13
|
+
require_relative "report"
|
|
14
|
+
|
|
15
|
+
module Branchproof
|
|
16
|
+
# Reconstructs one analysis from validated, offline report artifacts.
|
|
17
|
+
class Collation
|
|
18
|
+
SCHEMA_VERSIONS = %w[1.4 1.5 1.6].freeze
|
|
19
|
+
STATUS_ORDER = %w[passed skipped unknown failed].freeze
|
|
20
|
+
|
|
21
|
+
def initialize(inputs:, expected_shards: nil)
|
|
22
|
+
raise ArgumentError, "inputs must be a non-empty Array" unless inputs.is_a?(Array) && !inputs.empty?
|
|
23
|
+
if !expected_shards.nil? && (!expected_shards.is_a?(Array) || expected_shards.empty? ||
|
|
24
|
+
!expected_shards.all? { |id| nonempty_string?(id) } ||
|
|
25
|
+
expected_shards.uniq.length != expected_shards.length)
|
|
26
|
+
raise ArgumentError, "expected_shards must be a non-empty array of unique IDs or nil"
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
@inputs = inputs
|
|
30
|
+
@expected_shards = expected_shards&.dup
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def call
|
|
34
|
+
shards = validated_shards
|
|
35
|
+
unique_shards = unique_artifacts(shards).sort_by { |shard| [shard.fetch(:id), shard.fetch(:digest)] }
|
|
36
|
+
reference = unique_shards.first
|
|
37
|
+
validate_compatibility!(unique_shards, reference)
|
|
38
|
+
validate_union_storage!(unique_shards, reference.fetch(:limits))
|
|
39
|
+
evidence = merged_evidence(unique_shards, reference)
|
|
40
|
+
missing = @expected_shards ? @expected_shards - unique_shards.map { |shard| shard.fetch(:id) } : []
|
|
41
|
+
collection_status = if @expected_shards.nil?
|
|
42
|
+
"unknown"
|
|
43
|
+
elsif missing.empty?
|
|
44
|
+
"complete"
|
|
45
|
+
else
|
|
46
|
+
"incomplete"
|
|
47
|
+
end
|
|
48
|
+
baseline, failed_or_error = merged_baseline(unique_shards, collection_status)
|
|
49
|
+
analysis = if failed_or_error
|
|
50
|
+
nil
|
|
51
|
+
else
|
|
52
|
+
Analyzer.new(inventory: reference.fetch(:inventory), evidence: evidence,
|
|
53
|
+
limits: reference.fetch(:limits), reachability: reference.fetch(:reachability)).call
|
|
54
|
+
end
|
|
55
|
+
analysis = Marshal.load(Marshal.dump(analysis)) if analysis
|
|
56
|
+
completeness = effective_completeness(unique_shards, evidence, analysis, collection_status)
|
|
57
|
+
analysis[:completeness] = analysis.fetch(:completeness).merge(completeness) if analysis
|
|
58
|
+
evidence[:completeness] = completeness
|
|
59
|
+
evidence[:run_ids] = evidence.fetch(:run_ids).sort
|
|
60
|
+
minima = analysis ? calculated_minima(analysis, evidence, reference) : []
|
|
61
|
+
report = build_report(reference, evidence, analysis, minima, baseline, unique_shards, missing,
|
|
62
|
+
collection_status)
|
|
63
|
+
validate_output!(report)
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
private
|
|
67
|
+
|
|
68
|
+
def validated_shards
|
|
69
|
+
ids = {}
|
|
70
|
+
@inputs.map do |input|
|
|
71
|
+
raise ArgumentError, "each input must be an object" unless input.is_a?(Hash)
|
|
72
|
+
|
|
73
|
+
document = stringify(input[:document] || input["document"])
|
|
74
|
+
raise ArgumentError, "input document must be an object" unless document.is_a?(Hash)
|
|
75
|
+
|
|
76
|
+
SavedReport.new(document).validate!
|
|
77
|
+
schema = document.fetch("schema_version")
|
|
78
|
+
unless SCHEMA_VERSIONS.include?(schema)
|
|
79
|
+
raise ArgumentError,
|
|
80
|
+
"collation accepts raw report schemas 1.4 through 1.6"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
digest = Digest::SHA256.hexdigest(canonical_json(document))
|
|
84
|
+
path = input[:path] || input["path"]
|
|
85
|
+
raise ArgumentError, "input path must be a string" unless path.is_a?(String)
|
|
86
|
+
|
|
87
|
+
id = input[:id] || input["id"]
|
|
88
|
+
id = digest if id.nil? && @expected_shards.nil?
|
|
89
|
+
raise ArgumentError, "input shard ID is required" unless nonempty_string?(id)
|
|
90
|
+
raise ArgumentError, "unexpected shard ID #{id.inspect}" if @expected_shards && !@expected_shards.include?(id)
|
|
91
|
+
raise ArgumentError, "duplicate shard ID #{id.inspect}" if ids.key?(id) && ids[id] != digest
|
|
92
|
+
|
|
93
|
+
ids[id] = digest
|
|
94
|
+
metadata = document["run_metadata"]
|
|
95
|
+
validate_reconstruction_metadata!(document, metadata)
|
|
96
|
+
validate_baseline_tests!(document, metadata)
|
|
97
|
+
{
|
|
98
|
+
id: id, path: path, paths: [path], document: document, digest: digest,
|
|
99
|
+
run_ids: document.fetch("run_ids"), inventory: symbolize(document.fetch("source_inventory")),
|
|
100
|
+
snapshot: document.fetch("observations"), baseline: document.fetch("baseline"),
|
|
101
|
+
completeness: effective_input_completeness(document), metadata: metadata,
|
|
102
|
+
limits: Limits.normalize(metadata.fetch("limits")), level: metadata.fetch("requested_level"),
|
|
103
|
+
reachability: metadata.fetch("reachability"), minimum: document.dig("coverage_policy", "minimum"),
|
|
104
|
+
changed_scope: document["changed_scope"],
|
|
105
|
+
minimum_changed: document.dig("changed_coverage_policy", "minimum_changed") || {}
|
|
106
|
+
}
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def validate_reconstruction_metadata!(document, metadata)
|
|
111
|
+
raise ArgumentError, "collation requires run_metadata" unless metadata.is_a?(Hash)
|
|
112
|
+
|
|
113
|
+
unless [1, 2, 3].include?(metadata["requested_level"])
|
|
114
|
+
raise ArgumentError, "collation requires requested_level 1, 2, or 3"
|
|
115
|
+
end
|
|
116
|
+
unless [true, false].include?(metadata["reachability"])
|
|
117
|
+
raise ArgumentError, "collation requires Boolean reachability metadata"
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
limits = metadata["limits"]
|
|
121
|
+
valid_limits = limits.is_a?(Hash) && limits.keys.sort == Limits::KEYS.map(&:to_s).sort &&
|
|
122
|
+
limits.values.all? { |value| value.is_a?(Integer) && value.positive? }
|
|
123
|
+
raise ArgumentError, "collation requires complete limits metadata" unless valid_limits
|
|
124
|
+
raise ArgumentError, "collation requires project and framework metadata" unless
|
|
125
|
+
nonempty_string?(metadata["project_kind"]) && nonempty_string?(metadata["project_root"]) &&
|
|
126
|
+
nonempty_string?(metadata["framework"])
|
|
127
|
+
raise ArgumentError, "collation requires a current Branchproof version" unless
|
|
128
|
+
document["tool_version"] == Branchproof::VERSION
|
|
129
|
+
raise ArgumentError, "collation requires the current criterion version" unless
|
|
130
|
+
document["criterion_version"] == Evidence::CRITERION_VERSION
|
|
131
|
+
raise ArgumentError, "collation requires the current Ruby runtime" unless document["runtime"] == RUBY_DESCRIPTION
|
|
132
|
+
unless document["coverage_policy"].is_a?(Hash) &&
|
|
133
|
+
document["coverage_policy"]["minimum"].is_a?(Hash)
|
|
134
|
+
raise ArgumentError,
|
|
135
|
+
"collation requires complete policy metadata"
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
run_ids = document["run_ids"]
|
|
139
|
+
unless run_ids.is_a?(Array) && !run_ids.empty? && run_ids.all? { |id| nonempty_string?(id) } &&
|
|
140
|
+
run_ids.uniq.length == run_ids.length && document.dig("observations", "run_ids") == run_ids
|
|
141
|
+
raise ArgumentError, "collation requires matching nonempty report and observation run IDs"
|
|
142
|
+
end
|
|
143
|
+
|
|
144
|
+
validate_baseline_counts!(document.fetch("baseline"))
|
|
145
|
+
if %w[1.5 1.6].include?(document["schema_version"]) && !document["changed_scope"].is_a?(Hash)
|
|
146
|
+
raise ArgumentError, "collation requires changed scope metadata"
|
|
147
|
+
end
|
|
148
|
+
return unless document["schema_version"] == "1.6"
|
|
149
|
+
|
|
150
|
+
raise ArgumentError, "collation requires changed policy metadata" unless
|
|
151
|
+
document["changed_coverage_policy"].is_a?(Hash) &&
|
|
152
|
+
document["changed_coverage_policy"]["minimum_changed"].is_a?(Hash)
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
def validate_baseline_counts!(baseline)
|
|
156
|
+
unless %w[PASSED FAILED ERROR INCOMPLETE].include?(baseline["status"])
|
|
157
|
+
raise ArgumentError, "baseline status is unsupported for collation"
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
%w[executed_tests failed_tests skipped_tests].each do |field|
|
|
161
|
+
value = baseline[field]
|
|
162
|
+
raise ArgumentError, "baseline #{field} must be a nonnegative integer" unless value.is_a?(Integer) && value >= 0
|
|
163
|
+
end
|
|
164
|
+
raise ArgumentError, "baseline failed and skipped counts exceed executed tests" if
|
|
165
|
+
baseline.fetch("failed_tests") + baseline.fetch("skipped_tests") > baseline.fetch("executed_tests")
|
|
166
|
+
|
|
167
|
+
passed_with_failures = baseline.fetch("failed_tests").positive?
|
|
168
|
+
zero_test_pass = baseline.fetch("executed_tests").zero?
|
|
169
|
+
return unless baseline["status"] == "PASSED" && (passed_with_failures || zero_test_pass)
|
|
170
|
+
|
|
171
|
+
raise ArgumentError, "PASSED baseline cannot contain failures or zero tests"
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
def validate_baseline_tests!(document, metadata)
|
|
175
|
+
baseline_tests = document.dig("baseline", "tests")
|
|
176
|
+
observations_list = document.dig("observations", "tests")
|
|
177
|
+
observations = observations_list.to_h { |test| [test.fetch("id"), test] }
|
|
178
|
+
baseline_by_id = Array(baseline_tests).to_h { |test| [test.fetch("id"), test] }
|
|
179
|
+
baseline_by_id.each do |id, test|
|
|
180
|
+
observed = observations[test.fetch("id")]
|
|
181
|
+
next unless observed
|
|
182
|
+
|
|
183
|
+
unless normalized_test_identity(test, metadata["project_root"]) ==
|
|
184
|
+
normalized_test_identity(observed, metadata["project_root"])
|
|
185
|
+
raise ArgumentError, "baseline test identity differs from observation for #{id.inspect}"
|
|
186
|
+
end
|
|
187
|
+
if test.fetch("status") != observed.fetch("status")
|
|
188
|
+
raise ArgumentError, "baseline test status differs from observation for #{id.inspect}"
|
|
189
|
+
end
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
statuses = observations_list.map { |test| test.fetch("status") }
|
|
193
|
+
invalid = statuses - %w[passed failed skipped unknown running]
|
|
194
|
+
unless invalid.empty?
|
|
195
|
+
raise ArgumentError,
|
|
196
|
+
"observed tests contain unsupported statuses: #{invalid.uniq.join(", ")}"
|
|
197
|
+
end
|
|
198
|
+
|
|
199
|
+
baseline = document.fetch("baseline")
|
|
200
|
+
observed_failed = statuses.count("failed")
|
|
201
|
+
observed_skipped = statuses.count("skipped")
|
|
202
|
+
if observed_failed > baseline.fetch("failed_tests") || observed_skipped > baseline.fetch("skipped_tests")
|
|
203
|
+
raise ArgumentError, "observed test failures or skips exceed baseline counts"
|
|
204
|
+
end
|
|
205
|
+
if baseline.fetch("status") == "PASSED" && statuses.any? { |status| !%w[passed skipped].include?(status) }
|
|
206
|
+
raise ArgumentError, "PASSED baseline cannot contain failed, unknown, or running observed tests"
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
def unique_artifacts(shards)
|
|
211
|
+
by_digest = {}
|
|
212
|
+
by_run = {}
|
|
213
|
+
shards.each do |shard|
|
|
214
|
+
if (prior = by_digest[shard.fetch(:digest)])
|
|
215
|
+
if prior.fetch(:id) != shard.fetch(:id)
|
|
216
|
+
raise ArgumentError, "the same report artifact cannot satisfy two shard IDs"
|
|
217
|
+
end
|
|
218
|
+
|
|
219
|
+
prior[:paths] |= shard.fetch(:paths)
|
|
220
|
+
next
|
|
221
|
+
end
|
|
222
|
+
shard.fetch(:run_ids).each do |run_id|
|
|
223
|
+
if (prior = by_run[run_id])
|
|
224
|
+
raise ArgumentError, "conflicting run-ID overlap between #{prior.inspect} and #{shard.fetch(:id).inspect}"
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
by_run[run_id] = shard.fetch(:id)
|
|
228
|
+
end
|
|
229
|
+
by_digest[shard.fetch(:digest)] = shard
|
|
230
|
+
end
|
|
231
|
+
by_digest.values
|
|
232
|
+
end
|
|
233
|
+
|
|
234
|
+
def validate_compatibility!(shards, reference)
|
|
235
|
+
semantic_inventory = semantic(reference.fetch(:inventory))
|
|
236
|
+
keys = %i[limits level reachability minimum minimum_changed changed_scope]
|
|
237
|
+
shards.drop(1).each do |shard|
|
|
238
|
+
raise ArgumentError, "source inventories differ" unless semantic(shard.fetch(:inventory)) == semantic_inventory
|
|
239
|
+
|
|
240
|
+
keys.each do |key|
|
|
241
|
+
raise ArgumentError, "shard #{key} settings differ" unless shard.fetch(key) == reference.fetch(key)
|
|
242
|
+
end
|
|
243
|
+
%w[schema_version tool_version criterion_version runtime].each do |key|
|
|
244
|
+
raise ArgumentError, "shard #{key} metadata differs" unless shard.fetch(:document).fetch(key) ==
|
|
245
|
+
reference.fetch(:document).fetch(key)
|
|
246
|
+
end
|
|
247
|
+
%w[project_kind framework framework_version rspec_rails_version rails_version].each do |key|
|
|
248
|
+
next if reference.fetch(:metadata)[key] == shard.fetch(:metadata)[key]
|
|
249
|
+
|
|
250
|
+
raise ArgumentError, "shard #{key} metadata differs"
|
|
251
|
+
end
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
def merged_evidence(shards, reference)
|
|
256
|
+
run_ids = shards.flat_map { |shard| shard.fetch(:run_ids) }
|
|
257
|
+
raise ArgumentError, "run IDs must be unique across shards" unless run_ids.uniq.length == run_ids.length
|
|
258
|
+
|
|
259
|
+
first_run_id = run_ids.first
|
|
260
|
+
merger = Evidence.new(inventory: reference.fetch(:inventory), limits: reference.fetch(:limits),
|
|
261
|
+
run_id: first_run_id)
|
|
262
|
+
shards.each do |shard|
|
|
263
|
+
result = merger.merge(snapshot: shard.fetch(:snapshot))
|
|
264
|
+
raise ArgumentError, "cannot merge shard evidence: #{result[:reason]}" unless result[:status] == "merged"
|
|
265
|
+
end
|
|
266
|
+
snapshot = symbolize(merger.snapshot)
|
|
267
|
+
normalize_repeated_tests!(snapshot, shards)
|
|
268
|
+
snapshot
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def validate_union_storage!(shards, limits)
|
|
272
|
+
tests = shards.flat_map { |shard| shard.fetch(:snapshot).fetch("tests").map { |test| test.fetch("id") } }.uniq
|
|
273
|
+
raise ArgumentError, "collation union exceeds tests_per_run limit (#{limits[:tests_per_run]})" if
|
|
274
|
+
tests.length > limits[:tests_per_run]
|
|
275
|
+
|
|
276
|
+
vectors = shards.flat_map { |shard| shard.fetch(:snapshot).fetch("vectors") }.uniq { |vector| vector.fetch("id") }
|
|
277
|
+
vectors_by_decision = vectors.group_by { |vector| vector.fetch("decision_id") }
|
|
278
|
+
overfull = vectors_by_decision.find { |_decision, items| items.length > limits[:vectors_per_decision] }
|
|
279
|
+
if overfull
|
|
280
|
+
raise ArgumentError,
|
|
281
|
+
"collation union exceeds vectors_per_decision limit (#{limits[:vectors_per_decision]})"
|
|
282
|
+
end
|
|
283
|
+
owners = vectors.sum do |vector|
|
|
284
|
+
Array(vector["test_ids"]).uniq.length + (vector.fetch("unattributed_count", 0).positive? ? 1 : 0)
|
|
285
|
+
end
|
|
286
|
+
return unless owners > limits[:owner_associations_per_run]
|
|
287
|
+
|
|
288
|
+
raise ArgumentError,
|
|
289
|
+
"collation union exceeds owner_associations_per_run limit (#{limits[:owner_associations_per_run]})"
|
|
290
|
+
end
|
|
291
|
+
|
|
292
|
+
def effective_input_completeness(document)
|
|
293
|
+
sections = [document.fetch("completeness"), document.dig("observations", "completeness")]
|
|
294
|
+
sections << document.dig("analysis", "completeness") if document["analysis"].is_a?(Hash)
|
|
295
|
+
%w[observation attribution analysis].to_h do |field|
|
|
296
|
+
[field, sections.all? { |section| section.is_a?(Hash) && section[field] == true }]
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def effective_completeness(shards, evidence, analysis, collection_status)
|
|
301
|
+
input_complete = %w[observation attribution analysis].to_h do |field|
|
|
302
|
+
[field, shards.all? { |shard| shard.fetch(:completeness).fetch(field) }]
|
|
303
|
+
end
|
|
304
|
+
evidence_complete = symbolize(evidence.fetch(:completeness))
|
|
305
|
+
analysis_complete = analysis ? symbolize(analysis.fetch(:completeness)) : {}
|
|
306
|
+
result = %i[observation attribution analysis].to_h do |field|
|
|
307
|
+
[field, input_complete.fetch(field.to_s) && evidence_complete[field] == true &&
|
|
308
|
+
(!analysis || analysis_complete[field] == true)]
|
|
309
|
+
end
|
|
310
|
+
result[:observation] = false unless collection_status == "complete"
|
|
311
|
+
result
|
|
312
|
+
end
|
|
313
|
+
|
|
314
|
+
def normalize_repeated_tests!(snapshot, shards)
|
|
315
|
+
by_id = {}
|
|
316
|
+
locations = {}
|
|
317
|
+
shards.each do |shard|
|
|
318
|
+
symbolize(shard.fetch(:metadata)["test_locations"] || {}).each do |id, location|
|
|
319
|
+
if locations.key?(id) && locations[id] != location
|
|
320
|
+
raise ArgumentError, "conflicting test location for #{id.inspect}"
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
locations[id] = location
|
|
324
|
+
end
|
|
325
|
+
shard.fetch(:snapshot).fetch("tests").each do |raw|
|
|
326
|
+
test = symbolize(raw)
|
|
327
|
+
id = test.fetch(:id).to_s
|
|
328
|
+
if (prior_entry = by_id[id])
|
|
329
|
+
prior = prior_entry.fetch(:test)
|
|
330
|
+
unless test_identity(prior, prior_entry.fetch(:shard)) == test_identity(test, shard)
|
|
331
|
+
raise ArgumentError, "conflicting test identity for #{id.inspect}"
|
|
332
|
+
end
|
|
333
|
+
|
|
334
|
+
prior[:status] = worse_status(prior[:status], test[:status])
|
|
335
|
+
else
|
|
336
|
+
by_id[id] = { test: test, shard: shard }
|
|
337
|
+
end
|
|
338
|
+
end
|
|
339
|
+
end
|
|
340
|
+
snapshot[:tests].each do |test|
|
|
341
|
+
test[:status] = by_id.fetch(test.fetch(:id).to_s).fetch(:test).fetch(:status)
|
|
342
|
+
end
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
def test_identity(test, shard)
|
|
346
|
+
normalized_test_identity(test, shard.fetch(:metadata)["project_root"])
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
def normalized_test_identity(test, root)
|
|
350
|
+
identity = symbolize(test).except(:status, :phase_counts)
|
|
351
|
+
source = identity[:source]
|
|
352
|
+
if source.is_a?(Hash) && source[:path].is_a?(String)
|
|
353
|
+
absolute = File.expand_path(source[:path])
|
|
354
|
+
expanded_root = File.expand_path(root.to_s)
|
|
355
|
+
prefix = "#{expanded_root}#{File::SEPARATOR}"
|
|
356
|
+
source[:path] = normalized_path(absolute, prefix, expanded_root)
|
|
357
|
+
end
|
|
358
|
+
if identity[:source_path].is_a?(String)
|
|
359
|
+
absolute = File.expand_path(identity[:source_path])
|
|
360
|
+
expanded_root = File.expand_path(root.to_s)
|
|
361
|
+
prefix = "#{expanded_root}#{File::SEPARATOR}"
|
|
362
|
+
identity[:source_path] = normalized_path(absolute, prefix, expanded_root)
|
|
363
|
+
end
|
|
364
|
+
identity
|
|
365
|
+
end
|
|
366
|
+
|
|
367
|
+
def normalized_path(absolute, prefix, root)
|
|
368
|
+
return absolute unless absolute.start_with?(prefix)
|
|
369
|
+
|
|
370
|
+
Pathname.new(absolute).relative_path_from(Pathname.new(root)).to_s
|
|
371
|
+
end
|
|
372
|
+
|
|
373
|
+
def worse_status(left, right)
|
|
374
|
+
left_status = STATUS_ORDER.include?(left.to_s) ? left.to_s : "unknown"
|
|
375
|
+
right_status = STATUS_ORDER.include?(right.to_s) ? right.to_s : "unknown"
|
|
376
|
+
STATUS_ORDER[[STATUS_ORDER.index(left_status), STATUS_ORDER.index(right_status)].max]
|
|
377
|
+
end
|
|
378
|
+
|
|
379
|
+
def merged_baseline(shards, collection_status)
|
|
380
|
+
baselines = shards.map { |shard| shard.fetch(:baseline) }
|
|
381
|
+
failed = baselines.sum { |baseline| baseline.fetch("failed_tests", 0) }
|
|
382
|
+
skipped = baselines.sum { |baseline| baseline.fetch("skipped_tests", 0) }
|
|
383
|
+
executed = baselines.sum { |baseline| baseline.fetch("executed_tests", 0) }
|
|
384
|
+
error = baselines.any? { |baseline| baseline["status"] == "ERROR" }
|
|
385
|
+
failed_status = baselines.any? { |baseline| baseline["status"] == "FAILED" } || failed.positive?
|
|
386
|
+
incomplete = collection_status != "complete" || baselines.any? do |baseline|
|
|
387
|
+
baseline["status"] == "INCOMPLETE" || baseline["finalized"] != true
|
|
388
|
+
end || shards.any? { |shard| shard.fetch(:completeness).values.any?(&:!) }
|
|
389
|
+
status = if error
|
|
390
|
+
"ERROR"
|
|
391
|
+
elsif incomplete
|
|
392
|
+
"INCOMPLETE"
|
|
393
|
+
elsif failed_status
|
|
394
|
+
"FAILED"
|
|
395
|
+
else
|
|
396
|
+
"PASSED"
|
|
397
|
+
end
|
|
398
|
+
all_tests = shards.flat_map { |shard| shard.fetch(:snapshot).fetch("tests") }
|
|
399
|
+
summarized = all_tests.group_by { |test| test.fetch("id") }.map do |_id, records|
|
|
400
|
+
first = symbolize(records.first)
|
|
401
|
+
first[:status] = records.map { |record| record.fetch("status") }.reduce { |a, b| worse_status(a, b) }
|
|
402
|
+
first[:phase_counts] = records.map { |record| symbolize(record).fetch(:phase_counts, {}) }
|
|
403
|
+
.reduce({}) { |acc, counts| acc.merge(counts) { |_key, a, b| a.to_i + b.to_i } }
|
|
404
|
+
first
|
|
405
|
+
end
|
|
406
|
+
baseline = { status: status, finalized: status != "ERROR" && status != "INCOMPLETE",
|
|
407
|
+
executed_tests: executed, failed_tests: failed, skipped_tests: skipped, tests: summarized }
|
|
408
|
+
[baseline, error || failed_status]
|
|
409
|
+
end
|
|
410
|
+
|
|
411
|
+
def calculated_minima(analysis, evidence, reference)
|
|
412
|
+
return [] if reference.fetch(:level) < 2
|
|
413
|
+
|
|
414
|
+
ids = Array(reference.fetch(:inventory)[:decisions]).map { |decision| decision[:id] }
|
|
415
|
+
minimizer = Minimizer.new(analysis: analysis, evidence: evidence, limits: reference.fetch(:limits))
|
|
416
|
+
result = ids.filter_map { |id| minimizer.call(objective: :vectors, decision_ids: [id]) }
|
|
417
|
+
result += ids.filter_map { |id| minimizer.call(objective: :tests, decision_ids: [id]) }
|
|
418
|
+
result << minimizer.call(objective: :tests, decision_ids: ids)
|
|
419
|
+
result
|
|
420
|
+
end
|
|
421
|
+
|
|
422
|
+
def build_report(reference, evidence, analysis, minima, baseline, shards, missing, collection_status)
|
|
423
|
+
diagnostics = shards.flat_map do |shard|
|
|
424
|
+
Array(shard.fetch(:document)["diagnostics"]).map { symbolize(_1) }
|
|
425
|
+
end
|
|
426
|
+
unless collection_status == "complete"
|
|
427
|
+
diagnostics << { code: "collation_incomplete", severity: "warning",
|
|
428
|
+
message: "collection status is #{collection_status}" }
|
|
429
|
+
end
|
|
430
|
+
metadata = aggregate_metadata(shards)
|
|
431
|
+
baseline[:diagnostics] = diagnostics
|
|
432
|
+
report = Report.new(inventory: reference.fetch(:inventory), evidence: evidence, analysis: analysis,
|
|
433
|
+
minima: minima, baseline: baseline, diagnostics: diagnostics,
|
|
434
|
+
level: reference.fetch(:level), run_metadata: metadata,
|
|
435
|
+
changed_scope: reference.fetch(:changed_scope),
|
|
436
|
+
minimum: reference.fetch(:minimum), minimum_changed: reference.fetch(:minimum_changed))
|
|
437
|
+
output = StringIO.new
|
|
438
|
+
report.write(io: output, format: :json)
|
|
439
|
+
document = JSON.parse(output.string)
|
|
440
|
+
document["schema_version"] = "1.7"
|
|
441
|
+
document["run_ids"] = shards.flat_map { |shard| shard.fetch(:run_ids) }.sort
|
|
442
|
+
document["collation"] = {
|
|
443
|
+
"version" => "1.0", "collection_status" => collection_status,
|
|
444
|
+
"expected_shards" => @expected_shards, "missing_shards" => missing,
|
|
445
|
+
"shards" => shards.map { |shard| provenance(shard) }
|
|
446
|
+
}
|
|
447
|
+
document
|
|
448
|
+
end
|
|
449
|
+
|
|
450
|
+
def aggregate_metadata(shards)
|
|
451
|
+
metadata = symbolize(shards.first.fetch(:metadata))
|
|
452
|
+
%w[source_patterns exclude_patterns test_patterns test_files selected_test_files selected_example_ids
|
|
453
|
+
selected_source_files excluded_files].each do |field|
|
|
454
|
+
values = shards.flat_map { |shard| Array(shard.fetch(:metadata)[field]) }
|
|
455
|
+
metadata[field.to_sym] = values.uniq.sort unless values.empty?
|
|
456
|
+
end
|
|
457
|
+
metadata[:test_locations] = shards.each_with_object({}) do |shard, locations|
|
|
458
|
+
locations.merge!(symbolize(shard.fetch(:metadata)["test_locations"] || {}))
|
|
459
|
+
end.sort.to_h
|
|
460
|
+
metadata[:seed] = nil
|
|
461
|
+
metadata[:runner_args] = []
|
|
462
|
+
%w[project_root captured_at].each do |field|
|
|
463
|
+
values = shards.map { |shard| shard.fetch(:metadata)[field] }.uniq
|
|
464
|
+
metadata[field.to_sym] = values.one? ? values.first : "multiple"
|
|
465
|
+
end
|
|
466
|
+
metadata
|
|
467
|
+
end
|
|
468
|
+
|
|
469
|
+
def provenance(shard)
|
|
470
|
+
{ "id" => shard.fetch(:id), "paths" => shard.fetch(:paths).sort,
|
|
471
|
+
"report_digest" => shard.fetch(:digest), "run_ids" => shard.fetch(:run_ids).sort,
|
|
472
|
+
"baseline" => shard.fetch(:baseline).slice(
|
|
473
|
+
"status", "executed_tests", "failed_tests", "skipped_tests", "finalized"
|
|
474
|
+
),
|
|
475
|
+
"completeness" => shard.fetch(:completeness), "run_metadata" => shard.fetch(:metadata) }
|
|
476
|
+
end
|
|
477
|
+
|
|
478
|
+
def validate_output!(document)
|
|
479
|
+
SavedReport.new(document).validate!
|
|
480
|
+
end
|
|
481
|
+
|
|
482
|
+
def semantic(inventory)
|
|
483
|
+
data = stringify(inventory)
|
|
484
|
+
data.delete("root")
|
|
485
|
+
Array(data["source_units"]).each do |unit|
|
|
486
|
+
unit.delete("absolute_path")
|
|
487
|
+
unit.delete("real_path")
|
|
488
|
+
end
|
|
489
|
+
data
|
|
490
|
+
end
|
|
491
|
+
|
|
492
|
+
def symbolize(value)
|
|
493
|
+
case value
|
|
494
|
+
when Hash then value.to_h { |key, item| [key.to_sym, symbolize(item)] }
|
|
495
|
+
when Array then value.map { symbolize(_1) }
|
|
496
|
+
else value
|
|
497
|
+
end
|
|
498
|
+
end
|
|
499
|
+
|
|
500
|
+
def stringify(value)
|
|
501
|
+
case value
|
|
502
|
+
when Hash then value.to_h { |key, item| [key.to_s, stringify(item)] }
|
|
503
|
+
when Array then value.map { stringify(_1) }
|
|
504
|
+
else value
|
|
505
|
+
end
|
|
506
|
+
end
|
|
507
|
+
|
|
508
|
+
def canonical_json(value)
|
|
509
|
+
JSON.generate(canonical(value))
|
|
510
|
+
end
|
|
511
|
+
|
|
512
|
+
def canonical(value)
|
|
513
|
+
case value
|
|
514
|
+
when Hash then value.keys.sort.to_h { |key| [key, canonical(value.fetch(key))] }
|
|
515
|
+
when Array then value.map { canonical(_1) }
|
|
516
|
+
else value
|
|
517
|
+
end
|
|
518
|
+
end
|
|
519
|
+
|
|
520
|
+
def nonempty_string?(value)
|
|
521
|
+
value.is_a?(String) && !value.empty?
|
|
522
|
+
end
|
|
523
|
+
end
|
|
524
|
+
end
|
|
525
|
+
# rubocop:enable Metrics/AbcSize, Metrics/BlockLength, Metrics/ClassLength, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/ParameterLists, Metrics/PerceivedComplexity
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "json"
|
|
4
|
+
|
|
5
|
+
module Branchproof
|
|
6
|
+
# Resolves supplied reports against a manifest without reading report files.
|
|
7
|
+
class CollationManifest
|
|
8
|
+
attr_reader :expected_ids, :declared_paths, :declared_identities, :inputs
|
|
9
|
+
|
|
10
|
+
def self.path_identity(path)
|
|
11
|
+
resolve_path_identity(File.expand_path(path.to_s), [])
|
|
12
|
+
end
|
|
13
|
+
|
|
14
|
+
def self.resolve_path_identity(path, visited_symlinks)
|
|
15
|
+
absolute_path = File.expand_path(path)
|
|
16
|
+
return resolve_symlink_identity(absolute_path, visited_symlinks) if File.symlink?(absolute_path)
|
|
17
|
+
return File.realpath(absolute_path) if File.exist?(absolute_path)
|
|
18
|
+
|
|
19
|
+
resolve_missing_identity(absolute_path, visited_symlinks)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def self.resolve_symlink_identity(path, visited_symlinks)
|
|
23
|
+
raise ArgumentError, "symlink cycle while resolving path: #{path}" if visited_symlinks.include?(path)
|
|
24
|
+
|
|
25
|
+
target = File.expand_path(File.readlink(path), File.dirname(path))
|
|
26
|
+
resolve_path_identity(target, visited_symlinks + [path])
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
def self.resolve_missing_identity(path, visited_symlinks)
|
|
30
|
+
parent = File.dirname(path)
|
|
31
|
+
return path if parent == path
|
|
32
|
+
|
|
33
|
+
File.join(resolve_path_identity(parent, visited_symlinks), File.basename(path))
|
|
34
|
+
end
|
|
35
|
+
private_class_method :resolve_path_identity, :resolve_symlink_identity, :resolve_missing_identity
|
|
36
|
+
|
|
37
|
+
def initialize(path:, supplied_paths:)
|
|
38
|
+
validate_arguments!(path, supplied_paths)
|
|
39
|
+
@path = File.expand_path(path)
|
|
40
|
+
@manifest_directory = File.dirname(@path)
|
|
41
|
+
declarations = resolve_declarations(validate_document(read_document))
|
|
42
|
+
validate_declaration_uniqueness!(declarations)
|
|
43
|
+
assign_declarations(declarations)
|
|
44
|
+
@inputs = supplied_paths.map { |supplied_path| resolve_input(supplied_path, declarations).freeze }.freeze
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
private
|
|
48
|
+
|
|
49
|
+
def validate_arguments!(path, supplied_paths)
|
|
50
|
+
raise ArgumentError, "manifest path must be a nonempty string" unless nonempty_string?(path)
|
|
51
|
+
return if supplied_paths.is_a?(Array) && supplied_paths.all? { |item| nonempty_string?(item) }
|
|
52
|
+
|
|
53
|
+
raise ArgumentError, "supplied paths must be an array of nonempty strings"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def read_document
|
|
57
|
+
JSON.parse(File.binread(@path))
|
|
58
|
+
rescue JSON::ParserError => e
|
|
59
|
+
raise ArgumentError, "invalid collation manifest JSON: #{e.message}"
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
def assign_declarations(declarations)
|
|
63
|
+
@expected_ids = declarations.map { |declaration| declaration.fetch(:id) }.freeze
|
|
64
|
+
@declared_paths = declarations.map { |declaration| declaration.fetch(:path) }.freeze
|
|
65
|
+
@declared_identities = declarations.map { |declaration| declaration.fetch(:identity) }.freeze
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def validate_document(document)
|
|
69
|
+
raise ArgumentError, "manifest must be an object with a nonempty shards array" unless valid_root?(document)
|
|
70
|
+
|
|
71
|
+
document.fetch("shards").map do |shard|
|
|
72
|
+
validate_shard(shard)
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def valid_root?(document)
|
|
77
|
+
document.is_a?(Hash) && document.keys == ["shards"] &&
|
|
78
|
+
document["shards"].is_a?(Array) && !document["shards"].empty?
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def validate_shard(shard)
|
|
82
|
+
valid = shard.is_a?(Hash) && shard.keys.sort == %w[id report] &&
|
|
83
|
+
nonempty_string?(shard["id"]) && nonempty_string?(shard["report"])
|
|
84
|
+
raise ArgumentError, "each manifest shard must contain nonempty id and report strings" unless valid
|
|
85
|
+
|
|
86
|
+
{ id: shard.fetch("id"), report: shard.fetch("report") }
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def resolve_declarations(shards)
|
|
90
|
+
shards.map do |shard|
|
|
91
|
+
path = File.expand_path(shard.fetch(:report), @manifest_directory)
|
|
92
|
+
{ id: shard.fetch(:id), path: path, identity: self.class.path_identity(path) }
|
|
93
|
+
end
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def validate_declaration_uniqueness!(declarations)
|
|
97
|
+
ids = declarations.map { |declaration| declaration.fetch(:id) }
|
|
98
|
+
raise ArgumentError, "manifest shard IDs must be unique" unless ids.uniq.length == ids.length
|
|
99
|
+
|
|
100
|
+
declarations.combination(2) do |left, right|
|
|
101
|
+
next unless paths_alias?(left.fetch(:path), left.fetch(:identity), right.fetch(:path), right.fetch(:identity))
|
|
102
|
+
|
|
103
|
+
raise ArgumentError, "manifest shard paths must be unique across IDs"
|
|
104
|
+
end
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def resolve_input(supplied_path, declarations)
|
|
108
|
+
path = File.expand_path(supplied_path)
|
|
109
|
+
identity = self.class.path_identity(path)
|
|
110
|
+
declaration = declarations.find do |candidate|
|
|
111
|
+
paths_alias?(path, identity, candidate.fetch(:path), candidate.fetch(:identity))
|
|
112
|
+
end
|
|
113
|
+
raise ArgumentError, "supplied report is not declared by manifest: #{supplied_path}" unless declaration
|
|
114
|
+
|
|
115
|
+
{ id: declaration.fetch(:id), path: path }
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def paths_alias?(left_path, left_identity, right_path, right_identity)
|
|
119
|
+
left_path == right_path || left_identity == right_identity ||
|
|
120
|
+
(File.exist?(left_path) && File.exist?(right_path) && File.identical?(left_path, right_path))
|
|
121
|
+
rescue SystemCallError
|
|
122
|
+
left_path == right_path || left_identity == right_identity
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def nonempty_string?(value)
|
|
126
|
+
value.is_a?(String) && !value.empty?
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
end
|