branchproof 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +18 -0
- data/LICENSE +201 -0
- data/README.md +144 -8
- data/doc/Branchproof/Comparison.md +26 -0
- data/doc/Branchproof/ComparisonReport.md +19 -0
- data/doc/Branchproof/CoverageIndex.md +19 -0
- data/doc/Branchproof/FocusedReport.md +16 -0
- data/doc/Branchproof/Report.md +8 -1
- data/doc/Branchproof/SavedReport.md +32 -0
- data/doc/Branchproof.md +6 -1
- data/doc/CHANGELOG.md +18 -0
- data/doc/README.md +144 -8
- data/exe/branchproof +6 -0
- data/lib/branchproof/analyzer.rb +1 -1
- data/lib/branchproof/cli.rb +158 -7
- data/lib/branchproof/comparison.rb +363 -0
- data/lib/branchproof/comparison_report.rb +119 -0
- data/lib/branchproof/coverage_index.rb +187 -0
- data/lib/branchproof/focused_report.rb +205 -0
- data/lib/branchproof/report.rb +196 -12
- data/lib/branchproof/saved_report.rb +366 -0
- data/lib/branchproof/source.rb +1 -0
- data/lib/branchproof/version.rb +1 -1
- data/lib/branchproof.rb +5 -0
- data/llms.txt +6 -1
- data/sig/branchproof.rbs +30 -1
- metadata +14 -1
|
@@ -0,0 +1,363 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# rubocop:disable Metrics/ClassLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
4
|
+
require "json"
|
|
5
|
+
|
|
6
|
+
module Branchproof
|
|
7
|
+
# Compares two complete report documents without loading or executing the project.
|
|
8
|
+
class Comparison
|
|
9
|
+
SCHEMA_VERSION = "1.0"
|
|
10
|
+
SUPPORTED_REPORT_SCHEMAS = %w[1.0 1.1].freeze
|
|
11
|
+
CRITERION_VERSION = "masking_occurrence_v1"
|
|
12
|
+
|
|
13
|
+
def initialize(before:, after:)
|
|
14
|
+
@before = before
|
|
15
|
+
@after = after
|
|
16
|
+
@contexts = {}.compare_by_identity
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def call
|
|
20
|
+
errors = validate_documents
|
|
21
|
+
return invalid_result(errors) unless errors.empty?
|
|
22
|
+
|
|
23
|
+
before_sources = sources(@before)
|
|
24
|
+
after_sources = sources(@after)
|
|
25
|
+
changed_paths = before_sources.keys & after_sources.keys
|
|
26
|
+
changed_paths.reject! { |path| digest(before_sources[path]) == digest(after_sources[path]) }
|
|
27
|
+
before_conditions = condition_map(@before)
|
|
28
|
+
after_conditions = condition_map(@after)
|
|
29
|
+
matched_ids = before_conditions.keys & after_conditions.keys
|
|
30
|
+
matched_ids.select! { |id| compatible_condition?(before_conditions[id], after_conditions[id], changed_paths) }
|
|
31
|
+
changes = matched_ids.filter_map { |id| condition_change(id, before_conditions[id], after_conditions[id]) }
|
|
32
|
+
reasons = comparability_reasons(changed_paths)
|
|
33
|
+
reasons.concat(metadata_requirements)
|
|
34
|
+
reasons << "legacy report is missing comparison context" if [@before, @after].any? do |document|
|
|
35
|
+
value(document, :schema_version).to_s == "1.0" && value(document, :run_metadata).nil?
|
|
36
|
+
end
|
|
37
|
+
complete = complete_run?(@before) && complete_run?(@after)
|
|
38
|
+
reasons << "run baselines are not successful finalized complete reports" unless complete
|
|
39
|
+
reasons << "no comparable conditions" if matched_ids.empty?
|
|
40
|
+
if changes.any? { |change| [change["status_before"], change["status_after"]].include?("UNKNOWN") }
|
|
41
|
+
reasons << "condition proof status unavailable"
|
|
42
|
+
end
|
|
43
|
+
if before_sources.keys.sort != after_sources.keys.sort
|
|
44
|
+
reasons << "source population differs; whole-source comparison is partial"
|
|
45
|
+
end
|
|
46
|
+
if matched_ids.length != before_conditions.length || matched_ids.length != after_conditions.length
|
|
47
|
+
reasons << "unmatched conditions; exact-match scope is partial"
|
|
48
|
+
end
|
|
49
|
+
status = reasons.empty? ? "complete" : "comparison incomplete"
|
|
50
|
+
lost = changes.count { |change| change["change"] == "lost proof" }
|
|
51
|
+
|
|
52
|
+
{
|
|
53
|
+
"schema_version" => SCHEMA_VERSION,
|
|
54
|
+
"criterion_version" => CRITERION_VERSION,
|
|
55
|
+
"status" => status,
|
|
56
|
+
"comparison_status" => status,
|
|
57
|
+
"reasons" => reasons,
|
|
58
|
+
"before" => summary(@before),
|
|
59
|
+
"after" => summary(@after),
|
|
60
|
+
"context" => context_changes,
|
|
61
|
+
"matching" => { "before_conditions" => before_conditions.length,
|
|
62
|
+
"after_conditions" => after_conditions.length,
|
|
63
|
+
"matched_conditions" => matched_ids.length,
|
|
64
|
+
"denominator" => matched_ids.length,
|
|
65
|
+
"before_proven" => changes.count { |change| change["status_before"] == "PROVEN" },
|
|
66
|
+
"after_proven" => changes.count { |change| change["status_after"] == "PROVEN" } },
|
|
67
|
+
"changed_sources" => changed_paths.sort.map do |path|
|
|
68
|
+
{ "relative_path" => path, "reason" => "source changed / condition comparison unavailable" }
|
|
69
|
+
end,
|
|
70
|
+
"newly_in_report" => (after_sources.keys - before_sources.keys).sort,
|
|
71
|
+
"no_longer_in_report" => (before_sources.keys - after_sources.keys).sort,
|
|
72
|
+
"changes" => changes.sort_by do |change|
|
|
73
|
+
[change["relative_path"].to_s, change["line"].to_i, change["condition_id"]]
|
|
74
|
+
end,
|
|
75
|
+
"regressions" => lost,
|
|
76
|
+
"regression" => lost.positive? && status == "complete"
|
|
77
|
+
}
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
private
|
|
81
|
+
|
|
82
|
+
def validate_documents
|
|
83
|
+
errors = []
|
|
84
|
+
{ "before" => @before, "after" => @after }.each do |label, document|
|
|
85
|
+
errors << "#{label} report must be an object" unless document.is_a?(Hash)
|
|
86
|
+
next unless document.is_a?(Hash)
|
|
87
|
+
|
|
88
|
+
schema = value(document, :schema_version).to_s
|
|
89
|
+
criterion = value(document, :criterion_version).to_s
|
|
90
|
+
errors << "#{label} report has unsupported schema" unless SUPPORTED_REPORT_SCHEMAS.include?(schema)
|
|
91
|
+
errors << "#{label} report has incompatible criterion" unless criterion == CRITERION_VERSION
|
|
92
|
+
errors << "#{label} source_inventory must be an object" unless value(document, :source_inventory).is_a?(Hash)
|
|
93
|
+
end
|
|
94
|
+
errors.uniq
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def invalid_result(errors)
|
|
98
|
+
{ "schema_version" => SCHEMA_VERSION, "status" => "comparison incomplete",
|
|
99
|
+
"comparison_status" => "comparison incomplete", "reasons" => errors,
|
|
100
|
+
"before" => summary(@before), "after" => summary(@after),
|
|
101
|
+
"matching" => { "before_conditions" => 0, "after_conditions" => 0,
|
|
102
|
+
"matched_conditions" => 0, "denominator" => 0 },
|
|
103
|
+
"changed_sources" => [], "newly_in_report" => [], "no_longer_in_report" => [],
|
|
104
|
+
"changes" => [], "regressions" => 0, "regression" => false }
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def sources(document)
|
|
108
|
+
units = value(value(document, :source_inventory), :source_units)
|
|
109
|
+
Array(units).each_with_object({}) do |source, result|
|
|
110
|
+
path = value(source, :relative_path).to_s
|
|
111
|
+
result[path] = source unless path.empty?
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def condition_map(document)
|
|
116
|
+
data = context(document)
|
|
117
|
+
data[:decisions].values.each_with_object({}) do |decision, result|
|
|
118
|
+
next if value(decision, :support_status).to_s == "UNSUPPORTED"
|
|
119
|
+
|
|
120
|
+
Array(value(decision, :conditions)).each do |condition|
|
|
121
|
+
id = value(condition, :id).to_s
|
|
122
|
+
unless id.empty?
|
|
123
|
+
result[id] = { condition: condition, row: data[:conditions][id], decision: decision,
|
|
124
|
+
result: data[:results][id] }
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def compatible_condition?(before, after, changed_paths)
|
|
131
|
+
before_path = source_path(@before, before[:decision])
|
|
132
|
+
after_path = source_path(@after, after[:decision])
|
|
133
|
+
return false if before_path.nil? || changed_paths.include?(before_path) || before_path != after_path
|
|
134
|
+
|
|
135
|
+
before_digest = digest(sources(@before)[before_path])
|
|
136
|
+
after_digest = digest(sources(@after)[after_path])
|
|
137
|
+
return false unless before_digest && before_digest == after_digest
|
|
138
|
+
|
|
139
|
+
%i[index expression byte_start byte_length].all? do |key|
|
|
140
|
+
value(before[:condition], key) == value(after[:condition], key)
|
|
141
|
+
end && %i[id source_id expression context byte_start byte_length tree].all? do |key|
|
|
142
|
+
Records.normalize(value(before[:decision], key)) == Records.normalize(value(after[:decision], key))
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def condition_change(id, before, after)
|
|
147
|
+
old_status = status(before[:result], @before)
|
|
148
|
+
new_status = status(after[:result], @after)
|
|
149
|
+
if new_status == "NOT_PROVEN" && !complete_run?(@after) &&
|
|
150
|
+
context(@after)[:vectors].values.none? { |vector| value(vector, :decision_id) == value(after[:decision], :id) }
|
|
151
|
+
new_status = "UNKNOWN"
|
|
152
|
+
end
|
|
153
|
+
kind = if old_status == "PROVEN" && new_status == "NOT_PROVEN"
|
|
154
|
+
"lost proof"
|
|
155
|
+
elsif old_status == "NOT_PROVEN" && new_status == "PROVEN"
|
|
156
|
+
"gained proof"
|
|
157
|
+
elsif old_status == new_status
|
|
158
|
+
"unchanged"
|
|
159
|
+
else
|
|
160
|
+
"unknown"
|
|
161
|
+
end
|
|
162
|
+
return if kind.nil?
|
|
163
|
+
|
|
164
|
+
decision = after[:decision]
|
|
165
|
+
source_path = source_path(@after, decision)
|
|
166
|
+
row = { "condition_id" => id, "change" => kind, "status_before" => old_status,
|
|
167
|
+
"status_after" => new_status, "expression" => value(after[:condition], :expression),
|
|
168
|
+
"relative_path" => source_path, "line" => value(after[:condition], :line),
|
|
169
|
+
"index" => value(after[:condition], :index),
|
|
170
|
+
"previous_witness" => witness(@before, before[:result], before[:decision]),
|
|
171
|
+
"current_witness" => witness(@after, after[:result], after[:decision]) }
|
|
172
|
+
row["previous_witness_owners"] = row["previous_witness"].to_a.flat_map { |item| item["tests"] }.uniq
|
|
173
|
+
row["current_witness_owners"] = row["current_witness"].to_a.flat_map { |item| item["tests"] }.uniq
|
|
174
|
+
row["owner_context"] = owner_context(before[:result], before[:decision])
|
|
175
|
+
row
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
def context(document)
|
|
179
|
+
@contexts[document] ||= begin
|
|
180
|
+
index = CoverageIndex.new(document: document)
|
|
181
|
+
tests = Array(value(value(document, :observations), :tests)).to_h { |test| [value(test, :id).to_s, test] }
|
|
182
|
+
rows = index.tests.to_h { |test| [test[:id].to_s, test] }
|
|
183
|
+
keys = tests.to_h do |id, test|
|
|
184
|
+
parts = %i[adapter class_name method_name].map { |key| value(test, key) }
|
|
185
|
+
parts << rows.dig(id, :relative_path)
|
|
186
|
+
[id, parts.all? { |part| part.is_a?(String) && !part.empty? } ? parts : nil]
|
|
187
|
+
end
|
|
188
|
+
results = Array(value(value(document, :analysis), :decisions)).flat_map do |decision|
|
|
189
|
+
Array(value(decision, :condition_results))
|
|
190
|
+
end
|
|
191
|
+
{ conditions: index.conditions.to_h { |row| [row[:id].to_s, row] }, tests: tests, test_rows: rows,
|
|
192
|
+
test_keys: keys, key_owners: keys.keys.group_by { |id| keys[id] },
|
|
193
|
+
vectors: Array(value(value(document, :observations), :vectors)).to_h do |vector|
|
|
194
|
+
[value(vector, :id).to_s, vector]
|
|
195
|
+
end,
|
|
196
|
+
decisions: Array(value(value(document, :source_inventory), :decisions)).to_h do |decision|
|
|
197
|
+
[value(decision, :id).to_s, decision]
|
|
198
|
+
end,
|
|
199
|
+
results: results.to_h { |result| [value(result, :condition_id).to_s, result] } }
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def owner_context(result, decision)
|
|
204
|
+
before = context(@before)
|
|
205
|
+
after = context(@after)
|
|
206
|
+
Array(value(result, :canonical_pair)).flat_map do |vector_id|
|
|
207
|
+
vector = before[:vectors][vector_id.to_s]
|
|
208
|
+
next [] unless vector
|
|
209
|
+
|
|
210
|
+
Array(value(vector, :test_ids)).sort.map do |id|
|
|
211
|
+
key = before[:test_keys][id.to_s]
|
|
212
|
+
unique = key && before[:key_owners][key]&.length == 1 && after[:key_owners][key]&.length == 1
|
|
213
|
+
current_id = unique ? after[:key_owners][key].first : nil
|
|
214
|
+
observed = current_id && after[:vectors].values.any? do |current|
|
|
215
|
+
value(current, :decision_id) == value(decision, :id) &&
|
|
216
|
+
value(current,
|
|
217
|
+
:values) == value(vector, :values) && value(current, :outcome) == value(vector, :outcome) &&
|
|
218
|
+
Array(value(current, :test_ids)).include?(current_id)
|
|
219
|
+
end
|
|
220
|
+
test_status = current_id ? "present in current run" : "not observed in current run (no unique test match)"
|
|
221
|
+
{ "label" => test_display(@before, id), "current_label" => current_id && test_display(@after, current_id),
|
|
222
|
+
"test_status" => test_status,
|
|
223
|
+
"observation_status" => observed ? "observed in current run" : "not observed in current run",
|
|
224
|
+
"values" => value(vector, :values), "outcome" => value(vector, :outcome) }
|
|
225
|
+
end
|
|
226
|
+
end
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def witness(document, result, _decision)
|
|
230
|
+
pair = value(result, :canonical_pair)
|
|
231
|
+
return nil unless pair.is_a?(Array)
|
|
232
|
+
|
|
233
|
+
pair.map do |id|
|
|
234
|
+
vector = context(document)[:vectors][id.to_s]
|
|
235
|
+
{ "vector_id" => id, "values" => value(vector, :values), "outcome" => value(vector, :outcome),
|
|
236
|
+
"tests" => Array(value(vector, :test_ids)).map { |test_id| test_display(document, test_id) }.uniq.sort }
|
|
237
|
+
end
|
|
238
|
+
end
|
|
239
|
+
|
|
240
|
+
def test_display(document, id)
|
|
241
|
+
row = context(document)[:test_rows][id.to_s]
|
|
242
|
+
return "unknown test (location unavailable)" unless row
|
|
243
|
+
|
|
244
|
+
path = row[:relative_path]
|
|
245
|
+
location = path && row[:line] ? "#{path}:#{row[:line]}" : "location unavailable"
|
|
246
|
+
"#{row[:name]} (#{location})"
|
|
247
|
+
end
|
|
248
|
+
|
|
249
|
+
def context_changes
|
|
250
|
+
changes = []
|
|
251
|
+
if metadata(@before, :seed) != metadata(@after, :seed)
|
|
252
|
+
changes << "seed differs: #{metadata(@before, :seed).inspect} -> #{metadata(@after, :seed).inspect}"
|
|
253
|
+
end
|
|
254
|
+
if metadata(@before, :test_files) != metadata(@after, :test_files)
|
|
255
|
+
changes << "expanded test files differ under the recorded selection patterns"
|
|
256
|
+
end
|
|
257
|
+
if context(@before)[:test_keys].values.sort_by(&:to_s) != context(@after)[:test_keys].values.sort_by(&:to_s)
|
|
258
|
+
changes << "test population differs"
|
|
259
|
+
end
|
|
260
|
+
changes << "Deltas describe observed runs; nondeterminism and external environment changes are not ruled out."
|
|
261
|
+
changes
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
def comparability_reasons(changed_paths)
|
|
265
|
+
reasons = []
|
|
266
|
+
reasons << "source files changed: #{changed_paths.sort.join(", ")}" unless changed_paths.empty?
|
|
267
|
+
%i[criterion_version project_kind runner_args source_patterns test_patterns limits runtime].each do |key|
|
|
268
|
+
next if key == :criterion_version
|
|
269
|
+
|
|
270
|
+
left = metadata(@before, key)
|
|
271
|
+
right = metadata(@after, key)
|
|
272
|
+
if key == :runner_args
|
|
273
|
+
left = comparable_runner_args(left)
|
|
274
|
+
right = comparable_runner_args(right)
|
|
275
|
+
end
|
|
276
|
+
reasons << "#{key} differs" if !left.nil? && !right.nil? && left != right
|
|
277
|
+
end
|
|
278
|
+
reasons << "source selection differs" if source_selection(@before) != source_selection(@after)
|
|
279
|
+
reasons
|
|
280
|
+
end
|
|
281
|
+
|
|
282
|
+
def source_selection(document)
|
|
283
|
+
value(value(document, :run_metadata), :source_patterns)
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
def metadata_requirements
|
|
287
|
+
[@before, @after].each_with_index.filter_map do |document, index|
|
|
288
|
+
metadata = value(document, :run_metadata)
|
|
289
|
+
required = %i[project_kind source_patterns test_patterns test_files runner_args limits]
|
|
290
|
+
missing = required.reject { |key| metadata.is_a?(Hash) && !value(metadata, key).nil? }
|
|
291
|
+
missing << :runtime unless value(document, :runtime).is_a?(String) && !value(document, :runtime).empty?
|
|
292
|
+
if missing.empty?
|
|
293
|
+
nil
|
|
294
|
+
else
|
|
295
|
+
"#{index.zero? ? "before" : "after"} report missing run metadata: #{missing.join(", ")}"
|
|
296
|
+
end
|
|
297
|
+
end
|
|
298
|
+
end
|
|
299
|
+
|
|
300
|
+
def metadata(document, key)
|
|
301
|
+
value(value(document, :run_metadata), key) || value(document, key)
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def complete_run?(document)
|
|
305
|
+
baseline = value(document, :baseline) || {}
|
|
306
|
+
completeness = value(document, :completeness) || {}
|
|
307
|
+
analysis = value(document, :analysis)
|
|
308
|
+
return false unless value(baseline, :status).to_s.upcase == "PASSED" && value(baseline, :finalized) == true
|
|
309
|
+
return false unless analysis.is_a?(Hash)
|
|
310
|
+
|
|
311
|
+
[completeness, value(analysis, :completeness),
|
|
312
|
+
value(value(document, :observations), :completeness)].all? do |section|
|
|
313
|
+
%i[observation attribution analysis].all? { |key| value(section, key) == true }
|
|
314
|
+
end
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
def summary(document)
|
|
318
|
+
{ "run_id" => Array(value(value(document, :observations), :run_ids)).first,
|
|
319
|
+
"status" => value(value(document, :baseline), :status), "metrics" => value(document, :metrics),
|
|
320
|
+
"completeness" => value(document, :completeness), "run_metadata" => value(document, :run_metadata) }
|
|
321
|
+
end
|
|
322
|
+
|
|
323
|
+
def source_path(document, decision)
|
|
324
|
+
units = value(value(document, :source_inventory), :source_units)
|
|
325
|
+
source = Array(units).find { |item| value(item, :source_id).to_s == value(decision, :source_id).to_s }
|
|
326
|
+
value(source, :relative_path) || value(decision, :relative_path)
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
def digest(source) = value(source, :digest) || value(source, :content_digest)
|
|
330
|
+
|
|
331
|
+
def comparable_runner_args(args)
|
|
332
|
+
skip_next = false
|
|
333
|
+
Array(args).each_with_object([]) do |arg, result|
|
|
334
|
+
text = arg.to_s
|
|
335
|
+
if skip_next
|
|
336
|
+
skip_next = false
|
|
337
|
+
next
|
|
338
|
+
end
|
|
339
|
+
if ["--seed", "-s"].include?(text)
|
|
340
|
+
skip_next = true
|
|
341
|
+
next
|
|
342
|
+
end
|
|
343
|
+
next if text.start_with?("--seed=")
|
|
344
|
+
|
|
345
|
+
result << arg
|
|
346
|
+
end
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
def status(result, _document = nil)
|
|
350
|
+
text = value(result, :status).to_s.upcase
|
|
351
|
+
return text unless text.empty?
|
|
352
|
+
|
|
353
|
+
"UNKNOWN"
|
|
354
|
+
end
|
|
355
|
+
|
|
356
|
+
def value(hash, key)
|
|
357
|
+
return nil unless hash.respond_to?(:key?)
|
|
358
|
+
|
|
359
|
+
hash.key?(key) ? hash[key] : hash[key.to_s]
|
|
360
|
+
end
|
|
361
|
+
end
|
|
362
|
+
end
|
|
363
|
+
# rubocop:enable Metrics/ClassLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
@@ -0,0 +1,119 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# rubocop:disable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
4
|
+
|
|
5
|
+
require "json"
|
|
6
|
+
|
|
7
|
+
module Branchproof
|
|
8
|
+
# Renders the offline document returned by Comparison.
|
|
9
|
+
class ComparisonReport
|
|
10
|
+
def initialize(document:)
|
|
11
|
+
raise ArgumentError, "comparison document must be a Hash" unless document.is_a?(Hash)
|
|
12
|
+
|
|
13
|
+
@document = document
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def write(io:, format:)
|
|
17
|
+
format = format.to_sym
|
|
18
|
+
raise ArgumentError, "format must be :terminal or :json" unless %i[terminal json].include?(format)
|
|
19
|
+
|
|
20
|
+
io.write(format == :json ? JSON.generate(@document) : terminal_document)
|
|
21
|
+
nil
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
def exit_code(fail_on_regression: false)
|
|
25
|
+
return 2 if status != "complete"
|
|
26
|
+
return 1 if fail_on_regression && regression?
|
|
27
|
+
|
|
28
|
+
0
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
def terminal_document
|
|
34
|
+
lines = ["Branchproof comparison", "Status: #{status}"]
|
|
35
|
+
reasons.each { |reason| lines << "Reason: #{reason}" }
|
|
36
|
+
Array(value(:context)).each { |item| lines << "Context: #{item}" }
|
|
37
|
+
lines << "Conditions: #{matching.fetch("matched_conditions",
|
|
38
|
+
0)} matched (denominator #{matching.fetch("denominator", 0)})"
|
|
39
|
+
lines << "Before: #{summary_status(before)}; After: #{summary_status(after)}"
|
|
40
|
+
lines << "Full-run totals: before #{totals(before)}; after #{totals(after)}"
|
|
41
|
+
before_total = "#{matching["before_proven"] || 0}/#{matching["denominator"] || 0}"
|
|
42
|
+
after_total = "#{matching["after_proven"] || 0}/#{matching["denominator"] || 0}"
|
|
43
|
+
lines << "Exact-match totals: before #{before_total} proven; after #{after_total} proven"
|
|
44
|
+
lines << "No comparable conditions." if matching.fetch("matched_conditions", 0).zero?
|
|
45
|
+
changed_sources.each { |source| lines << "Source changed: #{source["relative_path"]}" }
|
|
46
|
+
newly.each { |path| lines << "Newly in report: #{path}" }
|
|
47
|
+
removed.each { |path| lines << "No longer in report: #{path}" }
|
|
48
|
+
rendered_changes = changes.reject { |change| change["change"] == "unchanged" }
|
|
49
|
+
if rendered_changes.empty?
|
|
50
|
+
lines << "No comparable condition changes."
|
|
51
|
+
else
|
|
52
|
+
rendered_changes.each do |change|
|
|
53
|
+
location = if change["relative_path"].to_s.empty?
|
|
54
|
+
"location unavailable"
|
|
55
|
+
else
|
|
56
|
+
"#{change["relative_path"]}:#{change["line"] || "condition line unavailable"}"
|
|
57
|
+
end
|
|
58
|
+
lines << "#{change["change"].capitalize}: #{location} (condition #{change["index"]}) #{change["expression"]}"
|
|
59
|
+
next unless change["change"] == "lost proof"
|
|
60
|
+
|
|
61
|
+
owners = Array(change["previous_witness"]).flat_map { |witness| Array(witness["tests"]) }.uniq
|
|
62
|
+
lines << " Previous owners: #{owners.empty? ? "unavailable" : owners.join(", ")}"
|
|
63
|
+
Array(change["previous_witness"]).each do |witness|
|
|
64
|
+
outcome = witness["outcome"] ? "T" : "F"
|
|
65
|
+
owners = Array(witness["tests"]).join(", ")
|
|
66
|
+
lines << " Previous witness: [#{signs(witness["values"])}] => #{outcome} — #{owners}"
|
|
67
|
+
end
|
|
68
|
+
context = Array(change["owner_context"]).map do |item|
|
|
69
|
+
"#{item["label"]}: #{item["test_status"]}; [#{signs(item["values"])}] => " \
|
|
70
|
+
"#{item["outcome"] ? "T" : "F"} #{item["observation_status"]}"
|
|
71
|
+
end
|
|
72
|
+
lines << " #{context.empty? ? "not observed in current run" : context.join(", ")}"
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
lines.join("\n") << "\n"
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def status = value(:status).to_s
|
|
79
|
+
|
|
80
|
+
def signs(values)
|
|
81
|
+
Array(values).map do |item|
|
|
82
|
+
if item.nil?
|
|
83
|
+
"-"
|
|
84
|
+
else
|
|
85
|
+
(item ? "T" : "F")
|
|
86
|
+
end
|
|
87
|
+
end.join
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def totals(summary)
|
|
91
|
+
metrics = value_from(summary, :metrics) || {}
|
|
92
|
+
"#{value_from(metrics,
|
|
93
|
+
:proven) || "unknown"}/#{value_from(metrics, :eligible_conditions) || "unknown"} conditions proven"
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
def regression? = value(:regression) == true || value(:regressions).to_i.positive?
|
|
97
|
+
def reasons = Array(value(:reasons))
|
|
98
|
+
def matching = value(:matching) || {}
|
|
99
|
+
def before = value(:before) || {}
|
|
100
|
+
def after = value(:after) || {}
|
|
101
|
+
def changes = Array(value(:changes))
|
|
102
|
+
def changed_sources = Array(value(:changed_sources))
|
|
103
|
+
def newly = Array(value(:newly_in_report))
|
|
104
|
+
def removed = Array(value(:no_longer_in_report))
|
|
105
|
+
|
|
106
|
+
def summary_status(summary)
|
|
107
|
+
value_from(summary, :status).to_s.upcase.then { |item| item.empty? ? "unknown" : item }
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def value(key) = value_from(@document, key)
|
|
111
|
+
|
|
112
|
+
def value_from(hash, key)
|
|
113
|
+
return nil unless hash.respond_to?(:key?)
|
|
114
|
+
|
|
115
|
+
hash.key?(key) ? hash[key] : hash[key.to_s]
|
|
116
|
+
end
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
# rubocop:enable Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# rubocop:disable Metrics/ClassLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity, Metrics/BlockLength
|
|
4
|
+
|
|
5
|
+
require "pathname"
|
|
6
|
+
|
|
7
|
+
module Branchproof
|
|
8
|
+
# Derives condition- and test-oriented rows from one report document.
|
|
9
|
+
class CoverageIndex
|
|
10
|
+
attr_reader :conditions, :tests
|
|
11
|
+
|
|
12
|
+
def initialize(document:)
|
|
13
|
+
@document = symbolize(document || {})
|
|
14
|
+
@inventory = symbolize(@document[:source_inventory] || @document[:inventory] || {})
|
|
15
|
+
@evidence = symbolize(@document[:observations] || @document[:evidence] || {})
|
|
16
|
+
@analysis = symbolize(@document[:analysis] || {})
|
|
17
|
+
@baseline = symbolize(@document[:baseline] || {})
|
|
18
|
+
@vectors = records(@evidence, :vectors)
|
|
19
|
+
@vectors_by_decision = @vectors.group_by { |vector| vector[:decision_id].to_s }
|
|
20
|
+
@vectors_by_id = @vectors.to_h { |vector| [vector[:id].to_s, vector] }
|
|
21
|
+
@analysis_by_decision = records(@analysis, :decisions).to_h { |decision| [decision[:decision_id].to_s, decision] }
|
|
22
|
+
@source_units_by_id = records(@inventory, :source_units).to_h { |unit| [unit[:source_id].to_s, unit] }
|
|
23
|
+
@test_locations = symbolize(@document[:run_metadata] || {})[:test_locations] || {}
|
|
24
|
+
@tests_by_id = {}
|
|
25
|
+
(records(@evidence, :tests) + records(@baseline, :tests)).each do |test|
|
|
26
|
+
id = test[:id].to_s
|
|
27
|
+
@tests_by_id[id] = (@tests_by_id[id] || {}).merge(test) unless id.empty?
|
|
28
|
+
end
|
|
29
|
+
@conditions = build_conditions.freeze
|
|
30
|
+
@tests = build_tests.freeze
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
private
|
|
34
|
+
|
|
35
|
+
def build_conditions
|
|
36
|
+
decisions = records(@inventory, :decisions)
|
|
37
|
+
rows = decisions.flat_map do |decision|
|
|
38
|
+
next [] if decision[:support_status].to_s == "UNSUPPORTED"
|
|
39
|
+
|
|
40
|
+
source = @source_units_by_id[decision[:source_id].to_s] || {}
|
|
41
|
+
result_map = records(analysis_for(decision), :condition_results).to_h do |result|
|
|
42
|
+
[result[:condition_id].to_s, result]
|
|
43
|
+
end
|
|
44
|
+
records(decision, :conditions).map do |condition|
|
|
45
|
+
condition = symbolize(condition)
|
|
46
|
+
result = symbolize(result_map[condition[:id].to_s] || {})
|
|
47
|
+
pair = Array(result[:canonical_pair])
|
|
48
|
+
vectors = @vectors_by_decision[decision[:id].to_s] || []
|
|
49
|
+
observations = { true => [], false => [], short_circuited: [], unattributed: [] }
|
|
50
|
+
vectors.each do |vector|
|
|
51
|
+
value = vector_values(vector)[condition[:index].to_i]
|
|
52
|
+
if value.nil?
|
|
53
|
+
observations[:short_circuited] << vector
|
|
54
|
+
elsif value == true
|
|
55
|
+
observations[true] << vector
|
|
56
|
+
elsif value == false
|
|
57
|
+
observations[false] << vector
|
|
58
|
+
end
|
|
59
|
+
observations[:unattributed] << vector if vector[:unattributed_count].to_i.positive?
|
|
60
|
+
end
|
|
61
|
+
{
|
|
62
|
+
id: condition[:id], decision_id: decision[:id], index: condition[:index],
|
|
63
|
+
expression: condition[:expression],
|
|
64
|
+
line: condition[:line], column: condition[:column], relative_path: relative_path(source[:relative_path]),
|
|
65
|
+
decision_expression: decision[:expression],
|
|
66
|
+
status: @document[:analysis] ? (result[:status] || "NOT_PROVEN") : "NOT CALCULATED",
|
|
67
|
+
observed_true: test_ids(observations[true]), observed_false: test_ids(observations[false]),
|
|
68
|
+
short_circuited: test_ids(observations[:short_circuited]),
|
|
69
|
+
unattributed: observations[:unattributed].sum { |vector| vector[:unattributed_count].to_i },
|
|
70
|
+
witness_vector_ids: pair, witness_owners: pair.flat_map { |id| owners_for(id) }.uniq.sort,
|
|
71
|
+
witness_owner_details: pair.flat_map { |id| owner_details_for(id) }.uniq.sort,
|
|
72
|
+
vectors: observations, constraint_result: result[:constraint_result],
|
|
73
|
+
supporting_set: supporting_set_for(decision[:id]), unexecuted: vectors.empty?
|
|
74
|
+
}
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
rows.sort_by do |row|
|
|
78
|
+
[row[:relative_path].to_s, row[:line].to_i, row[:column].to_i,
|
|
79
|
+
row[:decision_id].to_s, row[:index].to_i]
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def build_tests
|
|
84
|
+
by_id = @tests_by_id.transform_values do |test|
|
|
85
|
+
{ id: test[:id], name: display_name(test), relative_path: test_location(test)[0], line: test_location(test)[1],
|
|
86
|
+
status: test[:status], phases: Array(test[:phase_counts]).to_h.keys.sort,
|
|
87
|
+
observations: [], owns_witness: false }
|
|
88
|
+
end
|
|
89
|
+
@conditions.each do |condition|
|
|
90
|
+
condition[:vectors].each do |kind, vectors|
|
|
91
|
+
next if kind == :unattributed
|
|
92
|
+
|
|
93
|
+
vectors.each do |vector|
|
|
94
|
+
Array(vector[:test_ids]).each do |id|
|
|
95
|
+
row = by_id[id.to_s] ||= { id: id, name: id, relative_path: nil, line: nil, status: "unknown",
|
|
96
|
+
phases: [], observations: [], owns_witness: false }
|
|
97
|
+
phase_map = symbolize(vector[:phases_by_test] || {})
|
|
98
|
+
phases = phase_map[id.to_s] || phase_map[id.to_sym] || []
|
|
99
|
+
row[:phases] |= Array(phases).map(&:to_s)
|
|
100
|
+
row[:observations] << { condition_id: condition[:id], expression: condition[:expression],
|
|
101
|
+
relative_path: condition[:relative_path], line: condition[:line],
|
|
102
|
+
value: kind == :short_circuited ? nil : (kind == true), kind: kind,
|
|
103
|
+
phases: phases.map(&:to_s).sort,
|
|
104
|
+
owns_witness: condition[:witness_vector_ids].include?(vector[:id]) }
|
|
105
|
+
row[:owns_witness] = true if condition[:witness_vector_ids].include?(vector[:id])
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
by_id.each_value do |row|
|
|
111
|
+
row[:observations].uniq!
|
|
112
|
+
row[:phases] = row[:phases].map(&:to_s).uniq.sort
|
|
113
|
+
end
|
|
114
|
+
by_id.values.sort_by { |row| [row[:name].to_s, row[:relative_path].to_s, row[:line].to_i, row[:id].to_s] }
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def analysis_for(decision)
|
|
118
|
+
@analysis_by_decision[decision[:id].to_s] || {}
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def supporting_set_for(decision_id)
|
|
122
|
+
Array(@document[:minima] || @document["minima"]).filter_map do |minimum|
|
|
123
|
+
minimum = symbolize(minimum)
|
|
124
|
+
next unless minimum[:objective].to_s == "tests"
|
|
125
|
+
next unless Array(minimum[:scope_decision_ids]).map(&:to_s).include?(decision_id.to_s)
|
|
126
|
+
|
|
127
|
+
Array(minimum[:selected_ids]).map(&:to_s).sort
|
|
128
|
+
end.first || []
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def owners_for(vector_id)
|
|
132
|
+
vector = @vectors_by_id[vector_id.to_s]
|
|
133
|
+
Array(vector && vector[:test_ids]).map { |id| display_name(@tests_by_id[id.to_s] || { id: id }) }
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def owner_details_for(vector_id)
|
|
137
|
+
vector = @vectors_by_id[vector_id.to_s]
|
|
138
|
+
Array(vector && vector[:test_ids]).map do |id|
|
|
139
|
+
test = @tests_by_id[id.to_s] || { id: id }
|
|
140
|
+
name = display_name(test)
|
|
141
|
+
path, line = test_location(test)
|
|
142
|
+
path && line ? "#{name} (#{path}:#{line})" : name
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def test_ids(vectors) = vectors.flat_map { |vector| Array(vector[:test_ids]) }.map(&:to_s).uniq.sort
|
|
147
|
+
def vector_values(vector) = Array(vector[:values])
|
|
148
|
+
def records(hash, key) = Array(hash[key] || hash[key.to_s]).map { |item| symbolize(item) }
|
|
149
|
+
|
|
150
|
+
def display_name(test)
|
|
151
|
+
return "#{test[:class_name]}##{test[:method_name]}" if test[:class_name] && test[:method_name]
|
|
152
|
+
|
|
153
|
+
(test[:name] || test[:id]).to_s
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def test_location(test)
|
|
157
|
+
saved = symbolize(@test_locations[test[:id].to_s] || @test_locations[test[:id].to_sym] || {})
|
|
158
|
+
return [relative_path(saved[:relative_path]), saved[:line]] unless saved.empty?
|
|
159
|
+
|
|
160
|
+
source = symbolize(test[:source] || {})
|
|
161
|
+
path = source[:relative_path] || source[:path] || test[:source_path]
|
|
162
|
+
[relative_path(path), source[:line] || test[:line] || test[:source_line]]
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
def relative_path(path)
|
|
166
|
+
return nil if path.nil? || path.to_s.empty?
|
|
167
|
+
return path.to_s unless Pathname.new(path.to_s).absolute?
|
|
168
|
+
|
|
169
|
+
root = @inventory[:root] || symbolize(@document[:run_metadata] || {})[:project_root]
|
|
170
|
+
return nil unless root && Pathname.new(root.to_s).absolute?
|
|
171
|
+
|
|
172
|
+
Pathname.new(path.to_s).relative_path_from(Pathname.new(root.to_s)).to_s
|
|
173
|
+
rescue ArgumentError
|
|
174
|
+
nil
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def symbolize(value)
|
|
178
|
+
case value
|
|
179
|
+
when Hash then value.each_with_object({}) { |(key, item), result| result[key.to_sym] = symbolize(item) }
|
|
180
|
+
when Array then value.map { |item| symbolize(item) }
|
|
181
|
+
else value
|
|
182
|
+
end
|
|
183
|
+
end
|
|
184
|
+
end
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# rubocop:enable Metrics/ClassLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/MethodLength, Metrics/PerceivedComplexity, Metrics/BlockLength
|