necropsy 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +26 -0
- data/MEASUREMENTS.md +27 -0
- data/README.md +182 -17
- data/bench/README.md +92 -0
- data/bench/audit.rb +113 -0
- data/bench/audits/0.2.1/audit.json +501 -0
- data/bench/audits/0.2.1/audit.md +55 -0
- data/bench/audits/0.2.1/baseline_performance.yml +16 -0
- data/bench/audits/0.2.1/config.yml +42 -0
- data/bench/audits/0.2.1/review.yml +56 -0
- data/bench/audits/0.2.1/review_queue.yml +3651 -0
- data/bench/corpora/v1/README.md +30 -0
- data/bench/corpora/v1/labels.yml +37 -0
- data/bench/corpora/v1/manifest.yml +63 -0
- data/bench/corpora/v1/rubocop.necropsy.yml +7 -0
- data/bench/corpora/v1/self.necropsy.yml +8 -0
- data/bench/corpora/v1/tools/debride.yml +42 -0
- data/bench/corpora/v1/tools/spoom.yml +12 -0
- data/bench/corpora/v1/tools/type_aware.yml +11 -0
- data/bench/golden/v1/candidate_union.json +21918 -0
- data/bench/golden/v1/metadata.json +12 -0
- data/bench/golden/v1/reports/dynamic_evidence.json +146 -0
- data/bench/golden/v1/reports/plain_ruby.json +135 -0
- data/bench/golden/v1/reports/rails.json +176 -0
- data/bench/golden/v1/reports/rubocop_1_75_0.json +26848 -0
- data/bench/golden/v1/reports/self.json +3449 -0
- data/bench/review_queue.rb +35 -0
- data/bench/run.rb +31 -0
- data/bench/schema/candidate-union-v1.schema.json +70 -0
- data/docs/impv_implementation_matrix.md +179 -0
- data/docs/migrations/0.2.1.md +57 -0
- data/docs/migrations/0.3.0.md +207 -0
- data/docs/migrations/0.4.0.md +13 -0
- data/docs/necropsy_performance_adr.md +43 -0
- data/docs/necropsy_scope_decisions.md +58 -0
- data/docs/necropsy_type_facts_adr.md +22 -0
- data/gemfiles/prism_min.gemfile +9 -0
- data/gemfiles/prism_min.gemfile.lock +50 -0
- data/lib/necropsy/analyzer.rb +121 -2
- data/lib/necropsy/analyzers/dynamic/coverage_collector.rb +69 -17
- data/lib/necropsy/analyzers/dynamic/coverage_importer.rb +99 -9
- data/lib/necropsy/analyzers/dynamic/coverband_importer.rb +69 -299
- data/lib/necropsy/analyzers/dynamic/coverband_payload_set.rb +149 -0
- data/lib/necropsy/analyzers/dynamic/observation_policy.rb +96 -0
- data/lib/necropsy/analyzers/dynamic/redis_input_limits.rb +121 -0
- data/lib/necropsy/analyzers/dynamic/redis_nonblocking_io.rb +94 -0
- data/lib/necropsy/analyzers/dynamic/redis_payload_loader.rb +165 -0
- data/lib/necropsy/analyzers/dynamic/redis_transport.rb +217 -0
- data/lib/necropsy/analyzers/dynamic/runtime_reference.rb +96 -0
- data/lib/necropsy/analyzers/dynamic/trace_point_collector.rb +133 -23
- data/lib/necropsy/analyzers/dynamic/trace_point_importer.rb +3 -1
- data/lib/necropsy/analyzers/legacy_result_adapter.rb +226 -0
- data/lib/necropsy/analyzers/static/cha.rb +33 -73
- data/lib/necropsy/analyzers/static/name_resolution.rb +133 -12
- data/lib/necropsy/analyzers/static/rta.rb +237 -29
- data/lib/necropsy/ast_scanner/call_recording.rb +149 -30
- data/lib/necropsy/ast_scanner/call_site_creation.rb +54 -0
- data/lib/necropsy/ast_scanner/definition_creation.rb +43 -0
- data/lib/necropsy/ast_scanner/dsl_macros.rb +431 -41
- data/lib/necropsy/ast_scanner/method_definitions.rb +232 -50
- data/lib/necropsy/ast_scanner/references.rb +43 -9
- data/lib/necropsy/ast_scanner/ruby_semantics.rb +101 -15
- data/lib/necropsy/ast_scanner/traversal.rb +234 -71
- data/lib/necropsy/ast_scanner/value_definitions.rb +23 -13
- data/lib/necropsy/ast_scanner.rb +67 -6
- data/lib/necropsy/bench/candidate_union.rb +555 -0
- data/lib/necropsy/bench/claim_gate.rb +112 -0
- data/lib/necropsy/bench/evaluator.rb +329 -16
- data/lib/necropsy/bench/finding_facts.rb +152 -0
- data/lib/necropsy/bench/precision_gate.rb +144 -0
- data/lib/necropsy/bench/release_audit/adversarial_runner.rb +56 -0
- data/lib/necropsy/bench/release_audit/artifact_writer.rb +112 -0
- data/lib/necropsy/bench/release_audit/config_validator.rb +112 -0
- data/lib/necropsy/bench/release_audit/git_snapshot.rb +36 -0
- data/lib/necropsy/bench/release_audit/performance_gate.rb +165 -0
- data/lib/necropsy/bench/release_audit/run_provenance.rb +133 -0
- data/lib/necropsy/bench/release_audit.rb +360 -0
- data/lib/necropsy/bench/report_normalizer.rb +140 -0
- data/lib/necropsy/bench/review_queue.rb +154 -0
- data/lib/necropsy/bench/safety_mutation_harness.rb +59 -0
- data/lib/necropsy/bench/seed_runner.rb +408 -0
- data/lib/necropsy/bounded_canonicalizer.rb +218 -0
- data/lib/necropsy/cache/scan_cache.rb +85 -17
- data/lib/necropsy/call_site_identity.rb +54 -0
- data/lib/necropsy/cli.rb +220 -33
- data/lib/necropsy/clock.rb +40 -0
- data/lib/necropsy/confidence/scorer.rb +103 -58
- data/lib/necropsy/configuration.rb +224 -21
- data/lib/necropsy/convention_rules.rb +138 -0
- data/lib/necropsy/definition_identity/canonical_digest.rb +278 -0
- data/lib/necropsy/definition_identity.rb +37 -0
- data/lib/necropsy/diagnostics.rb +176 -36
- data/lib/necropsy/embedded_ruby.rb +55 -0
- data/lib/necropsy/entry_points/plain.rb +111 -10
- data/lib/necropsy/entry_points/rails.rb +322 -41
- data/lib/necropsy/entry_points/test.rb +6 -1
- data/lib/necropsy/flow_interpreter.rb +460 -0
- data/lib/necropsy/graph/blocker_matching.rb +338 -0
- data/lib/necropsy/graph/call_graph.rb +1099 -109
- data/lib/necropsy/graph/definition_index.rb +149 -0
- data/lib/necropsy/graph/dynamic_evidence_tracking.rb +206 -0
- data/lib/necropsy/graph/evidence_store.rb +213 -0
- data/lib/necropsy/graph/resolution_store.rb +497 -0
- data/lib/necropsy/graph_self_check.rb +79 -0
- data/lib/necropsy/guardrail/baseline.rb +350 -13
- data/lib/necropsy/guardrail/quarantine.rb +94 -9
- data/lib/necropsy/load_graph.rb +206 -0
- data/lib/necropsy/models.rb +878 -13
- data/lib/necropsy/performance_profiler.rb +108 -0
- data/lib/necropsy/project.rb +327 -25
- data/lib/necropsy/reachability/engine.rb +54 -13
- data/lib/necropsy/reference_barrier.rb +458 -0
- data/lib/necropsy/report.rb +113 -4
- data/lib/necropsy/reporter.rb +431 -15
- data/lib/necropsy/runner.rb +233 -18
- data/lib/necropsy/runtime_feedback.rb +136 -0
- data/lib/necropsy/semantics_matrix.rb +153 -0
- data/lib/necropsy/type_facts.rb +53 -0
- data/lib/necropsy/version.rb +1 -1
- data/lib/necropsy/why_not_explanation.rb +436 -0
- data/lib/necropsy/why_not_renderer.rb +197 -0
- data/lib/necropsy/world_policy.rb +90 -0
- data/lib/necropsy.rb +35 -2
- data/schema/necropsy-report-v2.schema.json +366 -0
- metadata +85 -1
|
@@ -0,0 +1,555 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'digest'
|
|
4
|
+
require 'date'
|
|
5
|
+
require 'yaml'
|
|
6
|
+
|
|
7
|
+
module Necropsy
|
|
8
|
+
module Bench
|
|
9
|
+
class CandidateUnion
|
|
10
|
+
LABELS = %w[dead alive external unknown].freeze
|
|
11
|
+
HIGH_CONFIDENCES = %w[high certain].freeze
|
|
12
|
+
MAX_ENTRIES = 100_000
|
|
13
|
+
MAX_INPUT_BYTES = 16 * 1024 * 1024
|
|
14
|
+
MAX_STRING_BYTES = 4_096
|
|
15
|
+
|
|
16
|
+
def initialize(manifest:, repository_root:, reports:, diagnostics:)
|
|
17
|
+
@manifest = manifest
|
|
18
|
+
@repository_root = repository_root
|
|
19
|
+
@reports = reports
|
|
20
|
+
@diagnostics = diagnostics
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def call
|
|
24
|
+
validate_inputs!
|
|
25
|
+
load_necropsy_candidates
|
|
26
|
+
load_external_candidates
|
|
27
|
+
discard_diagnostic_only_rows
|
|
28
|
+
apply_labels
|
|
29
|
+
fill_tool_results
|
|
30
|
+
result = {
|
|
31
|
+
'schema_version' => 1,
|
|
32
|
+
'tool_runs' => tool_runs.sort.to_h,
|
|
33
|
+
'summary' => summary,
|
|
34
|
+
'identity' => {
|
|
35
|
+
'primary' => 'definition_id',
|
|
36
|
+
'legacy_fallback' => 'id',
|
|
37
|
+
'legacy_mapping' => 'logical candidates and labels apply to every matching physical definition'
|
|
38
|
+
},
|
|
39
|
+
'candidates' => candidates.values.sort_by { |candidate| candidate_sort_key(candidate) }
|
|
40
|
+
}
|
|
41
|
+
validate_result!(result)
|
|
42
|
+
result
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
attr_reader :manifest, :repository_root, :reports, :diagnostics
|
|
48
|
+
|
|
49
|
+
def validate_inputs!
|
|
50
|
+
raise Error, 'Benchmark manifest must be a mapping' unless manifest.is_a?(Hash)
|
|
51
|
+
raise Error, 'Normalized reports must be a mapping' unless reports.is_a?(Hash)
|
|
52
|
+
raise Error, "Benchmark reports exceed #{MAX_ENTRIES} corpora" if reports.length > MAX_ENTRIES
|
|
53
|
+
|
|
54
|
+
reports.each do |corpus, report|
|
|
55
|
+
validate_identifier!(corpus, 'corpus')
|
|
56
|
+
validate_mapping!(report, "report #{corpus}")
|
|
57
|
+
validate_array!(report['findings'], "report #{corpus} findings")
|
|
58
|
+
report['findings'].each { |finding| validate_finding!(finding, corpus) }
|
|
59
|
+
identities = report['findings'].map { |finding| finding_identity(finding) }
|
|
60
|
+
duplicate = identities.tally.find { |_identity, count| count > 1 }
|
|
61
|
+
raise Error, "report #{corpus} has duplicate candidate identity #{duplicate.first}" if duplicate
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def validate_finding!(finding, corpus)
|
|
66
|
+
validate_mapping!(finding, "report #{corpus} finding")
|
|
67
|
+
validate_identifier!(finding['id'], "report #{corpus} finding id")
|
|
68
|
+
validate_identifier!(finding['definition_id'], 'definition_id') if finding.key?('definition_id')
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def candidates
|
|
72
|
+
@candidates ||= {}
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def tool_runs
|
|
76
|
+
@tool_runs ||= {
|
|
77
|
+
'necropsy' => {
|
|
78
|
+
'status' => 'generated',
|
|
79
|
+
'version' => manifest.dig('tools', 'necropsy', 'version'),
|
|
80
|
+
'provenance' => { 'kind' => 'generated_normalized_report' }
|
|
81
|
+
}.compact
|
|
82
|
+
}
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def load_necropsy_candidates
|
|
86
|
+
reports.each do |corpus, report|
|
|
87
|
+
report.fetch('findings').each do |finding|
|
|
88
|
+
candidate = candidate_for(corpus, finding_identity(finding), finding)
|
|
89
|
+
actionable = finding.fetch('candidate') { actionable_state?(finding.fetch('state')) }
|
|
90
|
+
candidate['tool_results']['necropsy'] = {
|
|
91
|
+
'candidate' => actionable,
|
|
92
|
+
'diagnostic' => !actionable,
|
|
93
|
+
'state' => finding.fetch('state'),
|
|
94
|
+
'confidence' => finding.fetch('confidence'),
|
|
95
|
+
'category' => finding['category'],
|
|
96
|
+
'loc' => finding['loc'],
|
|
97
|
+
'unknown' => finding['unknown'],
|
|
98
|
+
'rule_hits' => finding['rule_hits'],
|
|
99
|
+
'risk_flags' => finding['risk_flags']
|
|
100
|
+
}.compact
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
def actionable_state?(state)
|
|
106
|
+
%w[unreachable unused candidate].include?(state.to_s)
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def discard_diagnostic_only_rows
|
|
110
|
+
candidates.delete_if do |_key, candidate|
|
|
111
|
+
candidate.fetch('tool_results').values.none? { |result| result['candidate'] == true }
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def candidate_source_fields(source)
|
|
116
|
+
{
|
|
117
|
+
'path' => source['path'],
|
|
118
|
+
'line' => source['line'],
|
|
119
|
+
'end_line' => source['end_line'],
|
|
120
|
+
'loc' => source['loc'],
|
|
121
|
+
'category' => source['category']
|
|
122
|
+
}.compact
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def merge_candidate_source(candidate, source)
|
|
126
|
+
candidate_source_fields(source).each do |key, value|
|
|
127
|
+
candidate[key] = value unless candidate.key?(key)
|
|
128
|
+
end
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def tool_metrics
|
|
132
|
+
tool_runs.sort.to_h do |tool, run|
|
|
133
|
+
next [tool, { 'status' => 'skipped' }] if run['status'] == 'skipped'
|
|
134
|
+
|
|
135
|
+
selected = candidates.values.select { |candidate| candidate.dig('tool_results', tool, 'candidate') == true }
|
|
136
|
+
[tool, candidate_metrics(selected)]
|
|
137
|
+
end
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def candidate_metrics(selected)
|
|
141
|
+
reviewed = selected.select { |candidate| determinate_label?(candidate) }
|
|
142
|
+
true_positives = reviewed.count { |candidate| candidate.dig('label', 'value') == 'dead' }
|
|
143
|
+
known_positives = known_positive_entries
|
|
144
|
+
recalled = known_positives.count { |candidate| known_positive_recalled?(candidate, selected) }
|
|
145
|
+
measured_loc = selected.filter_map { |candidate| candidate['loc'] }
|
|
146
|
+
{
|
|
147
|
+
'candidate_precision' => reviewed.empty? ? nil : ratio(true_positives, reviewed.length),
|
|
148
|
+
'precision_status' => precision_status(selected, reviewed),
|
|
149
|
+
'candidate_count' => selected.length,
|
|
150
|
+
'candidate_loc' => measured_loc.sum,
|
|
151
|
+
'candidate_loc_measured_count' => measured_loc.length,
|
|
152
|
+
'reviewed_high_candidate_count' => reviewed_high_candidate_count(selected),
|
|
153
|
+
'known_positive_recall' => known_positives.empty? ? nil : ratio(recalled, known_positives.length),
|
|
154
|
+
'known_positive_count' => known_positives.length,
|
|
155
|
+
'reviewed_candidate_count' => reviewed.length,
|
|
156
|
+
'by_category' => category_metrics(selected, known_positives),
|
|
157
|
+
'by_corpus' => corpus_metrics(selected, known_positives),
|
|
158
|
+
'macro_average' => macro_average(selected, known_positives)
|
|
159
|
+
}
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def corpus_metrics(selected, known_positives)
|
|
163
|
+
corpora = (selected + known_positives).map { |candidate| candidate.fetch('corpus') }.uniq.sort
|
|
164
|
+
corpora.to_h do |corpus|
|
|
165
|
+
corpus_selected = selected.select { |candidate| candidate['corpus'] == corpus }
|
|
166
|
+
corpus_reviewed = corpus_selected.select { |candidate| determinate_label?(candidate) }
|
|
167
|
+
corpus_known = known_positives.select { |candidate| candidate['corpus'] == corpus }
|
|
168
|
+
true_positives = corpus_reviewed.count { |candidate| candidate.dig('label', 'value') == 'dead' }
|
|
169
|
+
recalled = corpus_known.count { |candidate| known_positive_recalled?(candidate, corpus_selected) }
|
|
170
|
+
[corpus, {
|
|
171
|
+
'candidate_precision' => corpus_reviewed.empty? ? nil : ratio(true_positives, corpus_reviewed.length),
|
|
172
|
+
'known_positive_recall' => corpus_known.empty? ? nil : ratio(recalled, corpus_known.length),
|
|
173
|
+
'candidate_count' => corpus_selected.length,
|
|
174
|
+
'reviewed_candidate_count' => corpus_reviewed.length,
|
|
175
|
+
'known_positive_count' => corpus_known.length
|
|
176
|
+
}]
|
|
177
|
+
end
|
|
178
|
+
end
|
|
179
|
+
|
|
180
|
+
def macro_average(selected, known_positives)
|
|
181
|
+
rows = corpus_metrics(selected, known_positives).values
|
|
182
|
+
precision = rows.filter_map { |row| row['candidate_precision'] }
|
|
183
|
+
recall = rows.filter_map { |row| row['known_positive_recall'] }
|
|
184
|
+
{
|
|
185
|
+
'candidate_precision' => mean(precision),
|
|
186
|
+
'candidate_precision_corpora' => precision.length,
|
|
187
|
+
'known_positive_recall' => mean(recall),
|
|
188
|
+
'known_positive_recall_corpora' => recall.length
|
|
189
|
+
}
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
def mean(values)
|
|
193
|
+
return if values.empty?
|
|
194
|
+
|
|
195
|
+
ratio(values.sum, values.length)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
def category_metrics(selected, known_positives)
|
|
199
|
+
categories = (selected + known_positives).map { |candidate| category_for(candidate) }.uniq.sort
|
|
200
|
+
categories.to_h do |category|
|
|
201
|
+
category_selected = selected.select { |candidate| category_for(candidate) == category }
|
|
202
|
+
category_reviewed = category_selected.select { |candidate| determinate_label?(candidate) }
|
|
203
|
+
category_known = known_positives.select { |candidate| category_for(candidate) == category }
|
|
204
|
+
true_positives = category_reviewed.count { |candidate| candidate.dig('label', 'value') == 'dead' }
|
|
205
|
+
recalled = category_known.count { |candidate| known_positive_recalled?(candidate, category_selected) }
|
|
206
|
+
measured_loc = category_selected.filter_map { |candidate| candidate['loc'] }
|
|
207
|
+
[category, {
|
|
208
|
+
'candidate_precision' => category_reviewed.empty? ? nil : ratio(true_positives, category_reviewed.length),
|
|
209
|
+
'precision_status' => precision_status(category_selected, category_reviewed),
|
|
210
|
+
'candidate_count' => category_selected.length,
|
|
211
|
+
'candidate_loc' => measured_loc.sum,
|
|
212
|
+
'candidate_loc_measured_count' => measured_loc.length,
|
|
213
|
+
'reviewed_high_candidate_count' => reviewed_high_candidate_count(category_selected),
|
|
214
|
+
'known_positive_recall' => category_known.empty? ? nil : ratio(recalled, category_known.length),
|
|
215
|
+
'known_positive_count' => category_known.length
|
|
216
|
+
}]
|
|
217
|
+
end
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
def category_for(candidate)
|
|
221
|
+
value = candidate.dig('label', 'category') || candidate['category']
|
|
222
|
+
value.to_s.empty? ? 'uncategorized' : value.to_s
|
|
223
|
+
end
|
|
224
|
+
|
|
225
|
+
def candidate_key(candidate)
|
|
226
|
+
[candidate['corpus'], candidate['definition_id'] || candidate['id']]
|
|
227
|
+
end
|
|
228
|
+
|
|
229
|
+
def candidate_sort_key(candidate)
|
|
230
|
+
[candidate['corpus'], candidate['id'], candidate['definition_id'].to_s]
|
|
231
|
+
end
|
|
232
|
+
|
|
233
|
+
def finding_identity(finding)
|
|
234
|
+
finding['definition_id'] || "logical:#{finding.fetch('id')}"
|
|
235
|
+
end
|
|
236
|
+
|
|
237
|
+
def candidates_matching(corpus, entry)
|
|
238
|
+
definition_id = entry['definition_id']
|
|
239
|
+
return [candidates[[corpus, definition_id]]].compact if definition_id
|
|
240
|
+
|
|
241
|
+
candidates.values.select { |candidate| candidate['corpus'] == corpus && candidate['id'] == entry['id'] }
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
def known_positive_recalled?(known_positive, selected)
|
|
245
|
+
selected.any? do |candidate|
|
|
246
|
+
next false unless candidate['corpus'] == known_positive['corpus']
|
|
247
|
+
|
|
248
|
+
if known_positive['definition_id']
|
|
249
|
+
candidate['definition_id'] == known_positive['definition_id']
|
|
250
|
+
else
|
|
251
|
+
candidate['id'] == known_positive['id']
|
|
252
|
+
end
|
|
253
|
+
end
|
|
254
|
+
end
|
|
255
|
+
|
|
256
|
+
def known_positive_entries
|
|
257
|
+
return @known_positive_entries if defined?(@known_positive_entries) && @known_positive_entries
|
|
258
|
+
|
|
259
|
+
candidates.values.select { |candidate| candidate.dig('label', 'value') == 'dead' }
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
def determinate_label?(candidate)
|
|
263
|
+
%w[dead alive external].include?(candidate.dig('label', 'value'))
|
|
264
|
+
end
|
|
265
|
+
|
|
266
|
+
def reviewed_high_candidate_count(selected)
|
|
267
|
+
selected.count do |candidate|
|
|
268
|
+
HIGH_CONFIDENCES.include?(candidate.dig('tool_results', 'necropsy', 'confidence').to_s) &&
|
|
269
|
+
determinate_label?(candidate)
|
|
270
|
+
end
|
|
271
|
+
end
|
|
272
|
+
|
|
273
|
+
def precision_status(selected, reviewed)
|
|
274
|
+
return 'no_candidates' if selected.empty?
|
|
275
|
+
return 'unreviewed' if reviewed.empty?
|
|
276
|
+
|
|
277
|
+
'measured'
|
|
278
|
+
end
|
|
279
|
+
|
|
280
|
+
def ratio(numerator, denominator)
|
|
281
|
+
(numerator.to_f / denominator).round(4)
|
|
282
|
+
end
|
|
283
|
+
|
|
284
|
+
def necropsy_diagnostics
|
|
285
|
+
by_corpus = reports.sort.to_h do |corpus, report|
|
|
286
|
+
[corpus, report['quality'] || legacy_quality(report)]
|
|
287
|
+
end
|
|
288
|
+
{
|
|
289
|
+
'by_corpus' => by_corpus,
|
|
290
|
+
'aggregate' => aggregate_quality(by_corpus.values)
|
|
291
|
+
}
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
def legacy_quality(report)
|
|
295
|
+
findings = report.fetch('findings')
|
|
296
|
+
blocked = findings.count { |finding| finding['state'] == 'blocked' }
|
|
297
|
+
{
|
|
298
|
+
'candidate_count' => findings.count { |finding| actionable_state?(finding['state']) },
|
|
299
|
+
'candidate_loc' => findings.filter_map { |finding| finding['loc'] }.sum,
|
|
300
|
+
'diagnostic_count' => findings.count { |finding| !actionable_state?(finding['state']) },
|
|
301
|
+
'blocked_count' => blocked,
|
|
302
|
+
'blocked_rate' => findings.empty? ? 0.0 : ratio(blocked, findings.length),
|
|
303
|
+
'unknown_finding_count' => findings.count { |finding| finding['unknown'] },
|
|
304
|
+
'unknown_finding_rate' => 0.0,
|
|
305
|
+
'resolution_counts' => { 'total' => 0, 'complete' => 0, 'partial' => 0, 'unknown' => 0 },
|
|
306
|
+
'unknown_resolution_rate' => 0.0,
|
|
307
|
+
'rule_counts' => {},
|
|
308
|
+
'risk_counts' => {},
|
|
309
|
+
'by_category' => {}
|
|
310
|
+
}
|
|
311
|
+
end
|
|
312
|
+
|
|
313
|
+
def aggregate_quality(quality_rows)
|
|
314
|
+
totals = %w[candidate_count candidate_loc diagnostic_count blocked_count unknown_finding_count].to_h do |key|
|
|
315
|
+
[key, quality_rows.sum { |quality| quality.fetch(key, 0) }]
|
|
316
|
+
end
|
|
317
|
+
finding_total = totals.fetch('candidate_count') + totals.fetch('diagnostic_count')
|
|
318
|
+
resolution_total = quality_rows.sum { |quality| quality.dig('resolution_counts', 'total').to_i }
|
|
319
|
+
unknown_resolutions = quality_rows.sum { |quality| quality.dig('resolution_counts', 'unknown').to_i }
|
|
320
|
+
totals.merge(
|
|
321
|
+
'blocked_rate' => finding_total.zero? ? 0.0 : ratio(totals.fetch('blocked_count'), finding_total),
|
|
322
|
+
'unknown_finding_rate' => finding_total.zero? ? 0.0 : ratio(totals.fetch('unknown_finding_count'), finding_total),
|
|
323
|
+
'unknown_resolution_rate' => resolution_total.zero? ? 0.0 : ratio(unknown_resolutions, resolution_total),
|
|
324
|
+
'rule_counts' => merge_counts(quality_rows, 'rule_counts'),
|
|
325
|
+
'risk_counts' => merge_counts(quality_rows, 'risk_counts')
|
|
326
|
+
)
|
|
327
|
+
end
|
|
328
|
+
|
|
329
|
+
def merge_counts(rows, key)
|
|
330
|
+
rows.each_with_object(Hash.new(0)) do |row, counts|
|
|
331
|
+
row.fetch(key, {}).each { |name, count| counts[name] += count }
|
|
332
|
+
end.sort.to_h
|
|
333
|
+
end
|
|
334
|
+
|
|
335
|
+
def load_external_candidates
|
|
336
|
+
manifest.fetch('tools', {}).sort.each do |tool, definition|
|
|
337
|
+
next if tool == 'necropsy'
|
|
338
|
+
|
|
339
|
+
path = snapshot_path(definition)
|
|
340
|
+
unless path && File.file?(path)
|
|
341
|
+
skip_tool(tool, definition)
|
|
342
|
+
next
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
validate_file_size!(path, "#{tool} candidate snapshot")
|
|
346
|
+
payload = YAML.safe_load_file(path, aliases: false)
|
|
347
|
+
validate_snapshot!(payload, tool)
|
|
348
|
+
payload.fetch('candidates').each do |entry|
|
|
349
|
+
validate_external_candidate!(entry, tool)
|
|
350
|
+
matches = candidates_matching(entry.fetch('corpus'), entry)
|
|
351
|
+
matches = [candidate_for(entry.fetch('corpus'), finding_identity(entry), entry)] if matches.empty?
|
|
352
|
+
matches.each do |candidate|
|
|
353
|
+
candidate['tool_results'][tool] = {
|
|
354
|
+
'candidate' => true,
|
|
355
|
+
'classification' => entry['classification'] || 'candidate',
|
|
356
|
+
'identity_match' => entry['definition_id'] ? 'physical' : 'legacy_logical_fallback'
|
|
357
|
+
}
|
|
358
|
+
end
|
|
359
|
+
end
|
|
360
|
+
tool_runs[tool] = {
|
|
361
|
+
'status' => 'snapshot',
|
|
362
|
+
'version' => definition['version'],
|
|
363
|
+
'snapshot_sha256' => Digest::SHA256.file(path).hexdigest,
|
|
364
|
+
'provenance' => payload['provenance']
|
|
365
|
+
}.compact
|
|
366
|
+
rescue KeyError, Psych::Exception => e
|
|
367
|
+
raise Error, "Invalid #{tool} candidate snapshot: #{e.message}"
|
|
368
|
+
end
|
|
369
|
+
end
|
|
370
|
+
|
|
371
|
+
def snapshot_path(definition)
|
|
372
|
+
env_path = ENV[definition['snapshot_env'].to_s] if definition['snapshot_env']
|
|
373
|
+
relative = env_path || definition['snapshot']
|
|
374
|
+
File.expand_path(relative, repository_root) if relative && !relative.empty?
|
|
375
|
+
end
|
|
376
|
+
|
|
377
|
+
def validate_snapshot!(payload, tool)
|
|
378
|
+
validate_mapping!(payload, "#{tool} candidate snapshot")
|
|
379
|
+
raise Error, 'schema_version must be 1' unless payload['schema_version'] == 1
|
|
380
|
+
raise Error, "tool must be #{tool}" unless payload['tool'] == tool
|
|
381
|
+
|
|
382
|
+
validate_array!(payload['candidates'], "#{tool} candidates")
|
|
383
|
+
|
|
384
|
+
provenance = payload['provenance']
|
|
385
|
+
raise Error, 'provenance must be a non-empty mapping' unless provenance.is_a?(Hash) && !provenance.empty?
|
|
386
|
+
end
|
|
387
|
+
|
|
388
|
+
def validate_external_candidate!(entry, tool)
|
|
389
|
+
validate_mapping!(entry, "#{tool} candidate")
|
|
390
|
+
validate_identifier!(entry['corpus'], "#{tool} candidate corpus")
|
|
391
|
+
validate_identifier!(entry['id'], "#{tool} candidate id")
|
|
392
|
+
validate_identifier!(entry['definition_id'], "#{tool} candidate definition_id") if entry.key?('definition_id')
|
|
393
|
+
end
|
|
394
|
+
|
|
395
|
+
def skip_tool(tool, definition)
|
|
396
|
+
command = Array(definition['command']).first || tool
|
|
397
|
+
reason = if executable?(command)
|
|
398
|
+
'candidate snapshot unavailable; live adapter is intentionally disabled for reproducibility'
|
|
399
|
+
else
|
|
400
|
+
"candidate snapshot unavailable and executable #{command.inspect} was not found"
|
|
401
|
+
end
|
|
402
|
+
message = "#{tool} skipped: #{reason}"
|
|
403
|
+
diagnostics << message
|
|
404
|
+
tool_runs[tool] = { 'status' => 'skipped', 'diagnostic' => message, 'version' => definition['version'] }
|
|
405
|
+
end
|
|
406
|
+
|
|
407
|
+
def executable?(command)
|
|
408
|
+
return File.executable?(command) if command.include?(File::SEPARATOR)
|
|
409
|
+
|
|
410
|
+
ENV.fetch('PATH', '').split(File::PATH_SEPARATOR).any? do |directory|
|
|
411
|
+
File.executable?(File.join(directory, command))
|
|
412
|
+
end
|
|
413
|
+
end
|
|
414
|
+
|
|
415
|
+
def candidate_for(corpus, id, source)
|
|
416
|
+
key = [corpus, id]
|
|
417
|
+
candidate = candidates[key] ||= {
|
|
418
|
+
'corpus' => corpus,
|
|
419
|
+
'id' => source.fetch('id'),
|
|
420
|
+
'definition_id' => source['definition_id'],
|
|
421
|
+
'tool_results' => {}
|
|
422
|
+
}.compact
|
|
423
|
+
merge_candidate_source(candidate, source)
|
|
424
|
+
candidate
|
|
425
|
+
end
|
|
426
|
+
|
|
427
|
+
def apply_labels
|
|
428
|
+
path = File.expand_path(manifest.fetch('labels'), repository_root)
|
|
429
|
+
validate_file_size!(path, 'benchmark labels')
|
|
430
|
+
payload = YAML.safe_load_file(path, aliases: false)
|
|
431
|
+
validate_mapping!(payload, 'benchmark labels')
|
|
432
|
+
labels = payload['labels']
|
|
433
|
+
validate_array!(labels, 'benchmark labels')
|
|
434
|
+
load_known_positives(payload)
|
|
435
|
+
minimum = Integer(manifest.fetch('minimum_reviewed_labels', 30))
|
|
436
|
+
raise Error, "Benchmark seed requires at least #{minimum} reviewed labels" if labels.length < minimum
|
|
437
|
+
|
|
438
|
+
labels.each do |entry|
|
|
439
|
+
reviewed_at = entry['reviewed_at'] || payload['reviewed_at']
|
|
440
|
+
source_revision = entry['source_revision'] || manifest.dig('corpora', entry['corpus'], 'revision')
|
|
441
|
+
validate_label(entry, reviewed_at: reviewed_at, source_revision: source_revision)
|
|
442
|
+
key = [entry.fetch('corpus'), entry['definition_id'] || entry.fetch('id')]
|
|
443
|
+
matches = candidates_matching(entry.fetch('corpus'), entry)
|
|
444
|
+
raise Error, "Benchmark label does not match a tool candidate: #{key.join(':')}" if matches.empty?
|
|
445
|
+
|
|
446
|
+
matches.each do |candidate|
|
|
447
|
+
raise Error, "Duplicate benchmark label: #{key.join(':')}" if candidate.key?('label')
|
|
448
|
+
|
|
449
|
+
candidate['label'] = entry.slice('value', 'rationale', 'reviewer', 'category').merge(
|
|
450
|
+
'reviewed_at' => reviewed_at,
|
|
451
|
+
'source_revision' => source_revision,
|
|
452
|
+
'identity_match' => entry['definition_id'] ? 'physical' : 'legacy_logical_fallback'
|
|
453
|
+
).compact
|
|
454
|
+
end
|
|
455
|
+
end
|
|
456
|
+
rescue KeyError, ArgumentError, Psych::Exception => e
|
|
457
|
+
raise Error, "Invalid benchmark labels: #{e.message}"
|
|
458
|
+
end
|
|
459
|
+
|
|
460
|
+
def load_known_positives(payload)
|
|
461
|
+
return unless payload.key?('known_positives')
|
|
462
|
+
|
|
463
|
+
entries = payload['known_positives']
|
|
464
|
+
validate_array!(entries, 'known-positive entries')
|
|
465
|
+
@known_positive_entries = entries.map do |entry|
|
|
466
|
+
validate_mapping!(entry, 'known-positive entry')
|
|
467
|
+
|
|
468
|
+
normalized = entry.transform_keys(&:to_s).slice('corpus', 'id', 'definition_id', 'category', 'rationale')
|
|
469
|
+
raise Error, 'Known-positive entries require corpus and id' if %w[corpus id].any? do |key|
|
|
470
|
+
normalized[key].to_s.empty?
|
|
471
|
+
end
|
|
472
|
+
|
|
473
|
+
validate_identifier!(normalized['corpus'], 'known-positive corpus')
|
|
474
|
+
validate_identifier!(normalized['id'], 'known-positive id')
|
|
475
|
+
validate_identifier!(normalized['definition_id'], 'known-positive definition_id') if normalized['definition_id']
|
|
476
|
+
raise Error, "Known positive #{candidate_key(normalized).join(':')} requires a rationale" if
|
|
477
|
+
normalized['rationale'].to_s.strip.empty?
|
|
478
|
+
|
|
479
|
+
normalized
|
|
480
|
+
end
|
|
481
|
+
duplicate = @known_positive_entries.map { |entry| candidate_key(entry) }.tally.find { |_key, count| count > 1 }
|
|
482
|
+
raise Error, "Duplicate known positive: #{duplicate.first.join(':')}" if duplicate
|
|
483
|
+
end
|
|
484
|
+
|
|
485
|
+
def validate_label(entry, reviewed_at:, source_revision:)
|
|
486
|
+
validate_mapping!(entry, 'benchmark label')
|
|
487
|
+
validate_identifier!(entry['corpus'], 'benchmark label corpus')
|
|
488
|
+
validate_identifier!(entry['id'], 'benchmark label id')
|
|
489
|
+
validate_identifier!(entry['definition_id'], 'benchmark label definition_id') if entry.key?('definition_id')
|
|
490
|
+
value = entry.fetch('value')
|
|
491
|
+
raise Error, "Invalid benchmark label #{value.inspect}" unless LABELS.include?(value)
|
|
492
|
+
raise Error, "Benchmark label #{entry['id']} requires a rationale" if entry['rationale'].to_s.strip.empty?
|
|
493
|
+
return unless manifest['label_provenance_required'] == true
|
|
494
|
+
|
|
495
|
+
raise Error, "Benchmark label #{entry['id']} requires a reviewer" if entry['reviewer'].to_s.strip.empty?
|
|
496
|
+
raise Error, "Benchmark label #{entry['id']} requires reviewed_at" if reviewed_at.to_s.strip.empty?
|
|
497
|
+
raise Error, "Benchmark label #{entry['id']} requires source_revision" if source_revision.to_s.strip.empty?
|
|
498
|
+
|
|
499
|
+
Date.iso8601(reviewed_at.to_s)
|
|
500
|
+
rescue Date::Error
|
|
501
|
+
raise Error, "Benchmark label #{entry['id']} reviewed_at must be an ISO 8601 date"
|
|
502
|
+
end
|
|
503
|
+
|
|
504
|
+
def validate_mapping!(value, label)
|
|
505
|
+
raise Error, "#{label} must be a mapping" unless value.is_a?(Hash)
|
|
506
|
+
end
|
|
507
|
+
|
|
508
|
+
def validate_array!(value, label)
|
|
509
|
+
raise Error, "#{label} must be an array" unless value.is_a?(Array)
|
|
510
|
+
raise Error, "#{label} exceeds #{MAX_ENTRIES} entries" if value.length > MAX_ENTRIES
|
|
511
|
+
end
|
|
512
|
+
|
|
513
|
+
def validate_identifier!(value, label)
|
|
514
|
+
raise Error, "#{label} must be a non-empty string" unless value.is_a?(String) && !value.empty?
|
|
515
|
+
raise Error, "#{label} exceeds #{MAX_STRING_BYTES} bytes" if value.bytesize > MAX_STRING_BYTES
|
|
516
|
+
end
|
|
517
|
+
|
|
518
|
+
def validate_file_size!(path, label)
|
|
519
|
+
raise Error, "#{label} exceeds #{MAX_INPUT_BYTES} bytes" if File.size(path) > MAX_INPUT_BYTES
|
|
520
|
+
end
|
|
521
|
+
|
|
522
|
+
def fill_tool_results
|
|
523
|
+
candidates.each_value do |candidate|
|
|
524
|
+
tool_runs.each do |tool, run|
|
|
525
|
+
result = { 'candidate' => false }
|
|
526
|
+
result = { 'candidate' => nil, 'status' => 'skipped' } if run['status'] == 'skipped'
|
|
527
|
+
candidate['tool_results'][tool] ||= result
|
|
528
|
+
end
|
|
529
|
+
candidate['tool_results'] = candidate['tool_results'].sort.to_h
|
|
530
|
+
end
|
|
531
|
+
end
|
|
532
|
+
|
|
533
|
+
def summary
|
|
534
|
+
labels = candidates.values.filter_map { |candidate| candidate.dig('label', 'value') }
|
|
535
|
+
{
|
|
536
|
+
'candidates' => candidates.length,
|
|
537
|
+
'reviewed' => labels.length,
|
|
538
|
+
'by_label' => labels.tally.sort.to_h,
|
|
539
|
+
'tool_metrics' => tool_metrics,
|
|
540
|
+
'necropsy_diagnostics' => necropsy_diagnostics
|
|
541
|
+
}
|
|
542
|
+
end
|
|
543
|
+
|
|
544
|
+
def validate_result!(result)
|
|
545
|
+
tools = result.fetch('tool_runs').keys
|
|
546
|
+
result.fetch('candidates').each do |candidate|
|
|
547
|
+
actual_tools = candidate.fetch('tool_results').keys
|
|
548
|
+
next if actual_tools == tools
|
|
549
|
+
|
|
550
|
+
raise Error, "Candidate #{candidate['corpus']}:#{candidate['id']} has incomplete tool results"
|
|
551
|
+
end
|
|
552
|
+
end
|
|
553
|
+
end
|
|
554
|
+
end
|
|
555
|
+
end
|
|
@@ -0,0 +1,112 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Necropsy
|
|
4
|
+
module Bench
|
|
5
|
+
# Final, configurable gate for a public accuracy claim. It is deliberately
|
|
6
|
+
# disabled unless a release config opts in with `claim_gate`.
|
|
7
|
+
class ClaimGate
|
|
8
|
+
ACTIONABLE_STATES = %w[unreachable unused candidate].freeze
|
|
9
|
+
|
|
10
|
+
def initialize(config:, reports:, summary:, adversarial_results:)
|
|
11
|
+
@config = config || {}
|
|
12
|
+
@reports = reports || {}
|
|
13
|
+
@summary = summary || {}
|
|
14
|
+
@adversarial_results = Array(adversarial_results)
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
def call
|
|
18
|
+
policy = normalized_policy
|
|
19
|
+
return { 'schema_version' => 1, 'enforced' => false, 'passed' => true } unless policy
|
|
20
|
+
|
|
21
|
+
checks = {
|
|
22
|
+
'corpus_count' => reports.length >= policy.fetch('minimum_corpora'),
|
|
23
|
+
'category_coverage' => category_coverage?(policy.fetch('required_categories')),
|
|
24
|
+
'high_candidates_explained' => unexplained_high_candidates.empty?,
|
|
25
|
+
'adversarial_suites' => adversarial_results.all? { |result| result['passed'] == true },
|
|
26
|
+
'candidate_precision' => precision_measured?,
|
|
27
|
+
'candidate_yield' => candidate_yield?
|
|
28
|
+
}
|
|
29
|
+
checks['reviewed_high_target'] = reviewed_high_count >= policy.fetch('minimum_reviewed_high')
|
|
30
|
+
{
|
|
31
|
+
'schema_version' => 1,
|
|
32
|
+
'enforced' => true,
|
|
33
|
+
'policy' => policy,
|
|
34
|
+
'checks' => checks,
|
|
35
|
+
'unexplained_high_candidates' => unexplained_high_candidates,
|
|
36
|
+
'reviewed_high_candidates' => reviewed_high_count,
|
|
37
|
+
'passed' => checks.values.all?
|
|
38
|
+
}
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
private
|
|
42
|
+
|
|
43
|
+
attr_reader :config, :reports, :summary, :adversarial_results
|
|
44
|
+
|
|
45
|
+
def normalized_policy
|
|
46
|
+
raw = config['claim_gate']
|
|
47
|
+
return unless raw
|
|
48
|
+
raise Error, 'claim_gate must be a mapping' unless raw.is_a?(Hash)
|
|
49
|
+
|
|
50
|
+
categories = Array(raw['required_categories'] || []).map(&:to_s).reject(&:empty?).uniq.sort
|
|
51
|
+
minimum_corpora = Integer(raw.fetch('minimum_corpora', 5))
|
|
52
|
+
minimum_reviewed_high = Integer(raw.fetch('minimum_reviewed_high', 0))
|
|
53
|
+
raise Error, 'claim_gate minimums must be non-negative' if minimum_corpora.negative? || minimum_reviewed_high.negative?
|
|
54
|
+
|
|
55
|
+
{
|
|
56
|
+
'minimum_corpora' => minimum_corpora,
|
|
57
|
+
'required_categories' => categories,
|
|
58
|
+
'minimum_reviewed_high' => minimum_reviewed_high
|
|
59
|
+
}
|
|
60
|
+
rescue ArgumentError, TypeError
|
|
61
|
+
raise Error, 'claim_gate minimums must be integers'
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
def category_coverage?(required)
|
|
65
|
+
return true if required.empty?
|
|
66
|
+
|
|
67
|
+
categories = reports.values.flat_map do |report|
|
|
68
|
+
Array(report['findings']).map { |finding| finding['category'].to_s }
|
|
69
|
+
end.uniq
|
|
70
|
+
required.all? { |category| categories.include?(category) }
|
|
71
|
+
end
|
|
72
|
+
|
|
73
|
+
def unexplained_high_candidates
|
|
74
|
+
reports.sort.flat_map do |corpus, report|
|
|
75
|
+
Array(report['findings']).filter_map do |finding|
|
|
76
|
+
next unless high_candidate?(finding)
|
|
77
|
+
next if Array(finding['reasons']).any? || Array(finding['rule_hits']).any? || Array(finding['blocker_kinds']).any?
|
|
78
|
+
|
|
79
|
+
{
|
|
80
|
+
'corpus' => corpus,
|
|
81
|
+
'id' => finding['id'],
|
|
82
|
+
'definition_id' => finding['definition_id'],
|
|
83
|
+
'path' => finding['path'],
|
|
84
|
+
'line' => finding['line']
|
|
85
|
+
}.compact
|
|
86
|
+
end
|
|
87
|
+
end.sort_by { |candidate| [candidate['corpus'], candidate['id'], candidate['definition_id'].to_s] }
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def high_candidate?(finding)
|
|
91
|
+
%w[high certain].include?(finding['confidence'].to_s) &&
|
|
92
|
+
(finding['candidate'] == true || ACTIONABLE_STATES.include?(finding['state'].to_s))
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
def reviewed_high_count
|
|
96
|
+
summary.dig('candidate_union', 'tool_metrics', 'necropsy', 'reviewed_high_candidate_count').to_i
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def necropsy_metrics
|
|
100
|
+
summary.dig('candidate_union', 'tool_metrics', 'necropsy') || {}
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
def precision_measured?
|
|
104
|
+
necropsy_metrics['precision_status'] == 'measured' && necropsy_metrics['candidate_precision'].is_a?(Numeric)
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
def candidate_yield?
|
|
108
|
+
necropsy_metrics['candidate_count'].to_i.positive? && necropsy_metrics['candidate_loc'].to_i.positive?
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|