necropsy 0.2.0 → 0.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (126) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +26 -0
  3. data/MEASUREMENTS.md +27 -0
  4. data/README.md +182 -17
  5. data/bench/README.md +92 -0
  6. data/bench/audit.rb +113 -0
  7. data/bench/audits/0.2.1/audit.json +501 -0
  8. data/bench/audits/0.2.1/audit.md +55 -0
  9. data/bench/audits/0.2.1/baseline_performance.yml +16 -0
  10. data/bench/audits/0.2.1/config.yml +42 -0
  11. data/bench/audits/0.2.1/review.yml +56 -0
  12. data/bench/audits/0.2.1/review_queue.yml +3651 -0
  13. data/bench/corpora/v1/README.md +30 -0
  14. data/bench/corpora/v1/labels.yml +37 -0
  15. data/bench/corpora/v1/manifest.yml +63 -0
  16. data/bench/corpora/v1/rubocop.necropsy.yml +7 -0
  17. data/bench/corpora/v1/self.necropsy.yml +8 -0
  18. data/bench/corpora/v1/tools/debride.yml +42 -0
  19. data/bench/corpora/v1/tools/spoom.yml +12 -0
  20. data/bench/corpora/v1/tools/type_aware.yml +11 -0
  21. data/bench/golden/v1/candidate_union.json +21918 -0
  22. data/bench/golden/v1/metadata.json +12 -0
  23. data/bench/golden/v1/reports/dynamic_evidence.json +146 -0
  24. data/bench/golden/v1/reports/plain_ruby.json +135 -0
  25. data/bench/golden/v1/reports/rails.json +176 -0
  26. data/bench/golden/v1/reports/rubocop_1_75_0.json +26848 -0
  27. data/bench/golden/v1/reports/self.json +3449 -0
  28. data/bench/review_queue.rb +35 -0
  29. data/bench/run.rb +31 -0
  30. data/bench/schema/candidate-union-v1.schema.json +70 -0
  31. data/docs/impv_implementation_matrix.md +179 -0
  32. data/docs/migrations/0.2.1.md +57 -0
  33. data/docs/migrations/0.3.0.md +207 -0
  34. data/docs/migrations/0.4.0.md +13 -0
  35. data/docs/necropsy_performance_adr.md +43 -0
  36. data/docs/necropsy_scope_decisions.md +58 -0
  37. data/docs/necropsy_type_facts_adr.md +22 -0
  38. data/gemfiles/prism_min.gemfile +9 -0
  39. data/gemfiles/prism_min.gemfile.lock +50 -0
  40. data/lib/necropsy/analyzer.rb +121 -2
  41. data/lib/necropsy/analyzers/dynamic/coverage_collector.rb +69 -17
  42. data/lib/necropsy/analyzers/dynamic/coverage_importer.rb +99 -9
  43. data/lib/necropsy/analyzers/dynamic/coverband_importer.rb +69 -299
  44. data/lib/necropsy/analyzers/dynamic/coverband_payload_set.rb +149 -0
  45. data/lib/necropsy/analyzers/dynamic/observation_policy.rb +96 -0
  46. data/lib/necropsy/analyzers/dynamic/redis_input_limits.rb +121 -0
  47. data/lib/necropsy/analyzers/dynamic/redis_nonblocking_io.rb +94 -0
  48. data/lib/necropsy/analyzers/dynamic/redis_payload_loader.rb +165 -0
  49. data/lib/necropsy/analyzers/dynamic/redis_transport.rb +217 -0
  50. data/lib/necropsy/analyzers/dynamic/runtime_reference.rb +96 -0
  51. data/lib/necropsy/analyzers/dynamic/trace_point_collector.rb +133 -23
  52. data/lib/necropsy/analyzers/dynamic/trace_point_importer.rb +3 -1
  53. data/lib/necropsy/analyzers/legacy_result_adapter.rb +226 -0
  54. data/lib/necropsy/analyzers/static/cha.rb +33 -73
  55. data/lib/necropsy/analyzers/static/name_resolution.rb +133 -12
  56. data/lib/necropsy/analyzers/static/rta.rb +237 -29
  57. data/lib/necropsy/ast_scanner/call_recording.rb +149 -30
  58. data/lib/necropsy/ast_scanner/call_site_creation.rb +54 -0
  59. data/lib/necropsy/ast_scanner/definition_creation.rb +43 -0
  60. data/lib/necropsy/ast_scanner/dsl_macros.rb +431 -41
  61. data/lib/necropsy/ast_scanner/method_definitions.rb +232 -50
  62. data/lib/necropsy/ast_scanner/references.rb +43 -9
  63. data/lib/necropsy/ast_scanner/ruby_semantics.rb +101 -15
  64. data/lib/necropsy/ast_scanner/traversal.rb +234 -71
  65. data/lib/necropsy/ast_scanner/value_definitions.rb +23 -13
  66. data/lib/necropsy/ast_scanner.rb +67 -6
  67. data/lib/necropsy/bench/candidate_union.rb +555 -0
  68. data/lib/necropsy/bench/claim_gate.rb +112 -0
  69. data/lib/necropsy/bench/evaluator.rb +329 -16
  70. data/lib/necropsy/bench/finding_facts.rb +152 -0
  71. data/lib/necropsy/bench/precision_gate.rb +144 -0
  72. data/lib/necropsy/bench/release_audit/adversarial_runner.rb +56 -0
  73. data/lib/necropsy/bench/release_audit/artifact_writer.rb +112 -0
  74. data/lib/necropsy/bench/release_audit/config_validator.rb +112 -0
  75. data/lib/necropsy/bench/release_audit/git_snapshot.rb +36 -0
  76. data/lib/necropsy/bench/release_audit/performance_gate.rb +165 -0
  77. data/lib/necropsy/bench/release_audit/run_provenance.rb +133 -0
  78. data/lib/necropsy/bench/release_audit.rb +360 -0
  79. data/lib/necropsy/bench/report_normalizer.rb +140 -0
  80. data/lib/necropsy/bench/review_queue.rb +154 -0
  81. data/lib/necropsy/bench/safety_mutation_harness.rb +59 -0
  82. data/lib/necropsy/bench/seed_runner.rb +408 -0
  83. data/lib/necropsy/bounded_canonicalizer.rb +218 -0
  84. data/lib/necropsy/cache/scan_cache.rb +85 -17
  85. data/lib/necropsy/call_site_identity.rb +54 -0
  86. data/lib/necropsy/cli.rb +220 -33
  87. data/lib/necropsy/clock.rb +40 -0
  88. data/lib/necropsy/confidence/scorer.rb +103 -58
  89. data/lib/necropsy/configuration.rb +224 -21
  90. data/lib/necropsy/convention_rules.rb +138 -0
  91. data/lib/necropsy/definition_identity/canonical_digest.rb +278 -0
  92. data/lib/necropsy/definition_identity.rb +37 -0
  93. data/lib/necropsy/diagnostics.rb +176 -36
  94. data/lib/necropsy/embedded_ruby.rb +55 -0
  95. data/lib/necropsy/entry_points/plain.rb +111 -10
  96. data/lib/necropsy/entry_points/rails.rb +322 -41
  97. data/lib/necropsy/entry_points/test.rb +6 -1
  98. data/lib/necropsy/flow_interpreter.rb +460 -0
  99. data/lib/necropsy/graph/blocker_matching.rb +338 -0
  100. data/lib/necropsy/graph/call_graph.rb +1099 -109
  101. data/lib/necropsy/graph/definition_index.rb +149 -0
  102. data/lib/necropsy/graph/dynamic_evidence_tracking.rb +206 -0
  103. data/lib/necropsy/graph/evidence_store.rb +213 -0
  104. data/lib/necropsy/graph/resolution_store.rb +497 -0
  105. data/lib/necropsy/graph_self_check.rb +79 -0
  106. data/lib/necropsy/guardrail/baseline.rb +350 -13
  107. data/lib/necropsy/guardrail/quarantine.rb +94 -9
  108. data/lib/necropsy/load_graph.rb +206 -0
  109. data/lib/necropsy/models.rb +878 -13
  110. data/lib/necropsy/performance_profiler.rb +108 -0
  111. data/lib/necropsy/project.rb +327 -25
  112. data/lib/necropsy/reachability/engine.rb +54 -13
  113. data/lib/necropsy/reference_barrier.rb +458 -0
  114. data/lib/necropsy/report.rb +113 -4
  115. data/lib/necropsy/reporter.rb +431 -15
  116. data/lib/necropsy/runner.rb +233 -18
  117. data/lib/necropsy/runtime_feedback.rb +136 -0
  118. data/lib/necropsy/semantics_matrix.rb +153 -0
  119. data/lib/necropsy/type_facts.rb +53 -0
  120. data/lib/necropsy/version.rb +1 -1
  121. data/lib/necropsy/why_not_explanation.rb +436 -0
  122. data/lib/necropsy/why_not_renderer.rb +197 -0
  123. data/lib/necropsy/world_policy.rb +90 -0
  124. data/lib/necropsy.rb +35 -2
  125. data/schema/necropsy-report-v2.schema.json +366 -0
  126. metadata +85 -1
@@ -0,0 +1,555 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'digest'
4
+ require 'date'
5
+ require 'yaml'
6
+
7
+ module Necropsy
8
+ module Bench
9
+ class CandidateUnion
10
+ LABELS = %w[dead alive external unknown].freeze
11
+ HIGH_CONFIDENCES = %w[high certain].freeze
12
+ MAX_ENTRIES = 100_000
13
+ MAX_INPUT_BYTES = 16 * 1024 * 1024
14
+ MAX_STRING_BYTES = 4_096
15
+
16
+ def initialize(manifest:, repository_root:, reports:, diagnostics:)
17
+ @manifest = manifest
18
+ @repository_root = repository_root
19
+ @reports = reports
20
+ @diagnostics = diagnostics
21
+ end
22
+
23
+ def call
24
+ validate_inputs!
25
+ load_necropsy_candidates
26
+ load_external_candidates
27
+ discard_diagnostic_only_rows
28
+ apply_labels
29
+ fill_tool_results
30
+ result = {
31
+ 'schema_version' => 1,
32
+ 'tool_runs' => tool_runs.sort.to_h,
33
+ 'summary' => summary,
34
+ 'identity' => {
35
+ 'primary' => 'definition_id',
36
+ 'legacy_fallback' => 'id',
37
+ 'legacy_mapping' => 'logical candidates and labels apply to every matching physical definition'
38
+ },
39
+ 'candidates' => candidates.values.sort_by { |candidate| candidate_sort_key(candidate) }
40
+ }
41
+ validate_result!(result)
42
+ result
43
+ end
44
+
45
+ private
46
+
47
+ attr_reader :manifest, :repository_root, :reports, :diagnostics
48
+
49
+ def validate_inputs!
50
+ raise Error, 'Benchmark manifest must be a mapping' unless manifest.is_a?(Hash)
51
+ raise Error, 'Normalized reports must be a mapping' unless reports.is_a?(Hash)
52
+ raise Error, "Benchmark reports exceed #{MAX_ENTRIES} corpora" if reports.length > MAX_ENTRIES
53
+
54
+ reports.each do |corpus, report|
55
+ validate_identifier!(corpus, 'corpus')
56
+ validate_mapping!(report, "report #{corpus}")
57
+ validate_array!(report['findings'], "report #{corpus} findings")
58
+ report['findings'].each { |finding| validate_finding!(finding, corpus) }
59
+ identities = report['findings'].map { |finding| finding_identity(finding) }
60
+ duplicate = identities.tally.find { |_identity, count| count > 1 }
61
+ raise Error, "report #{corpus} has duplicate candidate identity #{duplicate.first}" if duplicate
62
+ end
63
+ end
64
+
65
+ def validate_finding!(finding, corpus)
66
+ validate_mapping!(finding, "report #{corpus} finding")
67
+ validate_identifier!(finding['id'], "report #{corpus} finding id")
68
+ validate_identifier!(finding['definition_id'], 'definition_id') if finding.key?('definition_id')
69
+ end
70
+
71
+ def candidates
72
+ @candidates ||= {}
73
+ end
74
+
75
+ def tool_runs
76
+ @tool_runs ||= {
77
+ 'necropsy' => {
78
+ 'status' => 'generated',
79
+ 'version' => manifest.dig('tools', 'necropsy', 'version'),
80
+ 'provenance' => { 'kind' => 'generated_normalized_report' }
81
+ }.compact
82
+ }
83
+ end
84
+
85
+ def load_necropsy_candidates
86
+ reports.each do |corpus, report|
87
+ report.fetch('findings').each do |finding|
88
+ candidate = candidate_for(corpus, finding_identity(finding), finding)
89
+ actionable = finding.fetch('candidate') { actionable_state?(finding.fetch('state')) }
90
+ candidate['tool_results']['necropsy'] = {
91
+ 'candidate' => actionable,
92
+ 'diagnostic' => !actionable,
93
+ 'state' => finding.fetch('state'),
94
+ 'confidence' => finding.fetch('confidence'),
95
+ 'category' => finding['category'],
96
+ 'loc' => finding['loc'],
97
+ 'unknown' => finding['unknown'],
98
+ 'rule_hits' => finding['rule_hits'],
99
+ 'risk_flags' => finding['risk_flags']
100
+ }.compact
101
+ end
102
+ end
103
+ end
104
+
105
+ def actionable_state?(state)
106
+ %w[unreachable unused candidate].include?(state.to_s)
107
+ end
108
+
109
+ def discard_diagnostic_only_rows
110
+ candidates.delete_if do |_key, candidate|
111
+ candidate.fetch('tool_results').values.none? { |result| result['candidate'] == true }
112
+ end
113
+ end
114
+
115
+ def candidate_source_fields(source)
116
+ {
117
+ 'path' => source['path'],
118
+ 'line' => source['line'],
119
+ 'end_line' => source['end_line'],
120
+ 'loc' => source['loc'],
121
+ 'category' => source['category']
122
+ }.compact
123
+ end
124
+
125
+ def merge_candidate_source(candidate, source)
126
+ candidate_source_fields(source).each do |key, value|
127
+ candidate[key] = value unless candidate.key?(key)
128
+ end
129
+ end
130
+
131
+ def tool_metrics
132
+ tool_runs.sort.to_h do |tool, run|
133
+ next [tool, { 'status' => 'skipped' }] if run['status'] == 'skipped'
134
+
135
+ selected = candidates.values.select { |candidate| candidate.dig('tool_results', tool, 'candidate') == true }
136
+ [tool, candidate_metrics(selected)]
137
+ end
138
+ end
139
+
140
+ def candidate_metrics(selected)
141
+ reviewed = selected.select { |candidate| determinate_label?(candidate) }
142
+ true_positives = reviewed.count { |candidate| candidate.dig('label', 'value') == 'dead' }
143
+ known_positives = known_positive_entries
144
+ recalled = known_positives.count { |candidate| known_positive_recalled?(candidate, selected) }
145
+ measured_loc = selected.filter_map { |candidate| candidate['loc'] }
146
+ {
147
+ 'candidate_precision' => reviewed.empty? ? nil : ratio(true_positives, reviewed.length),
148
+ 'precision_status' => precision_status(selected, reviewed),
149
+ 'candidate_count' => selected.length,
150
+ 'candidate_loc' => measured_loc.sum,
151
+ 'candidate_loc_measured_count' => measured_loc.length,
152
+ 'reviewed_high_candidate_count' => reviewed_high_candidate_count(selected),
153
+ 'known_positive_recall' => known_positives.empty? ? nil : ratio(recalled, known_positives.length),
154
+ 'known_positive_count' => known_positives.length,
155
+ 'reviewed_candidate_count' => reviewed.length,
156
+ 'by_category' => category_metrics(selected, known_positives),
157
+ 'by_corpus' => corpus_metrics(selected, known_positives),
158
+ 'macro_average' => macro_average(selected, known_positives)
159
+ }
160
+ end
161
+
162
+ def corpus_metrics(selected, known_positives)
163
+ corpora = (selected + known_positives).map { |candidate| candidate.fetch('corpus') }.uniq.sort
164
+ corpora.to_h do |corpus|
165
+ corpus_selected = selected.select { |candidate| candidate['corpus'] == corpus }
166
+ corpus_reviewed = corpus_selected.select { |candidate| determinate_label?(candidate) }
167
+ corpus_known = known_positives.select { |candidate| candidate['corpus'] == corpus }
168
+ true_positives = corpus_reviewed.count { |candidate| candidate.dig('label', 'value') == 'dead' }
169
+ recalled = corpus_known.count { |candidate| known_positive_recalled?(candidate, corpus_selected) }
170
+ [corpus, {
171
+ 'candidate_precision' => corpus_reviewed.empty? ? nil : ratio(true_positives, corpus_reviewed.length),
172
+ 'known_positive_recall' => corpus_known.empty? ? nil : ratio(recalled, corpus_known.length),
173
+ 'candidate_count' => corpus_selected.length,
174
+ 'reviewed_candidate_count' => corpus_reviewed.length,
175
+ 'known_positive_count' => corpus_known.length
176
+ }]
177
+ end
178
+ end
179
+
180
+ def macro_average(selected, known_positives)
181
+ rows = corpus_metrics(selected, known_positives).values
182
+ precision = rows.filter_map { |row| row['candidate_precision'] }
183
+ recall = rows.filter_map { |row| row['known_positive_recall'] }
184
+ {
185
+ 'candidate_precision' => mean(precision),
186
+ 'candidate_precision_corpora' => precision.length,
187
+ 'known_positive_recall' => mean(recall),
188
+ 'known_positive_recall_corpora' => recall.length
189
+ }
190
+ end
191
+
192
+ def mean(values)
193
+ return if values.empty?
194
+
195
+ ratio(values.sum, values.length)
196
+ end
197
+
198
+ def category_metrics(selected, known_positives)
199
+ categories = (selected + known_positives).map { |candidate| category_for(candidate) }.uniq.sort
200
+ categories.to_h do |category|
201
+ category_selected = selected.select { |candidate| category_for(candidate) == category }
202
+ category_reviewed = category_selected.select { |candidate| determinate_label?(candidate) }
203
+ category_known = known_positives.select { |candidate| category_for(candidate) == category }
204
+ true_positives = category_reviewed.count { |candidate| candidate.dig('label', 'value') == 'dead' }
205
+ recalled = category_known.count { |candidate| known_positive_recalled?(candidate, category_selected) }
206
+ measured_loc = category_selected.filter_map { |candidate| candidate['loc'] }
207
+ [category, {
208
+ 'candidate_precision' => category_reviewed.empty? ? nil : ratio(true_positives, category_reviewed.length),
209
+ 'precision_status' => precision_status(category_selected, category_reviewed),
210
+ 'candidate_count' => category_selected.length,
211
+ 'candidate_loc' => measured_loc.sum,
212
+ 'candidate_loc_measured_count' => measured_loc.length,
213
+ 'reviewed_high_candidate_count' => reviewed_high_candidate_count(category_selected),
214
+ 'known_positive_recall' => category_known.empty? ? nil : ratio(recalled, category_known.length),
215
+ 'known_positive_count' => category_known.length
216
+ }]
217
+ end
218
+ end
219
+
220
+ def category_for(candidate)
221
+ value = candidate.dig('label', 'category') || candidate['category']
222
+ value.to_s.empty? ? 'uncategorized' : value.to_s
223
+ end
224
+
225
+ def candidate_key(candidate)
226
+ [candidate['corpus'], candidate['definition_id'] || candidate['id']]
227
+ end
228
+
229
+ def candidate_sort_key(candidate)
230
+ [candidate['corpus'], candidate['id'], candidate['definition_id'].to_s]
231
+ end
232
+
233
+ def finding_identity(finding)
234
+ finding['definition_id'] || "logical:#{finding.fetch('id')}"
235
+ end
236
+
237
+ def candidates_matching(corpus, entry)
238
+ definition_id = entry['definition_id']
239
+ return [candidates[[corpus, definition_id]]].compact if definition_id
240
+
241
+ candidates.values.select { |candidate| candidate['corpus'] == corpus && candidate['id'] == entry['id'] }
242
+ end
243
+
244
+ def known_positive_recalled?(known_positive, selected)
245
+ selected.any? do |candidate|
246
+ next false unless candidate['corpus'] == known_positive['corpus']
247
+
248
+ if known_positive['definition_id']
249
+ candidate['definition_id'] == known_positive['definition_id']
250
+ else
251
+ candidate['id'] == known_positive['id']
252
+ end
253
+ end
254
+ end
255
+
256
+ def known_positive_entries
257
+ return @known_positive_entries if defined?(@known_positive_entries) && @known_positive_entries
258
+
259
+ candidates.values.select { |candidate| candidate.dig('label', 'value') == 'dead' }
260
+ end
261
+
262
+ def determinate_label?(candidate)
263
+ %w[dead alive external].include?(candidate.dig('label', 'value'))
264
+ end
265
+
266
+ def reviewed_high_candidate_count(selected)
267
+ selected.count do |candidate|
268
+ HIGH_CONFIDENCES.include?(candidate.dig('tool_results', 'necropsy', 'confidence').to_s) &&
269
+ determinate_label?(candidate)
270
+ end
271
+ end
272
+
273
+ def precision_status(selected, reviewed)
274
+ return 'no_candidates' if selected.empty?
275
+ return 'unreviewed' if reviewed.empty?
276
+
277
+ 'measured'
278
+ end
279
+
280
+ def ratio(numerator, denominator)
281
+ (numerator.to_f / denominator).round(4)
282
+ end
283
+
284
+ def necropsy_diagnostics
285
+ by_corpus = reports.sort.to_h do |corpus, report|
286
+ [corpus, report['quality'] || legacy_quality(report)]
287
+ end
288
+ {
289
+ 'by_corpus' => by_corpus,
290
+ 'aggregate' => aggregate_quality(by_corpus.values)
291
+ }
292
+ end
293
+
294
+ def legacy_quality(report)
295
+ findings = report.fetch('findings')
296
+ blocked = findings.count { |finding| finding['state'] == 'blocked' }
297
+ {
298
+ 'candidate_count' => findings.count { |finding| actionable_state?(finding['state']) },
299
+ 'candidate_loc' => findings.filter_map { |finding| finding['loc'] }.sum,
300
+ 'diagnostic_count' => findings.count { |finding| !actionable_state?(finding['state']) },
301
+ 'blocked_count' => blocked,
302
+ 'blocked_rate' => findings.empty? ? 0.0 : ratio(blocked, findings.length),
303
+ 'unknown_finding_count' => findings.count { |finding| finding['unknown'] },
304
+ 'unknown_finding_rate' => 0.0,
305
+ 'resolution_counts' => { 'total' => 0, 'complete' => 0, 'partial' => 0, 'unknown' => 0 },
306
+ 'unknown_resolution_rate' => 0.0,
307
+ 'rule_counts' => {},
308
+ 'risk_counts' => {},
309
+ 'by_category' => {}
310
+ }
311
+ end
312
+
313
+ def aggregate_quality(quality_rows)
314
+ totals = %w[candidate_count candidate_loc diagnostic_count blocked_count unknown_finding_count].to_h do |key|
315
+ [key, quality_rows.sum { |quality| quality.fetch(key, 0) }]
316
+ end
317
+ finding_total = totals.fetch('candidate_count') + totals.fetch('diagnostic_count')
318
+ resolution_total = quality_rows.sum { |quality| quality.dig('resolution_counts', 'total').to_i }
319
+ unknown_resolutions = quality_rows.sum { |quality| quality.dig('resolution_counts', 'unknown').to_i }
320
+ totals.merge(
321
+ 'blocked_rate' => finding_total.zero? ? 0.0 : ratio(totals.fetch('blocked_count'), finding_total),
322
+ 'unknown_finding_rate' => finding_total.zero? ? 0.0 : ratio(totals.fetch('unknown_finding_count'), finding_total),
323
+ 'unknown_resolution_rate' => resolution_total.zero? ? 0.0 : ratio(unknown_resolutions, resolution_total),
324
+ 'rule_counts' => merge_counts(quality_rows, 'rule_counts'),
325
+ 'risk_counts' => merge_counts(quality_rows, 'risk_counts')
326
+ )
327
+ end
328
+
329
+ def merge_counts(rows, key)
330
+ rows.each_with_object(Hash.new(0)) do |row, counts|
331
+ row.fetch(key, {}).each { |name, count| counts[name] += count }
332
+ end.sort.to_h
333
+ end
334
+
335
+ def load_external_candidates
336
+ manifest.fetch('tools', {}).sort.each do |tool, definition|
337
+ next if tool == 'necropsy'
338
+
339
+ path = snapshot_path(definition)
340
+ unless path && File.file?(path)
341
+ skip_tool(tool, definition)
342
+ next
343
+ end
344
+
345
+ validate_file_size!(path, "#{tool} candidate snapshot")
346
+ payload = YAML.safe_load_file(path, aliases: false)
347
+ validate_snapshot!(payload, tool)
348
+ payload.fetch('candidates').each do |entry|
349
+ validate_external_candidate!(entry, tool)
350
+ matches = candidates_matching(entry.fetch('corpus'), entry)
351
+ matches = [candidate_for(entry.fetch('corpus'), finding_identity(entry), entry)] if matches.empty?
352
+ matches.each do |candidate|
353
+ candidate['tool_results'][tool] = {
354
+ 'candidate' => true,
355
+ 'classification' => entry['classification'] || 'candidate',
356
+ 'identity_match' => entry['definition_id'] ? 'physical' : 'legacy_logical_fallback'
357
+ }
358
+ end
359
+ end
360
+ tool_runs[tool] = {
361
+ 'status' => 'snapshot',
362
+ 'version' => definition['version'],
363
+ 'snapshot_sha256' => Digest::SHA256.file(path).hexdigest,
364
+ 'provenance' => payload['provenance']
365
+ }.compact
366
+ rescue KeyError, Psych::Exception => e
367
+ raise Error, "Invalid #{tool} candidate snapshot: #{e.message}"
368
+ end
369
+ end
370
+
371
+ def snapshot_path(definition)
372
+ env_path = ENV[definition['snapshot_env'].to_s] if definition['snapshot_env']
373
+ relative = env_path || definition['snapshot']
374
+ File.expand_path(relative, repository_root) if relative && !relative.empty?
375
+ end
376
+
377
+ def validate_snapshot!(payload, tool)
378
+ validate_mapping!(payload, "#{tool} candidate snapshot")
379
+ raise Error, 'schema_version must be 1' unless payload['schema_version'] == 1
380
+ raise Error, "tool must be #{tool}" unless payload['tool'] == tool
381
+
382
+ validate_array!(payload['candidates'], "#{tool} candidates")
383
+
384
+ provenance = payload['provenance']
385
+ raise Error, 'provenance must be a non-empty mapping' unless provenance.is_a?(Hash) && !provenance.empty?
386
+ end
387
+
388
+ def validate_external_candidate!(entry, tool)
389
+ validate_mapping!(entry, "#{tool} candidate")
390
+ validate_identifier!(entry['corpus'], "#{tool} candidate corpus")
391
+ validate_identifier!(entry['id'], "#{tool} candidate id")
392
+ validate_identifier!(entry['definition_id'], "#{tool} candidate definition_id") if entry.key?('definition_id')
393
+ end
394
+
395
+ def skip_tool(tool, definition)
396
+ command = Array(definition['command']).first || tool
397
+ reason = if executable?(command)
398
+ 'candidate snapshot unavailable; live adapter is intentionally disabled for reproducibility'
399
+ else
400
+ "candidate snapshot unavailable and executable #{command.inspect} was not found"
401
+ end
402
+ message = "#{tool} skipped: #{reason}"
403
+ diagnostics << message
404
+ tool_runs[tool] = { 'status' => 'skipped', 'diagnostic' => message, 'version' => definition['version'] }
405
+ end
406
+
407
+ def executable?(command)
408
+ return File.executable?(command) if command.include?(File::SEPARATOR)
409
+
410
+ ENV.fetch('PATH', '').split(File::PATH_SEPARATOR).any? do |directory|
411
+ File.executable?(File.join(directory, command))
412
+ end
413
+ end
414
+
415
+ def candidate_for(corpus, id, source)
416
+ key = [corpus, id]
417
+ candidate = candidates[key] ||= {
418
+ 'corpus' => corpus,
419
+ 'id' => source.fetch('id'),
420
+ 'definition_id' => source['definition_id'],
421
+ 'tool_results' => {}
422
+ }.compact
423
+ merge_candidate_source(candidate, source)
424
+ candidate
425
+ end
426
+
427
+ def apply_labels
428
+ path = File.expand_path(manifest.fetch('labels'), repository_root)
429
+ validate_file_size!(path, 'benchmark labels')
430
+ payload = YAML.safe_load_file(path, aliases: false)
431
+ validate_mapping!(payload, 'benchmark labels')
432
+ labels = payload['labels']
433
+ validate_array!(labels, 'benchmark labels')
434
+ load_known_positives(payload)
435
+ minimum = Integer(manifest.fetch('minimum_reviewed_labels', 30))
436
+ raise Error, "Benchmark seed requires at least #{minimum} reviewed labels" if labels.length < minimum
437
+
438
+ labels.each do |entry|
439
+ reviewed_at = entry['reviewed_at'] || payload['reviewed_at']
440
+ source_revision = entry['source_revision'] || manifest.dig('corpora', entry['corpus'], 'revision')
441
+ validate_label(entry, reviewed_at: reviewed_at, source_revision: source_revision)
442
+ key = [entry.fetch('corpus'), entry['definition_id'] || entry.fetch('id')]
443
+ matches = candidates_matching(entry.fetch('corpus'), entry)
444
+ raise Error, "Benchmark label does not match a tool candidate: #{key.join(':')}" if matches.empty?
445
+
446
+ matches.each do |candidate|
447
+ raise Error, "Duplicate benchmark label: #{key.join(':')}" if candidate.key?('label')
448
+
449
+ candidate['label'] = entry.slice('value', 'rationale', 'reviewer', 'category').merge(
450
+ 'reviewed_at' => reviewed_at,
451
+ 'source_revision' => source_revision,
452
+ 'identity_match' => entry['definition_id'] ? 'physical' : 'legacy_logical_fallback'
453
+ ).compact
454
+ end
455
+ end
456
+ rescue KeyError, ArgumentError, Psych::Exception => e
457
+ raise Error, "Invalid benchmark labels: #{e.message}"
458
+ end
459
+
460
+ def load_known_positives(payload)
461
+ return unless payload.key?('known_positives')
462
+
463
+ entries = payload['known_positives']
464
+ validate_array!(entries, 'known-positive entries')
465
+ @known_positive_entries = entries.map do |entry|
466
+ validate_mapping!(entry, 'known-positive entry')
467
+
468
+ normalized = entry.transform_keys(&:to_s).slice('corpus', 'id', 'definition_id', 'category', 'rationale')
469
+ raise Error, 'Known-positive entries require corpus and id' if %w[corpus id].any? do |key|
470
+ normalized[key].to_s.empty?
471
+ end
472
+
473
+ validate_identifier!(normalized['corpus'], 'known-positive corpus')
474
+ validate_identifier!(normalized['id'], 'known-positive id')
475
+ validate_identifier!(normalized['definition_id'], 'known-positive definition_id') if normalized['definition_id']
476
+ raise Error, "Known positive #{candidate_key(normalized).join(':')} requires a rationale" if
477
+ normalized['rationale'].to_s.strip.empty?
478
+
479
+ normalized
480
+ end
481
+ duplicate = @known_positive_entries.map { |entry| candidate_key(entry) }.tally.find { |_key, count| count > 1 }
482
+ raise Error, "Duplicate known positive: #{duplicate.first.join(':')}" if duplicate
483
+ end
484
+
485
+ def validate_label(entry, reviewed_at:, source_revision:)
486
+ validate_mapping!(entry, 'benchmark label')
487
+ validate_identifier!(entry['corpus'], 'benchmark label corpus')
488
+ validate_identifier!(entry['id'], 'benchmark label id')
489
+ validate_identifier!(entry['definition_id'], 'benchmark label definition_id') if entry.key?('definition_id')
490
+ value = entry.fetch('value')
491
+ raise Error, "Invalid benchmark label #{value.inspect}" unless LABELS.include?(value)
492
+ raise Error, "Benchmark label #{entry['id']} requires a rationale" if entry['rationale'].to_s.strip.empty?
493
+ return unless manifest['label_provenance_required'] == true
494
+
495
+ raise Error, "Benchmark label #{entry['id']} requires a reviewer" if entry['reviewer'].to_s.strip.empty?
496
+ raise Error, "Benchmark label #{entry['id']} requires reviewed_at" if reviewed_at.to_s.strip.empty?
497
+ raise Error, "Benchmark label #{entry['id']} requires source_revision" if source_revision.to_s.strip.empty?
498
+
499
+ Date.iso8601(reviewed_at.to_s)
500
+ rescue Date::Error
501
+ raise Error, "Benchmark label #{entry['id']} reviewed_at must be an ISO 8601 date"
502
+ end
503
+
504
+ def validate_mapping!(value, label)
505
+ raise Error, "#{label} must be a mapping" unless value.is_a?(Hash)
506
+ end
507
+
508
+ def validate_array!(value, label)
509
+ raise Error, "#{label} must be an array" unless value.is_a?(Array)
510
+ raise Error, "#{label} exceeds #{MAX_ENTRIES} entries" if value.length > MAX_ENTRIES
511
+ end
512
+
513
+ def validate_identifier!(value, label)
514
+ raise Error, "#{label} must be a non-empty string" unless value.is_a?(String) && !value.empty?
515
+ raise Error, "#{label} exceeds #{MAX_STRING_BYTES} bytes" if value.bytesize > MAX_STRING_BYTES
516
+ end
517
+
518
+ def validate_file_size!(path, label)
519
+ raise Error, "#{label} exceeds #{MAX_INPUT_BYTES} bytes" if File.size(path) > MAX_INPUT_BYTES
520
+ end
521
+
522
+ def fill_tool_results
523
+ candidates.each_value do |candidate|
524
+ tool_runs.each do |tool, run|
525
+ result = { 'candidate' => false }
526
+ result = { 'candidate' => nil, 'status' => 'skipped' } if run['status'] == 'skipped'
527
+ candidate['tool_results'][tool] ||= result
528
+ end
529
+ candidate['tool_results'] = candidate['tool_results'].sort.to_h
530
+ end
531
+ end
532
+
533
+ def summary
534
+ labels = candidates.values.filter_map { |candidate| candidate.dig('label', 'value') }
535
+ {
536
+ 'candidates' => candidates.length,
537
+ 'reviewed' => labels.length,
538
+ 'by_label' => labels.tally.sort.to_h,
539
+ 'tool_metrics' => tool_metrics,
540
+ 'necropsy_diagnostics' => necropsy_diagnostics
541
+ }
542
+ end
543
+
544
+ def validate_result!(result)
545
+ tools = result.fetch('tool_runs').keys
546
+ result.fetch('candidates').each do |candidate|
547
+ actual_tools = candidate.fetch('tool_results').keys
548
+ next if actual_tools == tools
549
+
550
+ raise Error, "Candidate #{candidate['corpus']}:#{candidate['id']} has incomplete tool results"
551
+ end
552
+ end
553
+ end
554
+ end
555
+ end
@@ -0,0 +1,112 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Necropsy
4
+ module Bench
5
+ # Final, configurable gate for a public accuracy claim. It is deliberately
6
+ # disabled unless a release config opts in with `claim_gate`.
7
+ class ClaimGate
8
+ ACTIONABLE_STATES = %w[unreachable unused candidate].freeze
9
+
10
+ def initialize(config:, reports:, summary:, adversarial_results:)
11
+ @config = config || {}
12
+ @reports = reports || {}
13
+ @summary = summary || {}
14
+ @adversarial_results = Array(adversarial_results)
15
+ end
16
+
17
+ def call
18
+ policy = normalized_policy
19
+ return { 'schema_version' => 1, 'enforced' => false, 'passed' => true } unless policy
20
+
21
+ checks = {
22
+ 'corpus_count' => reports.length >= policy.fetch('minimum_corpora'),
23
+ 'category_coverage' => category_coverage?(policy.fetch('required_categories')),
24
+ 'high_candidates_explained' => unexplained_high_candidates.empty?,
25
+ 'adversarial_suites' => adversarial_results.all? { |result| result['passed'] == true },
26
+ 'candidate_precision' => precision_measured?,
27
+ 'candidate_yield' => candidate_yield?
28
+ }
29
+ checks['reviewed_high_target'] = reviewed_high_count >= policy.fetch('minimum_reviewed_high')
30
+ {
31
+ 'schema_version' => 1,
32
+ 'enforced' => true,
33
+ 'policy' => policy,
34
+ 'checks' => checks,
35
+ 'unexplained_high_candidates' => unexplained_high_candidates,
36
+ 'reviewed_high_candidates' => reviewed_high_count,
37
+ 'passed' => checks.values.all?
38
+ }
39
+ end
40
+
41
+ private
42
+
43
+ attr_reader :config, :reports, :summary, :adversarial_results
44
+
45
+ def normalized_policy
46
+ raw = config['claim_gate']
47
+ return unless raw
48
+ raise Error, 'claim_gate must be a mapping' unless raw.is_a?(Hash)
49
+
50
+ categories = Array(raw['required_categories'] || []).map(&:to_s).reject(&:empty?).uniq.sort
51
+ minimum_corpora = Integer(raw.fetch('minimum_corpora', 5))
52
+ minimum_reviewed_high = Integer(raw.fetch('minimum_reviewed_high', 0))
53
+ raise Error, 'claim_gate minimums must be non-negative' if minimum_corpora.negative? || minimum_reviewed_high.negative?
54
+
55
+ {
56
+ 'minimum_corpora' => minimum_corpora,
57
+ 'required_categories' => categories,
58
+ 'minimum_reviewed_high' => minimum_reviewed_high
59
+ }
60
+ rescue ArgumentError, TypeError
61
+ raise Error, 'claim_gate minimums must be integers'
62
+ end
63
+
64
+ def category_coverage?(required)
65
+ return true if required.empty?
66
+
67
+ categories = reports.values.flat_map do |report|
68
+ Array(report['findings']).map { |finding| finding['category'].to_s }
69
+ end.uniq
70
+ required.all? { |category| categories.include?(category) }
71
+ end
72
+
73
+ def unexplained_high_candidates
74
+ reports.sort.flat_map do |corpus, report|
75
+ Array(report['findings']).filter_map do |finding|
76
+ next unless high_candidate?(finding)
77
+ next if Array(finding['reasons']).any? || Array(finding['rule_hits']).any? || Array(finding['blocker_kinds']).any?
78
+
79
+ {
80
+ 'corpus' => corpus,
81
+ 'id' => finding['id'],
82
+ 'definition_id' => finding['definition_id'],
83
+ 'path' => finding['path'],
84
+ 'line' => finding['line']
85
+ }.compact
86
+ end
87
+ end.sort_by { |candidate| [candidate['corpus'], candidate['id'], candidate['definition_id'].to_s] }
88
+ end
89
+
90
+ def high_candidate?(finding)
91
+ %w[high certain].include?(finding['confidence'].to_s) &&
92
+ (finding['candidate'] == true || ACTIONABLE_STATES.include?(finding['state'].to_s))
93
+ end
94
+
95
+ def reviewed_high_count
96
+ summary.dig('candidate_union', 'tool_metrics', 'necropsy', 'reviewed_high_candidate_count').to_i
97
+ end
98
+
99
+ def necropsy_metrics
100
+ summary.dig('candidate_union', 'tool_metrics', 'necropsy') || {}
101
+ end
102
+
103
+ def precision_measured?
104
+ necropsy_metrics['precision_status'] == 'measured' && necropsy_metrics['candidate_precision'].is_a?(Numeric)
105
+ end
106
+
107
+ def candidate_yield?
108
+ necropsy_metrics['candidate_count'].to_i.positive? && necropsy_metrics['candidate_loc'].to_i.positive?
109
+ end
110
+ end
111
+ end
112
+ end