necropsy 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +26 -0
- data/MEASUREMENTS.md +27 -0
- data/README.md +182 -17
- data/bench/README.md +92 -0
- data/bench/audit.rb +113 -0
- data/bench/audits/0.2.1/audit.json +501 -0
- data/bench/audits/0.2.1/audit.md +55 -0
- data/bench/audits/0.2.1/baseline_performance.yml +16 -0
- data/bench/audits/0.2.1/config.yml +42 -0
- data/bench/audits/0.2.1/review.yml +56 -0
- data/bench/audits/0.2.1/review_queue.yml +3651 -0
- data/bench/corpora/v1/README.md +30 -0
- data/bench/corpora/v1/labels.yml +37 -0
- data/bench/corpora/v1/manifest.yml +63 -0
- data/bench/corpora/v1/rubocop.necropsy.yml +7 -0
- data/bench/corpora/v1/self.necropsy.yml +8 -0
- data/bench/corpora/v1/tools/debride.yml +42 -0
- data/bench/corpora/v1/tools/spoom.yml +12 -0
- data/bench/corpora/v1/tools/type_aware.yml +11 -0
- data/bench/golden/v1/candidate_union.json +21918 -0
- data/bench/golden/v1/metadata.json +12 -0
- data/bench/golden/v1/reports/dynamic_evidence.json +146 -0
- data/bench/golden/v1/reports/plain_ruby.json +135 -0
- data/bench/golden/v1/reports/rails.json +176 -0
- data/bench/golden/v1/reports/rubocop_1_75_0.json +26848 -0
- data/bench/golden/v1/reports/self.json +3449 -0
- data/bench/review_queue.rb +35 -0
- data/bench/run.rb +31 -0
- data/bench/schema/candidate-union-v1.schema.json +70 -0
- data/docs/impv_implementation_matrix.md +179 -0
- data/docs/migrations/0.2.1.md +57 -0
- data/docs/migrations/0.3.0.md +207 -0
- data/docs/migrations/0.4.0.md +13 -0
- data/docs/necropsy_performance_adr.md +43 -0
- data/docs/necropsy_scope_decisions.md +58 -0
- data/docs/necropsy_type_facts_adr.md +22 -0
- data/gemfiles/prism_min.gemfile +9 -0
- data/gemfiles/prism_min.gemfile.lock +50 -0
- data/lib/necropsy/analyzer.rb +121 -2
- data/lib/necropsy/analyzers/dynamic/coverage_collector.rb +69 -17
- data/lib/necropsy/analyzers/dynamic/coverage_importer.rb +99 -9
- data/lib/necropsy/analyzers/dynamic/coverband_importer.rb +69 -299
- data/lib/necropsy/analyzers/dynamic/coverband_payload_set.rb +149 -0
- data/lib/necropsy/analyzers/dynamic/observation_policy.rb +96 -0
- data/lib/necropsy/analyzers/dynamic/redis_input_limits.rb +121 -0
- data/lib/necropsy/analyzers/dynamic/redis_nonblocking_io.rb +94 -0
- data/lib/necropsy/analyzers/dynamic/redis_payload_loader.rb +165 -0
- data/lib/necropsy/analyzers/dynamic/redis_transport.rb +217 -0
- data/lib/necropsy/analyzers/dynamic/runtime_reference.rb +96 -0
- data/lib/necropsy/analyzers/dynamic/trace_point_collector.rb +133 -23
- data/lib/necropsy/analyzers/dynamic/trace_point_importer.rb +3 -1
- data/lib/necropsy/analyzers/legacy_result_adapter.rb +226 -0
- data/lib/necropsy/analyzers/static/cha.rb +33 -73
- data/lib/necropsy/analyzers/static/name_resolution.rb +133 -12
- data/lib/necropsy/analyzers/static/rta.rb +237 -29
- data/lib/necropsy/ast_scanner/call_recording.rb +149 -30
- data/lib/necropsy/ast_scanner/call_site_creation.rb +54 -0
- data/lib/necropsy/ast_scanner/definition_creation.rb +43 -0
- data/lib/necropsy/ast_scanner/dsl_macros.rb +431 -41
- data/lib/necropsy/ast_scanner/method_definitions.rb +232 -50
- data/lib/necropsy/ast_scanner/references.rb +43 -9
- data/lib/necropsy/ast_scanner/ruby_semantics.rb +101 -15
- data/lib/necropsy/ast_scanner/traversal.rb +234 -71
- data/lib/necropsy/ast_scanner/value_definitions.rb +23 -13
- data/lib/necropsy/ast_scanner.rb +67 -6
- data/lib/necropsy/bench/candidate_union.rb +555 -0
- data/lib/necropsy/bench/claim_gate.rb +112 -0
- data/lib/necropsy/bench/evaluator.rb +329 -16
- data/lib/necropsy/bench/finding_facts.rb +152 -0
- data/lib/necropsy/bench/precision_gate.rb +144 -0
- data/lib/necropsy/bench/release_audit/adversarial_runner.rb +56 -0
- data/lib/necropsy/bench/release_audit/artifact_writer.rb +112 -0
- data/lib/necropsy/bench/release_audit/config_validator.rb +112 -0
- data/lib/necropsy/bench/release_audit/git_snapshot.rb +36 -0
- data/lib/necropsy/bench/release_audit/performance_gate.rb +165 -0
- data/lib/necropsy/bench/release_audit/run_provenance.rb +133 -0
- data/lib/necropsy/bench/release_audit.rb +360 -0
- data/lib/necropsy/bench/report_normalizer.rb +140 -0
- data/lib/necropsy/bench/review_queue.rb +154 -0
- data/lib/necropsy/bench/safety_mutation_harness.rb +59 -0
- data/lib/necropsy/bench/seed_runner.rb +408 -0
- data/lib/necropsy/bounded_canonicalizer.rb +218 -0
- data/lib/necropsy/cache/scan_cache.rb +85 -17
- data/lib/necropsy/call_site_identity.rb +54 -0
- data/lib/necropsy/cli.rb +220 -33
- data/lib/necropsy/clock.rb +40 -0
- data/lib/necropsy/confidence/scorer.rb +103 -58
- data/lib/necropsy/configuration.rb +224 -21
- data/lib/necropsy/convention_rules.rb +138 -0
- data/lib/necropsy/definition_identity/canonical_digest.rb +278 -0
- data/lib/necropsy/definition_identity.rb +37 -0
- data/lib/necropsy/diagnostics.rb +176 -36
- data/lib/necropsy/embedded_ruby.rb +55 -0
- data/lib/necropsy/entry_points/plain.rb +111 -10
- data/lib/necropsy/entry_points/rails.rb +322 -41
- data/lib/necropsy/entry_points/test.rb +6 -1
- data/lib/necropsy/flow_interpreter.rb +460 -0
- data/lib/necropsy/graph/blocker_matching.rb +338 -0
- data/lib/necropsy/graph/call_graph.rb +1099 -109
- data/lib/necropsy/graph/definition_index.rb +149 -0
- data/lib/necropsy/graph/dynamic_evidence_tracking.rb +206 -0
- data/lib/necropsy/graph/evidence_store.rb +213 -0
- data/lib/necropsy/graph/resolution_store.rb +497 -0
- data/lib/necropsy/graph_self_check.rb +79 -0
- data/lib/necropsy/guardrail/baseline.rb +350 -13
- data/lib/necropsy/guardrail/quarantine.rb +94 -9
- data/lib/necropsy/load_graph.rb +206 -0
- data/lib/necropsy/models.rb +878 -13
- data/lib/necropsy/performance_profiler.rb +108 -0
- data/lib/necropsy/project.rb +327 -25
- data/lib/necropsy/reachability/engine.rb +54 -13
- data/lib/necropsy/reference_barrier.rb +458 -0
- data/lib/necropsy/report.rb +113 -4
- data/lib/necropsy/reporter.rb +431 -15
- data/lib/necropsy/runner.rb +233 -18
- data/lib/necropsy/runtime_feedback.rb +136 -0
- data/lib/necropsy/semantics_matrix.rb +153 -0
- data/lib/necropsy/type_facts.rb +53 -0
- data/lib/necropsy/version.rb +1 -1
- data/lib/necropsy/why_not_explanation.rb +436 -0
- data/lib/necropsy/why_not_renderer.rb +197 -0
- data/lib/necropsy/world_policy.rb +90 -0
- data/lib/necropsy.rb +35 -2
- data/schema/necropsy-report-v2.schema.json +366 -0
- metadata +85 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: c6859ea7acfe1f2357759a87f5f2f2f2b1b487447d34f61d6778164c8362dc6e
|
|
4
|
+
data.tar.gz: 39f91eb66de04a292f749ba37fc2a9b4ba333ec799c1671716068f9d315f6d96
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: d7a3f31943091c4e9288d4705e65779f5165ad251fe8f440f3361f1ec1f675035b4d83f6425fbdb6b5b0e88ab1240800bf621d3bc78ad6e624f6c3e6e7eaca96
|
|
7
|
+
data.tar.gz: f1fa13456a999bb6897f938776dc3884197f01ffafdcab19ba416a6d41727980cb03906a5280cb48a60d245f50f714c61e12adb950134ee06ca71c202e417a55
|
data/CHANGELOG.md
CHANGED
|
@@ -4,6 +4,32 @@ All notable changes to Necropsy are documented in this file.
|
|
|
4
4
|
|
|
5
5
|
## Unreleased
|
|
6
6
|
|
|
7
|
+
No unreleased changes.
|
|
8
|
+
|
|
9
|
+
## 0.3.0 - 2026-08-13
|
|
10
|
+
|
|
11
|
+
- Preserve repeated and reopened Ruby definitions as distinct physical graph nodes while retaining logical method names for compatibility; ambiguous runtime references no longer silently select one definition.
|
|
12
|
+
- Record structured call-site resolution, scoped blockers, evidence grades, and analyzer provenance, with conservative adaptation for existing custom static analyzers.
|
|
13
|
+
- Store graph evidence once and expose exact, conservative, and scope-filtered observed views while preserving conservative reachability and the existing nested edge JSON.
|
|
14
|
+
- Separate runtime, test, and external roots; library mode now protects public and protected APIs from dead-code findings, with optional conservative roots for every production file.
|
|
15
|
+
- Separate analysis, repository reference, and report scopes so output filters cannot remove callers from the graph; reference-only Ruby definitions are not reported, and narrowed scans expose entry-point and symlink diagnostics.
|
|
16
|
+
- Block candidates that still appear in unparsed templates or configuration, with bounded file/line/snippet diagnostics and safe limits for common names, generated content, binary files, and oversized inputs.
|
|
17
|
+
- Add `why-not` human and versioned JSON diagnostics covering physical definitions, examined calls and resolutions, blockers, world policy, external text matches, analysis failures, enabled rules, and suggested evidence.
|
|
18
|
+
- Write physical-definition baseline schema v2 while continuing to read logical v1 baselines; CI now stops for an explicit review when a legacy entry maps to multiple definitions.
|
|
19
|
+
- Expose legacy and physical fingerprints additively in JSON and SARIF, and show both logical-symbol and physical-definition candidate views in benchmark output.
|
|
20
|
+
- Fail closed on analyzer, parser, source-revision, reference-scan, and unsafe RTA failures; expose explicit health status, strict-health policy, machine-readable failure reports, and graph self-checks.
|
|
21
|
+
- Harden Ruby semantic traversal and finite value flow for modifier definitions, default arguments, dynamic sends, short-circuit paths, unsupported control flow, reflective hooks, and receiver-aware core protocol calls.
|
|
22
|
+
- Add conservative Rails and framework conventions for generated methods, associations, enums, scopes, callbacks, ActionCable, ActiveJob, Sidekiq, GraphQL, serializers, ViewComponent, routes, ERB, inflections, and structural dependency detection.
|
|
23
|
+
- Add bounded and deterministic external-reference scanning with qualified-owner matching, streaming limits, byte/match/time budgets, unsafe-skip blockers, and safe repository discovery.
|
|
24
|
+
- Add deterministic NDJSON graph export, content-addressed cache metadata, reproducible runtime artifacts, performance distributions, macro-averaged evaluation, AST fuzzing, runtime target oracles, mutation safety harnesses, and a generated semantics matrix.
|
|
25
|
+
|
|
26
|
+
## 0.2.1 - 2026-08-04
|
|
27
|
+
|
|
28
|
+
- Prevent unsafe dead-code recommendations when runtime dispatch, source parsing, or an analyzer is incomplete; affected methods are now reported as low-confidence `blocked` findings with the reason and source location.
|
|
29
|
+
- Make the default analysis conservative: RTA no longer deletes broader static edges, runtime observations only prove liveness, and quarantine expiry requests review without increasing deadness confidence.
|
|
30
|
+
- Harden remote Coverband/Redis evidence loading with verified TLS, bounded DNS/connect/read/write/total deadlines, strict payload and RESP limits, safe deserialization, and credential-redacted errors.
|
|
31
|
+
- Add reproducible five-corpus release auditing with reviewed candidate transitions, safety-invariant and adversarial suites, provenance-bound artifacts, and fail-closed wall-time/RSS budgets.
|
|
32
|
+
|
|
7
33
|
## 0.2.0 - 2026-08-03
|
|
8
34
|
|
|
9
35
|
- **Breaking:** Reports now omit `low` confidence findings by default, substantially reducing noisy output. Pass `--min-confidence low` to retain the previous output.
|
data/MEASUREMENTS.md
CHANGED
|
@@ -2,6 +2,33 @@
|
|
|
2
2
|
|
|
3
3
|
Measurements are local wall-clock results. Compare runs made on the same machine and Ruby version; absolute timings are not portable.
|
|
4
4
|
|
|
5
|
+
The reproducible seed under `bench/corpora/v1` supersedes ad hoc commands for new comparisons.
|
|
6
|
+
`bundle exec ruby bench/run.rb` regenerates normalized findings, candidate-union results, corpus
|
|
7
|
+
summaries, wall time, and RSS measurements in one pass. RuboCop remains pinned to 1.75.0 and is
|
|
8
|
+
enabled by setting `NECROPSY_RUBOCOP_CORPUS`; an unavailable checkout is reported as a skip.
|
|
9
|
+
|
|
10
|
+
## 0.2.1 safety release audit
|
|
11
|
+
|
|
12
|
+
The versioned audit compares commit `51d490188ae9ad846b4c023f14e252ec624a2d5e`, the first
|
|
13
|
+
integrity-bound five-corpus snapshot, with `3831ce651ed331426413a767824955414529001d` on the same
|
|
14
|
+
Ruby 4.0.0/macOS arm64 environment. RSS is the benchmark process's current resident size after each
|
|
15
|
+
corpus, not a per-corpus isolated peak. Full provenance and candidate-level changes are stored in
|
|
16
|
+
`bench/audits/0.2.1/audit.json`.
|
|
17
|
+
|
|
18
|
+
| corpus | findings baseline/current | state changes | newly high | wall s baseline/current | RSS KiB baseline/current |
|
|
19
|
+
|---|---:|---:|---:|---:|---:|
|
|
20
|
+
| dynamic evidence | 4 / 4 | 0 | 0 | 0.001 / 0.003 | 52,720 / 51,200 |
|
|
21
|
+
| plain Ruby | 3 / 3 | 0 | 0 | 0.002 / 0.002 | 53,088 / 51,488 |
|
|
22
|
+
| Rails fixture | 4 / 4 | 0 | 0 | 0.004 / 0.004 | 54,272 / 52,656 |
|
|
23
|
+
| RuboCop 1.75.0 | 5,797 / 5,797 | 119 | 0 | 5.196 / 5.281 | 576,256 / 584,432 |
|
|
24
|
+
| self | 78 / 50 | 2 | 0 | 0.217 / 0.423 | 608,448 / 580,480 |
|
|
25
|
+
|
|
26
|
+
All seven release gates passed. Rails had no candidate changes. RuboCop retained the same candidate
|
|
27
|
+
count; its 119 state changes were conservative transitions to `blocked`, with all nine deterministic
|
|
28
|
+
stratified review samples accepted. The dynamic, parse, ambiguity, and remote-input adversarial
|
|
29
|
+
suites passed 69 examples in total. No new high-confidence candidate or confirmed reviewed false
|
|
30
|
+
positive was introduced.
|
|
31
|
+
|
|
5
32
|
| date | commit | target | nodes | findings | ratio | seconds |
|
|
6
33
|
|---|---|---|---:|---:|---:|---:|
|
|
7
34
|
| 2026-08-01 | baseline | necropsy | 678 | 246 | 36.3% | 4.33 |
|
data/README.md
CHANGED
|
@@ -5,12 +5,15 @@ collects method definitions with Prism, adds call-edge evidence from static and
|
|
|
5
5
|
optional dynamic analyzers, then runs reachability from framework and configured
|
|
6
6
|
entry points.
|
|
7
7
|
|
|
8
|
+
Machine-readable JSON reports use schema v2. The published contract is
|
|
9
|
+
[`schema/necropsy-report-v2.schema.json`](schema/necropsy-report-v2.schema.json).
|
|
10
|
+
|
|
8
11
|
Necropsy includes:
|
|
9
12
|
|
|
10
13
|
- Prism-based method collection for ordinary, singleton, delegated, aliased, forwarded, and dynamically defined methods
|
|
11
|
-
- static name resolution
|
|
14
|
+
- static name resolution and CHA, with rank-only RTA hints from classes instantiated in the scanned program
|
|
12
15
|
- Prism-backed Rails route parsing plus callback, view, component, migration, plain Ruby, and test-suite entry points
|
|
13
|
-
- `unreachable`, `unused`, and `test_only_reachable` classifications
|
|
16
|
+
- `unreachable`, `unused`, `blocked`, and `test_only_reachable` classifications
|
|
14
17
|
- confidence levels, compact JSON/YAML reports, SARIF/GitHub output, CI guardrails, dynamic collectors, and benchmarking
|
|
15
18
|
|
|
16
19
|
## Installation
|
|
@@ -29,6 +32,11 @@ Create a baseline:
|
|
|
29
32
|
bundle exec necropsy baseline --root .
|
|
30
33
|
```
|
|
31
34
|
|
|
35
|
+
New baselines use schema v2 and identify each physical method definition. Existing
|
|
36
|
+
schema v1 baselines remain readable, but `necropsy check` accepts only exact physical
|
|
37
|
+
matches. Run `necropsy baseline migrate` to use body/symbol/path hints and produce a
|
|
38
|
+
review report; ambiguous or legacy matches never pass an ordinary check silently.
|
|
39
|
+
|
|
32
40
|
Run a report:
|
|
33
41
|
|
|
34
42
|
```bash
|
|
@@ -46,11 +54,35 @@ nearby alive node, and unresolved dispatch notes:
|
|
|
46
54
|
```bash
|
|
47
55
|
bundle exec necropsy why 'MyService#call' --root .
|
|
48
56
|
bundle exec necropsy why 'MyService#call' --root . --format json
|
|
57
|
+
bundle exec necropsy why-not 'LegacyService#unused' --root .
|
|
58
|
+
bundle exec necropsy why-not 'LegacyService#unused' --root . --format json
|
|
49
59
|
bundle exec necropsy explain 'LegacyService#unused' --root .
|
|
50
60
|
```
|
|
51
61
|
|
|
52
62
|
`explain` shows every confidence score component and the final confidence
|
|
53
|
-
level.
|
|
63
|
+
level. `why-not` emits a refutable `necropsy.why-not.v1` artifact for candidates,
|
|
64
|
+
blocked findings, and test-only definitions. It includes examined call sites and
|
|
65
|
+
resolution statuses, rejected targets, blockers, world/root policy, non-Ruby
|
|
66
|
+
matches, parse/analyzer failures, enabled analyzers and type providers, artifact
|
|
67
|
+
digests, assumptions, risk flags, the recommended review action, and the next
|
|
68
|
+
evidence to collect. Every diagnostic collection reports total, returned, and
|
|
69
|
+
truncated counts, and nested metadata is bounded as well. Missing IDs return
|
|
70
|
+
partial-match suggestions. When a logical symbol ID matches multiple physical
|
|
71
|
+
definitions, all three diagnostic commands list every source location and an
|
|
72
|
+
executable command using its full definition ID.
|
|
73
|
+
|
|
74
|
+
For compatibility, the existing logical `id` and `fingerprint` remain stable.
|
|
75
|
+
Reports add `logical_fingerprint` and `physical_fingerprint`, while SARIF retains
|
|
76
|
+
the `necropsy` partial fingerprint and adds `necropsyPhysicalDefinition`. Reports
|
|
77
|
+
also include `symbol_id` and `definition_id`; human, GitHub, and SARIF findings
|
|
78
|
+
display or expose the full physical definition ID so reopened or duplicate methods
|
|
79
|
+
can be distinguished.
|
|
80
|
+
|
|
81
|
+
Runtime artifacts remain backward compatible: collectors keep the legacy
|
|
82
|
+
`nodes` and `edges` fields while adding structured `node_references` and edge
|
|
83
|
+
endpoint references with `definition_id`, `symbol_id`, `file`, and `line` when
|
|
84
|
+
available. Importers prefer structured references and continue to accept v1
|
|
85
|
+
logical-ID artifacts.
|
|
54
86
|
|
|
55
87
|
Fail CI only for new high-confidence findings:
|
|
56
88
|
|
|
@@ -58,6 +90,12 @@ Fail CI only for new high-confidence findings:
|
|
|
58
90
|
bundle exec necropsy check --root . --fail-on high
|
|
59
91
|
```
|
|
60
92
|
|
|
93
|
+
`check`, baseline writes, quarantine writes, and release benchmarks fail with status
|
|
94
|
+
3 when analysis health is incomplete. Use `--strict-health` to apply the same policy
|
|
95
|
+
to informational analysis/diagnostic commands. A known degraded reason can be
|
|
96
|
+
temporarily admitted by exact code, for example
|
|
97
|
+
`--allow-degraded=reference_scan_incomplete`; invalid health is never admitted.
|
|
98
|
+
|
|
61
99
|
Record dynamic evidence from a Ruby script:
|
|
62
100
|
|
|
63
101
|
```bash
|
|
@@ -80,12 +118,42 @@ Evaluate against a gold standard:
|
|
|
80
118
|
bundle exec necropsy bench --root . --gold-standard gold.yml --ablation
|
|
81
119
|
```
|
|
82
120
|
|
|
121
|
+
Benchmark JSON keeps the existing logical-ID metrics and adds `identity_views`.
|
|
122
|
+
The legacy view groups by `symbol_id`; the physical view lists every
|
|
123
|
+
`definition_id` and both fingerprints, so duplicate definitions cannot disappear
|
|
124
|
+
from review totals.
|
|
125
|
+
|
|
126
|
+
Benchmark precision is measured from actionable `unreachable`/`unused` findings only.
|
|
127
|
+
`blocked` and `test_only_reachable` remain visible diagnostics but are not removal candidates.
|
|
128
|
+
The additive `quality` and `by_category` objects report candidate precision/count/LOC,
|
|
129
|
+
known-positive recall, blocked and unknown rates, and rule/risk counts. A run with zero
|
|
130
|
+
candidates fails the evaluator's `candidate_yield` release check instead of receiving perfect
|
|
131
|
+
precision. The legacy `Report#dead_methods` API is unchanged; integrations that need the stricter
|
|
132
|
+
set can use `Report#actionable_candidates`.
|
|
133
|
+
|
|
134
|
+
Non-analyzer features can be compared with the same evaluator by supplying already analyzed
|
|
135
|
+
on/off reports. This keeps feature configuration outside the metric engine while producing a
|
|
136
|
+
physical-definition candidate diff and metric deltas:
|
|
137
|
+
|
|
138
|
+
```ruby
|
|
139
|
+
Necropsy::Bench::Evaluator.new(
|
|
140
|
+
report: feature_on_report,
|
|
141
|
+
gold_standard_path: 'gold.yml',
|
|
142
|
+
feature_ablation: {
|
|
143
|
+
'receiver_flow' => { on: feature_on_report, off: feature_off_report }
|
|
144
|
+
}
|
|
145
|
+
).call
|
|
146
|
+
```
|
|
147
|
+
|
|
83
148
|
JSON and YAML omit the full call graph by default. Add `--include-graph` when
|
|
84
149
|
nodes, edges, evidence, and entry points are needed in machine-readable output.
|
|
85
150
|
|
|
86
151
|
Example configuration:
|
|
87
152
|
|
|
88
153
|
```yaml
|
|
154
|
+
analysis:
|
|
155
|
+
world: application # application | library
|
|
156
|
+
load_roots: known # known | all
|
|
89
157
|
analyzers:
|
|
90
158
|
static: [name_resolution, cha, rta]
|
|
91
159
|
dynamic:
|
|
@@ -93,12 +161,24 @@ analyzers:
|
|
|
93
161
|
source: tmp/necropsy_coverage.yml
|
|
94
162
|
min_observation_days: 30
|
|
95
163
|
coverband:
|
|
96
|
-
source:
|
|
164
|
+
source: rediss://prod-redis:6379/2?key=coverband
|
|
165
|
+
connect_timeout: 5
|
|
166
|
+
read_timeout: 5
|
|
167
|
+
total_timeout: 15
|
|
168
|
+
max_response_bytes: 16777216
|
|
169
|
+
max_bulk_bytes: 8388608
|
|
170
|
+
max_array_elements: 100000
|
|
171
|
+
max_resp_depth: 16
|
|
172
|
+
max_keys: 1000
|
|
173
|
+
max_payload_depth: 64
|
|
97
174
|
trace_point:
|
|
98
175
|
source: tmp/necropsy_trace_point.yml
|
|
99
176
|
custom:
|
|
100
177
|
- class: "MyCompany::GraphqlEntryAnalyzer"
|
|
101
178
|
require: "config/necropsy/graphql_entry_analyzer"
|
|
179
|
+
trusted: true # executes Ruby in the Necropsy process
|
|
180
|
+
rta:
|
|
181
|
+
pruning: rank_only # legacy pruning marks analysis invalid and cannot pass check
|
|
102
182
|
cache:
|
|
103
183
|
enabled: true
|
|
104
184
|
path: .necropsy_cache/scan.json
|
|
@@ -109,6 +189,8 @@ implicit_callers:
|
|
|
109
189
|
owner_ancestors: ["RuboCop::Cop::Base"]
|
|
110
190
|
reason: "RuboCop Commissioner callback"
|
|
111
191
|
paths:
|
|
192
|
+
analyze: ["**/*.rb", "Rakefile", "**/*.rake", "bin/*", "exe/*", "*.gemspec"]
|
|
193
|
+
reference: ["**/*"]
|
|
112
194
|
exclude: ["app/legacy/**/*.rb"]
|
|
113
195
|
report:
|
|
114
196
|
include: ["app/**", "lib/**"]
|
|
@@ -121,44 +203,127 @@ ci:
|
|
|
121
203
|
baseline: .necropsy_baseline.yml
|
|
122
204
|
quarantine:
|
|
123
205
|
days: 30
|
|
206
|
+
expiry: warn # warn | fail | ignore
|
|
124
207
|
bench:
|
|
125
208
|
precision_threshold: 0.85
|
|
126
209
|
logging:
|
|
127
210
|
verbose: false
|
|
128
211
|
```
|
|
129
212
|
|
|
130
|
-
The scan cache is invalidated when
|
|
131
|
-
change.
|
|
213
|
+
The scan cache is invalidated when analyzed Ruby files, reference files, or
|
|
214
|
+
configuration values change.
|
|
215
|
+
|
|
216
|
+
`analysis.world: application` uses executable, framework, and configured roots.
|
|
217
|
+
Use `library` when callers may live outside the repository: every non-test
|
|
218
|
+
public or protected method becomes an `external` root, while private methods
|
|
219
|
+
remain eligible for review. `analysis.load_roots: all` is an opt-in conservative
|
|
220
|
+
mode that treats every non-test Ruby file top level as a runtime root when load
|
|
221
|
+
status cannot be established. Root domain, reason, and provenance are included
|
|
222
|
+
in graph JSON and `why` paths.
|
|
132
223
|
|
|
133
224
|
When a call receiver cannot be resolved exactly, Necropsy conservatively keeps
|
|
134
225
|
up to `resolution.ambiguity_limit` same-name candidates alive. The default of
|
|
135
|
-
four is based on the RuboCop 1.75.0 measurements in `MEASUREMENTS.md`.
|
|
226
|
+
four is based on the RuboCop 1.75.0 measurements in `MEASUREMENTS.md`. If a
|
|
227
|
+
runtime call has more candidates, Necropsy records a message- and owner-scoped
|
|
228
|
+
blocker instead of treating the empty target set as proof of deadness. Matching
|
|
229
|
+
definitions are reported as `blocked` with the call site, scope, and reason;
|
|
230
|
+
they never receive `high` or `certain` confidence. Unresolved calls found only
|
|
231
|
+
in test files remain diagnostic evidence and do not block production candidates.
|
|
136
232
|
|
|
137
233
|
Ruby VM hooks and common protocol methods receive lower confidence because
|
|
138
234
|
their callers may not appear in source. Add `implicit_callers` rules for
|
|
139
235
|
framework or application callbacks; `owner_ancestors` is optional.
|
|
140
236
|
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
237
|
+
Source discovery has three independent scopes. `paths.analyze` selects Ruby
|
|
238
|
+
definitions eligible for findings. `paths.reference` defaults to the whole
|
|
239
|
+
repository and keeps Ruby callers outside the analysis scope in the graph;
|
|
240
|
+
definitions found only there are never reported as dead-code findings. It also
|
|
241
|
+
inventories non-Ruby files for conservative reference checks. `report.include`
|
|
242
|
+
and `report.exclude` only filter output and never remove graph nodes or edges.
|
|
243
|
+
When `paths.analyze` is omitted, the existing conventional Ruby source set
|
|
244
|
+
(`*.rb`, `*.rake`, the root `Rakefile` and gem specifications, and Ruby
|
|
245
|
+
executables directly under `bin/` or `exe/`) remains the analysis scope.
|
|
246
|
+
|
|
247
|
+
`paths.exclude` continues to remove files from the analysis scope, while the
|
|
248
|
+
legacy `paths.include` key remains an alias for `paths.analyze`. Narrowing the
|
|
249
|
+
analysis scope can hide entry points, so reports include scope diagnostics and
|
|
250
|
+
Necropsy warns when detected executables, tests, routes, or task files are left
|
|
251
|
+
outside it. Repository discovery rejects symlinks and paths that resolve outside
|
|
252
|
+
the project root. If `paths.reference` excludes non-test Ruby files that may
|
|
253
|
+
contain callers, findings are conservatively `blocked` and the report lists the
|
|
254
|
+
excluded count and sample until the reference scope is expanded.
|
|
255
|
+
|
|
256
|
+
For otherwise actionable candidates, Necropsy also scans reference-scope files
|
|
257
|
+
that are not Ruby. Unexplained method names in templates, YAML, GraphQL SDL, and
|
|
258
|
+
scheduler configuration become `unparsed_external_reference` blockers with the
|
|
259
|
+
matching file, line, and snippet. Common names such as `call` or `run` require a
|
|
260
|
+
review more often because safety takes precedence over yield; benchmark results
|
|
261
|
+
identify formats that merit a dedicated parser. The scanner is a portable Ruby
|
|
262
|
+
fallback with no `rg` dependency. It ignores comments, generated/tool metadata,
|
|
263
|
+
and binary formats. Files from 1–16 MiB are streamed. Generated, unreadable,
|
|
264
|
+
larger, or budget-truncated runtime inputs are never treated as negative evidence:
|
|
265
|
+
they add a global blocker, degraded health, and bounded provenance diagnostics.
|
|
146
266
|
|
|
147
267
|
Dynamic inputs may provide `executed` or `nodes` entries with method IDs,
|
|
148
268
|
`edges` with `caller_id`/`callee_id`, and an `observation` hash. SARIF and
|
|
149
269
|
GitHub Actions annotations are available via `--format sarif` and
|
|
150
270
|
`--format github`.
|
|
151
271
|
|
|
272
|
+
Reports retain logical method names while the graph distinguishes repeated or
|
|
273
|
+
reopened definitions with physical `definition_id` values. Graph consumers and
|
|
274
|
+
custom analyzer authors upgrading to the structured resolution model should
|
|
275
|
+
follow the [0.3.0 migration guide](docs/migrations/0.3.0.md).
|
|
276
|
+
|
|
277
|
+
The adversarial disposition of every item in the implementation review is recorded
|
|
278
|
+
in the [148-item implementation matrix](docs/impv_implementation_matrix.md). Explicit
|
|
279
|
+
no-go decisions include measurable reconsideration gates rather than hidden backlog.
|
|
280
|
+
|
|
281
|
+
The graph keeps one interned evidence store and derives `exact`, `conservative`,
|
|
282
|
+
and scope-filtered `observed` edge views on demand. Normal analysis and the
|
|
283
|
+
serialized `edges` view use the conservative projection, preserving legacy
|
|
284
|
+
reachability. Serialized edges retain nested `evidences` and additionally
|
|
285
|
+
reference the top-level `evidence_records` through stable `evidence_ids`.
|
|
286
|
+
|
|
287
|
+
Runtime evidence is positive-only: executed methods and the endpoints of
|
|
288
|
+
observed edges are kept alive, while an unobserved method never becomes a new
|
|
289
|
+
finding and never receives higher confidence. Observation duration and
|
|
290
|
+
environment remain informational metadata. `min_observation_days` is a
|
|
291
|
+
read-compatibility no-op and cannot change candidates, classifications, or
|
|
292
|
+
confidence. The `unused` state remains only for reading and aggregating legacy
|
|
293
|
+
schemas; positive-only analysis does not produce new `unused` findings. Reports
|
|
294
|
+
include attempted, matched, and unmatched evidence counts plus a bounded
|
|
295
|
+
unmatched sample.
|
|
296
|
+
|
|
297
|
+
Schema v1 payloads may omit the analyzed source revision. Their positive
|
|
298
|
+
evidence is accepted for liveness for compatibility, but an omitted revision is
|
|
299
|
+
marked `source_revision_status: unknown` and is never treated as proof that
|
|
300
|
+
unobserved code is dead. When `source_revision` is supplied, it is retained in
|
|
301
|
+
the evidence scope so an exact projection can require a matching revision. Use
|
|
302
|
+
evidence recorded from the same checkout.
|
|
303
|
+
|
|
152
304
|
Coverband file, Redis string, and Redis hash exports are supported. Redis URLs
|
|
153
|
-
may include an ACL username and password
|
|
154
|
-
|
|
305
|
+
may include an ACL username and password. `rediss://` uses the system CA store,
|
|
306
|
+
TLS peer verification, SNI, and post-connect hostname verification; URI
|
|
307
|
+
credentials are redacted from loader errors. Connect, read, and total timeouts,
|
|
308
|
+
response and bulk bytes, RESP array size/depth, key count, and payload depth all
|
|
309
|
+
have the bounded defaults shown above and can be set in the Coverband analyzer
|
|
310
|
+
configuration. Rails route
|
|
155
311
|
entry point detection covers common `resources`, `resource`, `namespace`,
|
|
156
312
|
`scope`, `controller`, `concerns`, `draw`, `mount`, `root`, and verb route
|
|
157
313
|
forms.
|
|
158
314
|
|
|
159
|
-
`necropsy quarantine --write` adds
|
|
160
|
-
|
|
161
|
-
|
|
315
|
+
`necropsy quarantine --write` adds
|
|
316
|
+
`# necropsy:quarantine since=YYYY-MM-DD fingerprint=PHYSICAL_FINGERPRINT`.
|
|
317
|
+
The annotation applies only to the immediately following physical definition
|
|
318
|
+
when its fingerprint matches. Legacy annotations without a fingerprint require
|
|
319
|
+
review and can be upgraded in place with `quarantine --write`.
|
|
320
|
+
Use `--as-of YYYY-MM-DD` or `SOURCE_DATE_EPOCH` to make quarantine expiry,
|
|
321
|
+
annotation dates, and baseline timestamps reproducible.
|
|
322
|
+
Expiry never changes a finding's classification, score, or confidence. Instead,
|
|
323
|
+
the finding receives a `quarantine_review_required` diagnostic. The `check`
|
|
324
|
+
command warns by default; set `quarantine.expiry` to `fail` to make an expired
|
|
325
|
+
annotation fail CI, or to `ignore` to suppress the operational check. An invalid
|
|
326
|
+
`since` date is reported as `quarantine_invalid_date` without changing deadness.
|
|
162
327
|
|
|
163
328
|
## Development
|
|
164
329
|
|
data/bench/README.md
ADDED
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# Reproducible benchmark seed
|
|
2
|
+
|
|
3
|
+
Run the version 1 corpus from the repository root:
|
|
4
|
+
|
|
5
|
+
```shell
|
|
6
|
+
bundle exec ruby bench/run.rb
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
The command writes normalized reports, the candidate union, and a performance summary to
|
|
10
|
+
`tmp/necropsy-benchmark/v1`. The normalized reports omit machine-specific roots and timing data;
|
|
11
|
+
the separate summary records wall time, phase allocations, resolution-cache counts, and peak RSS
|
|
12
|
+
(or current RSS when the platform does not expose a process high-water mark). Its `golden.status` is `match`, `drift`, or `missing`, with every
|
|
13
|
+
changed deterministic artifact listed under `golden.differences`.
|
|
14
|
+
|
|
15
|
+
The manifest pins five corpus roles: this repository, RuboCop 1.75.0, a plain Ruby fixture, a Rails
|
|
16
|
+
fixture, and a positive dynamic-evidence fixture. The small fixtures are in `spec/fixtures`; set
|
|
17
|
+
`NECROPSY_RUBOCOP_CORPUS` to an existing RuboCop 1.75.0 checkout to enable that external corpus.
|
|
18
|
+
RuboCop is fixed to tag `v1.75.0` and commit `9c2bc8eb11e269f1cf47113041a1be3ff615f68b`;
|
|
19
|
+
a checkout at another commit fails before analysis. Missing external corpora are recorded as
|
|
20
|
+
explicit skips rather than downloaded automatically, but omission causes golden drift and a
|
|
21
|
+
nonzero exit once the pinned RuboCop result is present in the golden set. All five manifest entries
|
|
22
|
+
are required for golden updates; an incomplete run raises an error before touching existing golden
|
|
23
|
+
files. Pinned Git corpora must also have no tracked working-tree changes.
|
|
24
|
+
|
|
25
|
+
The comparison-tool YAML files are versioned review seeds, not claims from a fresh tool execution.
|
|
26
|
+
They exercise the candidate-union schema for Necropsy, Debride, Spoom, and a type-aware analyzer
|
|
27
|
+
without requiring those tools to be installed. Replace them with pinned output snapshots when a
|
|
28
|
+
comparison is rerun. A missing snapshot and executable produces an explicit skipped tool result.
|
|
29
|
+
|
|
30
|
+
Deterministic golden files live in `bench/golden/v1`. Drift is printed and exits nonzero. Updating
|
|
31
|
+
them requires an audit reason; `metadata.json` binds that reason to every artifact digest:
|
|
32
|
+
|
|
33
|
+
```shell
|
|
34
|
+
bundle exec ruby bench/run.rb --update-golden 'explain the intentional finding drift'
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Review `candidate_union.json` before accepting drift. Labels use `dead`, `alive`, `external`, or
|
|
38
|
+
`unknown`, and every reviewed label includes a rationale. A label that does not match an actual
|
|
39
|
+
candidate from a generated Necropsy report or a versioned comparison snapshot is rejected.
|
|
40
|
+
|
|
41
|
+
Normalized report schema version 1 remains compatible and now has additive measurement fields.
|
|
42
|
+
Each finding declares `candidate` versus `diagnostic`, physical `definition_id`, LOC, category,
|
|
43
|
+
unknown status, rule hits, risks, and blockers. The `quality` object provides the corresponding
|
|
44
|
+
counts and rates. Report include/exclude scope is applied before these fields are aggregated.
|
|
45
|
+
Blocked and test-only findings never become Necropsy candidates in `candidate_union.json`, though
|
|
46
|
+
they remain available under `summary.necropsy_diagnostics`.
|
|
47
|
+
|
|
48
|
+
`summary.tool_metrics` separates candidate-union precision from known-positive recall and reports
|
|
49
|
+
candidate count and measured LOC overall and by category. Add an optional `category` to reviewed
|
|
50
|
+
labels to define stable error strata. A separate top-level `known_positives` list in the labels
|
|
51
|
+
file may include methods missed by every tool; when omitted, reviewed `dead` labels are the
|
|
52
|
+
backward-compatible recall set. Each known positive requires its own rationale. `unknown` labels
|
|
53
|
+
are not treated as either true or false
|
|
54
|
+
positives. When a tool produces no candidates, precision is `null` with
|
|
55
|
+
`precision_status: no_candidates`; yield remains zero, so an empty result cannot claim 100%
|
|
56
|
+
precision. External snapshots without LOC remain valid and expose `candidate_loc_measured_count`
|
|
57
|
+
instead of inventing a method size.
|
|
58
|
+
|
|
59
|
+
## 0.2.1 safety release audit
|
|
60
|
+
|
|
61
|
+
The release audit compares all five current corpus reports with the first integrity-bound
|
|
62
|
+
five-corpus snapshot, reviews deterministic Rails/RuboCop differences, runs the dynamic, parse,
|
|
63
|
+
ambiguity, and remote-input adversarial suites, and enforces wall-time and RSS budgets:
|
|
64
|
+
|
|
65
|
+
```shell
|
|
66
|
+
NECROPSY_RUBOCOP_CORPUS=/path/to/rubocop-1.75.0 \
|
|
67
|
+
bundle exec ruby bench/audit.rb
|
|
68
|
+
```
|
|
69
|
+
|
|
70
|
+
Run it from a clean worktree. The command binds generated reports to the exact Git revision and to
|
|
71
|
+
the benchmark manifest and audit-config digests. It also records Ruby, OS, command, and RSS
|
|
72
|
+
measurement provenance, and fails closed when those values are incompatible with the saved
|
|
73
|
+
baseline. `--skip-benchmark` is intended only for an immediate rerun: it verifies the saved report
|
|
74
|
+
digests and all source provenance before accepting existing output. `--skip-adversarial` records
|
|
75
|
+
the suites as not run and cannot pass the release gate.
|
|
76
|
+
|
|
77
|
+
The reviewed input policy and baseline performance measurement live under
|
|
78
|
+
`bench/audits/0.2.1`. A passing run writes `audit.json` and `audit.md` there; neither an empty policy
|
|
79
|
+
nor an unknown review outcome is accepted.
|
|
80
|
+
|
|
81
|
+
The 1.0 public accuracy claim remains fail-closed until the independent review target is met. Generate
|
|
82
|
+
the deterministic, explicitly pending queue with:
|
|
83
|
+
|
|
84
|
+
```shell
|
|
85
|
+
bundle exec ruby bench/review_queue.rb
|
|
86
|
+
```
|
|
87
|
+
|
|
88
|
+
The committed `bench/audits/0.2.1/review_queue.yml` contains 300 actionable rows from all five
|
|
89
|
+
corpora, but it contains no labels, outcomes, or reviewer identities. Its `claim_gate_passed` value is
|
|
90
|
+
therefore `false`; editing the queue alone cannot satisfy the gate. Add independently reviewed labels
|
|
91
|
+
to the benchmark review input, rerun the audit, and publish precision and known-positive recall
|
|
92
|
+
separately.
|
data/bench/audit.rb
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require 'json'
|
|
4
|
+
require 'optparse'
|
|
5
|
+
require 'yaml'
|
|
6
|
+
require_relative '../lib/necropsy'
|
|
7
|
+
require_relative '../lib/necropsy/bench/seed_runner'
|
|
8
|
+
require_relative '../lib/necropsy/bench/release_audit'
|
|
9
|
+
require_relative '../lib/necropsy/bench/release_audit/adversarial_runner'
|
|
10
|
+
require_relative '../lib/necropsy/bench/release_audit/artifact_writer'
|
|
11
|
+
require_relative '../lib/necropsy/bench/release_audit/config_validator'
|
|
12
|
+
require_relative '../lib/necropsy/bench/release_audit/git_snapshot'
|
|
13
|
+
require_relative '../lib/necropsy/bench/release_audit/run_provenance'
|
|
14
|
+
|
|
15
|
+
root = File.expand_path('..', __dir__)
|
|
16
|
+
options = {
|
|
17
|
+
config: File.join(__dir__, 'audits/0.2.1/config.yml'),
|
|
18
|
+
benchmark_output: File.join(root, 'tmp/necropsy-benchmark/v1'),
|
|
19
|
+
audit_output: File.join(__dir__, 'audits/0.2.1'),
|
|
20
|
+
run_benchmark: true,
|
|
21
|
+
run_adversarial: true
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
OptionParser.new do |parser|
|
|
25
|
+
parser.banner = 'Usage: bundle exec ruby bench/audit.rb [options]'
|
|
26
|
+
parser.on('--config PATH', 'Audit configuration') { |path| options[:config] = path }
|
|
27
|
+
parser.on('--benchmark-output DIR', 'Current benchmark output') { |path| options[:benchmark_output] = path }
|
|
28
|
+
parser.on('--output DIR', 'Audit artifact output') { |path| options[:audit_output] = path }
|
|
29
|
+
parser.on('--skip-benchmark', 'Use an existing current benchmark run') { options[:run_benchmark] = false }
|
|
30
|
+
parser.on('--skip-adversarial', 'Do not execute adversarial suites') { options[:run_adversarial] = false }
|
|
31
|
+
end.parse!
|
|
32
|
+
|
|
33
|
+
config_path = File.expand_path(options.fetch(:config), root)
|
|
34
|
+
config = YAML.safe_load_file(config_path, aliases: false)
|
|
35
|
+
Necropsy::Bench::ReleaseAudit::ConfigValidator.new(config).validate!
|
|
36
|
+
|
|
37
|
+
manifest_path = File.join(root, 'bench/corpora/v1/manifest.yml')
|
|
38
|
+
benchmark_output = File.expand_path(options.fetch(:benchmark_output), root)
|
|
39
|
+
provenance_path = File.join(benchmark_output, 'run_metadata.json')
|
|
40
|
+
rubocop_corpus = ENV.fetch('NECROPSY_RUBOCOP_CORPUS', '<unset>')
|
|
41
|
+
measurement_command = "NECROPSY_RUBOCOP_CORPUS=#{rubocop_corpus} bundle exec ruby bench/run.rb"
|
|
42
|
+
provenance = Necropsy::Bench::ReleaseAudit::RunProvenance.new(
|
|
43
|
+
root: root,
|
|
44
|
+
manifest_path: manifest_path,
|
|
45
|
+
config_path: config_path,
|
|
46
|
+
output_dir: benchmark_output,
|
|
47
|
+
command: measurement_command
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
current_provenance = if options[:run_benchmark]
|
|
51
|
+
source = provenance.capture_source!
|
|
52
|
+
summary = Necropsy::Bench::SeedRunner.new(
|
|
53
|
+
manifest_path: manifest_path,
|
|
54
|
+
output_dir: benchmark_output
|
|
55
|
+
).call
|
|
56
|
+
provenance.complete(source, summary).tap do |payload|
|
|
57
|
+
provenance.write(provenance_path, payload)
|
|
58
|
+
end
|
|
59
|
+
else
|
|
60
|
+
provenance.load_and_validate!(provenance_path)
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
corpora = config.fetch('corpora')
|
|
64
|
+
current_reports = corpora.to_h do |corpus|
|
|
65
|
+
path = File.join(benchmark_output, 'reports', "#{corpus}.json")
|
|
66
|
+
[corpus, JSON.parse(File.read(path))]
|
|
67
|
+
end
|
|
68
|
+
current_summary = JSON.parse(File.read(File.join(benchmark_output, 'summary.json')))
|
|
69
|
+
baseline = config.fetch('baseline')
|
|
70
|
+
baseline_reports = Necropsy::Bench::ReleaseAudit::GitSnapshot.new(
|
|
71
|
+
root: root,
|
|
72
|
+
git_ref: baseline.fetch('git_ref'),
|
|
73
|
+
reports_path: baseline.fetch('reports_path')
|
|
74
|
+
).reports(corpora)
|
|
75
|
+
|
|
76
|
+
load_yaml = lambda do |relative|
|
|
77
|
+
YAML.safe_load_file(File.expand_path(relative, root), aliases: false) || {}
|
|
78
|
+
end
|
|
79
|
+
label_entries = Array(load_yaml.call(config.fetch('labels_path'))['labels'])
|
|
80
|
+
labels = label_entries.to_h { |entry| [[entry.fetch('corpus'), entry.fetch('id')], entry] }
|
|
81
|
+
reviews = Array(load_yaml.call(config.dig('review', 'path'))['reviews'])
|
|
82
|
+
baseline_performance = load_yaml.call(config.dig('performance', 'baseline_path'))
|
|
83
|
+
adversarial_results = if options[:run_adversarial]
|
|
84
|
+
Necropsy::Bench::ReleaseAudit::AdversarialRunner.new(
|
|
85
|
+
root: root,
|
|
86
|
+
suites: config.fetch('adversarial_suites')
|
|
87
|
+
).call
|
|
88
|
+
else
|
|
89
|
+
config.fetch('adversarial_suites').sort.map do |name, definition|
|
|
90
|
+
{ 'name' => name, 'command' => definition.fetch('command'), 'passed' => false,
|
|
91
|
+
'exit_status' => nil, 'duration_seconds' => 0.0, 'summary' => 'not run' }
|
|
92
|
+
end
|
|
93
|
+
end
|
|
94
|
+
|
|
95
|
+
audit = Necropsy::Bench::ReleaseAudit.new(
|
|
96
|
+
config: config,
|
|
97
|
+
baseline_reports: baseline_reports,
|
|
98
|
+
current_reports: current_reports,
|
|
99
|
+
current_summary: current_summary,
|
|
100
|
+
labels: labels,
|
|
101
|
+
reviews: reviews,
|
|
102
|
+
baseline_performance: baseline_performance,
|
|
103
|
+
adversarial_results: adversarial_results,
|
|
104
|
+
current_provenance: current_provenance
|
|
105
|
+
).call
|
|
106
|
+
paths = Necropsy::Bench::ReleaseAudit::ArtifactWriter.new(
|
|
107
|
+
audit: audit,
|
|
108
|
+
output_dir: File.expand_path(options.fetch(:audit_output), root)
|
|
109
|
+
).call
|
|
110
|
+
puts "release audit: #{audit.fetch('status')}"
|
|
111
|
+
puts "machine artifact: #{paths.fetch(:json)}"
|
|
112
|
+
puts "human artifact: #{paths.fetch(:markdown)}"
|
|
113
|
+
exit(audit.fetch('status') == 'pass' ? 0 : 1)
|