necropsy 0.1.0 → 0.3.0

data/lib/necropsy/bench/evaluator.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'yaml'
+require_relative 'finding_facts'
 
 module Necropsy
   module Bench
@@ -21,6 +22,7 @@ module Necropsy
         root: nil,
         config_path: nil,
         ablation: false,
+        feature_ablation: {},
         precision_threshold: 0.85,
         recall_threshold: nil
       )
@@ -30,15 +32,20 @@ module Necropsy
         @root = root
         @config_path = config_path
         @ablation = ablation
+        @feature_ablation = feature_ablation
         @precision_threshold = precision_threshold
         @recall_threshold = recall_threshold
       end
 
       def call
         result = metrics_for(report)
+        result['identity_views'] = identity_views(report)
+        result['quality'] = quality_metrics(report)
+        result['by_category'] = category_metrics(report)
         result['by_classification'] = grouped_metrics(:classification)
         result['by_confidence'] = grouped_metrics(:confidence)
         result['ablation'] = ablation_metrics if ablation && root
+        result['feature_ablation'] = feature_ablation_metrics unless feature_ablation.empty?
         result['release_criteria'] = release_criteria(result)
         result
       end
@@ -46,75 +53,203 @@ module Necropsy
       private
 
       attr_reader :report, :gold_standard_path, :min_confidence, :root, :config_path, :ablation, :precision_threshold,
-                  :recall_threshold
+                  :recall_threshold, :feature_ablation
 
-      def metrics_for(target_report, expected: gold_standard)
-        actual = target_report.dead_methods(min_confidence: min_confidence).to_set { |finding| finding.node.id }
+      def metrics_for(target_report, expected: gold_standard, recall_expected: expected)
+        actual = actionable_findings(target_report).to_set { |finding| finding.node.id }
         true_positive = actual & expected
         false_positive = actual - expected
-        false_negative = expected - actual
+        false_negative = recall_expected ? recall_expected - actual : Set.new
 
         precision = ratio(true_positive.length, actual.length)
-        recall = ratio(true_positive.length, expected.length)
-        f1 = (precision + recall).zero? ? 0.0 : (2 * precision * recall / (precision + recall))
+        recall = ratio((actual & recall_expected).length, recall_expected.length) if recall_expected
+        f1 = if recall
+               (precision + recall).zero? ? 0.0 : (2 * precision * recall / (precision + recall))
+             end
 
         {
           'precision' => precision.round(4),
-          'recall' => recall.round(4),
-          'f1' => f1.round(4),
+          'recall' => recall&.round(4),
+          'f1' => f1&.round(4),
           'true_positive' => true_positive.to_a.sort,
           'false_positive' => false_positive.to_a.sort,
           'false_negative' => false_negative.to_a.sort
         }
       end
 
-      def gold_standard(classification: nil)
-        entries = gold_entries
+      def gold_standard(classification: nil, confidence: nil)
         if classification
-          scoped = entries.select do |entry|
-            !entry.key?('classification') || entry['classification'] == classification.to_s
-          end
+          grouped = gold_entries_by(:classification)
+          scoped = grouped.fetch(nil, []) + grouped.fetch(classification.to_s, [])
           return ids_for(scoped)
         end
+        if confidence
+          scoped = gold_entries_by(:confidence).fetch(confidence.to_s, [])
+          return nil if scoped.empty?
 
-        ids_for(entries)
+          return ids_for(scoped)
+        end
+
+        ids_for(gold_entries)
       end
 
       def gold_entries
-        payload = YAML.load_file(gold_standard_path) || {}
-        entries = payload['dead_methods'] || payload['findings'] || payload
+        return @gold_entries if defined?(@gold_entries)
+
+        payload = gold_payload
+        entries = if payload.is_a?(Hash)
+                    payload['dead_methods'] || payload['findings'] || dead_labels(payload['labels']) || []
+                  else
+                    payload
+                  end
+        @gold_entries = normalize_entries(entries).select do |entry|
+          classification = entry['classification']
+          classification.nil? || FindingFacts::ACTIONABLE_CLASSIFICATIONS.include?(classification.to_sym)
+        end.freeze
+      end
+
+      def known_positive_entries
+        return @known_positive_entries if defined?(@known_positive_entries)
+
+        payload = gold_payload
+        entries = payload['known_positives'] || payload['known_positive_methods'] if payload.is_a?(Hash)
+        @known_positive_entries = (entries ? normalize_entries(entries).freeze : gold_entries)
+      end
+
+      def gold_entries_by(attribute)
+        @gold_entries_by ||= {}
+        @gold_entries_by[attribute] ||= gold_entries.group_by { |entry| entry[attribute.to_s] }.freeze
+      end
+
+      def gold_payload
+        @gold_payload ||= YAML.safe_load_file(gold_standard_path, aliases: true) || {}
+      end
+
+      def normalize_entries(entries)
         Array(entries).map do |entry|
           entry.is_a?(Hash) ? entry.transform_keys(&:to_s) : { 'id' => entry }
         end
       end
 
+      def dead_labels(entries)
+        return unless entries
+
+        Array(entries).select { |entry| entry.is_a?(Hash) && (entry['value'] || entry[:value]).to_s == 'dead' }
+      end
+
       def ids_for(entries)
         entries.map { |entry| entry['id'] || entry['node_id'] }.compact.to_set
       end
 
       def grouped_metrics(attribute)
-        report.findings.group_by { |finding| finding.public_send(attribute) }.transform_values do |findings|
-          expected_scope = attribute == :classification ? gold_standard(classification: findings.first.public_send(attribute)) : gold_standard
-          metrics_for(Report.new(root: report.root, graph: report.graph, findings: findings), expected: expected_scope)
+        report.reportable_findings.group_by { |finding| finding.public_send(attribute) }.transform_values do |findings|
+          value = findings.first.public_send(attribute)
+          grouped_report = Report.new(root: report.root, graph: report.graph, findings: findings)
+          if attribute == :classification
+            expected_scope = gold_standard(classification: value)
+            metrics_for(grouped_report, expected: expected_scope)
+          else
+            expected_scope = gold_standard(confidence: value)
+            if expected_scope
+              metrics_for(grouped_report, expected: expected_scope)
+            else
+              metrics_for(grouped_report, expected: gold_standard, recall_expected: nil)
+            end
+          end
         end.transform_keys(&:to_s)
       end
 
       def ablation_metrics
-        ABLATION_ANALYZERS.transform_values do |classes|
+        analyzer_metrics = ABLATION_ANALYZERS.transform_values do |classes|
           analyzers = classes.map(&:new)
-          metrics_for(Necropsy.analyze(root: root, config_path: config_path, analyzers: analyzers))
+          measurement_for(Necropsy.analyze(
+                            root: root,
+                            config_path: config_path,
+                            analyzers: analyzers,
+                            ignored_reference_paths: [gold_standard_path]
+                          ))
         end
+        analyzer_metrics.merge(rta_pruning_metrics)
+      end
+
+      def rta_pruning_metrics
+        analyzers = ABLATION_ANALYZERS.fetch('all_static').map(&:new)
+        rank_only = measurement_for(
+          Runner.new(
+            root: root,
+            config_path: config_path,
+            analyzers: analyzers,
+            ignored_reference_paths: [gold_standard_path]
+          ).analyze(rta_pruning: :rank_only)
+        )
+        legacy = measurement_for(
+          Runner.new(
+            root: root,
+            config_path: config_path,
+            analyzers: analyzers,
+            ignored_reference_paths: [gold_standard_path]
+          ).analyze(rta_pruning: :legacy)
+        )
+        rank_candidates = candidate_ids(rank_only)
+        legacy_candidates = candidate_ids(legacy)
+        difference = {
+          'only_in_rank_only' => (rank_candidates - legacy_candidates).sort,
+          'only_in_legacy' => (legacy_candidates - rank_candidates).sort
+        }
+        {
+          'rta_rank_only' => rank_only.merge('rta_pruning' => 'rank_only', 'candidate_diff' => difference),
+          'rta_legacy' => legacy.merge('rta_pruning' => 'legacy', 'candidate_diff' => difference)
+        }
+      end
+
+      def candidate_ids(metrics)
+        Set.new(metrics.fetch('true_positive') + metrics.fetch('false_positive'))
       end
 
       def ratio(numerator, denominator)
-        return 1.0 if denominator.zero?
+        return 0.0 if denominator.zero?
 
         numerator.to_f / denominator
       end
 
+      def identity_views(target_report)
+        findings = actionable_findings(target_report)
+        logical = findings.map { |finding| finding.node.symbol_id }.uniq.sort
+        physical = findings.sort_by do |finding|
+          [finding.node.symbol_id, finding.node.file, finding.node.line, finding.node.definition_id]
+        end.map do |finding|
+          {
+            'definition_id' => finding.node.definition_id,
+            'symbol_id' => finding.node.symbol_id,
+            'physical_fingerprint' => finding.physical_fingerprint,
+            'logical_fingerprint' => finding.logical_fingerprint,
+            'file' => finding.node.file,
+            'line' => finding.node.line
+          }
+        end
+        {
+          'legacy_logical' => {
+            'identity_key' => 'symbol_id',
+            'candidate_count' => logical.length,
+            'candidate_ids' => logical
+          },
+          'physical_definition' => {
+            'identity_key' => 'definition_id',
+            'candidate_count' => physical.length,
+            'candidates' => physical
+          }
+        }
+      end
+
       def release_criteria(result)
-        checks = { 'precision' => result['precision'] >= precision_threshold }
-        checks['recall'] = result['recall'] >= recall_threshold if recall_threshold
+        checks = {
+          'precision' => result.dig('quality', 'candidate_precision') >= precision_threshold,
+          'candidate_yield' => result.dig('quality', 'candidate_count').positive?
+        }
+        if recall_threshold
+          known_positive_recall = result.dig('quality', 'known_positive_recall')
+          checks['recall'] = !known_positive_recall.nil? && known_positive_recall >= recall_threshold
+        end
         {
           'precision_threshold' => precision_threshold,
           'recall_threshold' => recall_threshold,
@@ -122,6 +257,203 @@ module Necropsy
           'checks' => checks
         }
       end
+
+      def actionable_findings(target_report)
+        if target_report.respond_to?(:actionable_candidates)
+          target_report.actionable_candidates(min_confidence: min_confidence)
+        else
+          target_report.dead_methods(min_confidence: min_confidence).select { |finding| FindingFacts.actionable?(finding) }
+        end
+      end
+
+      def quality_metrics(target_report)
+        candidates = actionable_findings(target_report)
+        findings = target_report.reportable_findings
+        blocked = findings.select { |finding| finding.classification == :blocked }
+        unknown = findings.select { |finding| FindingFacts.unknown?(finding) }
+        resolutions = FindingFacts.resolution_counts(target_report)
+        {
+          'scope' => 'report',
+          'candidate_precision' => physical_precision(candidates, gold_entries),
+          'candidate_count' => candidates.length,
+          'candidate_loc' => candidates.sum { |finding| FindingFacts.loc(finding) },
+          'known_positive_recall' => physical_recall(candidates, known_positive_entries),
+          'known_positive_count' => known_positive_entries.length,
+          'diagnostic_count' => findings.count { |finding| !FindingFacts.actionable?(finding) },
+          'blocked_count' => blocked.length,
+          'blocked_rate' => FindingFacts.ratio(blocked.length, findings.length),
+          'unknown_finding_count' => unknown.length,
+          'unknown_finding_rate' => FindingFacts.ratio(unknown.length, findings.length),
+          'resolution_counts' => resolutions,
+          'unknown_resolution_rate' => FindingFacts.ratio(resolutions.fetch('unknown'), resolutions.fetch('total')),
+          'rule_counts' => FindingFacts.report_rule_counts(target_report),
+          'risk_counts' => FindingFacts.report_risk_counts(target_report)
+        }
+      end
+
+      def category_metrics(target_report)
+        candidates = actionable_findings(target_report)
+        findings = target_report.reportable_findings
+        expected = gold_entries
+        known = known_positive_entries
+        expected_by_category = expected.group_by { |entry| category_for_entry(entry) }
+        known_by_category = known.group_by { |entry| category_for_entry(entry) }
+        categories = candidates.map { |finding| category_for_finding(finding) } +
+                     findings.map { |finding| category_for_finding(finding) } +
+                     (expected + known).map { |entry| category_for_entry(entry) }
+        categories.uniq.sort.to_h do |category|
+          category_candidates = candidates.select { |finding| category_for_finding(finding) == category }
+          category_findings = findings.select { |finding| category_for_finding(finding) == category }
+          category_expected = expected_by_category.fetch(category, [])
+          category_known = known_by_category.fetch(category, [])
+          blocked = category_findings.select { |finding| finding.classification == :blocked }
+          unknown = category_findings.select { |finding| FindingFacts.unknown?(finding) }
+          [category, {
+            'candidate_precision' => physical_precision(category_candidates, category_expected),
+            'candidate_count' => category_candidates.length,
+            'candidate_loc' => category_candidates.sum { |finding| FindingFacts.loc(finding) },
+            'known_positive_recall' => physical_recall(category_candidates, category_known),
+            'known_positive_count' => category_known.length,
+            'finding_count' => category_findings.length,
+            'blocked_count' => blocked.length,
+            'blocked_rate' => FindingFacts.ratio(blocked.length, category_findings.length),
+            'unknown_count' => unknown.length,
+            'unknown_rate' => FindingFacts.ratio(unknown.length, category_findings.length),
+            'rule_counts' => FindingFacts.tally(
+              category_findings.flat_map { |finding| FindingFacts.rule_hits(finding) }
+            ),
+            'risk_counts' => FindingFacts.tally(
+              category_findings.flat_map { |finding| FindingFacts.risk_flags(finding) }
+            )
+          }]
+        end
+      end
+
+      def category_for_finding(finding)
+        entry = finding_identifiers(finding).filter_map { |identifier| gold_entry_index[identifier] }.min_by(&:first)&.last
+        entry ? category_for_entry(entry) : FindingFacts.category(finding)
+      end
+
+      def gold_entry_index
+        @gold_entry_index ||= (gold_entries + known_positive_entries).each_with_index.with_object({}) do |(entry, index), lookup|
+          identifier = entry_identifier(entry)
+          lookup[identifier] ||= [index, entry] if identifier
+        end.freeze
+      end
+
+      def category_for_entry(entry)
+        category = entry['category']
+        category.to_s.empty? ? 'uncategorized' : category.to_s
+      end
+
+      def physical_precision(candidates, expected_entries)
+        return 0.0 if candidates.empty?
+
+        expected_ids = expected_entries.filter_map { |entry| entry_identifier(entry) }.to_set
+        matches = candidates.count { |finding| expected_ids.intersect?(finding_identifiers(finding)) }
+        FindingFacts.ratio(matches, candidates.length)
+      end
+
+      def physical_recall(candidates, expected_entries)
+        return nil if expected_entries.empty?
+
+        candidate_ids = candidates.flat_map { |finding| finding_identifiers(finding) }.to_set
+        matches = expected_entries.count { |entry| candidate_ids.include?(entry_identifier(entry)) }
+        FindingFacts.ratio(matches, expected_entries.length)
+      end
+
+      def entry_identifier(entry)
+        entry['definition_id'] || entry['id'] || entry['node_id']
+      end
+
+      def finding_identifiers(finding)
+        [finding.node.definition_id, finding.node.symbol_id, finding.node.id].compact.uniq
+      end
+
+      def feature_ablation_metrics
+        feature_ablation.sort.to_h do |name, variants|
+          enabled = variant_report(variants, :on)
+          disabled = variant_report(variants, :off)
+          enabled_metrics = quality_metrics(enabled)
+          disabled_metrics = quality_metrics(disabled)
+          enabled_categories = category_metrics(enabled)
+          disabled_categories = category_metrics(disabled)
+          [name.to_s, {
+            'on' => enabled_metrics.merge('by_category' => enabled_categories),
+            'off' => disabled_metrics.merge('by_category' => disabled_categories),
+            'difference' => quality_difference(
+              enabled, disabled, enabled_metrics, disabled_metrics, enabled_categories, disabled_categories
+            )
+          }]
+        end
+      end
+
+      def variant_report(variants, key)
+        report = variants[key] || variants[key.to_s]
+        raise Error, "Feature ablation requires #{key.inspect} report" unless report
+
+        report
+      end
+
+      def quality_difference(enabled, disabled, enabled_metrics, disabled_metrics, enabled_categories,
+                             disabled_categories)
+        numeric_keys = %w[
+          candidate_count candidate_loc candidate_precision known_positive_recall
+          blocked_count blocked_rate unknown_finding_count unknown_finding_rate
+          unknown_resolution_rate
+        ]
+        difference = numeric_keys.to_h do |key|
+          [key, metric_delta(enabled_metrics[key], disabled_metrics[key])]
+        end
+        enabled_ids = physical_candidate_ids(enabled)
+        disabled_ids = physical_candidate_ids(disabled)
+        difference.merge(
+          'only_with_feature' => (enabled_ids - disabled_ids).sort,
+          'only_without_feature' => (disabled_ids - enabled_ids).sort,
+          'rule_counts' => count_difference(enabled_metrics['rule_counts'], disabled_metrics['rule_counts']),
+          'risk_counts' => count_difference(enabled_metrics['risk_counts'], disabled_metrics['risk_counts']),
+          'by_category' => category_difference(enabled_categories, disabled_categories)
+        )
+      end
+
+      def category_difference(enabled, disabled)
+        (enabled.keys | disabled.keys).sort.to_h do |category|
+          on = enabled.fetch(category, {})
+          off = disabled.fetch(category, {})
+          numeric_keys = %w[
+            candidate_precision candidate_count candidate_loc known_positive_recall
+            blocked_count blocked_rate unknown_count unknown_rate
+          ]
+          difference = numeric_keys.to_h { |key| [key, metric_delta(on[key] || 0, off[key] || 0)] }
+          [category, difference.merge(
+            'rule_counts' => count_difference(on.fetch('rule_counts', {}), off.fetch('rule_counts', {})),
+            'risk_counts' => count_difference(on.fetch('risk_counts', {}), off.fetch('risk_counts', {}))
+          )]
+        end
+      end
+
+      def measurement_for(target_report)
+        metrics_for(target_report).merge(
+          'quality' => quality_metrics(target_report),
+          'by_category' => category_metrics(target_report)
+        )
+      end
+
+      def physical_candidate_ids(target_report)
+        actionable_findings(target_report).to_set { |finding| finding.node.definition_id }
+      end
+
+      def metric_delta(enabled, disabled)
+        return nil if enabled.nil? || disabled.nil?
+
+        (enabled - disabled).round(4)
+      end
+
+      def count_difference(enabled, disabled)
+        (enabled.keys | disabled.keys).sort.to_h do |key|
+          [key, enabled.fetch(key, 0) - disabled.fetch(key, 0)]
+        end
+      end
     end
   end
 end

data/lib/necropsy/bench/finding_facts.rb

@@ -0,0 +1,152 @@
+# frozen_string_literal: true
+
+module Necropsy
+  module Bench
+    module FindingFacts
+      ACTIONABLE_CLASSIFICATIONS = %i[unreachable unused].freeze
+      UNKNOWN_BLOCKER_KINDS = %i[
+        analyzer_failure ambiguity_limit_exceeded dynamic_dispatch evidence_collision
+        dynamic_ancestry incomplete_analysis incomplete_source parse_failure parse_incomplete
+        partial_dispatch reference_scope_incomplete resolution_conflict resolution_invalid
+        unknown_dispatch unparsed_external_reference unsupported_refinement variable_eval
+      ].freeze
+      RULE_ROOT_REASONS = %i[
+        callback_registered rails_component rails_migration rails_route
+        rails_view_helper rails_view_reference
+      ].freeze
+      ORDINARY_DEFINITION_KINDS = %i[def defs define_method].freeze
+
+      module_function
+
+      def actionable?(finding)
+        ACTIONABLE_CLASSIFICATIONS.include?(finding.classification.to_sym)
+      end
+
+      def loc(finding)
+        start_line = positive_integer(finding.node.line)
+        end_line = positive_integer(finding.node.end_line) || start_line
+        return 0 unless start_line
+
+        [end_line - start_line + 1, 1].max
+      end
+
+      def unknown?(finding)
+        finding.blockers.any? { |blocker| unknown_blocker?(blocker) }
+      end
+
+      def risk_flags(finding)
+        node = finding.node
+        flags = []
+        flags << 'public_or_protected_visibility' if %i[public protected].include?(node.visibility.to_sym)
+        flags << 'generated_method' unless ORDINARY_DEFINITION_KINDS.include?(node.defined_via.to_sym)
+        flags << 'no_owner' if node.owner.to_s.empty?
+        flags << 'test_definition' if node.test
+        flags << 'analysis_incomplete' if finding.blockers.any?
+        flags << 'duplicate_or_redefinition' if finding.blockers.any? do |blocker|
+          blocker.kind.to_sym == :duplicate_definition
+        end
+        flags.uniq.sort
+      end
+
+      def rule_hits(finding)
+        evidence_rules = finding.evidences.flat_map do |evidence|
+          metadata = stringify_keys(evidence.metadata)
+          values = [metadata['rule_id'], metadata['rule'], metadata['rules']]
+          values.flatten.compact
+        end
+        blocker_rules = finding.blockers.filter_map do |blocker|
+          metadata = stringify_keys(blocker.metadata)
+          metadata['rule_id'] || metadata['rule']
+        end
+        (evidence_rules + blocker_rules).map(&:to_s).reject(&:empty?).uniq.sort
+      end
+
+      def category(finding)
+        explicit = explicit_category(finding)
+        return explicit if explicit
+
+        blocker = finding.blockers.first
+        return blocker.kind.to_s if blocker
+
+        defined_via = finding.node.defined_via.to_sym
+        return 'generated_method' unless ORDINARY_DEFINITION_KINDS.include?(defined_via)
+
+        finding.classification.to_s
+      end
+
+      def report_rule_counts(report)
+        finding_rules = report.reportable_findings.flat_map { |finding| rule_hits(finding) }
+        root_rules = report.graph.entry_points.filter_map do |root|
+          node = report.graph.nodes.exact(root.definition_id)
+          next unless node && report_path?(report, node.file)
+
+          root.reason.to_s if RULE_ROOT_REASONS.include?(root.reason.to_sym)
+        end
+        tally(finding_rules + root_rules)
+      end
+
+      def report_risk_counts(report)
+        tally(report.reportable_findings.flat_map { |finding| risk_flags(finding) })
+      end
+
+      def tally(values)
+        values.map(&:to_s).reject(&:empty?).tally.sort.to_h
+      end
+
+      def resolution_counts(report)
+        visible_call_site_ids = report.graph.call_sites.filter_map do |site|
+          site.call_site_id if report_path?(report, site.file)
+        end.to_set
+        records = report.graph.resolution_records.select do |record|
+          visible_call_site_ids.include?(record.resolution.call_site_id)
+        end
+        counts = records.map { |record| record.resolution.status.to_s }.tally
+        {
+          'total' => records.length,
+          'complete' => counts.fetch('complete', 0),
+          'partial' => counts.fetch('partial', 0),
+          'unknown' => counts.fetch('unknown', 0)
+        }
+      end
+
+      def ratio(numerator, denominator)
+        denominator.zero? ? 0.0 : (numerator.to_f / denominator).round(4)
+      end
+
+      def report_path?(report, path)
+        !report.respond_to?(:report_path?) || report.report_path?(path)
+      end
+      private_class_method :report_path?
+
+      def stringify_keys(value)
+        return {} unless value.respond_to?(:to_h)
+
+        value.to_h.transform_keys(&:to_s)
+      end
+      private_class_method :stringify_keys
+
+      def positive_integer(value)
+        integer = Integer(value, exception: false)
+        integer if integer&.positive?
+      end
+      private_class_method :positive_integer
+
+      def unknown_blocker?(blocker)
+        kind = blocker.kind.to_sym
+        UNKNOWN_BLOCKER_KINDS.include?(kind) || kind.to_s.include?('unknown') || kind.to_s.include?('partial')
+      end
+      private_class_method :unknown_blocker?
+
+      def explicit_category(finding)
+        records = finding.evidences + finding.blockers
+        records.each do |record|
+          metadata = stringify_keys(record.metadata)
+          value = metadata['benchmark_category'] || metadata['category']
+          return value.to_s unless value.to_s.empty?
+        end
+        nil
+      end
+      private_class_method :explicit_category
+    end
+  end
+end