necropsy 0.1.0

data/lib/necropsy/bench/evaluator.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+require 'yaml'
+
+module Necropsy
+  module Bench
+    class Evaluator
+      ABLATION_ANALYZERS = {
+        'name_resolution' => [Analyzers::Static::NameResolution],
+        'cha' => [Analyzers::Static::CHA],
+        'rta' => [Analyzers::Static::RTA],
+        'name_resolution+cha' => [Analyzers::Static::NameResolution, Analyzers::Static::CHA],
+        'name_resolution+rta' => [Analyzers::Static::NameResolution, Analyzers::Static::RTA],
+        'all_static' => [Analyzers::Static::NameResolution, Analyzers::Static::CHA, Analyzers::Static::RTA]
+      }.freeze
+
+      def initialize(
+        report:,
+        gold_standard_path:,
+        min_confidence: :low,
+        root: nil,
+        config_path: nil,
+        ablation: false,
+        precision_threshold: 0.85,
+        recall_threshold: nil
+      )
+        @report = report
+        @gold_standard_path = gold_standard_path
+        @min_confidence = min_confidence
+        @root = root
+        @config_path = config_path
+        @ablation = ablation
+        @precision_threshold = precision_threshold
+        @recall_threshold = recall_threshold
+      end
+
+      def call
+        result = metrics_for(report)
+        result['by_classification'] = grouped_metrics(:classification)
+        result['by_confidence'] = grouped_metrics(:confidence)
+        result['ablation'] = ablation_metrics if ablation && root
+        result['release_criteria'] = release_criteria(result)
+        result
+      end
+
+      private
+
+      attr_reader :report, :gold_standard_path, :min_confidence, :root, :config_path, :ablation, :precision_threshold,
+                  :recall_threshold
+
+      def metrics_for(target_report, expected: gold_standard)
+        actual = target_report.dead_methods(min_confidence: min_confidence).to_set { |finding| finding.node.id }
+        true_positive = actual & expected
+        false_positive = actual - expected
+        false_negative = expected - actual
+
+        precision = ratio(true_positive.length, actual.length)
+        recall = ratio(true_positive.length, expected.length)
+        f1 = (precision + recall).zero? ? 0.0 : (2 * precision * recall / (precision + recall))
+
+        {
+          'precision' => precision.round(4),
+          'recall' => recall.round(4),
+          'f1' => f1.round(4),
+          'true_positive' => true_positive.to_a.sort,
+          'false_positive' => false_positive.to_a.sort,
+          'false_negative' => false_negative.to_a.sort
+        }
+      end
+
+      def gold_standard(classification: nil)
+        entries = gold_entries
+        if classification
+          scoped = entries.select do |entry|
+            !entry.key?('classification') || entry['classification'] == classification.to_s
+          end
+          return ids_for(scoped)
+        end
+
+        ids_for(entries)
+      end
+
+      def gold_entries
+        payload = YAML.load_file(gold_standard_path) || {}
+        entries = payload['dead_methods'] || payload['findings'] || payload
+        Array(entries).map do |entry|
+          entry.is_a?(Hash) ? entry.transform_keys(&:to_s) : { 'id' => entry }
+        end
+      end
+
+      def ids_for(entries)
+        entries.map { |entry| entry['id'] || entry['node_id'] }.compact.to_set
+      end
+
+      def grouped_metrics(attribute)
+        report.findings.group_by { |finding| finding.public_send(attribute) }.transform_values do |findings|
+          expected_scope = attribute == :classification ? gold_standard(classification: findings.first.public_send(attribute)) : gold_standard
+          metrics_for(Report.new(root: report.root, graph: report.graph, findings: findings), expected: expected_scope)
+        end.transform_keys(&:to_s)
+      end
+
+      def ablation_metrics
+        ABLATION_ANALYZERS.transform_values do |classes|
+          analyzers = classes.map(&:new)
+          metrics_for(Necropsy.analyze(root: root, config_path: config_path, analyzers: analyzers))
+        end
+      end
+
+      def ratio(numerator, denominator)
+        return 1.0 if denominator.zero?
+
+        numerator.to_f / denominator
+      end
+
+      def release_criteria(result)
+        checks = { 'precision' => result['precision'] >= precision_threshold }
+        checks['recall'] = result['recall'] >= recall_threshold if recall_threshold
+        {
+          'precision_threshold' => precision_threshold,
+          'recall_threshold' => recall_threshold,
+          'passed' => checks.values.all?,
+          'checks' => checks
+        }
+      end
+    end
+  end
+end

data/lib/necropsy/cache/scan_cache.rb

@@ -0,0 +1,158 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'yaml'
+
+module Necropsy
+  module Cache
+    class ScanCache
+      VERSION = 3
+
+      def initialize(project:)
+        @project = project
+      end
+
+      def fetch(files)
+        return yield unless project.config.cache_enabled?
+
+        metadata = cache_metadata(files)
+        cached = read(metadata)
+        return cached if cached
+
+        result = yield
+        write(metadata, result)
+        result
+      rescue SystemCallError, Psych::Exception
+        yield
+      end
+
+      private
+
+      attr_reader :project
+
+      def read(metadata)
+        payload = load_payload
+        return nil unless payload
+        return nil unless payload['version'] == VERSION
+        return nil unless payload['metadata'] == metadata
+
+        deserialize_scan_result(payload.fetch('scan_result'))
+      rescue StandardError
+        nil
+      end
+
+      def load_payload
+        return nil unless File.exist?(path)
+
+        YAML.load_file(path)
+      end
+
+      def write(metadata, result)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, {
+          'version' => VERSION,
+          'metadata' => metadata,
+          'scan_result' => serialize_scan_result(result)
+        }.to_yaml)
+      rescue StandardError
+        nil
+      end
+
+      def path
+        @path ||= File.expand_path(project.config.cache_path, project.root)
+      end
+
+      def file_metadata(files)
+        files.each_with_object({}) do |file, metadata|
+          stat = File.stat(file)
+          metadata[project.relative_path(file)] = {
+            'size' => stat.size,
+            'mtime' => "#{stat.mtime.to_i}.#{stat.mtime.nsec}"
+          }
+        end
+      end
+
+      def cache_metadata(files)
+        {
+          'files' => file_metadata(files),
+          'configuration' => project.config.scan_cache_key
+        }
+      end
+
+      def serialize_scan_result(result)
+        {
+          'nodes' => result.nodes.map(&:to_h),
+          'call_sites' => result.call_sites.map(&:to_h),
+          'instantiated_classes' => result.instantiated_classes.to_a.sort,
+          'uncertainties' => result.uncertainties.transform_values { |messages| Array(messages).map(&:to_s) },
+          'class_infos' => result.class_infos.map(&:to_h),
+          'entrypoint_hints' => result.entrypoint_hints.map(&:to_h)
+        }
+      end
+
+      def deserialize_scan_result(data)
+        ScanResult.new(
+          nodes: Array(data['nodes']).map { |node| deserialize_node(node) },
+          call_sites: Array(data['call_sites']).map { |site| deserialize_call_site(site) },
+          instantiated_classes: Set.new(Array(data['instantiated_classes'])),
+          uncertainties: deserialize_uncertainties(data['uncertainties']),
+          class_infos: Array(data['class_infos']).map { |info| deserialize_class_info(info) },
+          entrypoint_hints: Array(data['entrypoint_hints']).map { |entry| deserialize_entry_point(entry) }
+        )
+      end
+
+      def deserialize_node(data)
+        Node.new(
+          id: data['id'],
+          kind: data['kind'].to_sym,
+          file: data['file'],
+          line: data['line'].to_i,
+          end_line: data['end_line'].to_i,
+          defined_via: data['defined_via'].to_sym,
+          owner: data['owner'],
+          name: data['name'],
+          test: data['test']
+        )
+      end
+
+      def deserialize_call_site(data)
+        CallSite.new(
+          caller_id: data['caller_id'],
+          message: data['message'],
+          receiver_kind: data['receiver_kind'].to_sym,
+          receiver_name: data['receiver_name'],
+          file: data['file'],
+          line: data['line'].to_i,
+          test: data['test'],
+          dynamic: data['dynamic'],
+          metadata: data['metadata'] || {}
+        )
+      end
+
+      def deserialize_class_info(data)
+        ClassInfo.new(
+          id: data['id'],
+          kind: data['kind'].to_sym,
+          file: data['file'],
+          line: data['line'],
+          superclass: data['superclass'],
+          superclass_candidates: Array(data['superclass_candidates']),
+          includes: Array(data['includes']),
+          prepends: Array(data['prepends']),
+          extends: Array(data['extends']),
+          dynamic: data['dynamic']
+        )
+      end
+
+      def deserialize_entry_point(data)
+        EntryPoint.new(node_id: data['node_id'], reason: data['reason'].to_sym)
+      end
+
+      def deserialize_uncertainties(data)
+        Hash.new { |hash, key| hash[key] = [] }.tap do |uncertainties|
+          (data || {}).each { |node_id, messages| uncertainties[node_id] = Array(messages) }
+        end
+      end
+    end
+  end
+end

data/lib/necropsy/cli.rb

@@ -0,0 +1,287 @@
+# frozen_string_literal: true
+
+require 'date'
+require 'English'
+require 'fileutils'
+require 'json'
+require 'optparse'
+require 'securerandom'
+require 'yaml'
+require 'necropsy'
+
+module Necropsy
+  class CLI
+    def self.run(argv)
+      new.run(argv)
+    end
+
+    def run(argv)
+      command = argv.first&.start_with?('-') ? 'analyze' : argv.shift || 'analyze'
+      options = default_options
+      parser = build_parser(options)
+      parser.parse!(argv)
+      apply_config_defaults(options)
+
+      case command
+      when 'analyze'
+        report = analyze(options)
+        puts Reporter.new(report).render(format: options[:format], min_confidence: options[:min_confidence])
+        0
+      when 'baseline'
+        report = analyze(options)
+        path = File.expand_path(options[:baseline], options[:root])
+        Guardrail::Baseline.write(report, path: path)
+        puts "Wrote #{path}"
+        0
+      when 'check'
+        check(options)
+      when 'quarantine'
+        quarantine(options)
+      when 'bench'
+        bench(options)
+      when 'record'
+        record(options, argv)
+      when 'coverage'
+        coverage(options, argv)
+      else
+        warn "Unknown command: #{command}"
+        warn parser
+        2
+      end
+    rescue OptionParser::ParseError, Error => e
+      warn e.message
+      2
+    end
+
+    private
+
+    def default_options
+      {
+        root: '.',
+        config: nil,
+        format: :human,
+        min_confidence: :low,
+        baseline: nil,
+        fail_on: nil,
+        diff_base: nil,
+        ratchet: false,
+        write: false,
+        gold_standard: nil,
+        output: 'tmp/necropsy_trace_point.yml',
+        sample_rate: 1.0,
+        ablation: false,
+        precision_threshold: nil,
+        recall_threshold: nil
+      }
+    end
+
+    def build_parser(options)
+      OptionParser.new do |parser|
+        parser.banner = 'Usage: necropsy COMMAND [options]'
+        parser.on('--root PATH', 'Project root') { |value| options[:root] = value }
+        parser.on('--config PATH', 'Configuration file') { |value| options[:config] = value }
+        parser.on('--format FORMAT', 'human, json, yaml, sarif, or github') { |value| options[:format] = value.to_sym }
+        parser.on('--min-confidence LEVEL', 'low, medium, high, or certain') do |value|
+          options[:min_confidence] = value.to_sym
+        end
+        parser.on('--baseline PATH', 'Baseline path') { |value| options[:baseline] = value }
+        parser.on('--fail-on LEVEL', 'CI failure threshold') { |value| options[:fail_on] = value.to_sym }
+        parser.on('--diff-base REV', 'Restrict reported findings to files changed since REV') do |value|
+          options[:diff_base] = value
+        end
+        parser.on('--ratchet', 'Fail if finding count grows beyond baseline count') { options[:ratchet] = true }
+        parser.on('--write', 'Write quarantine annotations') { options[:write] = true }
+        parser.on('--gold-standard PATH', 'Gold standard YAML for bench') { |value| options[:gold_standard] = value }
+        parser.on('--output PATH', 'Output path for record') { |value| options[:output] = value }
+        parser.on('--sample-rate RATE', Float, 'TracePoint sample rate for record') do |value|
+          options[:sample_rate] = value
+        end
+        parser.on('--ablation', 'Run bench across analyzer combinations') { options[:ablation] = true }
+        parser.on('--precision-threshold N', Float, 'Bench release precision threshold') do |value|
+          options[:precision_threshold] = value
+        end
+        parser.on('--recall-threshold N', Float, 'Bench release recall threshold') do |value|
+          options[:recall_threshold] = value
+        end
+        parser.on('-h', '--help', 'Show help') do
+          puts parser
+          exit 0
+        end
+      end
+    end
+
+    def analyze(options)
+      Necropsy.analyze(root: options[:root], config_path: options[:config])
+    end
+
+    def apply_config_defaults(options)
+      config = Configuration.load(root: File.expand_path(options[:root]), path: options[:config])
+      options[:baseline] ||= config.baseline_path
+      options[:fail_on] ||= config.fail_on
+    end
+
+    def check(options)
+      report = analyze(options)
+      findings = filtered_findings(report, options)
+      baseline_path = File.expand_path(options[:baseline], options[:root])
+      baseline = Guardrail::Baseline.load(baseline_path)
+      failures = findings.reject { |finding| baseline.include?(finding) }
+
+      if options[:ratchet] && findings.length > baseline.fingerprints.length
+        puts "Ratchet failed: #{findings.length} findings exceed baseline count #{baseline.fingerprints.length}"
+        return 1
+      end
+
+      if failures.any?
+        puts Reporter.new(Report.new(root: report.root, graph: report.graph, findings: failures)).render(
+          format: :human,
+          min_confidence: options[:fail_on]
+        )
+        return 1
+      end
+
+      puts 'Necropsy check passed'
+      0
+    end
+
+    def filtered_findings(report, options)
+      findings = report.dead_methods(min_confidence: options[:fail_on])
+      return findings unless options[:diff_base]
+
+      project = Project.new(root: File.expand_path(options[:root]), config: report_config(options))
+      changed = project.changed_files(options[:diff_base])
+      findings.select { |finding| changed.include?(finding.node.file) }
+    end
+
+    def report_config(options)
+      Configuration.load(root: File.expand_path(options[:root]), path: options[:config])
+    end
+
+    def quarantine(options)
+      report = analyze(options)
+      quarantine = Guardrail::Quarantine.new(report: report, root: File.expand_path(options[:root]))
+      if options[:write]
+        quarantine.write(min_confidence: options[:min_confidence])
+        puts 'Wrote quarantine annotations'
+      else
+        quarantine.suggestions(min_confidence: options[:min_confidence]).each do |suggestion|
+          finding = suggestion[:finding]
+          puts "#{suggestion[:path]}:#{suggestion[:line]} #{suggestion[:annotation]} #{finding.node.id}"
+        end
+      end
+      0
+    end
+
+    def bench(options)
+      raise Error, '--gold-standard is required for bench' unless options[:gold_standard]
+
+      report = analyze(options)
+      config = report_config(options)
+      result = Bench::Evaluator.new(
+        report: report,
+        gold_standard_path: options[:gold_standard],
+        min_confidence: options[:min_confidence],
+        root: options[:root],
+        config_path: options[:config],
+        ablation: options[:ablation],
+        precision_threshold: options[:precision_threshold] || config.bench_precision_threshold,
+        recall_threshold: options[:recall_threshold] || config.bench_recall_threshold
+      ).call
+      puts JSON.pretty_generate(result)
+      0
+    end
+
+    def record(options, argv)
+      script_argv = argv.dup
+      script_argv.shift if script_argv.first == 'ruby'
+      script = script_argv.shift
+      raise Error, 'record requires a Ruby script after --' unless script
+
+      output = File.expand_path(options[:output], options[:root])
+      FileUtils.mkdir_p(File.dirname(output))
+
+      previous_argv = ARGV.dup
+      ARGV.replace(script_argv)
+      Analyzers::Dynamic::TracePointCollector.record(
+        root: File.expand_path(options[:root]),
+        output: output,
+        sample_rate: options[:sample_rate]
+      ) do
+        load File.expand_path(script, options[:root])
+      end
+      puts "Wrote #{output}"
+      0
+    ensure
+      ARGV.replace(previous_argv) if previous_argv
+    end
+
+    def coverage(options, argv)
+      script_argv = argv.dup
+      raise Error, 'coverage requires a Ruby script or command after --' if script_argv.empty?
+
+      output = File.expand_path(options[:output].sub('trace_point', 'coverage'), options[:root])
+      FileUtils.mkdir_p(File.dirname(output))
+
+      return record_coverage_script(options, output, script_argv) if local_ruby_script?(options, script_argv)
+
+      run_coverage_command(options, output, script_argv)
+    end
+
+    def record_coverage_script(options, output, script_argv)
+      script, args = ruby_script_and_args(options, script_argv)
+      previous_argv = ARGV.dup
+      ARGV.replace(args)
+      Analyzers::Dynamic::CoverageCollector.record(root: File.expand_path(options[:root]), output: output) do
+        load File.expand_path(script, options[:root])
+      end
+      puts "Wrote #{output}"
+      0
+    ensure
+      ARGV.replace(previous_argv) if previous_argv
+    end
+
+    def run_coverage_command(options, output, script_argv)
+      run_id = SecureRandom.hex(16)
+      status = system(coverage_runtime_env(options, output, run_id), *script_argv)
+      puts "Wrote #{output}" if output_for_run?(output, run_id)
+      return 0 if status
+
+      $CHILD_STATUS&.exitstatus || 1
+    end
+
+    def local_ruby_script?(options, script_argv)
+      script, = ruby_script_and_args(options, script_argv)
+      script&.end_with?('.rb') && File.file?(File.expand_path(script, options[:root]))
+    end
+
+    def ruby_script_and_args(_options, script_argv)
+      return [script_argv[1], script_argv.drop(2)] if script_argv.first == 'ruby'
+
+      [script_argv.first, script_argv.drop(1)]
+    end
+
+    def coverage_runtime_env(options, output, run_id)
+      rubyopt = [ENV.fetch('RUBYOPT', nil), '-rnecropsy/coverage_runtime'].compact.reject(&:empty?).join(' ')
+      {
+        'NECROPSY_COVERAGE_ROOT' => File.expand_path(options[:root]),
+        'NECROPSY_COVERAGE_OUTPUT' => output,
+        'NECROPSY_COVERAGE_MERGE' => '1',
+        'NECROPSY_COVERAGE_RUN_ID' => run_id,
+        'RUBYOPT' => rubyopt,
+        'RUBYLIB' => rubylib
+      }
+    end
+
+    def rubylib
+      paths = [File.expand_path('..', __dir__), ENV.fetch('RUBYLIB', nil)].compact.reject(&:empty?)
+      paths.join(File::PATH_SEPARATOR)
+    end
+
+    def output_for_run?(output, run_id)
+      payload = YAML.load_file(output) || {}
+      payload.dig('observation', 'run_id') == run_id
+    rescue SystemCallError, Psych::Exception
+      false
+    end
+  end
+end