necropsy 0.1.0 → 0.2.0

data/lib/necropsy/cache/scan_cache.rb

@@ -1,12 +1,12 @@
 # frozen_string_literal: true
 
 require 'fileutils'
-require 'yaml'
+require 'json'
 
 module Necropsy
   module Cache
     class ScanCache
-      VERSION = 3
+      VERSION = 4
 
       def initialize(project:)
         @project = project
@@ -22,7 +22,8 @@ module Necropsy
         result = yield
         write(metadata, result)
         result
-      rescue SystemCallError, Psych::Exception
+      rescue SystemCallError, JSON::ParserError => e
+        warn_cache("Cache unavailable: #{e.message}")
         yield
       end
 
@@ -37,24 +38,26 @@ module Necropsy
         return nil unless payload['metadata'] == metadata
 
         deserialize_scan_result(payload.fetch('scan_result'))
-      rescue StandardError
+      rescue StandardError => e
+        warn_cache("Ignoring invalid cache: #{e.message}")
         nil
       end
 
       def load_payload
         return nil unless File.exist?(path)
 
-        YAML.load_file(path)
+        JSON.parse(File.read(path))
       end
 
       def write(metadata, result)
         FileUtils.mkdir_p(File.dirname(path))
-        File.write(path, {
-          'version' => VERSION,
-          'metadata' => metadata,
-          'scan_result' => serialize_scan_result(result)
-        }.to_yaml)
-      rescue StandardError
+        File.write(path, JSON.generate({
+                                         'version' => VERSION,
+                                         'metadata' => metadata,
+                                         'scan_result' => serialize_scan_result(result)
+                                       }))
+      rescue StandardError => e
+        warn_cache("Could not write cache: #{e.message}")
         nil
       end
 
@@ -111,7 +114,8 @@ module Necropsy
           defined_via: data['defined_via'].to_sym,
           owner: data['owner'],
           name: data['name'],
-          test: data['test']
+          test: data['test'],
+          visibility: (data['visibility'] || 'public').to_sym
         )
       end
 
@@ -153,6 +157,10 @@ module Necropsy
           (data || {}).each { |node_id, messages| uncertainties[node_id] = Array(messages) }
         end
       end
+
+      def warn_cache(message)
+        warn "Necropsy: #{message}" if project.config.verbose?
+      end
     end
   end
 end

data/lib/necropsy/cli.rb

@@ -5,6 +5,7 @@ require 'English'
 require 'fileutils'
 require 'json'
 require 'optparse'
+require 'rbconfig'
 require 'securerandom'
 require 'yaml'
 require 'necropsy'
@@ -20,12 +21,24 @@ module Necropsy
       options = default_options
       parser = build_parser(options)
       parser.parse!(argv)
+      if options[:help]
+        puts parser
+        return 0
+      end
+      if options[:version]
+        puts Necropsy::VERSION
+        return 0
+      end
       apply_config_defaults(options)
 
       case command
       when 'analyze'
         report = analyze(options)
-        puts Reporter.new(report).render(format: options[:format], min_confidence: options[:min_confidence])
+        puts Reporter.new(report).render(
+          format: options[:format],
+          min_confidence: options[:min_confidence],
+          include_graph: options[:include_graph]
+        )
         0
       when 'baseline'
         report = analyze(options)
@@ -43,12 +56,14 @@ module Necropsy
         record(options, argv)
       when 'coverage'
         coverage(options, argv)
+      when 'why', 'explain'
+        diagnose(command, options, argv)
       else
         warn "Unknown command: #{command}"
         warn parser
         2
       end
-    rescue OptionParser::ParseError, Error => e
+    rescue OptionParser::ParseError, Psych::Exception, Error => e
       warn e.message
       2
     end
@@ -60,18 +75,21 @@ module Necropsy
         root: '.',
         config: nil,
         format: :human,
-        min_confidence: :low,
+        min_confidence: Reporter::DEFAULT_MIN_CONFIDENCE,
         baseline: nil,
         fail_on: nil,
         diff_base: nil,
         ratchet: false,
         write: false,
         gold_standard: nil,
-        output: 'tmp/necropsy_trace_point.yml',
+        output: nil,
         sample_rate: 1.0,
         ablation: false,
         precision_threshold: nil,
-        recall_threshold: nil
+        recall_threshold: nil,
+        help: false,
+        version: false,
+        include_graph: false
       }
     end
 
@@ -80,12 +98,17 @@ module Necropsy
         parser.banner = 'Usage: necropsy COMMAND [options]'
         parser.on('--root PATH', 'Project root') { |value| options[:root] = value }
         parser.on('--config PATH', 'Configuration file') { |value| options[:config] = value }
-        parser.on('--format FORMAT', 'human, json, yaml, sarif, or github') { |value| options[:format] = value.to_sym }
+        parser.on('--format FORMAT', Reporter::FORMATS.map(&:to_s), 'Output format') do |value|
+          options[:format] = value.to_sym
+        end
+        parser.on('--include-graph', 'Include nodes and edges in JSON/YAML output') { options[:include_graph] = true }
         parser.on('--min-confidence LEVEL', 'low, medium, high, or certain') do |value|
-          options[:min_confidence] = value.to_sym
+          options[:min_confidence] = confidence_level(value)
         end
         parser.on('--baseline PATH', 'Baseline path') { |value| options[:baseline] = value }
-        parser.on('--fail-on LEVEL', 'CI failure threshold') { |value| options[:fail_on] = value.to_sym }
+        parser.on('--fail-on LEVEL', 'CI failure threshold') do |value|
+          options[:fail_on] = confidence_level(value)
+        end
         parser.on('--diff-base REV', 'Restrict reported findings to files changed since REV') do |value|
           options[:diff_base] = value
         end
@@ -94,6 +117,8 @@ module Necropsy
         parser.on('--gold-standard PATH', 'Gold standard YAML for bench') { |value| options[:gold_standard] = value }
         parser.on('--output PATH', 'Output path for record') { |value| options[:output] = value }
         parser.on('--sample-rate RATE', Float, 'TracePoint sample rate for record') do |value|
+          raise OptionParser::InvalidArgument, 'sample rate must be between 0.0 and 1.0' unless value.between?(0.0, 1.0)
+
           options[:sample_rate] = value
         end
         parser.on('--ablation', 'Run bench across analyzer combinations') { options[:ablation] = true }
@@ -104,9 +129,9 @@ module Necropsy
           options[:recall_threshold] = value
         end
         parser.on('-h', '--help', 'Show help') do
-          puts parser
-          exit 0
+          options[:help] = true
         end
+        parser.on('-v', '--version', 'Show version') { options[:version] = true }
       end
     end
 
@@ -114,6 +139,17 @@ module Necropsy
       Necropsy.analyze(root: options[:root], config_path: options[:config])
     end
 
+    def diagnose(command, options, argv)
+      node_id = argv.shift
+      raise Error, "#{command} requires a symbol ID" unless node_id
+      raise Error, "Unexpected arguments for #{command}: #{argv.join(' ')}" unless argv.empty?
+
+      diagnostics = Diagnostics.new(analyze(options))
+      payload = command == 'why' ? diagnostics.why(node_id) : diagnostics.explain(node_id)
+      puts diagnostics.render(payload, format: options[:format])
+      0
+    end
+
     def apply_config_defaults(options)
       config = Configuration.load(root: File.expand_path(options[:root]), path: options[:config])
       options[:baseline] ||= config.baseline_path
@@ -127,8 +163,9 @@ module Necropsy
       baseline = Guardrail::Baseline.load(baseline_path)
       failures = findings.reject { |finding| baseline.include?(finding) }
 
-      if options[:ratchet] && findings.length > baseline.fingerprints.length
-        puts "Ratchet failed: #{findings.length} findings exceed baseline count #{baseline.fingerprints.length}"
+      baseline_count = baseline.count_at_least(options[:fail_on])
+      if options[:ratchet] && findings.length > baseline_count
+        puts "Ratchet failed: #{findings.length} findings exceed baseline count #{baseline_count}"
         return 1
       end
 
@@ -192,34 +229,25 @@ module Necropsy
     end
 
     def record(options, argv)
-      script_argv = argv.dup
-      script_argv.shift if script_argv.first == 'ruby'
-      script = script_argv.shift
-      raise Error, 'record requires a Ruby script after --' unless script
+      command = argv.dup
+      raise Error, 'record requires a Ruby script or command after --' if command.empty?
 
-      output = File.expand_path(options[:output], options[:root])
+      output = File.expand_path(options[:output] || 'tmp/necropsy_trace_point.yml', options[:root])
       FileUtils.mkdir_p(File.dirname(output))
+      command = trace_command(options, command)
+      run_id = SecureRandom.hex(16)
+      status = system(trace_runtime_env(options, output, run_id), *command)
+      puts "Wrote #{output}" if output_for_run?(output, run_id)
+      return 0 if status
 
-      previous_argv = ARGV.dup
-      ARGV.replace(script_argv)
-      Analyzers::Dynamic::TracePointCollector.record(
-        root: File.expand_path(options[:root]),
-        output: output,
-        sample_rate: options[:sample_rate]
-      ) do
-        load File.expand_path(script, options[:root])
-      end
-      puts "Wrote #{output}"
-      0
-    ensure
-      ARGV.replace(previous_argv) if previous_argv
+      $CHILD_STATUS&.exitstatus || 1
     end
 
     def coverage(options, argv)
       script_argv = argv.dup
       raise Error, 'coverage requires a Ruby script or command after --' if script_argv.empty?
 
-      output = File.expand_path(options[:output].sub('trace_point', 'coverage'), options[:root])
+      output = File.expand_path(options[:output] || 'tmp/necropsy_coverage.yml', options[:root])
       FileUtils.mkdir_p(File.dirname(output))
 
       return record_coverage_script(options, output, script_argv) if local_ruby_script?(options, script_argv)
@@ -272,16 +300,44 @@ module Necropsy
       }
     end
 
+    def trace_runtime_env(options, output, run_id)
+      rubyopt = [ENV.fetch('RUBYOPT', nil), '-rnecropsy/trace_point_runtime'].compact.reject(&:empty?).join(' ')
+      {
+        'NECROPSY_TRACE_ROOT' => File.expand_path(options[:root]),
+        'NECROPSY_TRACE_OUTPUT' => output,
+        'NECROPSY_TRACE_SAMPLE_RATE' => options[:sample_rate].to_s,
+        'NECROPSY_TRACE_MERGE' => '1',
+        'NECROPSY_TRACE_RUN_ID' => run_id,
+        'RUBYOPT' => rubyopt,
+        'RUBYLIB' => rubylib
+      }
+    end
+
+    def trace_command(options, command)
+      script = command.first
+      path = File.expand_path(script, options[:root])
+      return [RbConfig.ruby, path, *command.drop(1)] if script.end_with?('.rb') && File.file?(path)
+
+      command
+    end
+
     def rubylib
       paths = [File.expand_path('..', __dir__), ENV.fetch('RUBYLIB', nil)].compact.reject(&:empty?)
       paths.join(File::PATH_SEPARATOR)
     end
 
     def output_for_run?(output, run_id)
-      payload = YAML.load_file(output) || {}
+      payload = YAML.safe_load_file(output, aliases: false) || {}
       payload.dig('observation', 'run_id') == run_id
     rescue SystemCallError, Psych::Exception
       false
     end
+
+    def confidence_level(value)
+      level = value.to_sym
+      return level if CONFIDENCE_LEVELS.key?(level)
+
+      raise OptionParser::InvalidArgument, "unknown confidence level: #{value}"
+    end
   end
 end

data/lib/necropsy/confidence/scorer.rb

@@ -5,10 +5,27 @@ require 'date'
 module Necropsy
   module Confidence
     class Scorer
+      RUBY_HOOKS = %w[
+        inherited included extended prepended method_added singleton_method_added
+        const_missing method_missing respond_to_missing?
+      ].freeze
+      RUBY_PROTOCOLS = %w[
+        == eql? hash <=> to_s to_str to_a to_h to_proc inspect each call coerce
+        succ initialize_copy marshal_dump marshal_load
+      ].freeze
+      BUILTIN_IMPLICIT_CALLERS = [
+        {
+          name_pattern: /^on_/,
+          owner_ancestors: ['RuboCop::Cop::Base'],
+          reason: 'RuboCop Commissioner dispatches on_* callbacks'
+        }
+      ].freeze
+
       def initialize(graph:, reachability:, project:)
         @graph = graph
         @reachability = reachability
         @project = project
+        @source_lines = {}
       end
 
       def findings
@@ -18,12 +35,13 @@ module Necropsy
           classification = classification_for(node)
           next unless classification
 
-          score, level, reasons = score_for(node, classification)
+          score, level, reasons, score_components = score_for(node, classification)
           Finding.new(
             node: node,
             classification: classification,
             confidence: level,
             score: score,
+            score_components: score_components,
             reasons: reasons,
             evidences: graph.incoming_edges(node.id).flat_map(&:evidences) + graph.alive_evidences(node.id)
           )
@@ -35,11 +53,13 @@ module Necropsy
       attr_reader :graph, :reachability, :project
 
       def classification_for(node)
-        if reachability.runtime_alive.include?(node.id)
-          return :unused if graph.dynamic_enabled? && !graph.dynamic_alive?(node.id)
+        return nil if graph.dynamic_alive?(node.id)
+
+        if reachability.runtime_paths.key?(node.id)
+          return :unused if graph.dynamic_enabled? && !generated_accessor?(node)
 
           nil
-        elsif reachability.test_alive.include?(node.id)
+        elsif reachability.test_paths.key?(node.id)
           :test_only_reachable
         else
           :unreachable
@@ -49,41 +69,42 @@ module Necropsy
       def score_for(node, classification)
         reasons = []
         score = base_score(classification)
+        components = [score_component("base(#{classification})", score, 'Base classification score')]
 
         if graph.uncertainties(node.id).any?
           score -= 0.35
+          components << score_component('near_metaprogramming', -0.35, 'Unresolved metaprogramming nearby')
           reasons << 'Lowered because this node is near unresolved metaprogramming.'
         end
 
         if graph.class_info(node.owner)&.dynamic
           score -= 0.25
+          components << score_component('dynamic_owner', -0.25, 'Owner defines dynamic dispatch')
           reasons << 'Lowered because the owner class defines dynamic dispatch.'
         end
 
         if generated_accessor?(node)
           score -= 0.2
+          components << score_component('generated_accessor', -0.2, 'Generated by an accessor DSL')
           reasons << 'Lowered because this node is generated by a Ruby accessor DSL.'
         end
 
-        static_count = graph.profiles.count { |profile| profile.kind == :static }
-        if static_count > 1
-          score += [static_count - 1, 3].min * 0.04
-          reasons << "Raised because #{static_count} static analyzers participated."
-        end
-
-        evidence_analyzers = graph.incoming_edges(node.id).flat_map(&:evidences).map(&:analyzer).uniq
-        if evidence_analyzers.length > 1
-          score += 0.08
-          reasons << 'Raised because multiple analyzers agree on incoming reachability evidence.'
+        if (reason = implicit_call_context(node))
+          score -= 0.4
+          components << score_component('implicit_caller', -0.4, reason)
+          reasons << "Lowered because #{reason}."
         end
 
         if classification == :unreachable && graph.dynamic_enabled? && !graph.dynamic_alive?(node.id)
           score += 0.25
+          components << score_component('absent_from_dynamic', 0.25, 'Absent from dynamic observations')
           reasons << 'Static unreachable and absent from dynamic observations.'
         end
 
         if classification == :unreachable && quarantine_expired?(node)
-          score = [score, 0.95].max
+          raised_score = [score, 0.95].max
+          components << score_component('expired_quarantine', raised_score - score, 'Expired without alive evidence')
+          score = raised_score
           reasons << 'Raised because quarantine annotation has expired without alive evidence.'
         end
 
@@ -91,20 +112,26 @@ module Necropsy
           days = observation_days
           if days && days < project.config.min_observation_days
             score -= 0.3
+            components << score_component('short_observation', -0.3, "Observed for only #{days} days")
             reasons << "Observation window is #{days} days, below the configured minimum."
           else
             reasons << 'Reachable statically but never observed dynamically.'
           end
         end
 
-        score = [[score, 0.0].max, 1.0].min
-        [score, level_for(score), reasons]
+        clamped_score = score.clamp(0.0, 1.0)
+        components << score_component('score_clamp', clamped_score - score, 'Clamped to the 0.0–1.0 range') if clamped_score != score
+        [clamped_score, level_for(clamped_score), reasons, components]
+      end
+
+      def score_component(name, value, details)
+        ScoreComponent.new(name: name, value: value.round(4), details: details)
       end
 
       def base_score(classification)
         case classification
         when :unreachable
-          graph.profiles.any? { |profile| profile.name == :rta } ? 0.78 : 0.62
+          0.62
         when :test_only_reachable
           0.55
         when :unused
@@ -118,6 +145,56 @@ module Necropsy
         %i[attr_reader attr_writer attr_accessor struct_new data_define].include?(node.defined_via)
       end
 
+      def implicit_call_reason(node)
+        return "#{node.name} is a Ruby hook invoked by the VM" if RUBY_HOOKS.include?(node.name)
+        return "#{node.name} is a Ruby protocol method invoked implicitly" if RUBY_PROTOCOLS.include?(node.name)
+
+        rule = implicit_caller_rules.find do |candidate|
+          candidate[:name_pattern].match?(node.name) && owner_matches?(node.owner, candidate[:owner_ancestors])
+        end
+        return unless rule
+
+        rule[:reason] || 'the method matches a configured implicit caller'
+      end
+
+      def implicit_call_context(node)
+        direct_reason = implicit_call_reason(node)
+        return direct_reason if direct_reason
+
+        root_id = implicit_paths[node.id]
+        "it is reachable from implicitly invoked #{root_id}" if root_id
+      end
+
+      def implicit_paths
+        @implicit_paths ||= begin
+          roots = graph.method_nodes.reject(&:test).select { |node| implicit_call_reason(node) }.map(&:id)
+          paths = roots.to_h { |node_id| [node_id, node_id] }
+          queue = roots.dup
+
+          until queue.empty?
+            node_id = queue.shift
+            graph.edges_from(node_id).each_key do |callee_id|
+              next if paths.key?(callee_id)
+
+              paths[callee_id] = paths.fetch(node_id)
+              queue << callee_id
+            end
+          end
+          paths
+        end
+      end
+
+      def implicit_caller_rules
+        BUILTIN_IMPLICIT_CALLERS + project.config.implicit_callers
+      end
+
+      def owner_matches?(owner, ancestors)
+        return true if ancestors.empty?
+        return false unless owner
+
+        ancestors.any? { |ancestor| graph.owner_reachable_from_ancestor?(owner, ancestor) }
+      end
+
       def level_for(score)
         return :certain if score >= 0.9
         return :high if score >= 0.7
@@ -145,7 +222,8 @@ module Necropsy
         path = File.join(project.root, node.file)
         return nil unless File.exist?(path)
 
-        window = File.readlines(path, chomp: true)[[node.line - 4, 0].max, 4] || []
+        lines = @source_lines[path] ||= File.readlines(path, chomp: true)
+        window = lines[[node.line - 4, 0].max, 4] || []
         annotation = window.find { |line| line.include?('necropsy:quarantine') && line.match?(/\bsince=/) }
         return nil unless annotation