necropsy 0.2.0 → 0.3.0

data/lib/necropsy/analyzers/dynamic/trace_point_collector.rb

@@ -4,50 +4,60 @@ require 'securerandom'
 require 'fileutils'
 require 'time'
 require 'yaml'
+require_relative '../../clock'
+require_relative 'runtime_reference'
+require_relative 'observation_policy'
 
 module Necropsy
   module Analyzers
     module Dynamic
       class TracePointCollector
-        def self.record(root:, output:, sample_rate: 1.0, &)
-          new(root: root, output: output, sample_rate: sample_rate).record(&)
+        def self.record(root:, output:, sample_rate: 1.0, clock: nil, random: nil, &)
+          new(root: root, output: output, sample_rate: sample_rate, clock: clock, random: random).record(&)
         end
 
-        def self.install_at_exit(root:, output:, sample_rate: 1.0, merge: false, run_id: nil)
-          new(root: root, output: output, sample_rate: sample_rate, merge: merge, run_id: run_id).install_at_exit
+        def self.install_at_exit(root:, output:, sample_rate: 1.0, merge: false, run_id: nil, clock: nil, random: nil)
+          new(
+            root: root, output: output, sample_rate: sample_rate, merge: merge, run_id: run_id,
+            clock: clock, random: random
+          ).install_at_exit
         end
 
-        def initialize(root:, output:, sample_rate: 1.0, merge: false, run_id: nil)
+        def initialize(root:, output:, sample_rate: 1.0, merge: false, run_id: nil, clock: nil, random: nil)
           @root = File.expand_path(root)
           @output = output
           @sample_rate = sample_rate.to_f
           raise Error, 'sample_rate must be between 0.0 and 1.0' unless @sample_rate.between?(0.0, 1.0)
 
           @nodes = {}
+          @node_references = {}
           @edges = {}
+          @edge_references = {}
           @stacks = {}.compare_by_identity
           @lock = Mutex.new
           @merge = merge
           @run_id = run_id
+          @clock = clock || -> { Clock.new.time }
+          @random = random || default_random
         end
 
         def record
-          started_at = Time.now.utc
+          started_at = current_time
           tracer = TracePoint.new(:call, :return) { |event| capture(event) }
           tracer.enable
           yield
         ensure
           tracer&.disable
-          write_payload(started_at: started_at, finished_at: Time.now.utc)
+          write_payload(started_at: started_at, finished_at: current_time)
         end
 
         def install_at_exit
-          started_at = Time.now.utc
+          started_at = current_time
           tracer = TracePoint.new(:call, :return) { |event| capture(event) }
           tracer.enable
           at_exit do
             tracer.disable
-            write_payload(started_at: started_at, finished_at: Time.now.utc)
+            write_payload(started_at: started_at, finished_at: current_time)
           rescue StandardError => e
             warn "Necropsy TracePoint collector failed: #{e.message}"
           end
@@ -55,7 +65,8 @@ module Necropsy
 
         private
 
-        attr_reader :root, :output, :sample_rate, :nodes, :edges, :stacks, :lock, :run_id
+        attr_reader :root, :output, :sample_rate, :nodes, :node_references, :edges, :edge_references, :stacks, :lock,
+                    :run_id, :clock, :random
 
         def merge?
           @merge
@@ -64,34 +75,52 @@ module Necropsy
         def capture(event)
           return unless project_path?(event.path)
 
-          node_id = node_id_for(event)
-          return unless node_id
+          reference = node_reference_for(event)
+          return unless reference
 
           lock.synchronize do
             stack = stacks[Thread.current] ||= []
             if event.event == :return
-              unwind_stack(stack, node_id)
+              unwind_stack(stack, reference)
               stacks.delete(Thread.current) if stack.empty?
               return
             end
 
-            sampled = sample_rate >= 1.0 || (sample_rate.positive? && SecureRandom.random_number < sample_rate)
-            caller_id = stack.last&.first if stack.last&.last
+            sampled = sample_rate >= 1.0 || (sample_rate.positive? && random.call < sample_rate)
+            caller_reference = stack.last&.first if stack.last&.last
             if sampled
-              nodes[node_id] = true
-              edges[[caller_id, node_id]] = true if caller_id
+              record_node(reference)
+              record_edge(caller_reference, reference) if caller_reference
             end
-            stack << [node_id, sampled]
+            stack << [reference, sampled]
           end
         end
 
-        def unwind_stack(stack, node_id)
-          index = stack.rindex { |frame| frame.first == node_id }
+        def current_time
+          clock.call.utc
+        end
+
+        def default_random
+          epoch = ENV.fetch(Clock::SOURCE_DATE_EPOCH, nil)
+          return SecureRandom.method(:random_number) if epoch.nil? || epoch.empty?
+
+          Random.new(Integer(epoch)).method(:rand)
+        rescue ArgumentError
+          SecureRandom.method(:random_number)
+        end
+
+        def unwind_stack(stack, reference)
+          reference_key = invocation_key(reference)
+          index = stack.rindex { |frame| invocation_key(frame.first) == reference_key }
           return unless index
 
           stack.slice!(index..)
         end
 
+        def invocation_key(reference)
+          reference.values_at('definition_id', 'symbol_id', 'file')
+        end
+
         def project_path?(path)
           return false unless path
 
@@ -105,6 +134,28 @@ module Necropsy
           "#{owner}#{separator}#{event.method_id}"
         end
 
+        def node_reference_for(event)
+          symbol_id = node_id_for(event)
+          return unless symbol_id
+
+          RuntimeReference.build(
+            symbol_id: symbol_id,
+            file: RuntimeReference.relative_file(root, event.path),
+            line: event.respond_to?(:lineno) ? event.lineno : nil
+          )
+        end
+
+        def record_node(reference)
+          nodes[reference.fetch('symbol_id')] = true
+          node_references[RuntimeReference.key(reference)] = reference
+        end
+
+        def record_edge(caller, callee)
+          edges[[caller.fetch('symbol_id'), callee.fetch('symbol_id')]] = true
+          key = [RuntimeReference.key(caller), RuntimeReference.key(callee)]
+          edge_references[key] = { 'caller_id' => caller.dup, 'callee_id' => callee.dup }
+        end
+
         def owner_and_separator(defined_class)
           name = defined_class.name
           return [name, '#'] if name
@@ -117,13 +168,24 @@ module Necropsy
 
         def write_payload(started_at:, finished_at:)
           payload = {
+            'schema_version' => 2,
+            'collector' => { 'name' => 'necropsy-trace-point', 'version' => Necropsy::VERSION },
+            'scope' => { 'sample_unit' => 'event', 'sample_rate' => sample_rate },
+            'quality' => { 'dropped_events' => 0, 'overflowed' => false },
             'nodes' => nodes.keys.sort,
-            'edges' => edges.keys.map { |caller_id, callee_id| { 'caller_id' => caller_id, 'callee_id' => callee_id } },
+            'node_references' => node_references.values.sort_by { |reference| RuntimeReference.sort_key(reference) },
+            'edges' => legacy_edges,
+            'edge_references' => structured_edges,
             'observation' => {
               'started_at' => started_at.iso8601,
               'finished_at' => finished_at.iso8601,
               'days' => [((finished_at - started_at) / 86_400.0).ceil, 1].max,
-              'sample_rate' => sample_rate
+              'sample_rate' => sample_rate,
+              'collector_overhead' => {
+                'wall_time_seconds' => [finished_at - started_at, 0].max.round(6),
+                'observed_nodes' => nodes.length,
+                'observed_edges' => edges.length
+              }
             }
           }
           payload['observation']['run_id'] = run_id if run_id
@@ -167,24 +229,72 @@ module Necropsy
           end
           left_observation = left.fetch('observation', {})
           right_observation = right.fetch('observation', {})
-          observation = left_observation.merge(right_observation)
+          observation = ObservationPolicy.compatible_merge(left_observation, right_observation)
           observation['started_at'] = [left_observation['started_at'], right_observation['started_at']].compact.min
           observation['finished_at'] = [left_observation['finished_at'], right_observation['finished_at']].compact.max
           observation['days'] = [left_observation['days'].to_i, right_observation['days'].to_i, 1].max
+          observation['collector_overhead'] = merge_overhead(left_observation, right_observation)
           observation['processes'] = process_count(left_observation) + process_count(right_observation)
           {
+            'schema_version' => [left['schema_version'], right['schema_version'], 2].compact.max,
+            'collector' => right['collector'] || left['collector'],
+            'scope' => right['scope'] || left['scope'],
+            'quality' => right['quality'] || left['quality'],
             'nodes' => (Array(left['nodes']) + Array(right['nodes'])).uniq.sort,
+            'node_references' => merge_node_references(left['node_references'], right['node_references']),
             'edges' => edges,
+            'edge_references' => merge_edge_references(left['edge_references'], right['edge_references']),
             'observation' => observation
           }
         end
 
+        def legacy_edges
+          edges.keys.sort.map { |caller_id, callee_id| { 'caller_id' => caller_id, 'callee_id' => callee_id } }
+        end
+
+        def structured_edges
+          edge_references.values.sort_by do |edge|
+            [RuntimeReference.sort_key(edge.fetch('caller_id')), RuntimeReference.sort_key(edge.fetch('callee_id'))]
+          end
+        end
+
+        def merge_node_references(left, right)
+          (Array(left) + Array(right))
+            .filter_map { |reference| RuntimeReference.normalize(reference) }
+            .uniq { |reference| RuntimeReference.key(reference) }
+            .sort_by { |reference| RuntimeReference.sort_key(reference) }
+        end
+
+        def merge_edge_references(left, right)
+          (Array(left) + Array(right)).filter_map do |edge|
+            next unless edge.is_a?(Hash)
+
+            caller = RuntimeReference.normalize(edge['caller_id'] || edge[:caller_id])
+            callee = RuntimeReference.normalize(edge['callee_id'] || edge[:callee_id])
+            { 'caller_id' => caller, 'callee_id' => callee } if caller && callee
+          end.uniq do |edge|
+            [RuntimeReference.key(edge.fetch('caller_id')), RuntimeReference.key(edge.fetch('callee_id'))]
+          end.sort_by do |edge|
+            [RuntimeReference.sort_key(edge.fetch('caller_id')), RuntimeReference.sort_key(edge.fetch('callee_id'))]
+          end
+        end
+
         def process_count(observation)
           return observation['processes'].to_i if observation['processes']
           return 0 if observation.empty?
 
           1
         end
+
+        def merge_overhead(left, right)
+          left_overhead = left.fetch('collector_overhead', {})
+          right_overhead = right.fetch('collector_overhead', {})
+          {
+            'wall_time_seconds' => left_overhead['wall_time_seconds'].to_f + right_overhead['wall_time_seconds'].to_f,
+            'observed_nodes' => left_overhead['observed_nodes'].to_i + right_overhead['observed_nodes'].to_i,
+            'observed_edges' => left_overhead['observed_edges'].to_i + right_overhead['observed_edges'].to_i
+          }
+        end
       end
     end
   end

data/lib/necropsy/analyzers/dynamic/trace_point_importer.rb

@@ -9,7 +9,9 @@ module Necropsy
             name: :trace_point,
             kind: :dynamic,
             soundness: :observational,
-            description: 'Imports method execution and caller-callee edges captured by Necropsy TracePoint recording.'
+            description: 'Imports method execution and caller-callee edges captured by Necropsy TracePoint recording.',
+            version: Necropsy::VERSION,
+            assumptions: %w[positive_observations_only tracepoint_call_events]
           )
         end
       end

data/lib/necropsy/analyzers/legacy_result_adapter.rb

@@ -0,0 +1,226 @@
+# frozen_string_literal: true
+
+module Necropsy
+  module Analyzers
+    class LegacyResultAdapter
+      MISSING = Object.new.freeze
+      private_constant :MISSING
+
+      def initialize(graph:, profile:)
+        @graph = graph
+        @profile = profile
+      end
+
+      def adapt(result)
+        return result unless legacy_static_result?(result)
+
+        edge_evidences = normalize_edge_evidences(result.edge_evidences)
+        alive_evidences = normalize_alive_evidences(result.alive_evidences)
+        wrapped_evidences = {}.compare_by_identity
+        result.edge_evidences.each { |edge| wrapped_evidences[edge.evidence] = true }
+        result.alive_evidences.each { |alive| wrapped_evidences[alive.evidence] = true }
+        evidences = result.evidences.reject { |record| wrapped_evidences.key?(record) }.map do |record|
+          normalize_evidence(
+            record,
+            relation: record.kind,
+            source: { 'type' => 'legacy_adapter', 'producer' => profile.name.to_s },
+            scope: {}
+          )
+        end
+        result.with(
+          edge_evidences: edge_evidences,
+          alive_evidences: alive_evidences,
+          resolutions: graph.call_sites.map { |site| resolution_for(site) },
+          evidences: EvidenceCollection.collect(evidences, edge_evidences, alive_evidences)
+        )
+      end
+
+      private
+
+      attr_reader :graph, :profile
+
+      def legacy_static_result?(result)
+        profile.kind.to_sym == :static && result.resolutions.nil?
+      end
+
+      def normalize_edge_evidences(edges)
+        @edges_by_call_site_id = Hash.new { |hash, key| hash[key] = [] }
+        edges.map do |edge|
+          site = matching_call_site(edge)
+          normalized = EdgeEvidence.new(
+            caller_id: edge.caller_id,
+            callee_id: edge.callee_id,
+            evidence: normalize_evidence(
+              edge.evidence,
+              relation: :call_edge,
+              source: legacy_edge_source(edge, site),
+              scope: legacy_edge_scope(edge, site)
+            )
+          )
+          @edges_by_call_site_id[site.call_site_id] << normalized if site
+          normalized
+        end
+      end
+
+      def normalize_alive_evidences(alive_evidences)
+        alive_evidences.map do |alive|
+          AliveEvidence.new(
+            node_id: alive.node_id,
+            evidence: normalize_evidence(
+              alive.evidence,
+              relation: :alive,
+              source: { 'type' => 'legacy_adapter', 'producer' => profile.name.to_s },
+              scope: { 'node_reference' => alive.node_id.to_s }
+            )
+          )
+        end
+      end
+
+      def normalize_evidence(record, relation:, source:, scope:)
+        normalized = if record.grade
+                       record
+                     else
+                       Evidence.new(
+                         analyzer: record.analyzer,
+                         kind: record.kind,
+                         weight: record.weight,
+                         details: record.details,
+                         metadata: record.metadata,
+                         producer: profile.name,
+                         producer_version: profile.version || 'unversioned',
+                         grade: legacy_evidence_grade,
+                         relation: relation,
+                         source: source,
+                         assumptions: profile.assumptions,
+                         scope: scope
+                       )
+                     end
+        return normalized if normalized.evidence_id
+
+        normalized.with(
+          evidence_id: EvidenceIdentity.generate(normalized.to_h.except('evidence_id'))
+        )
+      end
+
+      def legacy_evidence_grade
+        profile.soundness.to_sym == :conservative ? :conservative : :heuristic
+      end
+
+      def legacy_edge_source(edge, site)
+        {
+          'type' => 'legacy_adapter',
+          'producer' => profile.name.to_s,
+          'file' => site&.file,
+          'line' => site&.line,
+          'caller_reference' => edge.caller_id.to_s,
+          'callee_reference' => edge.callee_id.to_s
+        }.compact
+      end
+
+      def legacy_edge_scope(edge, site)
+        if site
+          return {
+            'call_site_id' => site.call_site_id,
+            'caller_definition_id' => site.caller_id,
+            'target_reference' => edge.callee_id.to_s
+          }
+        end
+
+        { 'caller_reference' => edge.caller_id.to_s, 'target_reference' => edge.callee_id.to_s }
+      end
+
+      def matching_call_site(edge)
+        metadata = edge.evidence.metadata
+        call_site_id = value(metadata, :call_site_id)
+        unless call_site_id.equal?(MISSING)
+          caller_id = physical_definition_id(edge.caller_id)
+          return unless caller_id
+
+          return unique_site do |site|
+            site.call_site_id == call_site_id.to_s && site.caller_id == caller_id
+          end
+        end
+
+        matching_legacy_call_site(edge, metadata)
+      end
+
+      def matching_legacy_call_site(edge, metadata)
+        caller_id = physical_definition_id(edge.caller_id)
+        return unless caller_id
+
+        metadata_caller = value(metadata, :caller_definition_id, :caller_id)
+        return if metadata_caller.equal?(MISSING)
+        return unless physical_definition_id(metadata_caller) == caller_id
+
+        signature = legacy_signature(metadata, caller_id)
+        return unless signature
+
+        unique_site do |site|
+          site.caller_id == caller_id && legacy_signature(site.to_h, caller_id) == signature
+        end
+      end
+
+      def legacy_signature(attributes, caller_id)
+        message = value(attributes, :message)
+        receiver_kind = value(attributes, :receiver_kind)
+        receiver_name = value(attributes, :receiver_name)
+        file = value(attributes, :file)
+        line = value(attributes, :line)
+        test = value(attributes, :test)
+        dynamic = value(attributes, :dynamic)
+        metadata = value(attributes, :metadata)
+        values = [message, receiver_kind, receiver_name, file, line, test, dynamic, metadata]
+        return if values.any? { |item| item.equal?(MISSING) }
+
+        CallSiteIdentity.legacy_id(
+          caller_definition_id: caller_id,
+          message: message,
+          receiver_kind: receiver_kind,
+          receiver_name: receiver_name,
+          file: file,
+          line: line,
+          test: test,
+          dynamic: dynamic,
+          metadata: metadata
+        )
+      end
+
+      def unique_site(&)
+        matches = graph.call_sites.select(&)
+        matches.first if matches.one?
+      end
+
+      def physical_definition_id(reference)
+        graph.nodes.lookup(reference.to_s).node&.graph_id
+      end
+
+      def resolution_for(site)
+        edges = @edges_by_call_site_id.fetch(site.call_site_id, [])
+        targets = edges.filter_map { |edge| physical_definition_id(edge.callee_id) }.uniq.sort
+        status = targets.empty? ? :unknown : :partial
+        ResolutionRecord.new(
+          resolution: Resolution.new(
+            call_site_id: site.call_site_id,
+            target_definition_ids: targets,
+            status: status,
+            unknown_scope: UnknownScope.new(scope_kind: :message, scope_value: site.message, match: :exact),
+            evidence_ids: edges.filter_map { |edge| edge.evidence.evidence_id }.uniq.sort
+          ),
+          producer: profile.name,
+          producer_version: profile.version || 'unversioned',
+          assumptions: profile.assumptions
+        )
+      end
+
+      def value(attributes, *keys)
+        return MISSING unless attributes.respond_to?(:key?)
+
+        keys.each do |key|
+          return attributes[key.to_s] if attributes.key?(key.to_s)
+          return attributes[key] if attributes.key?(key)
+        end
+        MISSING
+      end
+    end
+  end
+end

data/lib/necropsy/analyzers/static/cha.rb

@@ -5,25 +5,48 @@ module Necropsy
     module Static
       class CHA < Analyzer
         def analyze(graph, _project)
-          edge_evidences = graph.call_sites.flat_map do |site|
-            candidates(graph, site).map do |candidate|
+          edge_evidences = []
+          resolutions = graph.call_sites.map do |site|
+            lookup = graph.cha_method_lookup(site)
+            targets = lookup.targets
+            site_edges = targets.map do |candidate|
               EdgeEvidence.new(
                 caller_id: site.caller_id,
-                callee_id: candidate.id,
+                callee_id: candidate.graph_id,
                 evidence: evidence(
                   kind: :call_edge,
                   details: "CHA candidate at #{site.file}:#{site.line}",
-                  metadata: site.to_h
+                  metadata: site.to_h.merge('target_definition_id' => candidate.graph_id),
+                  grade: :conservative,
+                  relation: :call_edge,
+                  source: call_site_evidence_source(site).merge('target_definition_id' => candidate.graph_id),
+                  scope: call_site_evidence_scope(site).merge('target_definition_id' => candidate.graph_id)
                 )
               )
             end
+            edge_evidences.concat(site_edges)
+            status = if lookup.complete?
+                       :complete
+                     else
+                       (targets.empty? ? :unknown : :partial)
+                     end
+            resolution_record(
+              site,
+              targets,
+              site_edges,
+              status: status,
+              rejected_targets: lookup.complete? ? lookup.rejected_targets : [],
+              unknown_scope: graph.residual_scope_for(site)
+            )
           end
 
           AnalyzerResult.new(
             edge_evidences: edge_evidences,
             alive_evidences: [],
             uncertainties: {},
-            observation: {}
+            observation: {},
+            resolutions: resolutions,
+            evidences: result_evidences(edge_evidences)
           )
         end
 
@@ -32,77 +55,14 @@ module Necropsy
             name: :cha,
             kind: :static,
             soundness: :conservative,
-            description: 'Expands call targets through class hierarchy, descendants, and included/prepended modules.'
+            description: 'Expands call targets through class hierarchy, descendants, and included/prepended modules.',
+            version: Necropsy::VERSION,
+            assumptions: %w[closed_scanned_hierarchy loaded_ancestry]
           )
         end
 
-        def candidates(graph, site)
-          case site.receiver_kind
-          when :constant
-            constant_targets(graph, site)
-          when :instance
-            instance_targets(graph, site)
-          when :self, :implicit
-            self_targets(graph, site)
-          else
-            graph.resolve_call_site(site)
-          end.uniq(&:id)
-        end
-
-        private
-
-        def constant_targets(graph, site)
-          receiver_candidates(site).flat_map do |owner|
-            targets = owners_for_lookup(graph, owner, include_descendants: false).filter_map do |candidate_owner|
-              graph.nodes["#{candidate_owner}.#{site.message}"]
-            end
-            extended = Array(graph.class_info(owner)&.extends).filter_map do |mod|
-              graph.nodes["#{mod}##{site.message}"]
-            end
-            targets + extended
-          end
-        end
-
-        def instance_targets(graph, site)
-          receiver_candidates(site).flat_map do |owner|
-            owners_for_lookup(graph, owner, include_descendants: true).filter_map do |candidate_owner|
-              graph.nodes["#{candidate_owner}##{site.message}"]
-            end
-          end
-        end
-
-        def self_targets(graph, site)
-          caller = graph.nodes[site.caller_id]
-          return [] unless caller&.owner
-
-          separator = caller.kind == :singleton_method ? '.' : '#'
-          owners_for_lookup(graph, caller.owner, include_descendants: false).filter_map do |owner|
-            graph.nodes["#{owner}#{separator}#{site.message}"] || graph.nodes["#{owner}##{site.message}"]
-          end
-        end
-
-        def owners_for_lookup(graph, owner, include_descendants:)
-          owners = include_descendants ? graph.descendants_of(owner) : [owner]
-          owners.flat_map do |candidate|
-            info = graph.class_info(candidate)
-            modules = info ? info.prepends + info.includes + info.extends : []
-            [candidate, *modules, *ancestors(graph, candidate)]
-          end.uniq
-        end
-
-        def ancestors(graph, owner)
-          chain = []
-          current = graph.class_info(owner)&.superclass
-          while current && !chain.include?(current)
-            chain << current
-            current = graph.class_info(current)&.superclass
-          end
-          chain
-        end
-
-        def receiver_candidates(site)
-          candidates = site.metadata['receiver_candidates'] || site.metadata[:receiver_candidates]
-          Array(candidates).compact.empty? ? [site.receiver_name].compact : Array(candidates).compact
+        def capabilities
+          [:complete_resolution].freeze
         end
       end
     end