necropsy 0.1.0 → 0.2.0

data/lib/necropsy/entry_points/rails.rb

@@ -1,26 +1,40 @@
 # frozen_string_literal: true
 
+require 'prism'
+
 module Necropsy
   module EntryPoints
     class Rails
       ROUTE_VERBS = %w[get post put patch delete match].freeze
       RESTFUL_ACTIONS = %w[index show new create edit update destroy].freeze
       SINGULAR_ACTIONS = %w[show new create edit update destroy].freeze
+      IRREGULAR_PLURALS = { 'person' => 'people', 'man' => 'men', 'woman' => 'women', 'child' => 'children' }.freeze
       RouteContext = Struct.new(:modules, :resource, :controller, keyword_init: true)
 
       def apply(graph, project)
         return unless project.config.rails_enabled?
 
+        referenced_view_methods = view_method_names(project)
         graph.method_nodes.each do |node|
           case node.file
           when %r{\Aapp/jobs/}
             graph.add_entry_point(node.id, :job_perform) if node.name == 'perform'
           when %r{\Aapp/mailers/}
-            graph.add_entry_point(node.id, :mailer_action) if node.kind == :instance_method
+            if node.kind == :instance_method && node.visibility == :public
+              graph.add_entry_point(node.id,
+                                    :mailer_action)
+            end
           when %r{\Aapp/helpers/}
-            graph.add_entry_point(node.id, :rails_view_helper) if helper_referenced?(project, node.name)
+            graph.add_entry_point(node.id, :rails_view_helper) if referenced_view_methods.include?(node.name)
           when %r{\Aapp/components/}
             graph.add_entry_point(node.id, :rails_component) if component_entrypoint?(node)
+          when %r{\Adb/migrate/}
+            graph.add_entry_point(node.id, :rails_migration) if %w[change up down].include?(node.name)
+          end
+
+          if node.file.start_with?('app/') && !node.file.start_with?('app/helpers/', 'app/components/') &&
+             referenced_view_methods.include?(node.name)
+            graph.add_entry_point(node.id, :rails_view_reference)
           end
         end
 
@@ -50,49 +64,76 @@ module Necropsy
         return [] if seen[expanded]
 
         seen[expanded] = true
-        parse_routes(File.readlines(expanded), root: root, seen: seen, concerns: concerns)
+        source = File.read(expanded)
+        result = Prism.parse(source)
+        return [] if result.failure?
+
+        parse_route_statements(
+          result.value.statements,
+          source: source,
+          root: root,
+          seen: seen,
+          concerns: concerns,
+          context: RouteContext.new(modules: [], resource: nil)
+        )
+      rescue SystemCallError, EncodingError
+        []
       end
 
-      def parse_routes(lines, root:, seen:, concerns:, initial_context: RouteContext.new(modules: [], resource: nil))
-        contexts = [initial_context]
-        capture = nil
-        targets = []
+      def parse_route_statements(statements, source:, root:, seen:, concerns:, context:)
+        Array(statements&.body).flat_map do |statement|
+          parse_route_statement(statement, source: source, root: root, seen: seen, concerns: concerns,
+                                           context: context)
+        end.compact.uniq
+      end
 
-        lines.each do |line|
-          stripped = strip_route_comment(line).strip
-          next if stripped.empty?
+      def parse_route_statement(statement, source:, root:, seen:, concerns:, context:)
+        unless statement.is_a?(Prism::CallNode)
+          return statement.child_nodes.compact.flat_map do |child|
+            parse_route_statement(child, source: source, root: root, seen: seen, concerns: concerns, context: context)
+          end
+        end
 
-          if capture
-            capture[:depth] += block_openings(stripped)
-            capture[:depth] -= 1 if stripped == 'end'
+        call_source = route_call_source(statement, source)
+        if statement.name == :concern && statement.block
+          name = literal_route_argument(statement.arguments&.arguments&.first)
+          concerns[name] = [statement.block.body, source] if name
+          return []
+        end
 
-            if capture[:depth].zero?
-              concerns[capture[:name]] = capture[:lines]
-              capture = nil
-            else
-              capture[:lines] << stripped
-            end
-            next
-          end
+        targets = route_targets(call_source, context)
+        targets.concat(route_file_targets(call_source, root, seen, concerns))
+        targets.concat(concern_targets(call_source, context, root, seen, concerns))
+        return targets unless statement.block
+
+        child_context = nested_route_context(call_source, context)
+        targets.concat(
+          parse_route_statements(
+            statement.block.body,
+            source: source,
+            root: root,
+            seen: seen,
+            concerns: concerns,
+            context: child_context
+          )
+        )
+      end
 
-          if (match = stripped.match(/\bconcern\s+:([a-zA-Z_]\w*)\s+do\b/))
-            capture = { name: match[1], depth: 1, lines: [] }
-            next
-          end
+      def route_call_source(node, source)
+        finish = node.block ? node.block.opening_loc.start_offset : node.location.end_offset
+        source.byteslice(node.location.start_offset...finish).gsub(/\s+/, ' ').strip
+      end
 
-          if stripped == 'end'
-            contexts.pop if contexts.length > 1
-            next
-          end
+      def literal_route_argument(node)
+        return unless node.is_a?(Prism::SymbolNode) || node.is_a?(Prism::StringNode)
 
-          context = contexts.last
-          targets.concat(route_targets(stripped, context))
-          targets.concat(route_file_targets(stripped, root, seen, concerns))
-          targets.concat(concern_targets(stripped, context, root, seen, concerns))
-          push_context(contexts, stripped, context)
-        end
+        node.unescaped.to_s
+      end
 
-        targets.compact.uniq
+      def nested_route_context(line, context)
+        contexts = [context]
+        push_context(contexts, line, context)
+        contexts.last
       end
 
       def route_targets(line, context)
@@ -152,21 +193,15 @@ module Necropsy
         elsif (match = line.match(/\bscope\b.*\bmodule:\s+:?["']?([a-zA-Z_]\w*)/))
           contexts << RouteContext.new(modules: context.modules + [match[1]], resource: context.resource,
                                        controller: scoped_controller_option(line, context.controller))
-        elsif (match = line.match(%r{\bcontroller\s+:?["']?([a-zA-Z_][\w/]*)["']?\s+do\b}))
+        elsif (match = line.match(%r{\bcontroller\s+:?["']?([a-zA-Z_][\w/]*)["']?}))
           contexts << RouteContext.new(modules: context.modules, resource: context.resource, controller: match[1])
-        elsif line.match?(/\bscope\b.*\bcontroller:\s*.*do\b/)
-          contexts << RouteContext.new(modules: context.modules, resource: context.resource,
-                                       controller: scoped_controller_option(line, context.controller))
-        elsif (match = line.match(/\bresources\s+:([a-zA-Z_]\w*).*do\b/))
+        elsif (match = line.match(/\bresources\s+:([a-zA-Z_]\w*)/))
           contexts << RouteContext.new(modules: context.modules, resource: resource_controller(line, match[1]),
                                        controller: context.controller)
-        elsif (match = line.match(/\bresource\s+:([a-zA-Z_]\w*).*do\b/))
+        elsif (match = line.match(/\bresource\s+:([a-zA-Z_]\w*)/))
           contexts << RouteContext.new(modules: context.modules,
                                        resource: resource_controller(line, pluralize(match[1])), controller: context.controller)
-        elsif line.match?(/\b(?:member|collection)\s+do\b/)
-          contexts << RouteContext.new(modules: context.modules, resource: context.resource,
-                                       controller: context.controller)
-        elsif line.match?(/\b(?:constraints|defaults|scope)\b.*do\b/)
+        elsif line.match?(/\b(?:member|collection|constraints|defaults|scope)\b/)
           contexts << RouteContext.new(modules: context.modules, resource: context.resource,
                                        controller: scoped_controller_option(line, context.controller))
         end
@@ -181,8 +216,17 @@ module Necropsy
       def concern_targets(line, context, root, seen, concerns)
         target_context = concern_context(line, context)
         concern_names(line).flat_map do |name|
-          parse_routes(concerns.fetch(name, []), root: root, seen: seen, concerns: concerns,
-                                                 initial_context: target_context)
+          statements, source = concerns[name]
+          next [] unless statements && source
+
+          parse_route_statements(
+            statements,
+            source: source,
+            root: root,
+            seen: seen,
+            concerns: concerns,
+            context: target_context
+          )
         end
       end
 
@@ -219,7 +263,7 @@ module Necropsy
       def action_option(line, name)
         array_value = line[/\b#{name}:\s*(?:\[([^\]]+)\]|%i\[([^\]]+)\])/, 1] ||
                       line[/\b#{name}:\s*(?:\[([^\]]+)\]|%i\[([^\]]+)\])/, 2]
-        return array_value.scan(/:?["']?([a-zA-Z_]\w*)["']?/) if array_value
+        return array_value.scan(/:?["']?([a-zA-Z_]\w*)["']?/).flatten if array_value
 
         Array(line[/\b#{name}:\s+:?["']?([a-zA-Z_]\w*)/, 1])
       end
@@ -257,8 +301,9 @@ module Necropsy
       end
 
       def pluralize(name)
+        return IRREGULAR_PLURALS.fetch(name) if IRREGULAR_PLURALS.key?(name)
         return "#{name}es" if name.end_with?('s', 'x', 'z', 'ch', 'sh')
-        return "#{name.delete_suffix('y')}ies" if name.end_with?('y')
+        return "#{name.delete_suffix('y')}ies" if name.match?(/[^aeiou]y\z/)
 
         "#{name}s"
       end
@@ -269,17 +314,20 @@ module Necropsy
 
       def matching_route_nodes(graph, node_id)
         return [node_id] if graph.nodes.key?(node_id)
+        return [] unless node_id.include?('Controller#')
+
+        controller, action = node_id.split('#', 2)
+        expected_file = "app/controllers/#{underscore(controller.delete_suffix('Controller'))}_controller.rb"
+        graph.method_nodes.filter_map do |node|
+          next unless node.name == action && node.owner&.end_with?(controller)
+          next unless node.file == expected_file
 
-        suffix = "::#{node_id}"
-        graph.nodes.keys.select { |candidate| candidate.end_with?(suffix) }
+          node.id
+        end
       end
 
       def helper_referenced?(project, method_name)
-        view_files(project).any? do |path|
-          view_source(path).match?(/\b#{Regexp.escape(method_name)}\b/)
-        end
-      rescue SystemCallError, EncodingError
-        false
+        view_method_names(project).include?(method_name)
       end
 
       def view_files(project)
@@ -293,6 +341,17 @@ module Necropsy
             .lines.reject { |line| line.strip.start_with?('-#') }.join
       end
 
+      def view_method_names(project)
+        @view_method_names ||= {}
+        @view_method_names[project.root] ||= view_files(project).each_with_object(Set.new) do |path, names|
+          view_source(path).scan(/(?<![A-Za-z0-9_])([a-zA-Z_]\w*[!?=]?)(?![A-Za-z0-9_])/).each do |match|
+            names << match.first
+          end
+        rescue SystemCallError, EncodingError
+          next
+        end
+      end
+
       def component_entrypoint?(node)
         node.kind == :instance_method && %w[call render? before_render].include?(node.name)
       end
@@ -301,34 +360,11 @@ module Necropsy
         path.split('/').map { |part| part.split('_').map(&:capitalize).join }.join('::')
       end
 
-      def block_openings(line)
-        line.scan(/\bdo\b/).length
-      end
-
-      def strip_route_comment(line)
-        quote = nil
-        escaped = false
-        line.each_char.with_index do |char, index|
-          if escaped
-            escaped = false
-            next
-          end
-
-          if char == '\\'
-            escaped = true
-            next
-          end
-
-          if quote
-            quote = nil if char == quote
-            next
-          end
-
-          quote = char if ["'", '"'].include?(char)
-          return line[0...index] if char == '#'
-        end
-
-        line
+      def underscore(constant)
+        constant.gsub('::', '/')
+                .gsub(/([A-Z]+)([A-Z][a-z])/, '\\1_\\2')
+                .gsub(/([a-z\d])([A-Z])/, '\\1_\\2')
+                .downcase
       end
     end
   end

data/lib/necropsy/graph/call_graph.rb

@@ -5,24 +5,36 @@ module Necropsy
     attr_reader :nodes, :call_sites, :instantiated_classes, :entry_points, :profiles, :observation, :class_infos,
                 :entrypoint_hints
 
-    def initialize(scan_result)
+    def initialize(scan_result, ambiguity_limit: 4)
       @nodes = {}
-      @edges = Hash.new { |hash, key| hash[key] = {} }
+      @edges = {}
+      @incoming_edges = {}
       @call_sites = scan_result.call_sites
       @instantiated_classes = scan_result.instantiated_classes.dup
       @class_infos = scan_result.class_infos.to_h { |info| [info.id, info] }
       @entrypoint_hints = scan_result.entrypoint_hints
+      @ambiguity_limit = ambiguity_limit
       @entry_points = []
       @profiles = []
-      @uncertainties = scan_result.uncertainties
+      @uncertainties = scan_result.uncertainties.to_h do |node_id, messages|
+        [node_id, Array(messages).dup]
+      end
       @dynamic_alive = {}
       @observation = {}
+      @descendants = {}
       scan_result.nodes.each { |node| add_node(node) }
       retain_known_instantiated_classes
     end
 
     def add_node(node)
-      nodes[node.id] ||= node
+      return nodes[node.id] if nodes.key?(node.id)
+
+      @method_nodes = nil
+      @nodes_by_name = nil
+      @dispatch_cache = nil
+      @lookup_chain_cache = nil
+      @owner_ancestor_cache = nil
+      nodes[node.id] = node
     end
 
     def add_entry_point(node_id, reason)
@@ -34,8 +46,12 @@ module Necropsy
 
     def apply_result(result)
       result.edge_evidences.each { |edge| add_edge(edge.caller_id, edge.callee_id, edge.evidence) }
-      result.alive_evidences.each { |alive| add_alive(alive.node_id, alive.evidence) }
-      result.uncertainties.each { |node_id, messages| uncertainties[node_id].concat(Array(messages)) }
+      matched_alive = result.alive_evidences.count { |alive| add_alive(alive.node_id, alive.evidence) }
+      warn_unmatched_dynamic_evidence(result.alive_evidences.length) if matched_alive.zero?
+      result.uncertainties.each do |node_id, messages|
+        @uncertainties[node_id] ||= []
+        @uncertainties[node_id].concat(Array(messages))
+      end
       observation.merge!(result.observation) { |_key, left, right| merge_observation(left, right) }
     end
 
@@ -46,15 +62,19 @@ module Necropsy
     def add_edge(caller_id, callee_id, evidence)
       return unless nodes.key?(caller_id) && nodes.key?(callee_id)
 
+      @edges[caller_id] ||= {}
       @edges[caller_id][callee_id] ||= []
       @edges[caller_id][callee_id] << evidence
+      @incoming_edges[callee_id] ||= {}
+      @incoming_edges[callee_id][caller_id] = @edges[caller_id][callee_id]
     end
 
     def add_alive(node_id, evidence)
-      return unless nodes.key?(node_id)
+      return false unless nodes.key?(node_id)
 
       @dynamic_alive[node_id] ||= []
       @dynamic_alive[node_id] << evidence
+      true
     end
 
     def dynamic_alive?(node_id)
@@ -66,11 +86,11 @@ module Necropsy
     end
 
     def dynamic_enabled?
-      @dynamic_alive.any? || observation.any?
+      @dynamic_alive.any?
     end
 
     def edges_from(node_id)
-      @edges[node_id] || {}
+      @edges.fetch(node_id, {})
     end
 
     def edges
@@ -82,17 +102,23 @@ module Necropsy
     end
 
     def incoming_edges(node_id)
-      edges.select { |edge| edge.callee_id == node_id }
+      @incoming_edges.fetch(node_id, {}).map do |caller_id, evidences|
+        Edge.new(caller_id: caller_id, callee_id: node_id, evidences: evidences)
+      end
     end
 
     def uncertainties(node_id = nil)
       return @uncertainties unless node_id
 
-      @uncertainties[node_id] || []
+      @uncertainties.fetch(node_id, [])
     end
 
     def method_nodes
-      nodes.values.select(&:method?)
+      @method_nodes ||= nodes.values.select(&:method?)
+    end
+
+    def nodes_by_name
+      @nodes_by_name ||= method_nodes.group_by(&:name).freeze
     end
 
     def class_info(owner)
@@ -100,7 +126,9 @@ module Necropsy
     end
 
     def descendants_of(owner)
-      class_infos.keys.select { |candidate| candidate == owner || ancestor_chain(candidate).include?(owner) }
+      @descendants[owner] ||= class_infos.keys.select do |candidate|
+        candidate == owner || ancestor_chain(candidate).include?(owner)
+      end
     end
 
     def modules_for(owner)
@@ -111,15 +139,83 @@ module Necropsy
     end
 
     def candidate_nodes(message)
-      method_nodes.select { |node| node.name == message }
+      nodes_by_name.fetch(message, [])
+    end
+
+    def ambiguous_fallback_candidates(message)
+      candidates = candidate_nodes(message)
+      return candidates if candidates.one?
+      return [] if candidates.size > @ambiguity_limit
+
+      candidates
+    end
+
+    def ambiguous_resolution?
+      @ambiguity_limit > 1
+    end
+
+    def owner_reachable_from_ancestor?(owner, ancestor)
+      @owner_ancestor_cache ||= {}
+      key = [owner, ancestor]
+      return @owner_ancestor_cache[key] if @owner_ancestor_cache.key?(key)
+
+      @owner_ancestor_cache[key] = descendants_of(ancestor).any? do |descendant|
+        cached_lookup_chain(descendant).include?(owner)
+      end
     end
 
     def resolve_call_site(site, rta: false)
-      candidates = candidates_for_receiver(site)
+      candidates = rta ? rta_candidates_for_receiver(site) : candidates_for_receiver(site)
       candidates = candidates.select { |node| rta_candidate?(node, site) } if rta
       candidates
     end
 
+    def retain_rta_candidates(candidates, site)
+      candidates.select { |node| rta_candidate?(node, site) }
+    end
+
+    def reconcile_rta_result(result)
+      analyzed_sites = result.observation.dig('rta', 'analyzed_sites')
+      return unless analyzed_sites
+
+      analyzed_keys = analyzed_sites.to_set { |site| call_site_key(site) }
+      allowed = result.edge_evidences.each_with_object(Hash.new { |hash, key| hash[key] = Set.new }) do |edge, memo|
+        memo[call_site_key(edge.evidence.metadata)] << edge.callee_id
+      end
+
+      @edges.each_value do |callees|
+        callees.each do |callee_id, evidences|
+          evidences.reject! do |item|
+            next false unless %i[name_resolution cha].include?(item.analyzer)
+
+            key = call_site_key(item.metadata)
+            analyzed_keys.include?(key) && !allowed[key].include?(callee_id)
+          end
+        end
+        callees.delete_if { |_callee_id, evidences| evidences.empty? }
+      end
+      @edges.delete_if { |_caller_id, callees| callees.empty? }
+      rebuild_incoming_edges
+    end
+
+    def fallback_resolution?(site, resolved: nil)
+      resolved ||= resolve_call_site(site)
+      return false if resolved.empty?
+
+      case site.receiver_kind
+      when :constant
+        receiver_candidates(site).none? { |name| nodes.key?("#{name}.#{site.message}") }
+      when :instance
+        receiver_candidates(site).none? { |name| nodes.key?("#{name}##{site.message}") }
+      when :implicit
+        same_owner_candidates(site).empty?
+      when :unknown
+        true
+      else
+        false
+      end
+    end
+
     def to_h
       {
         'nodes' => nodes.values.map(&:to_h),
@@ -138,19 +234,30 @@ module Necropsy
       case site.receiver_kind
       when :constant
         exact = receiver_candidates(site).filter_map { |name| nodes["#{name}.#{site.message}"] }.first
-        exact ? [exact] : candidate_nodes(site.message)
+        exact ? [exact] : ambiguous_fallback_candidates(site.message)
       when :instance
         exact = receiver_candidates(site).filter_map { |name| nodes["#{name}##{site.message}"] }.first
-        exact ? [exact] : candidate_nodes(site.message)
+        exact ? [exact] : ambiguous_fallback_candidates(site.message)
       when :self
         same_owner_candidates(site)
+      when :super
+        super_candidates(site)
       when :implicit
-        same_owner_candidates(site).then { |matches| matches.empty? ? candidate_nodes(site.message) : matches }
+        same_owner_candidates(site).then do |matches|
+          matches.empty? ? ambiguous_fallback_candidates(site.message) : matches
+        end
       else
-        candidate_nodes(site.message)
+        ambiguous_fallback_candidates(site.message)
       end
     end
 
+    def rta_candidates_for_receiver(site)
+      exact = candidates_for_receiver(site)
+      return exact unless exact.empty?
+
+      candidate_nodes(site.message)
+    end
+
     def same_owner_candidates(site)
       caller = nodes[site.caller_id]
       return [] unless caller&.owner
@@ -162,6 +269,21 @@ module Necropsy
       ids.filter_map { |id| nodes[id] }
     end
 
+    def super_candidates(site)
+      caller = nodes[site.caller_id]
+      return [] unless caller&.owner
+
+      separator = caller.kind == :singleton_method ? '.' : '#'
+      owner = class_info(caller.owner)&.superclass
+      while owner
+        candidate = nodes["#{owner}#{separator}#{site.message}"]
+        return [candidate] if candidate
+
+        owner = class_info(owner)&.superclass
+      end
+      []
+    end
+
     def receiver_candidates(site)
       candidates = site.metadata['receiver_candidates'] || site.metadata[:receiver_candidates]
       Array(candidates).compact.empty? ? [site.receiver_name].compact : Array(candidates).compact
@@ -169,10 +291,39 @@ module Necropsy
 
     def rta_candidate?(node, site)
       return true unless node.kind == :instance_method
-      return true if node.owner == nodes[site.caller_id]&.owner
+
+      caller_owner = nodes[site.caller_id]&.owner
+      return true if site.receiver_kind == :super
+      return dispatched_instance_owner(caller_owner, site.message) == node.owner if %i[self implicit].include?(site.receiver_kind)
+
+      return true if node.owner == caller_owner
       return true if class_info(node.owner)&.dynamic
 
-      instantiated_classes.include?(node.owner)
+      instantiated_classes.any? { |owner| dispatched_instance_owner(owner, site.message) == node.owner }
+    end
+
+    def dispatched_instance_owner(owner, message)
+      @dispatch_cache ||= {}
+      key = [owner, message]
+      return @dispatch_cache[key] if @dispatch_cache.key?(key)
+
+      @dispatch_cache[key] = cached_lookup_chain(owner).find { |candidate| nodes.key?("#{candidate}##{message}") }
+    end
+
+    def cached_lookup_chain(owner)
+      @lookup_chain_cache ||= {}
+      @lookup_chain_cache[owner] ||= method_lookup_chain(owner)
+    end
+
+    def method_lookup_chain(owner, seen = Set.new)
+      return [] unless owner && seen.add?(owner)
+
+      info = class_info(owner)
+      return [owner] unless info
+
+      prepends = info.prepends.reverse.flat_map { |name| method_lookup_chain(name, seen) }
+      includes = info.includes.reverse.flat_map { |name| method_lookup_chain(name, seen) }
+      prepends + [owner] + includes + method_lookup_chain(info.superclass, seen)
     end
 
     def ancestor_chain(owner)
@@ -191,9 +342,40 @@ module Necropsy
       left.merge(right)
     end
 
+    def call_site_key(site)
+      metadata = site['metadata'] || site[:metadata] || {}
+      [
+        site['caller_id'] || site[:caller_id],
+        site['message'] || site[:message],
+        (site['receiver_kind'] || site[:receiver_kind])&.to_s,
+        site['receiver_name'] || site[:receiver_name],
+        site['file'] || site[:file],
+        site['line'] || site[:line],
+        Array(metadata['receiver_candidates'] || metadata[:receiver_candidates]).sort,
+        metadata['implicit_from'] || metadata[:implicit_from]
+      ]
+    end
+
+    def rebuild_incoming_edges
+      @incoming_edges = {}
+      @edges.each do |caller_id, callees|
+        callees.each do |callee_id, evidences|
+          @incoming_edges[callee_id] ||= {}
+          @incoming_edges[callee_id][caller_id] = evidences
+        end
+      end
+    end
+
     def retain_known_instantiated_classes
       known_owners = method_nodes.map(&:owner).compact.to_set
       instantiated_classes.select! { |name| known_owners.include?(name) }
     end
+
+    def warn_unmatched_dynamic_evidence(attempted)
+      return if attempted.zero?
+
+      warn "Necropsy ignored #{attempted} dynamic node IDs because none matched the scanned project; " \
+           'dynamic absence will not be used for unused classification.'
+    end
   end
 end