necropsy 0.2.0 → 0.3.0

data/lib/necropsy/diagnostics.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require 'json'
+require 'shellwords'
 
 module Necropsy
   class Diagnostics
@@ -13,34 +14,57 @@ module Necropsy
     end
 
     def why(node_id)
-      node = graph.nodes[node_id]
-      return missing_payload(node_id) unless node
+      lookup = graph.nodes.lookup(node_id)
+      return with_source_incompleteness(ambiguous_payload(lookup)) if lookup.ambiguous?
+      return with_source_incompleteness(missing_payload(node_id)) if lookup.missing?
 
-      runtime_path = reachability.witness(node_id)
-      return alive_payload(node, runtime_path, :runtime) if runtime_path
+      node = lookup.node
 
-      test_path = reachability.witness(node_id, kind: :test)
-      return alive_payload(node, test_path, :test) if test_path
+      graph_id = node.graph_id
+      runtime_path = reachability.witness(graph_id)
+      return with_source_incompleteness(alive_payload(node, runtime_path, :runtime)) if runtime_path
 
-      dead_payload(node)
+      external_path = reachability.witness(graph_id, kind: :external)
+      return with_source_incompleteness(alive_payload(node, external_path, :external)) if external_path
+
+      finding = finding_for(graph_id)
+      return with_source_incompleteness(dead_payload(node, finding)) if finding&.classification == :blocked
+
+      test_path = reachability.witness(graph_id, kind: :test)
+      return with_source_incompleteness(alive_payload(node, test_path, :test)) if test_path
+
+      with_source_incompleteness(dead_payload(node, finding))
     end
 
     def explain(node_id)
-      node = graph.nodes[node_id]
-      return missing_payload(node_id) unless node
+      lookup = graph.nodes.lookup(node_id)
+      return with_source_incompleteness(ambiguous_payload(lookup)) if lookup.ambiguous?
+      return with_source_incompleteness(missing_payload(node_id)) if lookup.missing?
 
-      finding = report.findings.find { |candidate| candidate.node.id == node_id }
-      return { 'status' => 'alive', 'node' => node.to_h } unless finding
+      node = lookup.node
 
-      {
+      finding = finding_for(node.graph_id)
+      return with_source_incompleteness({ 'status' => 'alive', 'node' => node.to_h }) unless finding
+
+      payload = {
         'status' => 'finding',
         'node' => node.to_h,
         'classification' => finding.classification.to_s,
         'confidence' => finding.confidence.to_s,
         'score' => finding.score,
         'components' => finding.score_components.map(&:to_h),
-        'reasons' => finding.reasons
+        'reasons' => finding.reasons,
+        'blockers' => finding.blockers.map(&:to_h)
       }
+      with_source_incompleteness(payload)
+    end
+
+    def why_not(node_id)
+      lookup = graph.nodes.lookup(node_id)
+      return with_source_incompleteness(ambiguous_payload(lookup)) if lookup.ambiguous?
+      return with_source_incompleteness(missing_payload(node_id)) if lookup.missing?
+
+      WhyNotExplanation.new(report).call(lookup.node)
     end
 
     def render(payload, format: :human)
@@ -71,39 +95,53 @@ module Necropsy
         'node' => node.to_h,
         'path' => path.each_with_index.map do |node_id, index|
           caller_id = index.positive? ? path[index - 1] : nil
-          path_step(node_id, caller_id)
+          path_step(node_id, caller_id, kind)
         end
       }
     end
 
-    def path_step(node_id, caller_id)
+    def path_step(node_id, caller_id, root_domain)
       step = { 'node' => graph.nodes.fetch(node_id).to_h }
-      entry = graph.entry_points.find { |candidate| candidate.node_id == node_id }
-      step['entry_reason'] = entry.reason.to_s if entry
+      roots = graph.entry_points.select do |candidate|
+        candidate.node_id == node_id && candidate.domain == root_domain
+      end.sort_by { |candidate| [candidate.reason.to_s, BoundedCanonicalizer.dump(candidate.evidence)] }
+      if roots.any?
+        step['entry_reason'] = roots.first.reason.to_s
+        step['root_domain'] = root_domain.to_s
+        step['root_provenance'] = roots.map(&:to_h)
+      end
       return step unless caller_id
 
       evidences = graph.edges_from(caller_id).fetch(node_id, [])
       step['edge'] = {
         'caller_id' => caller_id,
         'callee_id' => node_id,
+        'caller' => graph.nodes.fetch(caller_id).to_h,
+        'callee' => graph.nodes.fetch(node_id).to_h,
         'evidences' => evidences.map(&:to_h)
       }
       step
     end
 
-    def dead_payload(node)
-      finding = report.findings.find { |candidate| candidate.node.id == node.id }
+    def dead_payload(node, finding = finding_for(node.graph_id))
+      blockers = finding ? finding.blockers : graph.matching_blockers(node)
       {
-        'status' => 'dead',
+        'status' => finding&.classification == :blocked ? 'blocked' : 'dead',
         'node' => node.to_h,
         'classification' => finding&.classification&.to_s,
-        'nearest_alive' => nearest_alive(node.id),
-        'uncertainties' => graph.uncertainties(node.id)
+        'nearest_alive' => nearest_alive(node.graph_id),
+        'uncertainties' => graph.uncertainties(node.graph_id),
+        'blockers' => blockers.map(&:to_h)
       }
     end
 
+    def finding_for(node_id)
+      report.findings.find { |candidate| candidate.node.graph_id == node_id }
+    end
+
     def nearest_alive(node_id)
       kinds = reachability.runtime_alive.to_h { |id| [id, 'runtime'] }
+      reachability.external_alive.each { |id| kinds[id] ||= 'external' }
       reachability.test_alive.each { |id| kinds[id] ||= 'test' }
       visited = { node_id => 0 }
       queue = [node_id]
@@ -114,7 +152,17 @@ module Necropsy
           next if visited.key?(neighbor)
 
           distance = visited.fetch(current) + 1
-          return { 'node_id' => neighbor, 'kind' => kinds.fetch(neighbor), 'distance' => distance } if kinds.key?(neighbor)
+          if kinds.key?(neighbor)
+            node = graph.nodes.fetch(neighbor)
+            return {
+              'node_id' => neighbor,
+              'symbol_id' => node.symbol_id,
+              'definition_id' => node.definition_id,
+              'node' => node.to_h,
+              'kind' => kinds.fetch(neighbor),
+              'distance' => distance
+            }
+          end
 
           visited[neighbor] = distance
           queue << neighbor
@@ -123,6 +171,30 @@ module Necropsy
       nil
     end
 
+    def ambiguous_payload(lookup)
+      {
+        'status' => 'ambiguous',
+        'node_id' => lookup.identifier,
+        'definitions' => lookup.definitions.map do |definition|
+          {
+            'symbol_id' => definition.symbol_id,
+            'definition_id' => definition.definition_id,
+            'file' => definition.file,
+            'line' => definition.line,
+            'commands' => {
+              'why' => diagnostic_command('why', definition.definition_id),
+              'why_not' => diagnostic_command('why-not', definition.definition_id),
+              'explain' => diagnostic_command('explain', definition.definition_id)
+            }
+          }
+        end
+      }
+    end
+
+    def diagnostic_command(command, definition_id)
+      "bundle exec necropsy #{command} #{Shellwords.escape(definition_id)} --root #{Shellwords.escape(report.root)}"
+    end
+
     def neighbors(node_id)
       outgoing = graph.edges_from(node_id).keys
       incoming = graph.incoming_edges(node_id).map(&:caller_id)
@@ -131,27 +203,31 @@ module Necropsy
 
     def missing_payload(node_id)
       terms = [node_id, node_id.split(/[.#]/).last].compact.map(&:downcase).reject { |term| term.length < 2 }
-      suggestions = graph.nodes.keys.select do |candidate|
+      suggestions = graph.nodes.values.map(&:symbol_id).uniq.select do |candidate|
         terms.any? { |term| candidate.downcase[term] }
       end.sort.first(10)
       { 'status' => 'not_found', 'node_id' => node_id, 'suggestions' => suggestions }
     end
 
     def render_human(payload)
-      case payload.fetch('status')
-      when 'alive' then render_alive(payload)
-      when 'dead' then render_dead(payload)
-      when 'finding' then render_explanation(payload)
-      when 'not_found' then render_missing(payload)
-      else "#{payload.dig('node', 'id')} is alive and has no dead-code finding."
-      end
+      rendered = case payload.fetch('status')
+                 when 'alive' then render_alive(payload)
+                 when 'dead', 'blocked' then render_dead(payload)
+                 when 'finding' then render_explanation(payload)
+                 when 'why_not' then WhyNotRenderer.new(payload).render
+                 when 'ambiguous' then render_ambiguous(payload)
+                 when 'not_found' then render_missing(payload)
+                 else "#{node_reference(payload['node'])} is alive and has no dead-code finding."
+                 end
+      append_source_incompleteness(rendered, payload)
     end
 
     def render_alive(payload)
-      lines = ["Alive (#{payload.fetch('kind')}): #{payload.dig('node', 'id')}"]
+      lines = ["Alive (#{payload.fetch('kind')}): #{node_reference(payload.fetch('node'))}"]
       payload.fetch('path').each_with_index do |step, index|
         suffix = step['entry_reason'] ? " entry=#{step['entry_reason']}" : ''
-        lines << "  #{index}. #{step.dig('node', 'id')}#{suffix}"
+        suffix += " domain=#{step['root_domain']}" if step['root_domain']
+        lines << "  #{index}. #{node_reference(step.fetch('node'))}#{suffix}"
         Array(step.dig('edge', 'evidences')).each { |evidence| lines << render_evidence(evidence) }
       end
       lines.join("\n")
@@ -165,22 +241,25 @@ module Necropsy
     end
 
     def render_dead(payload)
-      lines = ["Dead: #{payload.dig('node', 'id')}"]
+      heading = payload['status'] == 'blocked' ? 'Blocked' : 'Dead'
+      lines = ["#{heading}: #{node_reference(payload.fetch('node'))}"]
       lines << "Classification: #{payload['classification']}" if payload['classification']
       nearest = payload['nearest_alive']
       lines << if nearest
-                 "Nearest alive: #{nearest['node_id']} (#{nearest['kind']}, distance #{nearest['distance']})"
+                 "Nearest alive: #{node_reference(nearest['node'])} " \
+                   "(#{nearest['kind']}, distance #{nearest['distance']})"
                else
                  'Nearest alive: none'
                end
       lines << 'Uncertainties: none' if payload.fetch('uncertainties').empty?
       payload.fetch('uncertainties').each { |message| lines << "Uncertainty: #{message}" }
+      append_blockers(lines, payload.fetch('blockers', []))
       lines.join("\n")
     end
 
     def render_explanation(payload)
       lines = [
-        "#{payload.dig('node', 'id')}: #{payload['classification']}",
+        "#{node_reference(payload.fetch('node'))}: #{payload['classification']}",
         "Confidence: #{payload['confidence']} (score #{format('%.2f', payload['score'])})"
       ]
       payload.fetch('components').each do |component|
@@ -190,13 +269,74 @@ module Necropsy
         )
       end
       lines << format('  %<name>-28s  %<value>0.2f', name: 'total', value: payload['score'])
+      append_blockers(lines, payload.fetch('blockers', []))
       lines.join("\n")
     end
 
+    def render_ambiguous(payload)
+      definitions = payload.fetch('definitions')
+      lines = [
+        "Ambiguous symbol ID: #{payload['node_id']}",
+        "Matched #{definitions.length} physical definitions:"
+      ]
+      definitions.each do |definition|
+        lines << "  #{definition['symbol_id']} [#{definition['definition_id']}] " \
+                 "#{definition['file']}:#{definition['line']}"
+        lines << "    why: #{definition.dig('commands', 'why')}"
+        lines << "    why-not: #{definition.dig('commands', 'why_not')}"
+        lines << "    explain: #{definition.dig('commands', 'explain')}"
+      end
+      lines.join("\n")
+    end
+
+    def append_blockers(lines, blockers)
+      blockers.each do |blocker|
+        metadata = blocker.fetch('metadata', {})
+        location = [metadata['file'], metadata['line']].compact.join(':')
+        caller = metadata['caller_id'] ? " caller=#{metadata['caller_id']}" : ''
+        lines << "Blocker: #{blocker['kind']} at #{location}#{caller}"
+        lines << "  Scope: #{blocker['scope_kind']}=#{blocker['scope_value'].inspect} message=#{metadata['message']}"
+        lines << "  Reason: #{blocker['reason']}"
+        lines << "  Match: #{metadata['snippet']}" if metadata['snippet']
+      end
+    end
+
     def render_missing(payload)
       lines = ["Node not found: #{payload['node_id']}"]
       lines << "Suggestions: #{payload['suggestions'].join(', ')}" unless payload['suggestions'].empty?
       lines.join("\n")
     end
+
+    def node_reference(node)
+      return 'unknown definition' unless node
+
+      symbol_id = node['symbol_id'] || node['id']
+      definition_id = node['definition_id'] || node['id']
+      "#{symbol_id} [#{definition_id}]"
+    end
+
+    def with_source_incompleteness(payload)
+      return payload if graph.incomplete_files.empty?
+
+      payload.merge('source_incompleteness' => graph.source_incompleteness)
+    end
+
+    def append_source_incompleteness(rendered, payload)
+      diagnostic = payload['source_incompleteness']
+      return rendered unless diagnostic
+
+      lines = [rendered, "Incomplete source files: #{diagnostic['incomplete_files']}"]
+      diagnostic.fetch('files').each do |file|
+        errors = file.fetch('errors')
+        if errors.empty?
+          lines << "  #{file['file']}:1 [#{file['status']}]"
+        else
+          errors.each do |error|
+            lines << "  #{error['file']}:#{error['line']} [#{error['type']}] #{error['message']}"
+          end
+        end
+      end
+      lines.join("\n")
+    end
   end
 end

data/lib/necropsy/embedded_ruby.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require 'prism'
+
+module Necropsy
+  module EmbeddedRuby
+    TAG_PATTERN = /<%(?![%#])[-=]?(.*?)-?%>/m
+
+    module_function
+
+    def extract(source)
+      cursor = 0
+      output = +''
+      source.to_enum(:scan, TAG_PATTERN).each do
+        match = Regexp.last_match
+        output << blank(source[cursor...match.begin(0)])
+        output << strip_ruby_comments(match[1])
+        cursor = match.end(0)
+      end
+      output << blank(source[cursor..])
+      output
+    end
+
+    def call_names(source)
+      result = Prism.parse(extract(source))
+      return Set.new if result.failure?
+
+      names = Set.new
+      pending = [result.value]
+      until pending.empty?
+        node = pending.pop
+        names << node.name.to_s if node.is_a?(Prism::CallNode) && node.name
+        pending.concat(node.child_nodes.compact)
+      end
+      names
+    end
+
+    def blank(source)
+      source.to_s.gsub(/[^\n]/, ' ')
+    end
+    private_class_method :blank
+
+    def strip_ruby_comments(source)
+      bytes = source.bytes
+      Prism.parse(source).comments.each do |comment|
+        location = comment.location
+        (location.start_offset...location.end_offset).each do |offset|
+          bytes[offset] = 32 unless bytes[offset] == 10
+        end
+      end
+      bytes.pack('C*').force_encoding(source.encoding)
+    end
+    private_class_method :strip_ruby_comments
+  end
+end

data/lib/necropsy/entry_points/plain.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require 'prism'
+
 module Necropsy
   module EntryPoints
     class Plain
@@ -13,41 +15,140 @@ module Necropsy
 
       def apply(graph, project)
         graph.nodes.values.select { |node| node.kind == :block_entry && !node.test }.each do |node|
-          next unless SCRIPT_PATTERNS.any? { |pattern| File.fnmatch?(pattern, node.file, File::FNM_PATHNAME) }
+          pattern = SCRIPT_PATTERNS.find { |candidate| File.fnmatch?(candidate, node.file, File::FNM_PATHNAME) }
+          next unless pattern
 
-          graph.add_entry_point(node.id, :main_script)
+          graph.add_entry_point(
+            node.graph_id,
+            :main_script,
+            domain: :runtime,
+            evidence: { 'type' => 'path_policy', 'pattern' => pattern, 'file' => node.file }
+          )
         end
 
         project.config.entry_point_patterns.each do |pattern|
-          graph.nodes.each_key do |node_id|
-            graph.add_entry_point(node_id, :public_api_declared) if File.fnmatch?(pattern, node_id)
+          graph.nodes.each_value do |node|
+            next if node.test
+            next unless File.fnmatch?(pattern, node.symbol_id)
+
+            graph.add_entry_point(
+              node.graph_id,
+              :public_api_declared,
+              domain: project.config.library_world? ? :external : :runtime,
+              evidence: { 'type' => 'configured_pattern', 'pattern' => pattern }
+            )
           end
         end
 
+        return if project.config.library_world?
+
         api_files = primary_library_files(project)
         graph.method_nodes.each do |node|
           next unless api_files.include?(node.file)
           next unless node.kind == :singleton_method && node.visibility == :public
 
-          graph.add_entry_point(node.id, :public_api_declared)
+          graph.add_entry_point(
+            node.graph_id,
+            :public_api_declared,
+            domain: :runtime,
+            evidence: { 'type' => 'gem_entry_file', 'file' => node.file }
+          )
         end
       end
 
       private
 
       def primary_library_files(project)
-        gem_names(project).flat_map do |name|
-          ["lib/#{name.tr('-', '/')}.rb", "lib/#{name.tr('-', '_')}.rb"]
+        gemspec_metadata(project).flat_map do |metadata|
+          metadata.fetch(:require_paths).flat_map do |require_path|
+            name = metadata.fetch(:name)
+            ["#{require_path}/#{name.tr('-', '/')}.rb", "#{require_path}/#{name.tr('-', '_')}.rb"]
+          end
         end.uniq.select { |relative| File.file?(File.join(project.root, relative)) }
       end
 
-      def gem_names(project)
-        Dir.glob(File.join(project.root, '*.gemspec')).filter_map do |path|
-          File.read(path)[/\.name\s*=\s*["']([^"']+)["']/, 1]
+      def gemspec_metadata(project)
+        project.reference_files.filter_map do |path|
+          relative = project.relative_path(path)
+          next unless relative.match?(%r{\A[^/]+\.gemspec\z})
+
+          parse_gemspec(File.read(path))
         rescue SystemCallError, EncodingError
           nil
         end
       end
+
+      def parse_gemspec(source)
+        result = Prism.parse(source)
+        return if result.failure?
+
+        specification = find_specification_call(result.value)
+        return unless specification&.block
+
+        parameter = required_block_parameter(specification.block)
+        return unless parameter.respond_to?(:name)
+
+        assignments = specification_assignments(specification.block.body, parameter.name)
+        name_node = first_argument(assignments[:name])
+        name = name_node.unescaped.to_s if name_node.is_a?(Prism::StringNode)
+        return if name.to_s.empty?
+
+        paths_node = first_argument(assignments[:require_paths])
+        paths = literal_string_array(paths_node)
+        { name: name, require_paths: paths.empty? ? ['lib'] : paths }
+      end
+
+      def required_block_parameter(block)
+        block_parameters = block.parameters
+        parameters = block_parameters.parameters if block_parameters
+        Array(parameters&.requireds).first
+      end
+
+      def first_argument(call)
+        arguments = call.arguments if call
+        Array(arguments&.arguments).first
+      end
+
+      def find_specification_call(root)
+        pending = [root]
+        until pending.empty?
+          node = pending.pop
+          return node if node.is_a?(Prism::CallNode) && node.name == :new &&
+                         constant_name(node.receiver) == 'Gem::Specification'
+
+          pending.concat(node.child_nodes.compact)
+        end
+      end
+
+      def specification_assignments(root, parameter_name)
+        pending = [root]
+        assignments = {}
+        until pending.empty?
+          node = pending.pop
+          if node.is_a?(Prism::CallNode) && node.receiver.is_a?(Prism::LocalVariableReadNode) &&
+             node.receiver.name == parameter_name
+            assignments[:name] = node if node.name == :name=
+            assignments[:require_paths] = node if node.name == :require_paths=
+          end
+          pending.concat(node.child_nodes.compact)
+        end
+        assignments
+      end
+
+      def literal_string_array(node)
+        return [] unless node.is_a?(Prism::ArrayNode) && !node.contains_splat?
+        return [] unless node.elements.all?(Prism::StringNode)
+
+        node.elements.map { |element| element.unescaped.to_s }.reject(&:empty?).uniq.sort
+      end
+
+      def constant_name(node)
+        case node
+        when Prism::ConstantReadNode then node.name.to_s
+        when Prism::ConstantPathNode
+          [constant_name(node.parent), node.name.to_s].compact.join('::')
+        end
+      end
     end
   end
 end