necropsy 0.2.0 → 0.3.0

data/lib/necropsy/ast_scanner/traversal.rb

@@ -4,12 +4,25 @@ module Necropsy
   class AstScanner
     private
 
+    def resolve_deferred_module_function_sources
+      deferred_module_functions.each do |copy_id, instance_id|
+        module_function_sources[copy_id] = nodes.filter_map do |definition|
+          definition.graph_id if definition.kind == :instance_method && definition.symbol_id == instance_id
+        end
+        signatures = module_function_sources.fetch(copy_id).filter_map { |source_id| method_signatures[source_id] }.uniq
+        method_signatures[copy_id] = signatures.first if signatures.one?
+      end
+    end
+
     def copy_module_function_call_sites
-      module_functions = nodes.select { |node| node.defined_via == :module_function }
-      copies = module_functions.flat_map do |node|
-        instance_id = "#{node.owner}##{node.name}"
-        call_sites.select { |site| site.caller_id == instance_id }.map do |site|
-          site.with(caller_id: node.id, metadata: site.metadata.merge('module_function' => true))
+      copies = module_function_sources.flat_map do |copy_id, source_ids|
+        source_ids.flat_map do |source_id|
+          call_sites.select { |site| site.caller_id == source_id }.map do |site|
+            derived_call_site(
+              site, derivation: :module_function, caller_id: copy_id,
+                    metadata: { 'module_function' => true }
+            )
+          end
         end
       end
       call_sites.concat(copies)
@@ -17,42 +30,49 @@ module Necropsy
 
     def scan_file(file)
       relative = project.relative_path(file)
-      root_id = "file:#{relative}"
       test = project.test_file?(file)
-      nodes << Node.new(
-        id: root_id,
+      context = Context.new(
+        namespace: nil,
+        lexical_nesting: [],
+        owner: nil,
+        current_caller_id: nil,
+        current_method_name: nil,
+        current_kind: :block_entry,
+        root_id: nil,
+        file: file,
+        relative_file: relative,
+        test: test,
+        singleton_scope: false,
+        visibility: :public,
+        module_function: false,
+        static_ancestry: true,
+        flow_result: nil
+      )
+      result = Prism.parse(File.read(file))
+      root = add_definition(
+        symbol_id: "file:#{relative}",
         kind: :block_entry,
-        file: relative,
-        line: 1,
-        end_line: 1,
+        source_node: result.value,
+        context: context,
         defined_via: :file,
         owner: nil,
         name: relative,
-        test: test,
         visibility: :public
       )
+      root_id = root.graph_id
+      context.current_caller_id = root_id
+      context.root_id = root_id
 
-      result = Prism.parse(File.read(file))
-      record_parse_errors(root_id, result) if result.failure?
-
-      visit(
-        result.value,
-        Context.new(
-          namespace: nil,
-          owner: nil,
-          current_caller_id: root_id,
-          current_kind: :block_entry,
-          root_id: root_id,
-          file: file,
-          relative_file: relative,
-          test: test,
-          singleton_scope: false,
-          visibility: :public,
-          module_function: false
-        )
-      )
-    rescue SystemCallError, EncodingError => e
-      uncertainties[root_id] << "Could not parse #{relative}: #{e.message}"
+      if result.failure?
+        file_statuses[relative] = :recovered
+        record_parse_errors(root_id, relative, result)
+      else
+        file_statuses[relative] = :complete
+      end
+
+      visit(result.value, context)
+    rescue DefinitionIdentity::CanonicalizationError, SystemStackError, SystemCallError, EncodingError => e
+      record_source_failure(root_id || "file:#{relative}", relative, e)
     end
 
     def visit(node, context)
@@ -74,15 +94,25 @@ module Necropsy
       when Prism::ConstantWriteNode
         visit_constant_write(node, context)
       else
-        node.child_nodes.compact.each { |child| visit(child, context) }
+        visit_generic_node(node, context)
       end
     end
 
+    def visit_generic_node(node, context)
+      return visit_children(node, context) unless ANCESTRY_CONTROL_FLOW_TYPES.include?(node.type)
+
+      nested_context = context.dup
+      nested_context.static_ancestry = false
+      visit_children(node, nested_context)
+    end
+
     def visit_namespace(node, context)
       namespace = qualify_constant(constant_name(node.constant_path), context.namespace)
       record_class_info(node, namespace, context)
+      visit(node.superclass, context) if node.respond_to?(:superclass) && node.superclass
       child_context = context.dup
       child_context.namespace = namespace
+      child_context.lexical_nesting = [namespace, *context.lexical_nesting].compact.uniq
       child_context.owner = namespace
       child_context.singleton_scope = false
       child_context.visibility = :public
@@ -92,93 +122,226 @@ module Necropsy
 
     def visit_def(node, context)
       owner = definition_owner(node, context)
-      return visit_children(node, context) unless owner
+      unless owner
+        visit(node.receiver, context) if node.receiver
+        record_semantic_blocker(
+          :dynamic_singleton_definition,
+          node,
+          context,
+          'singleton method receiver is not statically bounded',
+          suggested_action: :make_singleton_receiver_static,
+          force_global: true
+        )
+        visit_synthetic_body(node.body, context, :dynamic_singleton_method)
+        return
+      end
 
       kind = node.receiver || context.singleton_scope ? :singleton_method : :instance_method
       separator = kind == :singleton_method ? '.' : '#'
       id = "#{owner}#{separator}#{node.name}"
-      nodes << Node.new(
-        id: id,
+      definition = add_definition(
+        symbol_id: id,
         kind: kind,
-        file: context.relative_file,
-        line: node.location.start_line,
-        end_line: node.location.end_line,
+        source_node: node,
+        context: context,
         defined_via: :def,
         owner: owner,
         name: node.name.to_s,
-        test: context.test,
         visibility: node.receiver ? :public : context.visibility
       )
-      record_module_function_copy(node, context, owner) if kind == :instance_method && context.module_function
-      if node.name == :method_missing
-        uncertainties[id] << "#{owner} defines method_missing"
+      method_signatures[definition.graph_id] = method_signature(node.parameters)
+      record_module_function_copy(node, context, definition) if kind == :instance_method && context.module_function
+      if %i[method_missing respond_to_missing?].include?(node.name)
+        uncertainties[definition.graph_id] << "#{owner} defines #{node.name}"
         class_record(owner)[:dynamic] = true
       end
+      rule_hit = convention_rules.method_hit(
+        owner: owner,
+        method_name: node.name,
+        ancestors: convention_ancestors(owner),
+        frameworks: project.config.frameworks
+      )
+      if rule_hit
+        entrypoint_hints << EntryPoint.new(
+          node_id: definition.graph_id,
+          reason: :convention_callback,
+          evidence: { 'type' => 'convention_rule', **rule_hit }
+        )
+      end
 
       method_context = context.dup
       method_context.owner = owner
-      method_context.current_caller_id = id
+      method_context.current_caller_id = definition.graph_id
+      method_context.current_method_name = node.name.to_s
       method_context.current_kind = kind
       method_context.singleton_scope = false
       method_context.visibility = :public
       method_context.module_function = false
+      method_context.static_ancestry = false
+      method_context.flow_result = FlowInterpreter.new(
+        constant_resolver: lambda do |constant|
+          resolve_candidate_group(constant_candidates(constant, method_context.lexical_nesting))
+        end
+      ).analyze(node.body)
+      method_context.flow_result.issues.each do |issue|
+        uncertainties[definition.graph_id] << "Forward value analysis stopped at #{issue}; affected calls use conservative lookup"
+      end
+      visit_default_parameters(node.parameters, method_context)
       visit(node.body, method_context)
     end
 
+    def visit_default_parameters(parameters, context)
+      return unless parameters
+
+      defaults = [*Array(parameters.optionals), *Array(parameters.keywords)].filter_map do |parameter|
+        parameter.value if parameter.respond_to?(:value)
+      end
+      defaults.each { |default| visit(default, context) }
+    end
+
+    def convention_ancestors(owner)
+      ancestors = []
+      pending = convention_parent_candidates(class_data[owner])
+      until pending.empty?
+        current = pending.shift
+        next if ancestors.include?(current)
+
+        ancestors << current
+        pending.concat(convention_parent_candidates(class_data[current]))
+      end
+      ancestors
+    end
+
+    def convention_parent_candidates(data)
+      return [] unless data
+
+      Array(data[:superclass_candidates]) +
+        %i[includes prepends extends singleton_includes singleton_prepends].flat_map do |relation|
+          Array(data[relation]).flatten
+        end
+    end
+
     def visit_call(node, context)
-      return if handle_visibility(node, context)
-      return if handle_module_function(node, context)
-      return if handle_eval(node, context)
-      return if handle_define_singleton_method(node, context)
-      return if handle_define_method(node, context)
-      return if handle_attr_macro(node, context)
-      return if handle_delegate(node, context)
-      return if handle_forwardable(node, context)
-      return if handle_alias_method(node, context)
+      handlers = %i[
+        handle_unsupported_semantics handle_visibility handle_module_function handle_eval
+        handle_define_singleton_method handle_define_method handle_attr_macro handle_delegate
+        handle_forwardable handle_alias_method handle_method_removal
+      ]
+      handled = handlers.lazy.map { |handler| send(handler, node, context) }.find(&:itself)
+      if handled
+        raise TypeError, "semantic call handler must return AstScanner::CallTraversal, got #{handled.class}" unless
+          handled.is_a?(CallTraversal)
+
+        visit_call_children(
+          node,
+          context,
+          visit_receiver: handled.receiver,
+          visit_arguments: handled.arguments,
+          visit_block: handled.block
+        )
+        return
+      end
 
       handle_module_relation(node, context)
-      handle_rails_callback(node, context)
+      callback_block_consumed = handle_rails_callback(node, context)
+      handle_graphql_field(node, context)
+      if handle_generated_rails_methods(node, context)
+        visit_call_children(node, context, visit_block: false)
+        return
+      end
 
       record_instantiation(node, context)
       record_symbol_reference(node, context)
       record_symbol_to_proc(node, context)
-      site = build_call_site(node, context)
-      call_sites << site if site
-      if site&.dynamic
-        record_uncertainty(site)
-      elsif unresolved_dynamic_dispatch?(node)
-        record_uncertainty_at(node, context)
-      end
+      sites = build_call_sites(node, context).compact
+      call_sites.concat(sites)
+      sites.each { |site| record_uncertainty(site) if site.dynamic }
+      record_uncertainty_at(node, context) if sites.empty? && unresolved_dynamic_dispatch?(node)
 
-      visit_children(node, context)
+      visit_call_children(node, context, visit_block: !callback_block_consumed)
+    end
+
+    def visit_call_children(node, context, visit_receiver: true, visit_arguments: true, visit_block: true)
+      visit(node.receiver, context) if visit_receiver && node.receiver
+      arguments(node).each { |argument| visit(argument, context) } if visit_arguments
+      visit(node.block, context) if visit_block && node.block
+    end
+
+    def visit_synthetic_body(body, context, purpose)
+      return unless body
+
+      synthetic = add_definition(
+        symbol_id: "synthetic:#{purpose}:#{context.relative_file}:#{body.location.start_line}",
+        kind: :block_entry,
+        source_node: body,
+        context: context,
+        defined_via: purpose,
+        owner: context.owner,
+        name: purpose,
+        visibility: :public
+      )
+      body_context = context.dup
+      body_context.current_caller_id = synthetic.graph_id
+      body_context.current_method_name = nil
+      body_context.current_kind = :block_entry
+      body_context.static_ancestry = false
+      body_context.flow_result = nil
+      visit(body, body_context)
+      synthetic
     end
 
     def visit_super(node, context)
-      method_name = context.current_caller_id&.split(/[.#]/)&.last
+      method_name = context.current_method_name
       if method_name
-        call_sites << CallSite.new(
-          caller_id: context.current_caller_id,
+        add_scanned_call_site(
+          source_node: node,
+          context: context,
+          role: :super,
           message: method_name,
           receiver_kind: :super,
           receiver_name: context.owner,
-          file: context.relative_file,
-          line: node.location.start_line,
-          test: context.test,
           dynamic: false,
-          metadata: { 'super' => true }
+          metadata: {
+            'super' => true,
+            'zsuper' => node.is_a?(Prism::ForwardingSuperNode),
+            'arguments' => node.is_a?(Prism::ForwardingSuperNode) ? incomplete_arguments : call_arguments(node)
+          }
         )
       end
       visit_children(node, context)
     end
 
     def visit_singleton_class(node, context)
-      return visit_children(node, context) unless node.expression.is_a?(Prism::SelfNode) && context.owner
+      owner = singleton_class_owner(node.expression, context)
+      unless owner
+        visit(node.expression, context)
+        record_semantic_blocker(
+          :dynamic_singleton_definition,
+          node,
+          context,
+          'singleton class receiver is not statically bounded',
+          suggested_action: :make_singleton_receiver_static,
+          force_global: true
+        )
+        return
+      end
 
       singleton_context = context.dup
+      singleton_context.namespace = owner
+      singleton_context.owner = owner
       singleton_context.singleton_scope = true
       singleton_context.visibility = :public
       singleton_context.module_function = false
       visit(node.body, singleton_context)
     end
+
+    def singleton_class_owner(expression, context)
+      return context.owner if expression.is_a?(Prism::SelfNode)
+
+      name = constant_name(expression)
+      return unless name
+
+      resolve_candidate_group(constant_candidates(name, context.lexical_nesting))
+    end
   end
 end

data/lib/necropsy/ast_scanner/value_definitions.rb

@@ -14,31 +14,40 @@ module Necropsy
       data[:line] = node.location.start_line
       instantiated_classes << owner
 
-      symbol_arguments(node.value).each do |name|
-        nodes << Node.new(
-          id: "#{owner}##{name}",
+      value_arguments = arguments(node.value)
+      names = value_arguments.filter_map { |argument| literal_value(argument) if literal_name?(argument) }
+      if names.length != value_arguments.length
+        record_semantic_blocker(
+          :dynamic_generated_methods,
+          node.value,
+          context,
+          "#{node.value.name} has runtime-computed member names",
+          suggested_action: :make_member_name_literal,
+          scope_owner: owner
+        )
+      end
+
+      names.each do |name|
+        add_definition(
+          symbol_id: "#{owner}##{name}",
           kind: :instance_method,
-          file: context.relative_file,
-          line: node.location.start_line,
-          end_line: node.location.end_line,
+          source_node: node,
+          context: context,
           defined_via: node.value.name == :define ? :data_define : :struct_new,
           owner: owner,
           name: name,
-          test: context.test,
           visibility: :public
         )
         next if node.value.name == :define
 
-        nodes << Node.new(
-          id: "#{owner}##{name}=",
+        add_definition(
+          symbol_id: "#{owner}##{name}=",
           kind: :instance_method,
-          file: context.relative_file,
-          line: node.location.start_line,
-          end_line: node.location.end_line,
+          source_node: node,
+          context: context,
           defined_via: :struct_new,
           owner: owner,
           name: "#{name}=",
-          test: context.test,
           visibility: :public
         )
       end
@@ -51,6 +60,7 @@ module Necropsy
       block_context.singleton_scope = false
       block_context.visibility = :public
       block_context.module_function = false
+      block_context.static_ancestry = false
       visit(node.value.block.body, block_context)
     end
 

data/lib/necropsy/ast_scanner.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 require 'prism'
+require_relative 'ast_scanner/definition_creation'
+require_relative 'ast_scanner/call_site_creation'
 require_relative 'ast_scanner/traversal'
 require_relative 'ast_scanner/value_definitions'
 require_relative 'ast_scanner/method_definitions'
@@ -10,9 +12,29 @@ require_relative 'ast_scanner/references'
 require_relative 'ast_scanner/ruby_semantics'
 
 module Necropsy
-  ScanResult = Data.define(:nodes, :call_sites, :instantiated_classes, :uncertainties, :class_infos, :entrypoint_hints)
+  ScanResult = Data.define(
+    :nodes,
+    :call_sites,
+    :instantiated_classes,
+    :uncertainties,
+    :class_infos,
+    :entrypoint_hints,
+    :file_statuses,
+    :source_errors,
+    :source_domains,
+    :scope_diagnostics,
+    :method_signatures,
+    :semantic_blockers
+  ) do
+    def initialize(nodes:, call_sites:, instantiated_classes:, uncertainties:, class_infos:, entrypoint_hints:,
+                   file_statuses: {}, source_errors: [], source_domains: {}, scope_diagnostics: {},
+                   method_signatures: {}, semantic_blockers: [])
+      super
+    end
+  end
 
   class AstScanner
+    CallTraversal = Data.define(:receiver, :arguments, :block)
     ATTR_MACROS = %i[attr_reader attr_writer attr_accessor].freeze
     DYNAMIC_SENDS = %i[send public_send __send__].freeze
     MODULE_RELATION_MACROS = %i[include prepend extend].freeze
@@ -20,13 +42,24 @@ module Necropsy
       before_action after_action around_action
       before_validation after_validation before_save after_save around_save
       around_validation before_touch after_touch
+      before_enqueue around_enqueue after_enqueue before_perform around_perform after_perform
       before_create after_create around_create before_update after_update around_update
       before_destroy after_destroy around_destroy after_commit after_rollback after_initialize after_find
       after_create_commit after_update_commit after_destroy_commit after_save_commit
       validate rescue_from helper_method
     ].freeze
+    RAILS_GENERATED_METHOD_MACROS = %i[
+      enum store store_accessor attribute class_attribute mattr_reader mattr_writer mattr_accessor
+      cattr_reader cattr_writer cattr_accessor belongs_to has_one has_many scope
+    ].freeze
+    RAILS_RUNTIME_BLOCK_MACROS = %i[discard_on retry_on stream_for stream_from].freeze
+    SIDEKIQ_RUNTIME_BLOCK_MACROS = %i[sidekiq_retries_exhausted sidekiq_retry_in].freeze
     VISIBILITY_MACROS = %i[public protected private public_class_method private_class_method].freeze
     SYMBOL_REFERENCE_CALLS = %i[method respond_to? try try!].freeze
+    ANCESTRY_CONTROL_FLOW_TYPES = %i[
+      and_node block_node case_match_node case_node else_node for_node if_node in_node lambda_node
+      or_node rescue_modifier_node rescue_node until_node when_node while_node
+    ].freeze
     RAILS_BUILTIN_VALIDATORS = %w[
       absence acceptance allow_blank allow_nil comparison confirmation exclusion format if inclusion length message numericality
       on presence strict uniqueness unless
@@ -34,8 +67,10 @@ module Necropsy
 
     Context = Struct.new(
       :namespace,
+      :lexical_nesting,
       :owner,
       :current_caller_id,
+      :current_method_name,
       :current_kind,
       :root_id,
       :file,
@@ -44,37 +79,63 @@ module Necropsy
       :singleton_scope,
       :visibility,
       :module_function,
+      :static_ancestry,
+      :flow_result,
       keyword_init: true
     )
 
-    def initialize(project:, files:)
+    def initialize(project:, files:, source_domains: nil, scope_diagnostics: {})
       @project = project
       @files = files
+      @source_domains = source_domains || files.to_h { |file| [project.relative_path(file), :analyze] }
+      @scope_diagnostics = scope_diagnostics
       @nodes = []
       @call_sites = []
       @instantiated_classes = Set.new
       @uncertainties = Hash.new { |hash, key| hash[key] = [] }
       @class_data = {}
       @entrypoint_hints = []
+      @file_statuses = {}
+      @source_errors = []
+      @definition_ordinals = Hash.new(0)
+      @call_site_ordinals = Hash.new(0)
+      @module_function_sources = {}
+      @deferred_module_functions = {}
+      @method_signatures = {}
+      @semantic_blockers = []
       @factory_methods = project.config.factory_methods.to_set(&:to_s)
+      @convention_rules = ConventionRules.new
     end
 
     def scan
-      files.each { |file| scan_file(file) }
+      files.sort_by { |file| project.relative_path(file) }.each { |file| scan_file(file) }
+      resolve_deferred_module_function_sources
       copy_module_function_call_sites
       ScanResult.new(
-        nodes: nodes.reverse.uniq(&:id).reverse,
+        nodes: nodes,
         call_sites: call_sites,
         instantiated_classes: instantiated_classes,
         uncertainties: uncertainties,
         class_infos: class_infos,
-        entrypoint_hints: entrypoint_hints.uniq
+        entrypoint_hints: entrypoint_hints.uniq,
+        file_statuses: file_statuses,
+        source_errors: source_errors,
+        source_domains: source_domains,
+        scope_diagnostics: scope_diagnostics,
+        method_signatures: method_signatures,
+        semantic_blockers: semantic_blockers.sort_by { |blocker| BoundedCanonicalizer.dump(blocker.to_h) }
       )
     end
 
     private
 
+    def handled_call(receiver: true, arguments: true, block: false)
+      CallTraversal.new(receiver: receiver, arguments: arguments, block: block)
+    end
+
     attr_reader :project, :files, :nodes, :call_sites, :instantiated_classes, :uncertainties, :class_data,
-                :entrypoint_hints
+                :entrypoint_hints, :file_statuses, :source_errors, :definition_ordinals, :module_function_sources,
+                :deferred_module_functions, :call_site_ordinals, :source_domains, :scope_diagnostics,
+                :method_signatures, :semantic_blockers, :convention_rules
   end
 end