scully 0.0.1 → 0.1.0

data/lib/scully/frameworks/detector.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require 'prism'
+
+module Scully
+  module Frameworks
+    # Decides whether a file holds RSpec or Minitest tests: file content first, then its path.
+    module Detector
+      SKIP_REASON = 'no RSpec or Minitest test structure found'
+
+      RSPEC_HELPERS = %w[spec_helper rails_helper].freeze
+      MINITEST_HELPERS = %w[test_helper].freeze
+      # Every example group name the RSpec normalizer reads, so a file it can analyze is not skipped.
+      RSPEC_GROUPS = %i[describe context feature example_group shared_examples shared_examples_for shared_context
+                        fdescribe fcontext ffeature xdescribe xcontext xfeature].freeze
+      MINITEST_BASE = /(Test|TestCase)\z/
+
+      module_function
+
+      # Returns :rspec, :minitest, or nil when the file shows no test structure.
+      # +program+ is the parsed ProgramNode, or nil when the file could not be parsed.
+      def detect(path, program)
+        from_content(program) || from_path(path)
+      end
+
+      def from_content(program)
+        return unless program
+
+        statements = program.statements.body
+        requires = statements.filter_map { required_feature(it) }
+        return :rspec if rspec_describe?(statements) || requires.any? { RSPEC_HELPERS.include?(File.basename(it)) }
+        return :minitest if minitest_require?(requires) || minitest_class?(program)
+
+        :rspec if statements.any? { group_call?(it) }
+      end
+
+      def from_path(path)
+        segments = File.dirname(path).split('/')
+        return :rspec if segments.include?('spec')
+
+        :minitest if segments.include?('test')
+      end
+
+      def required_feature(node)
+        return unless node.is_a?(Prism::CallNode) && node.receiver.nil?
+        return unless %i[require require_relative].include?(node.name)
+
+        argument = node.arguments&.arguments&.first
+        argument.unescaped if argument.is_a?(Prism::StringNode)
+      end
+
+      # RSpec.describe and friends name RSpec explicitly; a bare describe may be minitest/spec.
+      def rspec_describe?(statements)
+        statements.any? { group_call?(it) && rspec_receiver?(it.receiver) }
+      end
+
+      def rspec_receiver?(receiver)
+        receiver.is_a?(Prism::ConstantReadNode) && receiver.name == :RSpec
+      end
+
+      def minitest_require?(requires)
+        requires.any? do
+          it == 'minitest' || it.start_with?('minitest/') || MINITEST_HELPERS.include?(File.basename(it))
+        end
+      end
+
+      def group_call?(node)
+        node.is_a?(Prism::CallNode) && RSPEC_GROUPS.include?(node.name) &&
+          (node.receiver.nil? || rspec_receiver?(node.receiver))
+      end
+
+      def minitest_class?(node)
+        return true if node.is_a?(Prism::ClassNode) && minitest_base?(node.superclass)
+
+        node.compact_child_nodes.any? { minitest_class?(it) }
+      end
+
+      def minitest_base?(superclass)
+        case superclass
+        when Prism::ConstantReadNode, Prism::ConstantPathNode then MINITEST_BASE.match?(superclass.name.to_s)
+        else false
+        end
+      end
+    end
+  end
+end

data/lib/scully/frameworks/helpers.rb

@@ -0,0 +1,215 @@
+# frozen_string_literal: true
+
+require 'prism'
+
+module Scully
+  module Frameworks
+    # Resolves calls to helper methods against the methods defined in the same file.
+    #
+    # A helper counts as an assertion when its name starts with `assert_` or `expect_`, or when
+    # it is defined in the file and its body makes an assertion, directly or through other
+    # helpers defined in the file.
+    class Helpers
+      CONVENTION = /\A(assert|expect)_/
+
+      # +program+ is the file's ProgramNode; +assertion+ is called with each CallNode and says
+      # whether it is an assertion in the file's framework.
+      def initialize(program, assertion:)
+        @assertion = assertion
+        @methods = {}
+        collect_methods(program)
+        @asserting = asserting_methods
+      end
+
+      # :convention, :same_file_with_assertion, :same_file_without_assertion or :unresolved.
+      def resolve(name)
+        return :convention if CONVENTION.match?(name)
+        return :unresolved unless @methods.key?(name)
+
+        @asserting.include?(name) ? :same_file_with_assertion : :same_file_without_assertion
+      end
+
+      def assertion?(name) = %i[convention same_file_with_assertion].include?(resolve(name))
+
+      def defined?(name) = @methods.key?(name)
+
+      private
+
+      # Later definitions replace earlier ones, as they do when Ruby loads the file.
+      def collect_methods(node)
+        @methods[node.name] = node if node.is_a?(Prism::DefNode)
+        node.compact_child_nodes.each { collect_methods(it) }
+      end
+
+      # Fixed point: start from methods that assert directly, then add methods that call one of
+      # them until nothing changes. Terminates on recursive helpers.
+      def asserting_methods
+        calls = @methods.transform_values { helper_calls_in(it) }
+        asserting = @methods.select { |_, node| asserts_directly?(node) }.keys.to_set
+        loop do
+          added = calls.select { |name, called| !asserting.include?(name) && called.intersect?(asserting) }.keys
+          return asserting if added.empty?
+
+          asserting.merge(added)
+        end
+      end
+
+      def asserts_directly?(node) = calls_in(node).any? { @assertion.call(it) || convention_call?(it) }
+
+      def helper_calls_in(node) = calls_in(node).select { it.receiver.nil? }.to_set(&:name)
+
+      def convention_call?(call) = call.receiver.nil? && CONVENTION.match?(call.name)
+
+      def calls_in(node, found = [])
+        node.compact_child_nodes.each do |child|
+          found << child if child.is_a?(Prism::CallNode)
+          calls_in(child, found) unless child.is_a?(Prism::DefNode)
+        end
+        found
+      end
+    end
+
+    # Reading of DSL calls shared by both normalizers.
+    module Syntax
+      module_function
+
+      def statements(node) = node.is_a?(Prism::StatementsNode) ? node.body : [node].compact
+
+      # `it "..."` up to its block: the call name and its arguments, with parentheses if any.
+      def definition(call)
+        call.message_loc.join(call.closing_loc || call.arguments&.location || call.message_loc)
+      end
+
+      # The description a group or test is written with; symbols and hashes are metadata.
+      def description(call)
+        case (argument = call.arguments&.arguments&.first)
+        when nil, Prism::KeywordHashNode, Prism::SymbolNode then nil
+        when Prism::StringNode then argument.unescaped
+        when Prism::InterpolatedStringNode then argument.parts.map(&:slice).join
+        else argument.slice
+        end
+      end
+
+      # The name a `let(:name)` or `subject(:name)` binds; a bare `subject` binds `subject`.
+      def binding_name(call)
+        argument = call.arguments&.arguments&.first
+        argument.is_a?(Prism::SymbolNode) ? argument.unescaped.to_sym : call.name
+      end
+
+      # The value a `let` or `subject` block returns: its last expression.
+      def block_value(call) = call.block.is_a?(Prism::BlockNode) ? statements(call.block.body).last : nil
+
+      # An expression that reads +name+, standing for the object a let or variable is bound to.
+      def identifier(name) = Prism.parse(name.to_s).value.statements.body.first
+
+      def arguments(call) = call.arguments&.arguments || []
+
+      # [method name, value] pairs from `name: value` arguments with symbol keys.
+      def keyword_pairs(call)
+        arguments(call).grep(Prism::KeywordHashNode).flat_map(&:elements).filter_map do |element|
+          next unless element.is_a?(Prism::AssocNode) && element.key.is_a?(Prism::SymbolNode)
+
+          [element.key.unescaped.to_sym, element.value]
+        end
+      end
+
+      # [call, its receiver, ...] while the receivers are calls: `a.b(1).c` gives [c, b, a].
+      def chain(node)
+        calls = []
+        while node.is_a?(Prism::CallNode)
+          calls << node
+          node = node.receiver
+        end
+        calls
+      end
+
+      # The method name a stub names with its first argument (`receive(:m)`, `stubs(:m)`), or nil.
+      def stubbed_name(call)
+        argument = arguments(call).first
+        argument.is_a?(Prism::SymbolNode) ? argument.unescaped.to_sym : nil
+      end
+
+      # Local and instance variables assigned in +node+, in source order: [[name, value], ...].
+      def assignments(node, found = [])
+        node&.compact_child_nodes&.each do |child|
+          next if child.is_a?(Prism::DefNode)
+
+          found << [child.name, child.value] if assignment?(child)
+          assignments(child, found)
+        end
+        found
+      end
+
+      def assignment?(node) = node.is_a?(Prism::LocalVariableWriteNode) || node.is_a?(Prism::InstanceVariableWriteNode)
+    end
+
+    # Builds TestDoubles for one file and one origin (:setup or :body).
+    class Doubles
+      def initialize(source, origin)
+        @source = source
+        @origin = origin
+      end
+
+      # +location+ is the stub's Prism location; +scope+ is the location of the block a stub is
+      # limited to, if any.
+      # +fields+ are the TestDouble's receiver, method_name and returns.
+      def build(location, stub_source: true, scope: nil, **fields)
+        TestDouble.new(span: @source.span(location), origin: @origin, scope: scope && @source.span(scope),
+                       stub_source:, **fields)
+      end
+    end
+
+    # Collects the assertions and candidate helper calls in a test body, in source order.
+    #
+    # +assertion+ is called with each CallNode and returns the Assertion fields other than the
+    # span (kind, comparison, subject, expected) when the call is an assertion, or nil. The
+    # assertion's subtree is not searched further. +ignored+ says whether a receiverless call
+    # name is framework vocabulary or a binding rather than a possible helper.
+    class Inventory
+      def initialize(source:, helpers:, assertion:, ignored:)
+        @source = source
+        @helpers = helpers
+        @assertion = assertion
+        @ignored = ignored
+      end
+
+      # Returns [assertions, helper_calls] for +body+ (a node or nil).
+      def scan(body)
+        @assertions = []
+        @helper_calls = []
+        visit(body) if body
+        [@assertions, @helper_calls]
+      end
+
+      private
+
+      def visit(node)
+        node.compact_child_nodes.each do |child|
+          next if child.is_a?(Prism::DefNode)
+          next if child.is_a?(Prism::CallNode) && assertion(child)
+
+          helper(child) if child.is_a?(Prism::CallNode)
+          visit(child)
+        end
+      end
+
+      def assertion(call)
+        fields = @assertion.call(call) or return false
+        @assertions << Assertion.new(span: @source.span(call.location), **fields)
+      end
+
+      def helper(call)
+        return unless call.receiver.nil? && !@ignored.call(call.name)
+
+        @helper_calls << HelperCall.new(name: call.name, span: @source.span(call.message_loc),
+                                        resolution: @helpers.resolve(call.name))
+        helper_assertion(call) if @helpers.assertion?(call.name)
+      end
+
+      def helper_assertion(call)
+        @assertions << Assertion.new(span: @source.span(call.location), kind: :helper, comparison: nil,
+                                     subject: nil, expected: nil)
+      end
+    end
+  end
+end

data/lib/scully/frameworks/minitest.rb

@@ -0,0 +1,305 @@
+# frozen_string_literal: true
+
+require 'prism'
+
+module Scully
+  module Frameworks
+    # Turns a Minitest file into TestCases: `def test_*` methods and Rails-style `test "..." do`
+    # blocks in test classes, and minitest/spec `it` blocks inside `describe`.
+    class Minitest
+      TEST_METHOD = /\Atest_/
+      SETUP_METHODS = { before_setup: :setup, setup: :setup, after_setup: :setup,
+                        before_teardown: :teardown, teardown: :teardown, after_teardown: :teardown }.freeze
+      SETUP_CALLS = { setup: :setup, teardown: :teardown, before: :before, after: :teardown, let: :let,
+                      subject: :subject }.freeze
+      GROUPS = %i[describe].freeze
+      EXAMPLES = %i[it specify].freeze
+
+      # The class, group names, setup, bindings and setup doubles visible at a point in the file.
+      Context = Data.define(:class_name, :names, :setup, :bindings, :doubles)
+      # Setup kinds whose code runs before the test body.
+      BEFORE = %i[setup before].freeze
+
+      include Syntax
+
+      def self.call(source) = new(source).test_cases
+
+      def initialize(source)
+        @source = source
+        @helpers = Helpers.new(source.result.value, assertion: ->(call) { !Assertions.fields(call).nil? })
+      end
+
+      def test_cases
+        @tests = []
+        walk(@source.result.value.statements,
+             Context.new(class_name: nil, names: [], setup: [], bindings: {}, doubles: []))
+        @tests
+      end
+
+      private
+
+      def walk(node, context)
+        statements(node).each { statement(it, context) }
+      end
+
+      def statement(node, context)
+        case node
+        when Prism::ClassNode then class_statement(node, context)
+        when Prism::ModuleNode then walk(node.body, context)
+        when Prism::DefNode then test_method(node, context) if context.class_name && TEST_METHOD.match?(node.name)
+        when Prism::CallNode then call_statement(node, context)
+        end
+      end
+
+      def class_statement(node, context)
+        return walk(node.body, context) unless Detector.minitest_base?(node.superclass)
+
+        name = node.constant_path.slice
+        walk(node.body, scope(node.body, context.with(class_name: name, names: [name])))
+      end
+
+      def call_statement(call, context)
+        return unless call.receiver.nil?
+
+        if GROUPS.include?(call.name) && call.block then group(call, context)
+        elsif EXAMPLES.include?(call.name) then spec_example(call, context)
+        elsif call.name == :test && context.class_name then rails_test(call, context)
+        end
+      end
+
+      def group(call, context)
+        walk(call.block.body, scope(call.block.body, context.with(names: [*context.names, *description(call)])))
+      end
+
+      # Adds the setup, bindings and setup doubles declared directly in a class or group body.
+      def scope(body, context)
+        own = statements(body).filter_map { [it, Scope.kind(it)] if Scope.kind(it) }
+        Scope.apply(context, own, Doubles.new(@source, :setup))
+             .then { it.with(setup: it.setup + own.map { |node, kind| setup_step(node, kind) }) }
+      end
+
+      def setup_step(node, kind) = SetupStep.new(kind:, span: span(node.location))
+
+      def span(location) = @source.span(location)
+
+      def test_method(node, context)
+        add(context, node.body, name: "#{context.class_name}##{node.name}",
+                                definition: node.def_keyword_loc.join(node.name_loc), extent: node.location,
+                                status: skip_call?(node.body) ? :skipped : :active)
+      end
+
+      def rails_test(call, context)
+        add(context, call.block&.body, name: "#{context.class_name}#test #{call.arguments&.arguments&.first&.slice}",
+                                       definition: definition(call), extent: call.location,
+                                       status: block_status(call, :pending))
+      end
+
+      # A minitest/spec `it` without a block is skipped by Minitest.
+      def spec_example(call, context)
+        description = description(call) || "example at line #{call.location.start_line}"
+        add(context, call.block&.body, name: [*context.names, description].join(' '),
+                                       definition: definition(call), extent: call.location,
+                                       status: block_status(call, :skipped))
+      end
+
+      def block_status(call, without_block)
+        return without_block unless call.block
+
+        skip_call?(call.block.body) ? :skipped : :active
+      end
+
+      def add(context, body, definition:, extent:, **fields)
+        assertions, helper_calls = inventory(context).scan(body)
+        @tests << TestCase.new(span: span(definition), extent: span(extent), framework: :minitest, setup: context.setup,
+                               assertions:, helper_calls:, bindings: context.bindings.merge(assignments(body).to_h),
+                               doubles: context.doubles + Stubs.scan(body, Doubles.new(@source, :body)), **fields)
+      end
+
+      # A `skip` call anywhere in the body skips the test.
+      def skip_call?(node)
+        return false unless node
+
+        node.compact_child_nodes.any? do |child|
+          next false if child.is_a?(Prism::DefNode)
+
+          (child.is_a?(Prism::CallNode) && child.receiver.nil? && child.name == :skip) || skip_call?(child)
+        end
+      end
+
+      def inventory(context)
+        Inventory.new(source: @source, helpers: @helpers, assertion: Assertions.method(:fields),
+                      ignored: ->(name) { Assertions::VOCABULARY.include?(name) || context.bindings.include?(name) })
+      end
+
+      # What a class or group's setup gives its tests: bindings and doubles.
+      module Scope
+        module_function
+
+        # [node, kind] pairs => +context+ with their bindings and doubles added.
+        def apply(context, own, doubles)
+          context.with(bindings: context.bindings.merge(*own.map { |node, kind| bindings(node, kind) }),
+                       doubles: context.doubles + own.flat_map { |node, kind| doubles(node, kind, doubles) })
+        end
+
+        # The setup kind a `def setup` or `setup do` declares, or nil.
+        def kind(node)
+          case node
+          when Prism::DefNode then SETUP_METHODS[node.name]
+          when Prism::CallNode then node.receiver.nil? ? SETUP_CALLS[node.name] : nil
+          end
+        end
+
+        # `let(:order) { ... }` binds `order`; setup binds the variables it assigns, such as `@order`.
+        def bindings(node, kind)
+          return { Syntax.binding_name(node) => Syntax.block_value(node) } if %i[let subject].include?(kind)
+
+          BEFORE.include?(kind) ? Syntax.assignments(body(node)).to_h : {}
+        end
+
+        def doubles(node, kind, doubles) = BEFORE.include?(kind) ? Stubs.scan(body(node), doubles) : []
+
+        def body(node) = node.is_a?(Prism::DefNode) ? node.body : node.block&.body
+      end
+
+      # Finds Minitest block stubs (`obj.stub(:m, v) do ... end`), mocha stubs and expectations
+      # (`obj.stubs(:m).returns(v)`, `obj.expects(:m)`), and `Minitest::Mock#expect`.
+      module Stubs
+        # mocha method => whether it is a stub source. `expects` can fail on its own.
+        MOCHA = { stubs: true, expects: false }.freeze
+
+        module_function
+
+        # The TestDoubles in +node+, in source order. +doubles+ is a Frameworks::Doubles.
+        def scan(node, doubles, found = [])
+          node&.compact_child_nodes&.each do |child|
+            next if child.is_a?(Prism::DefNode)
+
+            matched = mocha(child, doubles) || mock_expect(child, doubles)
+            next found.concat(matched) if matched
+
+            found.concat(block_stub(child, doubles))
+            scan(child, doubles, found)
+          end
+          found
+        end
+
+        # `obj.stubs(:m).with(1).returns(v)`: the chain is inside-out, so `stubs` sits under any
+        # `with` and `returns` calls. `stubs(m: v)` stubs several methods.
+        def mocha(node, doubles)
+          chain = Syntax.chain(node)
+          index = chain.index { MOCHA.key?(it.name) && it.receiver } or return
+          fields = fields(chain[index])
+          name = Syntax.stubbed_name(chain[index]) or return pairs(chain[index], node.location, doubles, fields)
+
+          [doubles.build(node.location, method_name: name, returns: returned(chain[...index]), **fields)]
+        end
+
+        def fields(call) = { receiver: call.receiver, stub_source: MOCHA[call.name] }
+
+        def pairs(call, location, doubles, fields)
+          Syntax.keyword_pairs(call).map do |name, value|
+            doubles.build(location, method_name: name, returns: value, **fields)
+          end
+        end
+
+        # The single value given to `returns`; several values are a sequence, not one value.
+        def returned(outer)
+          returns = outer.find { it.name == :returns } or return
+          values = Syntax.arguments(returns)
+          values.first if values.size == 1
+        end
+
+        # `mock.expect(:m, v)` is a message expectation, verified later.
+        def mock_expect(node, doubles)
+          return unless node.is_a?(Prism::CallNode) && node.name == :expect && node.receiver
+
+          name = Syntax.stubbed_name(node) or return
+          [doubles.build(node.location, receiver: node.receiver, method_name: name,
+                                        returns: Syntax.arguments(node)[1], stub_source: false)]
+        end
+
+        # `obj.stub(:m, v) do ... end` stubs only inside its block; the block itself is still scanned.
+        def block_stub(node, doubles)
+          name = block_stub?(node) && Syntax.stubbed_name(node) or return []
+
+          location = node.receiver.location.join(node.closing_loc || node.arguments.location)
+          [doubles.build(location, receiver: node.receiver, method_name: name, returns: Syntax.arguments(node)[1],
+                                   scope: node.block.location)]
+        end
+
+        def block_stub?(node)
+          node.is_a?(Prism::CallNode) && node.name == :stub && node.receiver && node.block.is_a?(Prism::BlockNode)
+        end
+      end
+
+      # Recognises Minitest assertions, expectations, mocha expectations and mock verification.
+      module Assertions
+        COMPARISONS = { assert_equal: :equality, assert_same: :identity, must_equal: :equality,
+                        must_be_same_as: :identity }.freeze
+        RAISE_CHECKS = %i[assert_raises must_raise].freeze
+        # `_(x)`, `value(x)` and `expect(x)` wrap the value a minitest/spec expectation checks.
+        WRAPPERS = %i[_ value expect].freeze
+
+        # Receiverless calls that are Minitest or Ruby vocabulary, never assertion helpers.
+        VOCABULARY = Set[
+          *GROUPS, *EXAMPLES, *SETUP_CALLS.keys, :test, :skip, *WRAPPERS, :puts, :p, :pp, :print, :raise, :fail,
+          :lambda, :proc, :loop, :sleep, :require, :require_relative, :format
+        ].freeze
+
+        module_function
+
+        # Assertion fields for an assertion, or nil when +call+ is not one.
+        def fields(call)
+          name = call.name.to_s
+          if call.receiver.nil? && (name.start_with?('assert', 'refute') || name == 'flunk')
+            assertion(call)
+          elsif name.start_with?('must_', 'wont_')
+            expectation(call)
+          elsif message_expectation?(call)
+            { kind: :message_expectation, comparison: nil, subject: nil, expected: nil }
+          end
+        end
+
+        # `assert_equal expected, actual`
+        def assertion(call)
+          arguments = call.arguments&.arguments || []
+          kind, comparison = classify(call.name)
+          { kind:, comparison:, subject: comparison && arguments[1], expected: comparison && arguments[0] }
+        end
+
+        # `_(actual).must_equal expected`, `actual.must_equal expected`
+        def expectation(call)
+          kind, comparison = classify(call.name)
+          { kind:, comparison:, subject: comparison && wrapped(call.receiver),
+            expected: comparison && call.arguments&.arguments&.first }
+        end
+
+        def wrapped(receiver)
+          return receiver unless receiver.is_a?(Prism::CallNode) && receiver.receiver.nil? &&
+                                 WRAPPERS.include?(receiver.name)
+
+          receiver.arguments&.arguments&.first
+        end
+
+        # mocha `obj.expects(:m)`, `mock.expect(:m, value)` and `mock.verify`
+        def message_expectation?(call)
+          return false if call.receiver.nil?
+
+          case call.name
+          when :expects then true
+          when :expect then call.arguments&.arguments&.first.is_a?(Prism::SymbolNode)
+          when :verify then call.arguments.nil?
+          else false
+          end
+        end
+
+        def classify(name)
+          return [:comparison, COMPARISONS[name]] if COMPARISONS.key?(name)
+          return [:raise_check, nil] if RAISE_CHECKS.include?(name)
+
+          [:other, nil]
+        end
+      end
+    end
+  end
+end