megatest 0.1.0

data/lib/megatest/runner.rb

@@ -0,0 +1,119 @@
+# frozen_string_literal: true
+
+# :stopdoc:
+
+module Megatest
+  class Runner
+    def initialize(config)
+      @config = config
+    end
+
+    def execute(test_case)
+      if test_case.tag(:isolated)
+        isolate(test_case) do
+          run(test_case)
+        end
+      else
+        run(test_case)
+      end
+    end
+
+    if Megatest.fork?
+      def isolate(test_case)
+        read, write = IO.pipe.each(&:binmode)
+        pid = Process.fork do
+          read.close
+          result = yield
+          Marshal.dump(result, write)
+          write.close
+          # We don't want to run at_exit hooks the app may have
+          # installed.
+          Process.exit!(0)
+        end
+        write.close
+        result = begin
+          Marshal.load(read)
+        rescue EOFError
+          TestCaseResult.new(test_case).lost
+        end
+        Process.wait(pid)
+        result
+      end
+    else
+      def isolate(test_case)
+        parent_read, child_write = IO.pipe.each(&:binmode)
+        child_read, parent_write = IO.pipe.each(&:binmode)
+        pid = Subprocess.spawn(child_read, child_write, "run_test")
+        child_read.close
+        child_write.close
+
+        Marshal.dump(@config, parent_write)
+        Marshal.dump(test_case.source_file, parent_write)
+        Marshal.dump(test_case.id, parent_write)
+        parent_write.close
+
+        result = begin
+          Marshal.load(parent_read)
+        rescue EOFError
+          TestCaseResult.new(test_case).lost
+        end
+        Process.wait(pid)
+        result
+      end
+    end
+
+    def run(test_case)
+      result = TestCaseResult.new(test_case)
+      runtime = Runtime.new(@config, test_case, result)
+      instance = test_case.klass.new(runtime)
+
+      # We always reset the seed before running any test as to have the most consistent
+      # result as possible, especially on retries.
+      Random.srand(@config.seed)
+
+      result.record_time do
+        ran = false
+        failed = false
+        recursive_callbacks(test_case.around_callbacks) do
+          ran = true
+          return result if runtime.record_failures { instance.before_setup }
+
+          test_case.each_setup_callback do |callback|
+            failed ||= runtime.record_failures(downlevel: 2) { instance.instance_exec(&callback) }
+          end
+          failed ||= runtime.record_failures { instance.setup }
+          failed ||= runtime.record_failures { instance.after_setup }
+
+          failed ||= test_case.execute(runtime, instance)
+
+          result.ensure_assertions unless @config.minitest_compatibility
+        ensure
+          runtime.record_failures do
+            instance.before_teardown
+          end
+          test_case.each_teardown_callback do |callback|
+            runtime.record_failures(downlevel: 2) do
+              instance.instance_exec(&callback)
+            end
+          end
+          runtime.record_failures do
+            instance.teardown
+          end
+          runtime.record_failures do
+            instance.after_teardown
+          end
+        end
+
+        result.did_not_run("Test wasn't run. Did an around block failed to yield?") unless ran
+      end
+    end
+
+    def recursive_callbacks(callbacks, &block)
+      if callback = callbacks.pop
+        callback.call(-> { recursive_callbacks(callbacks, &block) })
+      else
+        yield
+      end
+    end
+  end
+end

data/lib/megatest/runtime.rb

@@ -0,0 +1,168 @@
+# frozen_string_literal: true
+
+# :stopdoc:
+
+module Megatest
+  class Runtime
+    attr_reader :test_case, :result
+
+    def initialize(config, test_case, result)
+      @config = config
+      @test_case = test_case
+      @result = result
+      @asserting = false
+    end
+
+    support_locations = begin
+      error = StandardError.new
+      # Ruby 3.4: https://github.com/ruby/ruby/pull/10017
+      error.set_backtrace(caller_locations(1, 1))
+      true
+    rescue TypeError
+      false
+    end
+
+    if support_locations
+      def assert(uplevel: 1)
+        if @asserting
+          yield
+        else
+          @asserting = true
+          @result.assertions_count += 1
+          begin
+            yield
+          rescue Assertion => failure
+            if failure.backtrace.empty?
+              failure.set_backtrace(caller_locations(uplevel + 2))
+            end
+            raise
+          ensure
+            @asserting = false
+          end
+        end
+      end
+
+      def strip_backtrace(error, yield_file, yield_line, downlevel)
+        if backtrace = error.backtrace_locations
+          rindex = backtrace.rindex { |l| l.lineno == yield_line && l.path == yield_file }
+          backtrace = backtrace.slice(0..rindex)
+          backtrace.pop(downlevel) unless downlevel.zero?
+          error.set_backtrace(backtrace)
+        elsif backtrace = error.backtrace
+          yield_point = "#{yield_file}:#{yield_line}:"
+          rindex = backtrace.rindex { |l| l.start_with?(yield_point) }
+          backtrace = backtrace.slice(0..rindex)
+          backtrace.pop(downlevel) unless downlevel.zero?
+          error.set_backtrace(backtrace)
+        end
+
+        error
+      end
+    else
+      def assert(uplevel: 1)
+        if @asserting
+          yield
+        else
+          @asserting = true
+          @result.assertions_count += 1
+          begin
+            yield
+          rescue Assertion => failure
+            if failure.backtrace.empty?
+              failure.set_backtrace(caller(uplevel + 2))
+            end
+            raise
+          ensure
+            @asserting = false
+          end
+        end
+      end
+
+      def strip_backtrace(error, yield_file, yield_line, downlevel)
+        if backtrace = error.backtrace
+          yield_point = "#{yield_file}:#{yield_line}:"
+          rindex = backtrace.rindex { |l| l.start_with?(yield_point) }
+          backtrace = backtrace.slice(0..rindex)
+          backtrace.pop(downlevel) unless downlevel.zero?
+          error.set_backtrace(backtrace)
+        end
+
+        error
+      end
+    end
+
+    def msg(positional, keyword)
+      if positional.nil?
+        keyword
+      elsif !keyword.nil?
+        raise ArgumentError, "Can't pass both a positional and keyword assertion message"
+      else
+        positional # TODO: deprecation mecanism
+      end
+    end
+
+    def expect_no_failures
+      was_asserting = @asserting
+      @asserting = false
+      yield
+    rescue Assertion, *Megatest::IGNORED_ERRORS
+      raise # Exceptions we shouldn't rescue
+    rescue Exception => unexpected_error
+      raise UnexpectedError, unexpected_error, EMPTY_BACKTRACE
+    ensure
+      @asserting = was_asserting
+    end
+
+    EMPTY_BACKTRACE = [].freeze
+
+    def fail(user_message, *message)
+      message = build_message(message)
+      if user_message
+        user_message = user_message.call if user_message.respond_to?(:call)
+        user_message = String(user_message)
+        if message && !user_message.end_with?("\n")
+          user_message += "\n"
+        end
+        message = "#{user_message}#{message}"
+      end
+      raise(Assertion, message, EMPTY_BACKTRACE)
+    end
+
+    def build_message(strings)
+      return if strings.empty?
+
+      if (strings.size + strings.sum(&:size)) < 80
+        strings.join(" ")
+      else
+        strings.join("\n\n")
+      end
+    end
+
+    def minitest_compatibility?
+      @config.minitest_compatibility
+    end
+
+    def pp(object)
+      @config.pretty_print(object)
+    end
+
+    def diff(expected, actual)
+      @config.diff(expected, actual)
+    end
+
+    def record_failures(downlevel: 1, &block)
+      expect_no_failures(&block)
+    rescue Assertion => assertion
+      error = assertion
+      while error
+        error = strip_backtrace(error, __FILE__, __LINE__ - 4, downlevel + 2)
+        error = error.cause
+      end
+
+      @result.failures << Failure.new(assertion)
+      true
+    else
+      false
+    end
+  end
+end

data/lib/megatest/selector.rb

@@ -0,0 +1,293 @@
+# frozen_string_literal: true
+
+# :stopdoc:
+
+module Megatest
+  module Selector
+    class List
+      def initialize(loaders, filters)
+        @loaders = loaders
+        if loaders.empty?
+          @loaders = [Loader.new("test")]
+        end
+        @filters = filters
+      end
+
+      def main_paths
+        paths = @loaders.map(&:path)
+        paths.compact!
+        paths.uniq!
+        paths
+      end
+
+      def paths(random:)
+        paths = @loaders.reduce([]) do |paths_to_load, loader|
+          loader.append_paths(paths_to_load)
+        end
+
+        paths.uniq!
+        paths.sort!
+        paths.shuffle!(random: random) if random
+        paths
+      end
+
+      def select(registry, random:)
+        # If any of the selector points to an exact test or a subset of a suite,
+        # then each selector is responsible for shuffling the group of tests it selects,
+        # so that tests are shuffled inside groups, but groups are ordered.
+        test_cases = if @loaders.any?(&:partial?)
+          @loaders.reduce([]) do |tests_to_run, loader|
+            loader.append_tests(tests_to_run, registry, random: random)
+          end
+        else
+          # Otherwise, we do one big shuffle at the end, all groups are mixed.
+          test_cases = registry.test_cases
+          test_cases.sort!
+          test_cases.shuffle!(random: random) if random
+          test_cases
+        end
+
+        @filters.reduce(test_cases) do |cases, filter|
+          filter.select(cases)
+        end
+      end
+    end
+
+    class Loader
+      attr_reader :path
+
+      def initialize(path, filter = nil)
+        @path = File.expand_path(path)
+        if @directory = File.directory?(@path)
+          @path = File.join(@path, "/")
+          @paths = Megatest.glob(@path)
+        else
+          @paths = [@path]
+        end
+        @filter = filter
+      end
+
+      def partial?
+        !!@filter
+      end
+
+      def append_tests(tests_to_run, registry, random:)
+        test_cases = select(registry)
+        if partial?
+          test_cases.sort!
+          test_cases.shuffle!(random: random) if random
+        end
+        tests_to_run.concat(test_cases)
+      end
+
+      def append_paths(paths_to_load)
+        paths_to_load.concat(@paths)
+      end
+
+      def select(registry)
+        test_cases = if @directory
+          registry.test_cases.select do |test_case|
+            test_case.source_file.start_with?(@path)
+          end
+        else
+          registry.test_cases_by_path(@path)
+        end
+
+        if @filter
+          @filter.select(test_cases)
+        else
+          test_cases
+        end
+      end
+    end
+
+    class NegativeLoader
+      def initialize(loader)
+        @loader = loader
+      end
+
+      def partial?
+        @loader.partial?
+      end
+
+      def append_paths(paths_to_load)
+        if @loader.partial?
+          paths_to_load
+        else
+          paths_to_not_load = @loader.append_paths([])
+          paths_to_load - paths_to_not_load
+        end
+      end
+
+      def append_tests(tests_to_run, registry, random:)
+        tests_to_not_run = @loader.append_tests([], registry, random: nil)
+        tests_to_run - tests_to_not_run
+      end
+    end
+
+    class TagFilter
+      class << self
+        def parse(arg)
+          if match = arg.match(/\A@([\w-]+)(?:=(.*))?\z/)
+            new(match[1], match[2])
+          end
+        end
+      end
+
+      def initialize(tag, value)
+        @tag = tag.to_sym
+        @value = value
+      end
+
+      def select(test_cases)
+        if @value
+          test_cases.select do |test_case|
+            test_case.tag(@tag).to_s == @value
+          end
+        else
+          test_cases.select do |test_case|
+            test_case.tag(@tag)
+          end
+        end
+      end
+    end
+
+    class ExactLineFilter
+      class << self
+        def parse(arg)
+          if match = arg.match(/\A(\d+)(?:~(\d+))?\z/)
+            new(Integer(match[1]), match[2]&.to_i)
+          end
+        end
+      end
+
+      def initialize(line, index)
+        @line = line
+        @index = index
+      end
+
+      def select(test_cases)
+        test_cases = test_cases.sort { |a, b| b.source_line <=> a.source_line }
+        test_cases = test_cases.drop_while { |t| t.source_line > @line }
+
+        # Line not found, fallback to run the whole file?
+        return [] if test_cases.empty?
+
+        real_line = test_cases.first&.source_line
+        test_cases = test_cases.take_while { |t| t.source_line == real_line }
+
+        if @index
+          test_cases.select! { |t| t.index == @index }
+        end
+        test_cases
+      end
+    end
+
+    class NameMatchFilter
+      class << self
+        def parse(arg)
+          if match = arg.match(%r{\A/(.+)\z})
+            new(match[1])
+          end
+        end
+      end
+
+      def initialize(pattern)
+        @pattern = Regexp.new(pattern)
+      end
+
+      def select(test_cases)
+        test_cases.select do |t|
+          @pattern.match?(t.name) || @pattern.match?(t.id)
+        end
+      end
+    end
+
+    class NameFilter
+      class << self
+        def parse(arg)
+          if match = arg.match(/\A#(.+)\z/)
+            new(match[1])
+          end
+        end
+      end
+
+      def initialize(name)
+        @name = name
+      end
+
+      def select(test_cases)
+        test_cases.select do |t|
+          @name == t.name || @name == t.id
+        end
+      end
+    end
+
+    class NegativeFilter
+      def initialize(filter)
+        @filter = filter
+      end
+
+      def select(test_cases)
+        test_cases - @filter.select(test_cases)
+      end
+    end
+
+    FILTERS = [
+      ExactLineFilter,
+      TagFilter,
+      NameMatchFilter,
+      NameFilter,
+    ].freeze
+
+    class << self
+      def parse(argv)
+        if argv.empty?
+          return List.new([], [])
+        end
+
+        argv = argv.dup
+        loaders = []
+        filters = []
+
+        negative = false
+
+        until argv.empty?
+          case argument = argv.shift
+          when "!"
+            negative = true
+          else
+            loader_str, filter_str = argument.split(":", 2)
+            loader_str = nil if loader_str.empty?
+
+            filter = nil
+            if filter_str
+              FILTERS.each do |filter_class|
+                if filter = filter_class.parse(filter_str)
+                  break
+                end
+              end
+            end
+
+            if loader_str
+              loader = Loader.new(loader_str, filter)
+              if negative
+                loader = NegativeLoader.new(loader)
+                negative = false
+              end
+              loaders << loader
+            else
+              if negative
+                filter = NegativeFilter.new(filter)
+                negative = false
+              end
+              filters << filter
+            end
+          end
+        end
+
+        List.new(loaders, filters)
+      end
+    end
+  end
+end