megatest 0.1.0

data/lib/megatest/differ.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+# :stopdoc:
+
+module Megatest
+  class Differ
+    HEADER = "--- expected\n+++ actual\n\n"
+
+    def initialize(config)
+      @config = config
+    end
+
+    using Compat::ByteRIndex unless String.method_defined?(:byterindex)
+
+    def call(expected, actual)
+      if String === expected && String === actual
+        if multiline?(expected) || multiline?(actual)
+          multiline_string_diff(expected, actual)
+        else
+          single_line_string_diff(expected, actual)
+        end
+      elsif Array === expected && Array === actual
+        array_diff(expected, actual)
+      elsif Hash === expected && Hash === actual
+        hash_diff(expected, actual)
+      else
+        expected_inspect = pp(expected)
+        actual_inspect = pp(actual)
+
+        if multiline?(expected_inspect) || multiline?(actual_inspect)
+          object_diff(expected, expected_inspect, actual_inspect)
+        end
+      end
+    end
+
+    private
+
+    def pp(object)
+      @config.pretty_print(object)
+    end
+
+    def object_diff(expected, expected_inspect, actual_inspect)
+      differ = PatienceDiff::Differ.new(@config.colors)
+      diff = differ.diff_text(expected_inspect, actual_inspect)
+      render_diff(expected, diff)
+    end
+
+    def multiline_string_diff(expected, actual)
+      differ = PatienceDiff::Differ.new(@config.colors)
+
+      if expected.encoding != actual.encoding
+        expected = encoding_prefix(expected) << expected
+        actual = encoding_prefix(actual) << actual
+      end
+
+      if need_escape?(expected) || need_escape?(actual)
+        expected = escape_string(expected)
+        actual = escape_string(actual)
+      end
+
+      if expected.end_with?("\n") ^ actual.end_with?("\n")
+        expected = "#{expected}\n\\ No newline at end of string" unless expected.end_with?("\n")
+        actual = "#{actual}\n\\ No newline at end of string" unless actual.end_with?("\n")
+      end
+      diff = differ.diff_text(expected, actual)
+      render_diff(expected, diff)
+    end
+
+    def multiline?(string)
+      string.byterindex("\n", -1)
+    end
+
+    def render_diff(expected, diff)
+      if diff
+        "#{HEADER}#{diff.join}"
+      else
+        <<~TEXT
+          No visible difference in the #{expected.class}#inspect output.
+          You should look at the implementation of #== on #{expected.class.name} or its members.
+          #{pp(expected)}
+        TEXT
+      end
+    end
+
+    def encoding_prefix(string)
+      encoding_name = string.encoding == Encoding::BINARY ? "BINARY" : string.encoding.name
+      prefix = +"# encoding: #{encoding_name}\n"
+      prefix.force_encoding(string.encoding)
+    end
+
+    def escape_string(string)
+      string.b.split("\n").map { |line| line.inspect.byteslice(1..-2) }.join("\n")
+    end
+
+    def need_escape?(string)
+      (string.encoding == Encoding::BINARY && !string.ascii_only?) || !string.valid_encoding?
+    end
+
+    def single_line_string_diff(expected, actual)
+      "Expected: #{pp(expected)}\n  Actual: #{pp(actual)}"
+    end
+
+    def array_diff(expected, actual)
+      differ = PatienceDiff::Differ.new(@config.colors)
+      diff = differ.diff_sequences(array_sequence(expected), array_sequence(actual))
+      render_diff(expected, diff)
+    end
+
+    def array_sequence(array)
+      array = array.map { |e| "  ".dup << pp(e).chomp << ",\n" }
+      array.unshift("[\n")
+      array << "]\n"
+    end
+
+    def hash_diff(expected, actual)
+      differ = PatienceDiff::Differ.new(@config.colors)
+      expected_seq, actual_seq = hash_sort(expected, actual)
+      diff = differ.diff_sequences(hash_sequence(expected_seq), hash_sequence(actual_seq))
+      render_diff(expected, diff)
+    end
+
+    def hash_sort(expected, actual)
+      [expected.sort, actual.sort]
+    rescue ArgumentError
+      [expected, actual]
+    end
+
+    def hash_sequence(pairs)
+      pairs = pairs.map do |k, v|
+        "  ".dup << pp(k) << " => " << pp(v).chomp << ",\n"
+      end
+      pairs.unshift("{\n")
+      pairs << "}\n"
+    end
+  end
+end

data/lib/megatest/dsl.rb

@@ -0,0 +1,164 @@
+# frozen_string_literal: true
+
+module Megatest
+  # All the methods necessary to define test cases.
+  # Can be used directly to define test cases in modules
+  # for later inclusion.
+  #
+  # Example:
+  #
+  #  module SomeSharedTests
+  #    extend Megatest::DSL
+  #
+  #    setup do
+  #    end
+  #
+  #    test "the truth" do
+  #      assert_equal 4, 2 + 2
+  #    end
+  #  end
+  #
+  #  class SomeTest < Megatest::Test
+  #    include SomeSharedTests
+  #  end
+  #
+  #  class SomeOtherTest < Megatest::Test
+  #    include SomeSharedTests
+  #  end
+  module DSL
+    # :stopdoc:
+    class << self
+      def extended(mod)
+        super
+        if mod.is_a?(Class)
+          unless mod == ::Megatest::Test
+            raise ArgumentError, "Megatest::DSL should only be extended in modules"
+          end
+        else
+          ::Megatest.registry.shared_suite(mod)
+        end
+      end
+    end
+
+    using Compat::StartWith unless Symbol.method_defined?(:start_with?)
+
+    def method_added(name)
+      super
+      if name.start_with?("test_")
+        ::Megatest.registry.suite(self).register_test_case(name, instance_method(name), nil)
+      end
+    end
+
+    # :startdoc:
+
+    ##
+    # Define a test case.
+    #
+    # Example:
+    #
+    #  test "the truth" do
+    #    assert_equal 4, 2 + 2
+    #  end
+    #
+    # For ease of transition from other test frameworks, any method
+    # that starts by +test_+ is also considered a test:
+    #
+    # Example:
+    #
+    #  def test_the_truth
+    #    assert_equal 4, 2 + 2
+    #  end
+    def test(name, tags = nil, &block)
+      ::Megatest.registry.suite(self).register_test_case(-name, block, tags)
+    end
+
+    # Applies tags to all the test case of this suite
+    #
+    # Example:
+    #
+    #  class SomeTest < Megatest::Test
+    #    tag focus: true
+    #
+    #    test "something" do
+    #      assert_equal true, __test__.tag(:focus)
+    #    end
+    #
+    #    test "something else", focus: false do
+    #      assert_equal false, __test__.tag(:focus)
+    #    end
+    #  end
+    def tag(**kwargs)
+      ::Megatest.registry.suite(self).add_tags(kwargs)
+    end
+
+    ##
+    # Creates a context block, for logically grouping test cases.
+    # The context string will be prepended to all the test cases
+    # defined within the block.
+    #
+    # Example:
+    #
+    #  context "maths" do
+    #    test "the truth" do
+    #      assert_equal 4, 2 + 2
+    #    end
+    #
+    #    test "oddity" do
+    #      refute_predicate 4, odd?
+    #    end
+    #  end
+    #
+    # Setup and teardown callbacks are not allowed withing a context blocks,
+    # as it too easily lead to "write only" tests. It's only meant to help
+    # group test cases together.
+    #
+    # If you need a common setup procedure, just define a helper method, and explictly call it.
+    #
+    # Example:
+    #
+    #  context "admin user" do
+    #    def setup_admin_user
+    #      # ...
+    #    end
+    #
+    #    test "#admin?" do
+    #      user = setup_admin_user
+    #      assert_predicate user, :admin?
+    #    end
+    #
+    #    test "#can?(:delete_post)" do
+    #      user = setup_admin_user
+    #      assert user.can?(:delete_post)
+    #    end
+    #  end
+    def context(name, tags = nil, &block)
+      ::Megatest.registry.suite(self).with_context(name, tags, &block)
+    end
+
+    # Registers a block to be invoked before every test cases.
+    def setup(&block)
+      ::Megatest.registry.suite(self).on_setup(block)
+    end
+
+    # Registers a block to be invoked around every test cases.
+    # The block will recieve a Proc as first argument and MUST
+    # call it.
+    #
+    # Example:
+    #
+    #  around do |block|
+    #    do_something do
+    #      block.call
+    #    end
+    #  end
+    def around(&block)
+      ::Megatest.registry.suite(self).on_around(block)
+    end
+
+    # Registers a block to be invoked after every test cases,
+    # regardless of whether it passed or failed.
+    def teardown(&block)
+      ::Megatest.registry.suite(self).on_teardown(block)
+    end
+  end
+end

data/lib/megatest/executor.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+# :stopdoc:
+
+module Megatest
+  class Executor
+    class ExternalMonitor
+      def initialize(config)
+        require "rbconfig"
+        @config = config
+        spawn
+      end
+
+      def reap
+        if @pipe
+          @pipe.close
+          @pipe = nil
+          _, status = Process.waitpid2(@pid)
+          @pid = nil
+          status
+        end
+      end
+
+      def spawn
+        child_read, @pipe = IO.pipe
+        ready_pipe, child_write = IO.pipe
+        @pid = Process.spawn(
+          RbConfig.ruby,
+          File.expand_path("../queue_monitor.rb", __FILE__),
+          in: child_read,
+          out: child_write,
+        )
+        child_read.close
+        Marshal.dump(@config, @pipe)
+
+        # Check the process is alive.
+        if ready_pipe.wait_readable(10)
+          ready_pipe.gets
+          ready_pipe.close
+          Process.kill(0, @pid)
+        else
+          Process.kill(0, @pid)
+          Process.wait(@pid)
+          raise Error, "ExternalMonitor failed to boot"
+        end
+      end
+    end
+
+    attr_reader :wall_time
+
+    def initialize(config, out)
+      @config = config
+      @out = Output.new(out, colors: @config.colors)
+    end
+
+    def run(queue, reporters)
+      start_time = Megatest.now
+
+      @config.run_global_setup_callbacks
+      @config.run_job_setup_callbacks(nil)
+
+      monitor = ExternalMonitor.new(@config) if queue.respond_to?(:heartbeat)
+
+      reporters.each { |r| r.start(self, queue) }
+
+      runner = Runner.new(@config)
+
+      begin
+        while true
+          if test_case = queue.pop_test
+            reporters.each { |r| r.before_test_case(queue, test_case) }
+
+            result = runner.execute(test_case)
+
+            result = queue.record_result(result)
+            reporters.each { |r| r.after_test_case(queue, test_case, result) }
+
+            @config.circuit_breaker.record_result(result)
+            break if @config.circuit_breaker.break?
+          elsif queue.empty?
+            break
+          else
+            # There was no tests to pop, but not all tests are completed.
+            # So we stick around to pop tests that could be lost.
+            sleep(1)
+          end
+        end
+      rescue Interrupt
+        # Early exit
+      end
+
+      monitor&.reap
+
+      @wall_time = Megatest.now - start_time
+      reporters.each { |r| r.summary(self, queue, queue.summary) }
+
+      if @config.circuit_breaker.break?
+        @out.error("Exited early because too many failures were encountered")
+      end
+
+      queue.cleanup
+    end
+  end
+end

data/lib/megatest/multi_process.rb

@@ -0,0 +1,263 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require "socket"
+
+# :stopdoc:
+
+module Megatest
+  # Fairly experimental multi-process queue implementation.
+  # It's absolutely not resilient yet, if something goes a bit wrong
+  # in may fail in unexpected ways (e.g. hang or whatever).
+  # At this stage it's only here to uncover what in the design
+  # need to be refactored to make multi-processing and test
+  # distribution work well (See the TODOs).
+  module MultiProcess
+    class << self
+      def socketpair
+        UNIXSocket.socketpair(:SOCK_STREAM).map { |s| MessageSocket.new(s) }
+      end
+    end
+
+    class MessageSocket
+      def initialize(socket)
+        @socket = socket
+      end
+
+      def <<(message)
+        begin
+          @socket.write(Marshal.dump(message))
+        rescue Errno::EPIPE
+          return nil # Other side was closed
+        end
+        self
+      end
+
+      def read
+        Marshal.load(@socket)
+      rescue EOFError
+        nil # Other side was closed
+      end
+
+      def closed?
+        @socket.closed?
+      end
+
+      def close
+        @socket.close
+      end
+
+      def to_io
+        @socket
+      end
+    end
+
+    class ClientQueue
+      def initialize(socket, test_cases_index)
+        @socket = socket
+        @test_cases_index = test_cases_index
+      end
+
+      def pop_test
+        @socket << [:pop]
+        if test_id = @socket.read
+          @test_cases_index.fetch(test_id)
+        end
+      end
+
+      def record_result(result)
+        @socket << [:record, result]
+        @socket.read
+      end
+
+      def close
+        @socket.close
+      end
+
+      def to_io
+        @socket.to_io
+      end
+    end
+
+    class Job
+      def initialize(config, index)
+        @config = config
+        @index = index
+        @pid = nil
+        @child_socket, @parent_socket = MultiProcess.socketpair
+        @assigned_test = nil
+        @idle = false
+      end
+
+      def run(executor, parent_queue)
+        @pid = Process.fork do
+          @config.job_index = @index
+          @parent_socket.close
+          executor.after_fork_in_child(self)
+
+          queue = ClientQueue.new(@child_socket, parent_queue)
+          @config.run_job_setup_callbacks(@index)
+
+          runner = Runner.new(@config)
+
+          begin
+            while (test_case = queue.pop_test)
+              result = runner.execute(test_case)
+              result = queue.record_result(result)
+              @config.circuit_breaker.record_result(result)
+              break if @config.circuit_breaker.break?
+            end
+          rescue Interrupt
+          end
+          queue.close
+
+          # We don't want to run at_exit hooks the app may have
+          # installed.
+          Process.exit!(0)
+        end
+        @child_socket.close
+      end
+
+      def to_io
+        @parent_socket.to_io
+      end
+
+      def term
+        Process.kill(:TERM, @pid)
+      rescue Errno::ESRCH
+        # Already dead
+      end
+
+      def close
+        @parent_socket.close
+        @child_socket.close
+      end
+
+      def closed?
+        @parent_socket.closed?
+      end
+
+      def idle?
+        @idle
+      end
+
+      def process(queue, reporters)
+        if @idle
+          if @assigned_test = queue.pop_test
+            @idle = false
+            @parent_socket << @assigned_test&.id
+          end
+          return
+        end
+
+        message, *args = @parent_socket.read
+        case message
+        when nil
+          # Socket closed, child probably died
+          @parent_socket.close
+        when :pop
+          if @assigned_test = queue.pop_test
+            @parent_socket << @assigned_test&.id
+          else
+            @idle = true
+          end
+        when :record
+          result = queue.record_result(*args)
+          @assigned_test = nil
+          @parent_socket << result
+          reporters.each { |r| r.after_test_case(queue, nil, result) }
+          @config.circuit_breaker.record_result(result)
+        else
+          raise "Unexpected message: #{message.inspect}"
+        end
+      end
+
+      def on_exit(queue, reporters)
+        if @assigned_test
+          result = queue.record_lost_test(@assigned_test)
+          @assigned_test = nil
+          reporters.each { |r| r.after_test_case(queue, nil, result) }
+        end
+      end
+
+      def reap
+        Process.wait(@pid)
+      end
+    end
+
+    class InlineMonitor
+      def initialize(config, queue)
+        @config = config
+        @queue = queue
+        @last_heartbeat = 0
+      end
+
+      def tick
+        now = Megatest.now
+        if now - @last_heartbeat > @config.heartbeat_frequency && @queue.heartbeat
+          @last_heartbeat = now
+        end
+      end
+    end
+
+    class Executor
+      attr_reader :wall_time
+
+      def initialize(config, out)
+        @config = config
+        @out = Output.new(out, colors: config.colors)
+      end
+
+      def after_fork_in_child(active_job)
+        @jobs.each do |job|
+          job.close unless job == active_job
+        end
+      end
+
+      def run(queue, reporters)
+        start_time = Megatest.now
+        @config.run_global_setup_callbacks
+        @jobs = @config.jobs_count.times.map { |index| Job.new(@config, index) }
+
+        @config.before_fork_callbacks.each(&:call)
+        @jobs.each { |j| j.run(self, queue.test_cases_index) }
+
+        monitor = InlineMonitor.new(@config, queue) if queue.respond_to?(:heartbeat)
+
+        begin
+          while true
+            monitor&.tick
+            dead_jobs = @jobs.select(&:closed?).each { |j| j.on_exit(queue, reporters) }
+            @jobs -= dead_jobs
+            break if @jobs.empty?
+            break if queue.empty?
+
+            @jobs.select(&:idle?).each do |job|
+              job.process(queue, reporters)
+            end
+
+            reads, = IO.select(@jobs, nil, nil, 1)
+            reads&.each do |job|
+              job.process(queue, reporters)
+            end
+
+            break if @config.circuit_breaker.break?
+          end
+        rescue Interrupt
+          @jobs.each(&:term) # Early exit
+        end
+
+        @jobs.each(&:close)
+        @jobs.each(&:reap)
+        @wall_time = Megatest.now - start_time
+        reporters.each { |r| r.summary(self, queue, queue.summary) }
+
+        if @config.circuit_breaker.break?
+          @out.error("Exited early because too many failures were encountered")
+        end
+
+        queue.cleanup
+      end
+    end
+  end
+end