megatest 0.1.0

data/lib/megatest/backtrace.rb

@@ -0,0 +1,70 @@
+# frozen_string_literal: true
+
+module Megatest
+  class Backtrace
+    module InternalFilter
+      class << self
+        INTERNAL_PATHS = [
+          File.expand_path("../assertions.rb:", __FILE__).freeze,
+          File.expand_path("../runtime.rb:", __FILE__).freeze,
+        ].freeze
+
+        def call(backtrace)
+          backtrace.reject do |frame|
+            frame.start_with?(*INTERNAL_PATHS)
+          end
+        end
+      end
+    end
+
+    module RelativePathCleaner
+      class << self
+        def call(path)
+          Megatest.relative_path(path)
+        end
+      end
+    end
+
+    attr_accessor :filters, :formatters
+
+    def initialize
+      @filters = [InternalFilter]
+      @formatters = [RelativePathCleaner]
+      @full = false
+    end
+
+    def full!
+      @full = true
+    end
+
+    def clean(backtrace)
+      return backtrace if @full
+
+      format(filter(backtrace))
+    end
+
+    def filter(backtrace)
+      return backtrace if @full
+
+      if backtrace
+        filters.each do |filter|
+          backtrace = filter.call(backtrace)
+        end
+        backtrace
+      else
+        []
+      end
+    end
+
+    def format(backtrace)
+      return backtrace if @full
+
+      backtrace.map do |frame|
+        formatters.each do |formatter|
+          frame = formatter.call(frame)
+        end
+        frame
+      end
+    end
+  end
+end

data/lib/megatest/cli.rb

@@ -0,0 +1,249 @@
+# frozen_string_literal: true
+
+# :stopdoc:
+
+require "optparse"
+
+module Megatest
+  class CLI
+    InvalidArgument = Class.new(ArgumentError)
+
+    class << self
+      def run!
+        program_name = $PROGRAM_NAME
+        program_name = "megatest" if program_name == `command -v megatest`.strip
+        exit(new(program_name, $stdout, $stderr, ARGV, ENV).run)
+      end
+    end
+
+    undef_method :puts, :print # Should only use @out.puts or @err.puts
+
+    RUNNERS = {
+      "report" => :report,
+      "run" => :run,
+    }.freeze
+
+    def initialize(program_name, out, err, argv, env)
+      @out = out
+      @err = err
+      @argv = argv.dup
+      @processes = nil
+      @config = Config.new(env)
+      @program_name = @config.program_name = program_name
+      @runner = nil
+      @verbose = false
+      @junit = false
+    end
+
+    def run
+      configure
+      case @runner
+      when :report
+        report
+      when nil, :run
+        run_tests
+      else
+        raise InvalidArgument, "Parsing failure"
+      end
+    rescue InvalidArgument, OptionParser::ParseError => error
+      if error.is_a?(InvalidArgument)
+        @err.puts "invalid arguments: #{error.message}"
+      else
+        @err.puts error.message
+      end
+      @err.puts
+      @err.puts @parser
+      1
+    end
+
+    def configure
+      if @runner = RUNNERS[@argv.first]
+        @argv.shift
+      end
+
+      Megatest.config = @config
+      @parser = build_parser(@runner)
+      @parser.parse!(@argv)
+      @argv.shift if @argv.first == "--"
+      @config
+    end
+
+    def run_tests
+      queue = @config.build_queue
+
+      if queue.distributed?
+        raise InvalidArgument, "Distributed queues require a build-id" unless @config.build_id
+        raise InvalidArgument, "Distributed queues require a worker-id" unless @config.worker_id
+      elsif queue.sharded?
+        unless @config.valid_worker_index?
+          raise InvalidArgument, "Splitting the queue requires a worker-id lower than workers-count, got: #{@config.worker_id.inspect}"
+        end
+      end
+
+      @config.selectors = Selector.parse(@argv)
+      Megatest.load_config(@config)
+      Megatest.init(@config)
+      test_cases = Megatest.load_tests(@config)
+
+      if test_cases.empty?
+        @err.puts "No tests to run"
+        return 1
+      end
+
+      queue.populate(test_cases)
+      executor.run(queue, default_reporters)
+      queue.success? ? 0 : 1
+    end
+
+    def report
+      queue = @config.build_queue
+
+      raise InvalidArgument, "Only distributed queues can be summarized" unless queue.distributed?
+      raise InvalidArgument, "Distributed queues require a build-id" unless @config.build_id
+      raise InvalidArgument, @argv.join(" ") unless @argv.empty?
+
+      Megatest.load_config(@argv)
+
+      QueueReporter.new(@config, queue, @out).run(default_reporters) ? 0 : 1
+    end
+
+    private
+
+    def default_reporters
+      reporters = if @verbose || @config.ci
+        [
+          Reporters::VerboseReporter.new(@config, @out),
+        ]
+      else
+        [
+          Reporters::SimpleReporter.new(@config, @out),
+        ]
+      end
+
+      if @config.ci
+        reporters << Reporters::OrderReporter.new(@config, open_file("log/test_order.log"))
+      end
+
+      if @junit != false
+        junit_file = open_file(@junit || "log/junit.xml")
+        reporters << Reporters::JUnitReporter.new(@config, Megatest::Output.new(junit_file, colors: true))
+      end
+
+      reporters
+    end
+
+    def open_file(path)
+      File.open(path, "w+")
+    rescue Errno::ENOENT
+      mkdir_p(File.dirname(path))
+      retry
+    end
+
+    def mkdir_p(directory)
+      raise InvalidArgument if directory.empty?
+
+      Dir.mkdir(directory)
+    rescue Errno::ENOENT
+      mkdir_p(File.dirname(directory))
+      retry
+    rescue InvalidArgument
+      raise InvalidArgument, "Couldn't create directory: #{directory}"
+    end
+
+    def executor
+      if @config.jobs_count > 1
+        require "megatest/multi_process"
+        MultiProcess::Executor.new(@config, @out)
+      else
+        Executor.new(@config, @out)
+      end
+    end
+
+    def build_parser(runner)
+      runner = :run if runner.nil?
+      OptionParser.new do |opts|
+        case runner
+        when :report
+          opts.banner = "Usage: #{@program_name} report [options]"
+        when :run
+          opts.banner = "Usage: #{@program_name} run [options] [files or directories]"
+        else
+          opts.banner = "Usage: #{@program_name} command [options] [files or directories]"
+          opts.separator ""
+          opts.separator "Commands:"
+          opts.separator ""
+
+          opts.separator "\trun\t\tExecute the given tests."
+          opts.separator "\t\t\t  $ #{@program_name} test/integration/"
+          opts.separator "\t\t\t  $ #{@program_name} test/my_test.rb:42 test/another_test.rb:36"
+          opts.separator ""
+
+          opts.separator "\treport\t\tWait for the queue to be entirely processed and report the status"
+          opts.separator "\t\t\t  $ #{@program_name} report --queue redis://ci-queue.example.com --build-id $CI_BUILD_ID"
+          opts.separator ""
+        end
+
+        opts.separator ""
+        opts.separator "Options:"
+        opts.separator ""
+
+        opts.on("-b", "--backtrace", "Print full backtraces") do
+          @config.backtrace.full!
+        end
+
+        opts.on("-v", "--verbose", "Use the verbose reporter") do
+          @verbose = true
+        end
+
+        opts.on("--junit [PATH]", String, "Generate a junit.xml file") do |path|
+          @junit = path
+        end
+
+        if runner == :run
+          opts.on("--seed SEED", Integer, "The seed used to define run order") do |seed|
+            @config.seed = seed
+          end
+
+          opts.on("-j", "--jobs JOBS", Integer, "Number of processes to use") do |jobs|
+            @config.jobs_count = jobs
+          end
+
+          help = "Number of consecutive failures before exiting. Default to 1"
+          opts.on("-f", "--fail-fast [COUNT]", Integer, help) do |max|
+            @config.max_consecutive_failures = (max || 1)
+          end
+
+          opts.on("--max-retries COUNT", Integer, "How many times a given test may be retried") do |max_retries|
+            @config.max_retries = max_retries
+          end
+
+          opts.on("--retry-tolerance RATE", Float, "The proportion of tests that may be retried. e.g. 0.05 for 5% of retried tests") do |retry_tolerance|
+            @config.retry_tolerance = retry_tolerance
+          end
+        end
+
+        opts.separator ""
+        opts.separator "Test distribution and sharding:"
+        opts.separator ""
+
+        opts.on("--queue URL", String, "URL of queue server to use for test distribution. Default to $MEGATEST_QUEUE_URL") do |queue_url|
+          @config.queue_url = queue_url
+        end
+
+        opts.on("--build-id ID", String, "Unique identifier for the CI build") do |build_id|
+          @config.build_id = build_id
+        end
+
+        if runner == :run
+          opts.on("--worker-id ID", String, "Unique identifier for the CI job") do |worker_id|
+            @config.worker_id = worker_id
+          end
+
+          opts.on("--workers-count COUNT", Integer, "Number of CI jobs") do |workers_count|
+            @config.workers_count = workers_count
+          end
+        end
+      end
+    end
+  end
+end

data/lib/megatest/compat.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+# :stopdoc:
+
+module Megatest
+  module Compat
+    unless Enumerable.method_defined?(:filter_map) # RUBY_VERSION >= "2.7"
+      module FilterMap
+        refine Enumerable do
+          def filter_map(&block)
+            result = map(&block)
+            result.compact!
+            result
+          end
+        end
+      end
+    end
+
+    unless Symbol.method_defined?(:start_with?) # RUBY_VERSION >= "2.7"
+      module StartWith
+        refine Symbol do
+          def start_with?(*args)
+            to_s.start_with?(*args)
+          end
+        end
+      end
+    end
+
+    unless UnboundMethod.method_defined?(:bind_call) # RUBY_VERSION >= "2.7"
+      module BindCall
+        refine UnboundMethod do
+          def bind_call(receiver, *args, &block)
+            bind(receiver).call(*args, &block)
+          end
+        end
+      end
+    end
+
+    unless Enumerable.method_defined?(:tally) # RUBY_VERSION >= "2.7"
+      module Tally
+        refine Enumerable do
+          def tally(hash = {})
+            each do |element|
+              hash[element] = (hash[element] || 0) + 1
+            end
+            hash
+          end
+        end
+      end
+    end
+
+    unless Symbol.method_defined?(:name) # RUBY_VERSION >= "3.0"
+      module Name
+        refine Symbol do
+          alias_method :name, :to_s
+        end
+      end
+    end
+
+    unless String.method_defined?(:byterindex) # RUBY_VERSION >= "3.2"
+      module ByteRIndex
+        refine String do
+          def byterindex(matcher, offset = -1)
+            if encoding == Encoding::BINARY
+              rindex(matcher, offset)
+            else
+              b.rindex(matcher, offset)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/megatest/config.rb

@@ -0,0 +1,281 @@
+# frozen_string_literal: true
+
+# :stopdoc:
+
+module Megatest
+  class CircuitBreaker
+    def initialize(max)
+      @max = max
+      @consecutive_failures = 0
+    end
+
+    def record_result(result)
+      if result.bad?
+        @consecutive_failures += 1
+      elsif result.success?
+        @consecutive_failures = 0
+      end
+    end
+
+    def break?
+      @consecutive_failures >= @max
+    end
+  end
+
+  class CIService
+    @implementations = []
+
+    class << self
+      def inherited(base)
+        super
+        @implementations << base
+      end
+
+      def configure(config, env)
+        @implementations.each do |service|
+          service.new(env).configure(config)
+        end
+      end
+    end
+
+    attr_reader :env
+
+    def initialize(env)
+      @env = env
+    end
+
+    def configure(_config)
+      raise NotImplementedError
+    end
+
+    class CircleCI < self
+      def configure(config)
+        if env["CIRCLE_BUILD_URL"]
+          config.ci = true
+          config.build_id = env["CIRCLE_BUILD_URL"]
+          config.worker_id = env["CIRCLE_NODE_INDEX"]
+          config.workers_count = Integer(env["CIRCLE_NODE_TOTAL"])
+          config.seed = env["CIRCLE_SHA1"]&.first(4)&.to_i(16)
+        end
+      end
+    end
+
+    class Buildkite < self
+      def configure(config)
+        if env["BUILDKITE_BUILD_ID"]
+          config.ci = true
+          config.build_id = env["BUILDKITE_BUILD_ID"]
+          config.worker_id = env["BUILDKITE_PARALLEL_JOB"]
+          config.workers_count = env["BUILDKITE_PARALLEL_JOB_COUNT"]
+          config.seed = env["BUILDKITE_COMMIT"]&.first(4)&.to_i(16)
+        end
+      end
+    end
+
+    class Travis < self
+      def configure(config)
+        if env["TRAVIS_BUILD_ID"]
+          config.ci = true
+          config.build_id = env["TRAVIS_BUILD_ID"]
+          # Travis doesn't have builtin parallelization
+          # but CI_NODE_INDEX is what is used in their documentation
+          # https://docs.travis-ci.com/user/speeding-up-the-build#parallelizing-rspec-cucumber-and-minitest-on-multiple-vms
+          config.worker_id = env["CI_NODE_INDEX"]
+          config.workers_count = env["CI_NODE_TOTAL"]
+          config.seed = env["TRAVIS_COMMIT"]&.first(4)&.to_i(16)
+        end
+      end
+    end
+
+    class Heroku < self
+      def configure(config)
+        if env["HEROKU_TEST_RUN_ID"]
+          config.ci = true
+          config.build_id = env["HEROKU_TEST_RUN_ID"]
+          config.worker_id = env["CI_NODE_INDEX"]
+          config.workers_count = env["CI_NODE_TOTAL"]
+          config.seed = env["HEROKU_TEST_RUN_COMMIT_VERSION"]&.first(4)&.to_i(16)
+        end
+      end
+    end
+
+    class Megatest < self
+      def configure(config)
+        if env["CI"]
+          config.ci = true
+        end
+
+        if url = env["MEGATEST_QUEUE_URL"]
+          config.queue_url = url
+        end
+
+        if id = env["MEGATEST_BUILD_ID"]
+          config.build_id = id
+        end
+
+        if id = env["MEGATEST_WORKER_ID"]
+          config.worker_id = id
+        end
+
+        if seed = env["SEED"]
+          config.seed = seed
+        end
+      end
+    end
+  end
+
+  # :startdoc:
+
+  class << self
+    attr_writer :config
+
+    def config
+      yield @config if block_given?
+      @config
+    end
+  end
+
+  class Config
+    attr_accessor :queue_url, :retry_tolerance, :max_retries, :jobs_count, :job_index, :load_paths, :deprecations,
+                  :build_id, :heartbeat_frequency, :minitest_compatibility, :ci, :selectors
+    attr_reader :before_fork_callbacks, :global_setup_callbacks, :worker_setup_callbacks, :backtrace, :circuit_breaker, :seed,
+                :worker_id, :workers_count
+    attr_writer :differ, :pretty_printer, :program_name, :colors
+
+    def initialize(env)
+      @load_paths = ["test"] # For easier transition from other frameworks
+      @retry_tolerance = 0.0
+      @max_retries = 0
+      @deprecations = true
+      @full_backtrace = false
+      @queue_url = nil
+      @ci = false
+      @build_id = nil
+      @worker_id = nil
+      @workers_count = 1
+      @jobs_count = 1
+      @colors = nil # auto
+      @before_fork_callbacks = []
+      @global_setup_callbacks = []
+      @job_setup_callbacks = []
+      @heartbeat_frequency = 5
+      @backtrace = Backtrace.new
+      @program_name = nil
+      @circuit_breaker = CircuitBreaker.new(Float::INFINITY)
+      @seed = Random.rand(0xFFFF)
+      @differ = Differ.new(self)
+      @pretty_printer = PrettyPrint.new(self)
+      @minitest_compatibility = false
+      @selectors = nil
+      CIService.configure(self, env)
+    end
+
+    def program_name
+      @program_name || "megatest"
+    end
+
+    def worker_id=(id)
+      @worker_id = if id.is_a?(String) && /\A\d+\z/.match?(id)
+        Integer(id)
+      else
+        id
+      end
+    end
+
+    def workers_count=(count)
+      @workers_count = count ? Integer(count) : 1
+    end
+
+    def valid_worker_index?
+      worker_id.is_a?(Integer) && worker_id.positive? && worker_id < workers_count
+    end
+
+    def colors(io = nil)
+      case @colors
+      when true
+        Output::ANSIColors
+      when false
+        Output::NoColors
+      else
+        if io && !io.tty?
+          Output::NoColors
+        else
+          Output::ANSIColors
+        end
+      end
+    end
+
+    def max_consecutive_failures=(max)
+      @circuit_breaker = CircuitBreaker.new(max)
+    end
+
+    def diff(expected, actual)
+      @differ&.call(expected, actual)
+    end
+
+    def pretty_print(object)
+      @pretty_printer.pretty_print(object)
+    end
+    alias_method :pp, :pretty_print
+
+    # We always return a new generator with the same seed as to
+    # best reproduce remote builds locally if the same seed is given.
+    def random
+      Random.new(@seed)
+    end
+
+    def seed=(seed)
+      @seed = Integer(seed)
+    end
+
+    def build_queue
+      case @queue_url
+      when nil
+        Queue.build(self)
+      when /\Arediss?:/
+        require "megatest/redis_queue"
+        RedisQueue.build(self)
+      else
+        raise ArgumentError, "Unsupported queue type: #{@queue_url.inspect}"
+      end
+    end
+
+    def run_before_fork_callback
+      @before_fork_callback.each { |c| c.call(self) }
+    end
+
+    def before_fork(&block)
+      @before_fork_callbacks << block
+    end
+
+    def run_global_setup_callbacks
+      @global_setup_callbacks.each { |c| c.call(self) }
+    end
+
+    def global_setup(&block)
+      @global_setup_callbacks << block
+    end
+
+    def run_job_setup_callbacks(job_index)
+      @job_setup_callbacks.each { |c| c.call(self, job_index) }
+    end
+
+    def job_setup(&block)
+      @job_setup_callbacks << block
+    end
+
+    def retries?
+      @max_retries.positive?
+    end
+
+    def total_max_retries(size)
+      if @retry_tolerance.positive?
+        (size * @retry_tolerance).ceil
+      else
+        @max_retries * size
+      end
+    end
+  end
+
+  @config = Config.new({})
+end