megatest 0.1.0

data/lib/megatest/redis_queue.rb

@@ -0,0 +1,459 @@
+# frozen_string_literal: true
+
+gem "redis-client", ">= 0.22"
+require "redis-client"
+require "rbconfig"
+
+# :stopdoc:
+
+module Megatest
+  # Data structures
+  #
+  # Note: All keys are prefixed by `build:<@build_id>:`
+  #
+  # - "leader-status": String, either `setup` or `ready`
+  #
+  # - "queue": List, contains the test ids that haven't yet been poped.
+  #
+  # - "running": SortedSet, members are the test ids currently being processed.
+  #    Scores are the lease expiration timestamp. If the score is lower than
+  #    current time, the test was lost and should be re-assigned.
+  #
+  # - "processed": Set, members are the ids of test that were fully processed.
+  #
+  # - "owners": Hash, contains a mapping of currently being processed tests and the worker they are assigned to.
+  #    Keys are test ids, values are "worker:<@worker_id>:queue".
+  #
+  # - "worker:<@worker_id>:running": Set, tests ids currently held by a worker.
+  #
+  # - "worker:<@worker_id>:failures": List, all the ids of failed tests processed by a worker.
+  #     Used as the base for a new queue when retrying a job. May contain duplicates.
+  #
+  # - "results": List, inside are serialized TestCaseResult instances. Append only.
+  #
+  # - "requeues-count": Hash, keys are test ids, values are the number of time that particular test
+  #    was retried. There is also the special "___total___" key.
+  class RedisQueue < AbstractQueue
+    class ExternalHeartbeatMonitor
+      def initialize(queue)
+        @queue = queue
+      end
+    end
+
+    class << self
+      def build(config)
+        queue = new(config)
+        if queue.retrying?
+          queue = RetryQueue.build(config, queue)
+        end
+        queue
+      end
+    end
+
+    attr_reader :summary
+
+    def initialize(config, ttl: 24 * 60 * 60)
+      super(config)
+
+      @summary = Queue::Summary.new
+      @redis = RedisClient.new(
+        url: config.queue_url,
+        # We retry quite aggressively in case the network
+        # is spotty, we'd rather wait a bit than to crash
+        # a worker.
+        reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5],
+      )
+      @ttl = ttl
+      @load_timeout = 30 # TODO: configurable
+      @worker_id = config.worker_id
+      @build_id = config.build_id
+      @success = true
+      @leader = nil
+      @script_cache = {}
+      @leader = nil
+    end
+
+    def retrying?
+      @worker_id && !@redis.call("llen", key("worker", worker_id, "failures")).zero?
+    end
+
+    def failed_test_ids
+      test_ids = @redis.call("lrange", key("worker", worker_id, "failures"), 0, -1)&.uniq
+      test_ids.reverse!
+      test_ids
+    end
+
+    def cleanup
+      if @success
+        if @worker_id
+          @redis.call(
+            "del",
+            key("worker", worker_id, "running"),
+            key("worker", worker_id, "failures"),
+          )
+        else
+          @redis.call(
+            "del",
+            key("leader-status"),
+            key("queue"),
+            key("running"),
+            key("processed"),
+            key("owners"),
+            key("results"),
+            key("requeue-counts"),
+          )
+        end
+      end
+    rescue RedisClient::ConnectionError
+      false # Cleanup is best effort
+    end
+
+    HEARTBEAT = <<~'LUA'
+      local running_key = KEYS[1]
+      local processed_key = KEYS[2]
+      local owners_key = KEYS[3]
+      local worker_running_key = KEYS[4]
+
+      local worker_id = ARGV[1]
+      local current_time = ARGV[2]
+
+      local count = 0
+
+      local tests = redis.call('smembers', worker_running_key)
+      for index = 1, #tests do
+        local test = tests[index]
+
+        -- # already processed, we do not need to bump the timestamp
+        if redis.call('sismember', processed_key, test) == 0 then
+          -- # we're still the owner of the test, we can bump the timestamp
+          local owner_id = redis.call('hget', owners_key, test)
+          if owner_id == worker_id then
+            redis.call('zadd', running_key, current_time, test)
+            count = count + 1
+          end
+        end
+      end
+
+      return count
+    LUA
+
+    def heartbeat
+      eval_script(
+        HEARTBEAT,
+        keys: [
+          key("running"),
+          key("processed"),
+          key("owners"),
+          key("worker", worker_id, "running"),
+        ],
+        argv: [
+          worker_id,
+          Megatest.now,
+        ],
+      )
+      true
+    rescue RedisClient::ConnectionError
+      false # Heartbeat is best effort
+    end
+
+    def distributed?
+      true
+    end
+
+    def populated?
+      @redis.call("get", key("leader-status")) == "ready"
+    end
+
+    def leader?
+      @leader
+    end
+
+    def remaining_size
+      @redis.multi do |transaction|
+        transaction.call("llen", key("queue"))
+        transaction.call("zcard", key("running"))
+      end.inject(:+)
+    end
+
+    def empty?
+      remaining_size.zero?
+    end
+
+    RESERVE = <<~'LUA'
+      local queue_key = KEYS[1]
+      local running_key = KEYS[2]
+      local processed_key = KEYS[3]
+      local owners_key = KEYS[4]
+      local worker_running_key = KEYS[5]
+
+      local worker_id = ARGV[1]
+      local current_time = ARGV[2]
+      local timeout = ARGV[3]
+
+      -- # First we requeue all timed out tests
+      local lost_tests = redis.call('zrangebyscore', running_key, 0, current_time - timeout)
+      for _, test in ipairs(lost_tests) do
+        if redis.call('sismember', processed_key, test) == 0 then
+          local test = redis.call('rpush', queue_key, test)
+        end
+      end
+
+      local test = redis.call('rpop', queue_key)
+      if test then
+        redis.call('zadd', running_key, current_time, test)
+        redis.call('sadd', worker_running_key, test)
+        redis.call('hset', owners_key, test, worker_id)
+        return test
+      end
+
+      return nil
+    LUA
+
+    def reserve
+      load_script(RESERVE)
+      test_id, = eval_script(
+        RESERVE,
+        keys: [
+          key("queue"),
+          key("running"),
+          key("processed"),
+          key("owners"),
+          key("worker", worker_id, "running"),
+        ],
+        argv: [
+          worker_id,
+          Megatest.now,
+          @config.heartbeat_frequency * 2,
+        ],
+      )
+      test_id
+    end
+
+    def populate(test_cases)
+      super
+
+      leader_key_set, = @redis.pipelined do |pipeline|
+        pipeline.call("setnx", key("leader-status"), "setup")
+        pipeline.call("expire", key("leader-status"), @ttl)
+      end
+      @leader = leader_key_set == 1
+
+      if @leader
+        @redis.multi do |transaction|
+          transaction.call("lpush", key("queue"), test_cases.map(&:id)) unless test_cases.empty?
+          transaction.call("expire", key("queue"), @ttl)
+          transaction.call("set", key("leader-status"), "ready")
+        end
+      else
+        (@load_timeout * 10).times do
+          if populated?
+            break
+          else
+            sleep 0.1
+          end
+        end
+      end
+    end
+
+    def success?
+      @success
+    end
+
+    def pop_test
+      if test_id = reserve
+        test_cases_index.fetch(test_id)
+      end
+    end
+
+    ACKNOWLEDGE = <<~'LUA'
+      local running_key = KEYS[1]
+      local processed_key = KEYS[2]
+      local owners_key = KEYS[3]
+      local worker_running_key = KEYS[4]
+
+      local test = ARGV[1]
+
+      redis.call('zrem', running_key, test)
+      redis.call('srem', worker_running_key, test)
+      redis.call('hdel', owners_key, test) -- # Doesn't matter if it was reclaimed by another workers
+      return redis.call('sadd', processed_key, test)
+    LUA
+
+    def record_result(original_result)
+      result = original_result
+      if result.failed?
+        if attempt_to_retry(result)
+          result = result.retry
+        else
+          @success = false
+        end
+      end
+      @summary.record_result(result)
+
+      if result.retried?
+        @redis.pipelined do |pipeline|
+          pipeline.call("rpush", key("results"), result.dump)
+          pipeline.call("expire", key("results"), @ttl)
+        end
+      else
+        load_script(ACKNOWLEDGE)
+        @redis.pipelined do |pipeline|
+          eval_script(
+            ACKNOWLEDGE,
+            keys: [
+              key("running"),
+              key("processed"),
+              key("owners"),
+              key("worker", worker_id, "running"),
+            ],
+            argv: [result.test_id],
+            redis: pipeline,
+          )
+          if result.failed?
+            pipeline.call("rpush", key("worker", worker_id, "failures"), result.test_id)
+            pipeline.call("expire", key("worker", worker_id, "failures"), @ttl)
+          elsif result.success?
+            pipeline.call("lrem", key("worker", worker_id, "failures"), 0, result.test_id)
+          end
+          pipeline.call("rpush", key("results"), result.dump)
+          pipeline.call("expire", key("results"), @ttl)
+        end
+      end
+
+      result
+    end
+
+    def global_summary
+      if payloads = @redis.call("lrange", key("results"), 0, -1)
+        Queue::Summary.new(payloads.map { |p| TestCaseResult.load(p) })
+      else
+        Queue::Summary.new
+      end
+    end
+
+    private
+
+    REQUEUE = <<~'LUA'
+      local processed_key = KEYS[1]
+      local requeues_count_key = KEYS[2]
+      local queue_key = KEYS[3]
+      local running_key = KEYS[4]
+      local owners_key = KEYS[5]
+
+      local worker_id = ARGV[1]
+      local max_requeues = tonumber(ARGV[2])
+      local global_max_requeues = tonumber(ARGV[3])
+      local test = ARGV[4]
+      local index = ARGV[5]
+
+      if redis.call('hget', owners_key, test) == worker_id then
+         redis.call('hdel', owners_key, test)
+      end
+
+      if redis.call('sismember', processed_key, test) == 1 then
+        return false
+      end
+
+      local global_requeues = tonumber(redis.call('hget', requeues_count_key, '___total___'))
+      if global_requeues and global_requeues >= tonumber(global_max_requeues) then
+        return false
+      end
+
+      local requeues = tonumber(redis.call('hget', requeues_count_key, test))
+      if requeues and requeues >= max_requeues then
+        return false
+      end
+
+      redis.call('hincrby', requeues_count_key, '___total___', 1)
+      redis.call('hincrby', requeues_count_key, test, 1)
+
+      local pivot = redis.call('lrange', queue_key, -1 - index, 0 - index)[1]
+      if pivot then
+        redis.call('linsert', queue_key, 'BEFORE', pivot, test)
+      else
+        redis.call('lpush', queue_key, test)
+      end
+
+      redis.call('zrem', running_key, test)
+
+      return true
+    LUA
+
+    def attempt_to_retry(result)
+      return false unless @config.retries?
+
+      index = @config.random.rand(0..@redis.call("llen", key("queue")))
+      load_script(REQUEUE)
+      eval_script(
+        REQUEUE,
+        keys: [
+          key("processed"),
+          key("requeues-count"),
+          key("queue"),
+          key("running"),
+          key("owners"),
+        ],
+        argv: [
+          worker_id,
+          @config.max_retries,
+          @config.total_max_retries(@size),
+          result.test_id,
+          index,
+        ],
+      ) == 1
+    end
+
+    def eval_script(script, keys: [], argv: [], redis: @redis)
+      script_id = load_script(script)
+      result, = pipelined(redis) do |pipeline|
+        pipeline.call("evalsha", script_id, keys.size, keys, argv)
+        keys.each do |key|
+          pipeline.call("expire", key, @ttl)
+        end
+      end
+      result
+    end
+
+    def pipelined(redis, &block)
+      if redis.respond_to?(:pipelined)
+        redis.pipelined(&block)
+      else
+        yield redis
+      end
+    end
+
+    def load_script(script)
+      @scripts_cache ||= {}
+      @scripts_cache[script] ||= @redis.call("script", "load", script)
+    end
+
+    def key(*args)
+      ["build", @build_id, *args].join(":")
+    end
+
+    def worker_id
+      @worker_id or raise Error, "RedisQueue not configued with a worker id"
+    end
+
+    class RetryQueue < Queue
+      def initialize(config, global_queue)
+        super(config)
+        @global_queue = global_queue
+      end
+
+      def populate(test_cases)
+        super
+        failed_test_ids = @global_queue.failed_test_ids
+        @size = failed_test_ids.size
+        @queue = failed_test_ids.map { |id| @test_cases_index.fetch(id) }
+      end
+
+      def record_result(original_result)
+        result = super
+        if result.success?
+          @global_queue.record_result(result)
+        end
+      end
+    end
+  end
+end

data/lib/megatest/reporters.rb

@@ -0,0 +1,266 @@
+# frozen_string_literal: true
+
+# :stopdoc:
+
+module Megatest
+  module Reporters
+    class AbstractReporter
+      undef_method :puts, :print
+
+      def initialize(config, out)
+        @config = config
+        @out = Output.new(out, colors: config.colors)
+      end
+
+      def start(_executor, _queue)
+      end
+
+      def before_test_case(_queue, _test_case)
+      end
+
+      def after_test_case(_queue, _test_case, _result)
+      end
+
+      def summary(_executor, _queue, _summary)
+      end
+
+      private
+
+      LABELS = {
+        retried: "Retried",
+        error: "Error",
+        failure: "Failure",
+        skipped: "Skipped",
+      }.freeze
+
+      def render_failure(result, command: true)
+        str = "#{LABELS.fetch(result.status)}: #{result.test_id}\n"
+        str = if result.retried? || result.skipped?
+          @out.yellow(str)
+        else
+          @out.red(str)
+        end
+        str = +str
+        str << "\n"
+
+        if result.error?
+          str << @out.indent("#{result.failure.cause.name}: #{@out.colored(result.failure.cause.message)}\n")
+        elsif result.failed?
+          str << @out.indent(@out.colored(result.failure.message.to_s))
+        end
+        str << "\n" unless str.end_with?("\n")
+        str << "\n"
+
+        @config.backtrace.clean(result.failure.backtrace)&.each do |frame|
+          str << "  #{@out.cyan(frame)}\n"
+        end
+
+        if command
+          str << "\n"
+          str << @out.yellow(run_command(result))
+        end
+
+        str
+      end
+
+      def run_command(result)
+        "#{@config.program_name} #{Megatest.relative_path(result.test_location)}"
+      end
+    end
+
+    class SimpleReporter < AbstractReporter
+      def start(_executor, queue)
+        @out.puts("Running #{queue.size} test cases with --seed #{@config.seed}")
+        @out.puts
+      end
+
+      def after_test_case(_queue, _test_case, result)
+        if result.skipped?
+          @out.print(@out.yellow("S"))
+        elsif result.retried?
+          @out.print(@out.yellow("R"))
+        elsif result.error?
+          @out.print(@out.red("E"))
+        elsif result.failed?
+          @out.print(@out.red("F"))
+        else
+          @out.print(@out.green("."))
+        end
+      end
+
+      def summary(executor, _queue, summary)
+        @out.puts
+        @out.puts
+
+        failures = summary.failures.reject(&:skipped?)
+        unless failures.empty?
+          failures = failures.sort_by(&:test_id)
+          failures.each_with_index do |result, index|
+            @out.print "  #{index + 1}) "
+            @out.puts render_failure(result)
+            @out.puts
+          end
+        end
+
+        if (wall_time = executor.wall_time.to_f) > 0.0
+          @out.puts format(
+            "Finished in %.2fs, %d cases/s, %d assertions/s, %.2fs tests runtime.",
+            wall_time,
+            (summary.runs_count / wall_time).to_i,
+            (summary.assertions_count / wall_time).to_i,
+            summary.total_time,
+          )
+        end
+
+        @out.puts format(
+          "Ran %d cases, %d assertions, %d failures, %d errors, %d retries, %d skips",
+          summary.runs_count,
+          summary.assertions_count,
+          summary.failures_count,
+          summary.errors_count,
+          summary.retries_count,
+          summary.skips_count,
+        )
+      end
+    end
+
+    class VerboseReporter < SimpleReporter
+      def before_test_case(_queue, test_case)
+        @out.print("#{test_case.id} = ")
+      end
+
+      def after_test_case(_queue, _test_case, result)
+        super
+        @out.puts
+        if result.bad?
+          @out.puts @out.colored(render_failure(result))
+        end
+      end
+    end
+
+    class OrderReporter < AbstractReporter
+      def before_test_case(_queue, test_case)
+        @out.puts(test_case.id)
+      end
+    end
+
+    class JUnitReporter < AbstractReporter
+      def summary(executor, _queue, summary)
+        @depth = 0
+        @out.puts(%{<?xml version="1.0" encoding="UTF-8"?>})
+
+        results_by_suite = summary.results.map { |r| r.test_id.split("#", 2) << r }.group_by(&:first)
+
+        tag(:testsuites, { time: executor.wall_time }) do
+          results_by_suite.each do |testsuite, named_results|
+            render_test_suite(testsuite, named_results)
+          end
+        end
+      end
+
+      private
+
+      def attr_escape(string)
+        if string.include?('"')
+          string.gsub('"', "&quot;")
+        else
+          string
+        end
+      end
+
+      def cdata(string)
+        string = string.gsub("]]>", "] ]>") if string.include?("]]>")
+        "<![CDATA[#{string}]]>"
+      end
+
+      using Compat::Tally unless Enumerable.method_defined?(:tally)
+
+      def render_test_suite(testsuite, named_results)
+        results = named_results.map(&:last)
+        statuses = results.map(&:status).tally
+
+        attributes = {
+          name: testsuite,
+          filepath: Megatest.relative_path(results.first.test_location.split(":", 2).first),
+          tests: results.size,
+          assertions: results.sum(&:assertions_count),
+          time: results.sum { |r| r.duration || 0.0 },
+          failures: statuses.fetch(:failure, 0),
+          errors: statuses.fetch(:error, 0),
+          skipped: statuses.fetch(:skipped, 0) + statuses.fetch(:retried, 0),
+        }
+
+        tag(:testsuite, attributes) do
+          named_results.each do |(_, testcase, result)|
+            render_test_case(testsuite, testcase, result)
+          end
+        end
+      end
+
+      def render_test_case(testsuite, testcase, result)
+        file, line = result.test_location.split(":", 2)
+        line.sub!(/~.*/, "")
+        file = Megatest.relative_path(file)
+
+        attributes = {
+          name: testcase,
+          classname: testsuite,
+          file: file,
+          line: line,
+          assertions: result.assertions_count,
+          time: result.duration || 0.0,
+          "run-command": run_command(result),
+        }
+
+        if result.success?
+          tag(:test_case, attributes)
+        elsif result.skipped? || result.retried?
+          tag(:test_case, attributes) do
+            tag(:skipped, { message: result.failure.message })
+          end
+        else
+          tag(:test_case, attributes) do
+            if result.error?
+              tag_name = :error
+              message = result.failure.message
+            else
+              tag_name = :failure
+              message = "Assertion Failure"
+            end
+            tag(tag_name, { type: result.failure.name, message: message }, text: cdata(render_failure(result, command: false)))
+          end
+        end
+      end
+
+      def tag(name, attrs, text: nil)
+        indent
+
+        @out << "<#{name}"
+        attrs&.each do |attr, value|
+          unless value.nil?
+            @out << %{ #{attr}="#{attr_escape(value.to_s)}"}
+          end
+        end
+
+        if block_given?
+          @out.puts(">")
+          @depth += 1
+          yield
+          @depth -= 1
+          indent
+          @out.puts("</#{name}>")
+        elsif text
+          @out.print(">")
+          @out.print(text)
+          @out.puts("</#{name}>")
+        else
+          @out.puts("/>")
+        end
+      end
+
+      def indent
+        @depth.times { @out << "  " }
+      end
+    end
+  end
+end