deflake 0.1.0

data/lib/deflake/minitest_tests.rb

@@ -0,0 +1,161 @@
+# frozen_string_literal: true
+
+require "minitest"
+require_relative "result_file"
+
+module Deflake
+  # The tests of a Minitest process: which of them to run, in which order, and where each is declared.
+  # A test is known by "Class#test_name", the class by its name, and a second class
+  # with the same name, as two describe blocks with one description give, by "Class (2)"
+  module MinitestTests
+    # Checks what a whole class leaves behind, beyond the leaks of its tests: the state a
+    # class-level hook such as minitest-hooks' before_all sets up and does not put back
+    module ClassLeaks
+      define_method(Minitest::VERSION.to_i < 6 ? :run : :run_suite) do |reporter, options = {}|
+        recorder = MinitestReporter.current
+        # Tests that run in parallel would mix their leaks with those of other classes
+        order = respond_to?(:run_order) ? run_order : test_order
+        return super(reporter, options) unless recorder && order != :parallel
+
+        recorder.around_class(self) { super(reporter, options) }
+      end
+    end
+
+    class << self
+      # Test ids asked for through DEFLAKE_ORDER or DEFLAKE_ONLY, in the order to run them.
+      # The variables are removed, so a test that starts deflake itself does not pass them on
+      def requested
+        order = ENV.delete(ResultFile::ORDER_KEY)
+        only = ENV.delete(ResultFile::ONLY_KEY)
+        return ResultFile.unpack_ids(File.read(order)) if order
+
+        ResultFile.unpack_ids(only) if only
+      end
+
+      # Makes Minitest run exactly these tests in exactly this order. Minitest runs all tests of
+      # a class together, so the ids of a recorded order come class by class. Each class still
+      # seeds the random numbers as Minitest does, so code that uses rand behaves as in a whole run
+      def run_only(ids)
+        order = by_class(ids)
+        order.each do |klass, methods|
+          original = klass.method(:runnable_methods)
+          klass.define_singleton_method(:runnable_methods) do
+            original.call
+            methods
+          end
+        end
+        only_these(order.keys)
+      end
+
+      # Only Minitest's own run gets these classes: afterwards the tests see every class again,
+      # as tests of a Minitest extension that reset and list them do
+      def only_these(suites)
+        # Minitest shuffles the classes before it runs them
+        suites.define_singleton_method(:shuffle) { |*| self }
+        OnlyThese.suites = suites
+        Minitest::Runnable.singleton_class.prepend(OnlyThese)
+      end
+
+      # Gives the classes to run once, then Minitest's own list again
+      module OnlyThese
+        class << self
+          attr_accessor :suites
+        end
+
+        def runnables
+          suites = OnlyThese.suites
+          return super unless suites
+
+          OnlyThese.suites = nil
+          suites
+        end
+      end
+
+      # Class => its methods among these ids, in their order
+      def by_class(ids)
+        tests = {}
+        each_test { |klass, method_name, id| tests[id] = [klass, method_name] }
+        ids.filter_map { |id| tests[id] }.group_by(&:first).transform_values { |pairs| pairs.map(&:last) }
+      end
+
+      # Remembers where in this file each test is declared, before the file loads. A test that a
+      # macro or Rails' test "..." defines has its method's source elsewhere
+      def watch_declarations(path)
+        Minitest::Runnable.singleton_class.prepend(declarations_hook(path, declared))
+      end
+
+      def declarations_hook(path, found)
+        target = File.expand_path(path).b
+        Module.new do
+          define_method(:method_added) do |name|
+            place = name.to_s.start_with?("test_") && MinitestTests.declared_at(caller_locations, target)
+            found[self][name.to_s] = [path, place.lineno] if place
+            super(name)
+          end
+        end
+      end
+
+      # The line of this file that declares the test: in the body of a class or a describe block,
+      # not inside a helper method that defines tests, such as Rails' test. None for a test in
+      # another file this one requires: the line of the require does not declare it
+      def declared_at(locations, target)
+        first = locations.index { |location| in_file?(location, target) } or return
+        return if locations.first(first).any? { |location| location.label.to_s.include?("<top (required)>") }
+
+        here = locations.select { |location| in_file?(location, target) }
+        here.find { |location| location.label.to_s.include?("<") } || here.first
+      end
+
+      def in_file?(location, target)
+        file = location.absolute_path || location.path
+        file && File.expand_path(file).b == target
+      end
+
+      # Class => { test method => [file, line] }
+      def declared
+        @declared ||= Hash.new { |hash, klass| hash[klass] = {} }.compare_by_identity
+      end
+
+      # Writes every test with the file and line it is declared at, without running any.
+      # A test in a class without a name has no id
+      def list(path)
+        Minitest::Runnable.runnables.each do |klass|
+          prefix = prefix(klass)
+          klass.runnable_methods.each do |method_name|
+            file, line = declared[klass][method_name] || klass.instance_method(method_name).source_location
+            ResultFile.append(path, listed: prefix && "#{prefix}##{method_name}", file: file, line: line)
+          end
+        end
+      end
+
+      def each_test
+        Minitest::Runnable.runnables.each do |klass|
+          prefix = prefix(klass) or next
+          klass.runnable_methods.each { |method_name| yield klass, method_name, "#{prefix}##{method_name}" }
+        end
+      end
+
+      # "Class", "Class (2)" for the second class with that name, nil for a class without a name
+      def prefix(klass)
+        prefixes[klass]
+      end
+
+      # Classes with this name, in the order they were defined
+      def named(name)
+        prefixes.keys.select { |klass| klass.name == name }
+      end
+
+      private
+
+      def prefixes
+        @prefixes ||= begin
+          seen = Hash.new(0)
+          Minitest::Runnable.runnables.select(&:name).to_h do |klass|
+            count = seen[klass.name] += 1
+            [klass, count == 1 ? klass.name : "#{klass.name} (#{count})"]
+          end.compare_by_identity
+        end
+      end
+    end
+  end
+end

data/lib/deflake/outcome.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Deflake
+  module Execution
+    # What one test process left: rows (nil when it wrote none), results of tests that cannot run
+    # by name (unnamed), errors outside of tests it told about, its status and the end of its output
+    Outcome = Struct.new(:rows, :unnamed, :outside, :status, :output) do
+      # Failures of tests that cannot run by name, such as those in a class without a name
+      def unnamed_failures = unnamed.reject { |row| row["passed"] }
+
+      # Whether any test ran, even one that cannot run by name
+      def ran? = rows.any? || unnamed.any?
+
+      def failed_test? = rows.any? { |row| row["passed"] == false }
+
+      def leaks = outside.flat_map { |row| row["leaks"] || [] }
+    end
+  end
+end

data/lib/deflake/result_file.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+module Deflake
+  # A file where a test process records one JSON line per test: a "started" line
+  # when the test begins and a result line when it ends. The test process writes it without
+  # the json library, which would add methods such as Object#to_json to the tests it runs
+  module ResultFile
+    # Name of the environment variable that tells a test process where to write
+    ENV_KEY = "DEFLAKE_OUTPUT"
+    # Path of a file with test ids to run in exactly this order instead of the whole file
+    ORDER_KEY = "DEFLAKE_ORDER"
+    # Test ids to run, like ORDER_KEY but for a few ids passed directly (Minitest)
+    ONLY_KEY = "DEFLAKE_ONLY"
+    # Path of the file whose tests to list, with where each is defined, instead of running them (Minitest)
+    LIST_KEY = "DEFLAKE_LIST"
+    # Every variable that tells a test process what to do: a test that starts deflake itself
+    # must not pass them on
+    KEYS = [ENV_KEY, ORDER_KEY, ONLY_KEY, LIST_KEY].freeze
+    # Characters JSON escapes with a short form; other control characters become \u00XX
+    ESCAPES = { '"' => '\\"', "\\" => "\\\\", "\n" => "\\n", "\r" => "\\r", "\t" => "\\t" }.freeze
+
+    module_function
+
+    def append(path, **fields)
+      File.open(path, "a") { |file| file.puts(generate(clean(fields))) }
+    end
+
+    # Creates the file, so a run where no test wrote a line still tells that the process got going
+    def start(path)
+      File.open(path, "a", &:itself)
+    end
+
+    # A process killed in the middle of a write leaves a broken last line, which is skipped
+    def read(path)
+      require "json"
+      File.readlines(path, mode: "r:UTF-8").filter_map do |line|
+        JSON.parse(line)
+      rescue JSON::ParserError
+        nil
+      end
+    end
+
+    # Test ids as one line of text, for an environment variable or a file
+    def pack_ids(ids)
+      [Marshal.dump(Array(ids).map { |id| clean(id) })].pack("m0")
+    end
+
+    def unpack_ids(text)
+      Marshal.load(text.unpack1("m0")) # rubocop:disable Security/MarshalLoad
+    end
+
+    # Error messages can carry bytes that are not valid UTF-8, which JSON refuses. A string in
+    # another encoding, such as Latin-1, is converted, not just relabelled
+    def clean(value)
+      case value
+      when String then clean_string(value)
+      when Array then value.map { |item| clean(item) }
+      when Hash then value.to_h { |key, item| [key.to_s, clean(item)] }
+      else value
+      end
+    end
+
+    def clean_string(value)
+      if value.valid_encoding? && ![Encoding::UTF_8, Encoding::BINARY, Encoding::US_ASCII].include?(value.encoding)
+        value.encode(Encoding::UTF_8, invalid: :replace, undef: :replace)
+      else
+        String.new(value, encoding: Encoding::UTF_8).scrub
+      end
+    end
+
+    # JSON of strings, numbers, true, false, nil, arrays and hashes, which is all a results line holds
+    def generate(value)
+      case value
+      when Array then "[#{value.map { |item| generate(item) }.join(",")}]"
+      when Hash then "{#{value.map { |key, item| "#{quote(key.to_s)}:#{generate(item)}" }.join(",")}}"
+      when nil, true, false, Integer, Float then scalar(value)
+      else quote(value.to_s)
+      end
+    end
+
+    def scalar(value)
+      value.nil? || (value.is_a?(Float) && !value.finite?) ? "null" : value.to_s
+    end
+
+    def quote(text)
+      "\"#{text.gsub(/["\\\x00-\x1f]/) { |char| ESCAPES[char] || format("\\u%04x", char.ord) }}\""
+    end
+  end
+end

data/lib/deflake/rspec.rb

@@ -0,0 +1,90 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+require_relative "rspec_formatter"
+require_relative "leaks"
+
+if Gem::Version.new(RSpec::Core::Version::STRING) < Gem::Version.new("3.3")
+  abort "deflake needs RSpec 3.3 or newer, this is #{RSpec::Core::Version::STRING}"
+end
+
+module Deflake
+  # What deflake changes in RSpec inside a test process it started
+  module RSpecRun
+    # deflake runs every example: a run stopped at the first failure would hide the others,
+    # whether the project asks for it in .rspec or in spec_helper.rb. Nor is the profile printed,
+    # which would fill the end of the output where errors outside of examples are looked for
+    module Settings
+      # rubocop:disable Naming/PredicateMethod -- RSpec's own names
+      def fail_fast = false
+      def profile_examples = false
+      def profile_examples? = false
+      # rubocop:enable Naming/PredicateMethod
+    end
+
+    # Errors outside of examples, such as in an after(:context) hook, go to deflake's results
+    # whatever the formatters print around them
+    module Reporter
+      def notify_non_example_exception(exception, context_description)
+        RSpecFormatter.outside(context_description, exception)
+        super
+      end
+    end
+
+    module_function
+
+    # The examples deflake asked for through a file, which holds more of them than a command line could
+    def only(config, path)
+      ResultFile.unpack_ids(File.read(path)).group_by { |id| id[/\A(.+)\[[^\]]+\]\z/, 1] }.each do |file, ids|
+        config.filter_manager.add_ids(file, ids.map { |id| id[/\[([^\]]+)\]\z/, 1] }) if file
+      end
+    end
+
+    # The leaks of a top-level group that none of its examples left: the state a before(:context)
+    # hook sets up and does not put back
+    def around_group(group, before)
+      return unless before
+
+      snapshot, reported = before
+      leaks = Leaks.own(Leaks.since(snapshot), RSpecFormatter.reported.drop(reported))
+      RSpecFormatter.group_leaks(group.description, leaks) if leaks.any?
+    end
+
+    def top_level?(group)
+      group.superclass.equal?(RSpec::Core::ExampleGroup)
+    end
+  end
+end
+
+RSpec.configure do |config|
+  if (path = ENV.delete(Deflake::ResultFile::ENV_KEY))
+    # Here, not with --format: formatters given in SPEC_OPTS replace those on the command line
+    Deflake::RSpecFormatter.path = path
+    config.add_formatter(Deflake::RSpecFormatter)
+    config.singleton_class.prepend(Deflake::RSpecRun::Settings)
+    RSpec::Core::Reporter.prepend(Deflake::RSpecRun::Reporter)
+    order = ENV.delete(Deflake::ResultFile::ORDER_KEY)
+    Deflake::RSpecRun.only(config, order) if order
+
+    config.prepend_before(:context) do
+      if Deflake::RSpecRun.top_level?(self.class)
+        self.class.metadata[:deflake_group] = [Deflake::Leaks.snapshot, Deflake::RSpecFormatter.reported.size]
+      end
+    end
+    config.append_after(:context) do
+      Deflake::RSpecRun.around_group(self.class, self.class.metadata.delete(:deflake_group))
+    end
+  end
+
+  # Declared before the project's own hooks, so it wraps them all and a rolled
+  # back transaction or a travel_back in an after hook is not reported as a leak
+  config.around do |example|
+    before = Deflake::Leaks.snapshot
+    begin
+      example.run
+    ensure
+      # Also when a hook inside fails after the example ran
+      example.metadata[:deflake_leaks] = Deflake::Leaks.since(before)
+    end
+  end
+end

data/lib/deflake/rspec_formatter.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+require_relative "result_file"
+
+module Deflake
+  # Writes the result of every example to a file, one JSON line per example
+  class RSpecFormatter
+    ::RSpec::Core::Formatters.register self, :start, :example_started, :example_passed, :example_failed,
+                                       :example_pending
+
+    # Failures of expectations, whose message says it all, as Minitest's assertions do
+    EXPECTATIONS = %w[RSpec::Expectations::ExpectationNotMetError
+                      RSpec::Expectations::MultipleExpectationsNotMetError
+                      RSpec::Mocks::MockExpectationError].freeze
+
+    class << self
+      # Where to write, taken from the environment before the tests can see it
+      attr_accessor :path
+
+      # Every leak the examples reported so far
+      def reported
+        @reported ||= []
+      end
+
+      # An error outside of examples, such as in an after(:context) hook: its class and message
+      # without the backtrace, which would split one error into many
+      def outside(context, exception)
+        ResultFile.append(path, outside: "#{context}\n#{describe(exception)}")
+      rescue StandardError
+        nil
+      end
+
+      def group_leaks(group, leaks)
+        ResultFile.append(path, outside: nil, leaks: leaks.map { |leak| "#{group}, outside its examples: #{leak}" })
+      end
+
+      # The class too, so errors of two classes with one message are not one error. The message
+      # may be anything an exception class returns, nil included
+      def describe(exception)
+        message = exception.message.to_s.strip
+        EXPECTATIONS.include?(exception.class.name) ? message : "#{exception.class}: #{message}"
+      rescue StandardError, ScriptError
+        exception.class.name.to_s
+      end
+    end
+
+    def initialize(_output)
+      @path = self.class.path
+    end
+
+    # The file exists once RSpec starts, so a run where no example wrote a line still tells that
+    # the process got going
+    def start(_notification)
+      ResultFile.start(@path)
+    end
+
+    def example_started(notification)
+      ResultFile.append(@path, started: notification.example.id, name: notification.example.full_description)
+    end
+
+    def example_passed(notification)
+      record(notification.example, nil)
+    end
+
+    def example_failed(notification)
+      record(notification.example, self.class.describe(notification.example.exception))
+    end
+
+    # An example marked pending that failed as expected passed: its run counts.
+    # One that skip or xit left out is skipped, and only its leaks count
+    def example_pending(notification)
+      example = notification.example
+      return record(example, nil) unless example.execution_result.example_skipped?
+
+      ResultFile.append(@path, id: example.id, name: example.full_description, skipped: true,
+                               seed: ::RSpec.configuration.seed, leaks: leaks(example))
+    end
+
+    private
+
+    def record(example, error)
+      ResultFile.append(@path, id: example.id, name: example.full_description, passed: error.nil?,
+                               seed: ::RSpec.configuration.seed, error: error, leaks: leaks(example))
+    end
+
+    def leaks(example)
+      (example.metadata[:deflake_leaks] || []).tap { |leaks| self.class.reported.concat(leaks) }
+    end
+  end
+end

data/lib/deflake/run.rb

@@ -0,0 +1,11 @@
+# frozen_string_literal: true
+
+module Deflake
+  # Result of a single test run, with messages about the global state it leaked.
+  # group: what the error is grouped by instead of its text, when the text changes from run to run
+  Run = Data.define(:seed, :passed, :error, :leaks, :group) do
+    def initialize(seed:, passed:, error:, leaks: [], group: nil)
+      super
+    end
+  end
+end

data/lib/deflake/runner.rb

@@ -0,0 +1,143 @@
+# frozen_string_literal: true
+
+module Deflake
+  # Runs a test file many times, each in a fresh process with a new seed
+  class Runner
+    # Seeds Minitest and RSpec accept
+    SEEDS = 0xFFFF
+
+    # only: run just one test, by its Summary#id; jobs: how many processes at once
+    def initialize(path, runs:, only: nil, jobs: 1, adapter: Adapters.for(path))
+      raise ArgumentError, "runs must be positive" unless runs.positive?
+      raise ArgumentError, "jobs must be positive" unless jobs.positive?
+
+      @path = path
+      @count = runs
+      @only = only
+      @jobs = jobs
+      @adapter = adapter
+    end
+
+    # Ids of every test that started in the last call, even those left out of its result
+    attr_reader :seen
+    # Every summary of the last call, the quiet ones too, and every test's name by its id
+    attr_reader :all, :names
+
+    # Returns a hash of test id => Summary. Runs where a test skipped itself count only for its
+    # leaks, and a test that skipped itself every time and leaked nothing is left out.
+    # Errors outside of any test, in a run of the whole file, show up as Execution::OUTSIDE
+    def call
+      rows = in_parallel(seeds) { |seed, worker| execute(seed, worker_env(worker)) }
+      check_some_ran(rows)
+      check_some_tests(rows)
+      # The id, not the name: RSpec examples can share a description
+      remember(rows.group_by { |row| row["id"] })
+      @all.reject { |id, summary| quiet?(id, summary) }
+    end
+
+    private
+
+    def remember(groups)
+      @seen = groups.keys
+      @all = groups.to_h { |id, group| [id, summarize(id, group)] }
+      @names = @all.transform_values(&:name)
+    end
+
+    # A run that broke before its first test is reported among the others, unless every run did
+    def check_some_ran(rows)
+      silent = rows.select { |row| row.key?("output") }
+      return if silent.empty? || silent.size < rows.size
+
+      raise Execution::NoResults.new(ResultFile.clean(@path), silent.first["output"], silent.first["ending"])
+    end
+
+    # A file without tests, or whose tests a filter of the project leaves out, is a mistake, as a
+    # line without a test is: a passing report would hide it
+    def check_some_tests(rows)
+      return if @only || rows.any? { |row| row["id"] != Execution::OUTSIDE || row["ran"] }
+      return unless rows.all? { |row| quiet_outside?(row) }
+
+      raise Error, "no tests ran in #{ResultFile.clean(@path)} (no tests in it, or a filter leaves them all out)"
+    end
+
+    def quiet_outside?(row)
+      row["passed"] && row["leaks"].empty?
+    end
+
+    # Each run gets a seed of its own while there are enough of them
+    def seeds
+      return Array.new(@count) { rand(SEEDS) } if @count > SEEDS
+
+      (0...SEEDS).to_a.sample(@count)
+    end
+
+    def execute(seed, env)
+      return Execution.whole_file(@adapter, @path, seed: seed, env: env) unless @only
+
+      Execution.some(@adapter, @path, seed: seed, only: @only, env: env)
+    end
+
+    # Threads only wait for the test processes, so they do not fight over Ruby's lock
+    def in_parallel(seeds, &block)
+      queue = Queue.new(seeds).tap(&:close)
+      workers = Array.new([@jobs, seeds.size].min) do |worker|
+        Thread.new do
+          # The error reaches the caller through Thread#value, no need to print it here too
+          Thread.current.report_on_exception = false
+          drain(queue, worker, &block)
+        end
+      end
+      workers.flat_map(&:value)
+    ensure
+      stop(workers)
+    end
+
+    # On an error or an interrupt the other workers stop too, killing their test processes
+    def stop(workers)
+      workers&.each(&:kill)&.each do |worker|
+        worker.join
+      rescue StandardError
+        nil
+      end
+    end
+
+    def drain(queue, worker)
+      rows = []
+      while (seed = queue.pop)
+        rows.concat(yield(seed, worker))
+      end
+      rows
+    end
+
+    # Same convention as parallel_tests: "", "2", "3"... so each process can use its own database
+    def worker_env(worker)
+      return {} if @jobs == 1
+
+      { "TEST_ENV_NUMBER" => worker.zero? ? "" : (worker + 1).to_s }
+    end
+
+    # Leaks outside of tests, such as those of a before(:all) hook, are not quiet either
+    def quiet?(id, summary)
+      return summary.failures.zero? && summary.leaks.empty? if id == Execution::OUTSIDE
+
+      summary.total.zero? && summary.leaks.empty?
+    end
+
+    def summarize(id, rows)
+      skipped, ran = rows.partition { |row| row["skipped"] }
+      runs = ran.map do |row|
+        Run.new(seed: row["seed"], passed: row["passed"], error: row["error"], leaks: row.fetch("leaks", []),
+                group: row["group"])
+      end
+      Summary.new(runs, id: id, name: name_of(rows), skipped: skipped.size,
+                        skipped_leaks: skipped.flat_map { |row| row.fetch("leaks", []) })
+    end
+
+    # RSpec names a one-liner only once its matcher runs, so a passing run has the full name
+    def name_of(rows)
+      named = rows.select { |row| row["passed"] }
+      named = rows if named.empty?
+      named.map { |row| row["name"] }.tally.max_by(&:last).first
+    end
+  end
+end

data/lib/deflake/summary.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Deflake
+  # Summary of all runs of a single test
+  class Summary
+    # How to run only this test: "Class#test_name" or an RSpec example id
+    attr_reader :id
+    # Readable test name
+    attr_reader :name
+    # In how many runs the test skipped itself; those runs are not in total
+    attr_reader :skipped
+
+    # skipped_leaks: leaks of the runs where the test skipped itself
+    def initialize(runs, id: nil, name: id, skipped: 0, skipped_leaks: [])
+      @runs = runs
+      @id = id
+      @name = name
+      @skipped = skipped
+      @skipped_leaks = skipped_leaks
+    end
+
+    def total
+      @runs.size
+    end
+
+    def failures
+      @runs.count { |run| !run.passed }
+    end
+
+    def failure_rate
+      total.zero? ? 0.0 : failures.fdiv(total)
+    end
+
+    # Messages that differ only in the addresses inside inspected objects ("#<User:0x000055...>"),
+    # which change in every process, are one error. Other hexadecimal numbers are kept. Runs with
+    # a group, such as a crash whose output depends on the seed, show the error of the first one
+    def errors
+      groups = @runs.reject(&:passed).group_by { |run| run.group || normalize(run.error) }.values
+      groups.each_with_object({}) do |runs, all|
+        (all[normalize(runs.first.error)] ||= []).concat(runs.map(&:seed))
+      end
+    end
+
+    # Leak message => in how many runs it happened, skipped runs included
+    def leaks
+      (@runs.flat_map(&:leaks) + @skipped_leaks).tally
+    end
+
+    private
+
+    def normalize(message)
+      message&.gsub(/(#<(?:[A-Z]\w*::)*[^\s>:]+:)0x\h{8,16}/, '\10x...')
+    end
+  end
+end