deflake 0.1.0

data/lib/deflake/cli.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+require "optparse"
+
+module Deflake
+  # Command-line entry point
+  class CLI
+    REPORTS = { "text" => TextReport, "json" => JsonReport }.freeze
+
+    def initialize(argv, out: $stdout, err: $stderr)
+      @argv = argv.dup
+      @out = out
+      @err = err
+      @options = { runs: 10, jobs: 1, format: "text", timeout: nil }
+    end
+
+    # Returns the process exit code
+    def run
+      dispatch
+    rescue OptionParser::ParseError, Error => e
+      complain(e.message, usage: e.is_a?(OptionParser::ParseError))
+    rescue Interrupt
+      interrupted
+    rescue StandardError => e
+      # 1 would mean flaky tests were found, so anything unexpected is a 2 as well
+      complain("unexpected error, please report it: #{e.class}: #{e.message}\n  #{e.backtrace&.first(5)&.join("\n  ")}")
+    end
+
+    private
+
+    def dispatch
+      case parse_command
+      when :version then @out.puts "deflake #{VERSION}"
+      when :help then @out.puts parser
+      else return run_tests
+      end
+      0
+    end
+
+    # The test processes are stopped on the way out
+    def interrupted
+      @err.puts "deflake: interrupted"
+      130
+    end
+
+    def complain(message, usage: false)
+      @err.puts "deflake: #{message}"
+      @err.puts parser if usage
+      2
+    end
+
+    def parse_command
+      command = nil
+      define_options
+      parser.on("-v", "--version", "Print version and exit") { command = :version }
+      parser.on("-h", "--help", "Show this help and exit") { command = :help }
+      parser.parse!(@argv)
+      command || check_paths
+    end
+
+    def check_paths
+      raise OptionParser::MissingArgument, "TEST_PATH" if @argv.empty? || @argv.first.empty?
+      raise OptionParser::NeedlessArgument, "#{@argv.drop(1).join(" ")} (one TEST_PATH at a time)" if @argv.size > 1
+
+      :run
+    end
+
+    def define_options
+      count(:runs, "-n", "--runs N", "How many times to run the file (default: 10)")
+      count(:jobs, "-j", "--jobs N", "How many test processes to run at once (default: 1)")
+      count(:timeout, "-t", "--timeout SECONDS", "Stop a test process that runs longer (default: no limit)")
+      parser.on("-f", "--format FORMAT", REPORTS.keys, "Report format: text or json (default: text)") do |format|
+        @options[:format] = format
+      end
+    end
+
+    # Decimal, not Integer: Integer reads 010 as 8
+    def count(option, *switches)
+      parser.on(*switches[0, 2], OptionParser::DecimalInteger, switches[2]) { |n| @options[option] = positive(n) }
+    end
+
+    def positive(number)
+      raise OptionParser::InvalidArgument, "#{number} (must be 1 or more)" unless number.positive?
+
+      number
+    end
+
+    def run_tests
+      path, line = target
+      raise Error, "no such file: #{path}" unless File.file?(path)
+
+      Execution.timeout = @options[:timeout]
+      investigation = Investigation.new(path, line: line, runs: @options[:runs], jobs: @options[:jobs])
+      diagnoses = investigation.call
+      report(diagnoses, line, investigation)
+    end
+
+    # A file whose name itself ends in ":12" is that file, not line 12 of another. The path is
+    # taken as UTF-8 whatever the locale, as names in the report are
+    def target
+      argument = ResultFile.clean(@argv.first)
+      return [argument, nil] if File.file?(argument)
+
+      path, line = argument.match(/\A(.+?)(?::(\d+))?\z/m).captures
+      [path, line&.to_i]
+    end
+
+    # A test on the line that never started is an error: something stopped every run before it,
+    # or the file's own filters leave it out
+    def report(diagnoses, line, investigation)
+      not_run = investigation.not_run
+      if not_run.any?
+        @err.puts "deflake: #{not_run.join(", ")} never ran in a run of the whole file " \
+                  "(a filter of the file leaves it out, or something stops every run before it)"
+      end
+      # As with a plain run, tests that all skipped themselves are not an error
+      @err.puts "deflake: #{skipped(line)}, nothing to report" if diagnoses.empty? && not_run.empty?
+      print_report(diagnoses, not_run: not_run, shared: investigation.shared_names)
+      diagnoses.all?(&:clean?) && not_run.empty? ? 0 : 1
+    end
+
+    # A reader that stops early, such as head, is not an error of deflake's
+    def print_report(diagnoses, **)
+      REPORTS.fetch(@options[:format]).new(@out).print(diagnoses, **)
+    rescue Errno::EPIPE
+      nil
+    end
+
+    def skipped(line)
+      line ? "the test at line #{line} skipped itself in every run" : "every test skipped itself"
+    end
+
+    def parser
+      @parser ||= OptionParser.new("Usage: deflake TEST_PATH[:LINE] [options]")
+    end
+  end
+end

data/lib/deflake/culprits.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+module Deflake
+  # Tests that make another test fail when they run before it, and a command
+  # that shows the failure. ids is empty when bisect could not reproduce the failure,
+  # or when the test failed with that seed with no other test before it (alone)
+  Culprits = Data.define(:ids, :command, :names, :alone, :hidden, :group_wide) do
+    # names: readable test names, the ids themselves by default.
+    # group_wide: culprits that other tests of their group stand in for, as when the
+    # state comes from a hook of the group, such as before(:context), and not from the test.
+    # hidden: the failure did not happen again, but tests that cannot run by name ran in the file
+    def initialize(ids:, command:, names: ids, alone: false, hidden: false, group_wide: [])
+      super
+    end
+
+    def found? = ids.any?
+  end
+end

data/lib/deflake/diagnosis.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+module Deflake
+  # Everything deflake found out about one test.
+  # isolated_runs and culprits are nil when there was nothing to investigate,
+  # and isolated_runs also when the failure could not run alone: errors outside of tests,
+  # or a test whose runs alone all broke before it started
+  Diagnosis = Data.define(:name, :id, :file_runs, :isolated_runs, :culprits) do
+    # :stable, :failed (could not run alone), :fails_on_its_own, :order_dependent, or :random:
+    # it passed every time alone, and the failure did not happen again with the same seed and order,
+    # with every test of the file that ran before it
+    def verdict
+      return :stable if file_runs.failures.zero?
+      return :failed if isolated_runs.nil?
+      return :fails_on_its_own if fails_alone?
+
+      culprits.nil? || culprits.found? || culprits.hidden ? :order_dependent : :random
+    end
+
+    # It failed in a run alone, or with no test before it in a bisect
+    def fails_alone?
+      isolated_runs.failures.positive? || culprits&.alone || false
+    end
+
+    # Nothing failed and nothing leaked
+    def clean?
+      verdict == :stable && file_runs.leaks.empty?
+    end
+  end
+end

data/lib/deflake/execution.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require "tmpdir"
+require_relative "outcome"
+
+module Deflake
+  # Runs a test file once in a fresh process and returns one row per test that started, in run order
+  module Execution
+    # Id of the row that stands for errors outside of any test, such as in an after(:context) hook
+    OUTSIDE = "(outside of tests)"
+    # Runs the tests of a process one at a time
+    SERIAL = { "PARALLEL_WORKERS" => "1", "MT_CPU" => "1" }.freeze
+    # Raised when a test process wrote no results at all; output is what it printed,
+    # ending how the process ended, such as "timed out after 5 seconds"
+    class NoResults < Error
+      attr_reader :output, :ending
+
+      # success: whether the process itself exited with success
+      def initialize(target, output, ending = nil, success: false)
+        @output = output
+        @ending = ending
+        @success = success
+        super("#{target} produced no test results#{" (the test process #{ending})" if ending}:\n#{output}")
+      end
+
+      def success? = @success
+    end
+
+    class << self
+      # Seconds a test process may run before it is killed, nil for no limit
+      attr_accessor :timeout
+
+      # only: what to run instead of the whole file; env: extra environment variables for the test process
+      def call(adapter, path, seed:, only: nil, env: {})
+        complete(run(adapter, path, seed, only, env), only || path).rows
+      end
+
+      # The ids of the whole file's tests in the order they ran, and whether tests that cannot run
+      # by name, such as those in a class without a name, ran too
+      def order(adapter, path, seed:)
+        process = complete(run(adapter, path, seed, nil, {}), path)
+        [process.rows.map { |row| row["id"] }, process.unnamed.any?]
+      end
+
+      # Tries again when the test process wrote no results, as a flaky boot does now and then
+      def retrying(attempts = 3)
+        yield
+      rescue NoResults
+        attempts -= 1
+        retry if attempts.positive?
+        raise
+      end
+
+      # Runs the whole file and adds an OUTSIDE row telling whether the process failed on its own.
+      # A process that wrote no results at all is one failed OUTSIDE row carrying its output
+      def whole_file(adapter, path, seed:, env: {})
+        process = run(adapter, path, seed, nil, env)
+        return [no_results_row(seed, process)] unless process.rows
+
+        process.rows + [outside_row(process, seed)]
+      end
+
+      # Runs only some tests. As in whole_file, a process that wrote no results is a failed OUTSIDE
+      # row, and so is one where none of these tests started: only a failure that cannot run by
+      # name, such as a class-level hook, may have run
+      def some(adapter, path, seed:, only:, env: {})
+        process = run(adapter, path, seed, only, env)
+        process.rows.nil? || process.rows.empty? ? [no_results_row(seed, process)] : process.rows
+      end
+
+      def no_results_row(seed, process)
+        message = "the test process #{ending(process.status)} before any test finished"
+        failures = outside_failures(process)
+        error = failures ? "#{message}:\n#{failures}" : with_tail(message, process.output)
+        { "id" => OUTSIDE, "name" => "errors outside of tests", "passed" => false, "seed" => seed,
+          "error" => error, "output" => process.output, "ending" => ending(process.status), "leaks" => [] }
+      end
+
+      def ending(status) = Child.ending(status)
+
+      def with_tail(message, output) = Tail.with(message, output)
+
+      private
+
+      def complete(process, target)
+        return process if process.rows
+
+        raise NoResults.new(ResultFile.clean(Array(target).join(" ")), process.output, ending(process.status),
+                            success: process.status.success?)
+      end
+
+      def run(adapter, path, seed, only, env)
+        Dir.mktmpdir do |dir|
+          output = File.join(dir, "results.jsonl")
+          log = File.join(dir, "output.log")
+          # Rails' parallelize and Minitest's parallelize_me! would run tests at once, in no fixed
+          # order, and the leaks of one would show up in the others. Variables a test process of an
+          # outer deflake got are not passed on
+          env = ResultFile::KEYS.to_h { |key| [key, nil] }
+                                .merge(env, adapter.env(only), ResultFile::ENV_KEY => output, **SERIAL)
+          status = Child.run(env, adapter.command(path, seed: seed, only: only), log, timeout)
+          finish(File.exist?(output) ? ResultFile.read(output) : nil, seed, status, ResultFile.clean(Tail.read(log)))
+        end
+      end
+
+      # A test that started and never finished took its process down with it, so it counts as failed.
+      # A failure that cannot be run alone by name, such as a test in a class without a name or a
+      # class-level hook, is told about as an error outside of tests. A process that wrote nothing,
+      # or failed before any test, has no rows
+      def finish(rows, seed, status, output)
+        return Outcome.new(nil, [], [], status, output) unless rows
+
+        outside, rows = rows.partition { |row| row.key?("outside") }
+        unnamed, rows = rows.partition { |row| row.key?("unnamed") }
+        outcome = Outcome.new(tests(rows, seed, status, output), unnamed, outside, status, output)
+        outcome.rows = nil unless outcome.rows.any? || outcome.unnamed_failures.any? || status.success?
+        outcome
+      end
+
+      def tests(rows, seed, status, output)
+        started, finished = without_repeats(rows).partition { |row| row.key?("started") }
+        finished + lost(started, finished, seed, status, output)
+      end
+
+      # One result each time a test starts: RSpec may report an example again, without starting it,
+      # when its group breaks afterwards
+      def without_repeats(rows)
+        running = Hash.new(0)
+        rows.select do |row|
+          next running[row["started"]] += 1 if row.key?("started")
+          next true unless row.key?("id")
+
+          running[row["id"]].positive?.tap { |started| running[row["id"]] -= 1 if started }
+        end
+      end
+
+      def lost(started, finished, seed, status, output)
+        done = finished.to_h { |row| [row["id"], true] }
+        message = "the test process #{ending(status)} during this test"
+        started.reject { |row| done[row["started"]] }.uniq { |row| row["started"] }.map do |row|
+          # What the output shows before the crash changes with the order of the tests
+          { "id" => row["started"], "name" => row["name"], "passed" => false, "seed" => seed,
+            "error" => with_tail(message, output), "group" => message,
+            "timed_out" => status.is_a?(TimedOut), "leaks" => [] }
+        end
+      end
+
+      # A process that failed while every test passed broke somewhere else, for example in a hook.
+      # Errors outside of tests that the test framework told about count even when a test failed.
+      # ran: whether any test ran, even one that cannot run by name
+      def outside_row(process, seed)
+        error = outside_failures(process)
+        passed = error.nil? && (process.status.success? || process.failed_test?)
+        { "id" => OUTSIDE, "name" => "errors outside of tests", "passed" => passed, "seed" => seed,
+          "error" => (passed ? nil : error || outside_tail(process)), "leaks" => process.leaks, "ran" => process.ran? }
+      end
+
+      def outside_tail(process)
+        with_tail("the test process #{ending(process.status)} although no test failed", process.output)
+      end
+
+      # Sorted, as the order they happened in changes with the seed
+      def outside_failures(process)
+        failures = process.unnamed_failures.map { |row| "#{row["unnamed"]}: #{row["error"]}" }.uniq.sort
+        failures.unshift("a failure deflake cannot run alone by name:") if failures.any?
+        text = (process.outside.filter_map { |row| row["outside"] }.uniq.sort + failures).join("\n")
+        text unless text.empty?
+      end
+    end
+
+    TimedOut = Child::TimedOut
+  end
+end

data/lib/deflake/investigation.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+module Deflake
+  # Runs a test file many times, then reruns every failing test on its own
+  # to tell a flaky test from one broken by its neighbours, and bisects the latter
+  class Investigation
+    # How many failing seeds bisect tries before it gives up
+    SEEDS = 3
+
+    # line: look only at the test on this line, but still run the whole file
+    def initialize(path, runs:, line: nil, jobs: 1, adapter: Adapters.for(path))
+      @path = path
+      @runs = runs
+      @line = line
+      @jobs = jobs
+      @adapter = adapter
+      @names = {}
+    end
+
+    # Ids of the tests on the line that never started in any run, as when an earlier test
+    # always takes the process down
+    attr_reader :not_run
+
+    # Returns an array of Diagnosis, sorted by test name. It is empty when every test skipped itself.
+    # With a line, errors outside of tests still show up: they may be why the test never ran
+    def call
+      # Before the runs, so a line without a test is reported right away
+      focus = @line && @adapter.focus(@path, @line)
+      summaries = focused(focus)
+      summaries.values.sort_by { |summary| [summary.name, summary.id] }.map { |summary| diagnose(summary) }
+    end
+
+    # Names that more than one test of the file has, so a report tells them apart by their ids
+    def shared_names
+      @names.values.tally.select { |_, count| count > 1 }.keys
+    end
+
+    private
+
+    # Summaries of the whole file, only of the tests in focus and errors outside of tests when there is one
+    def focused(focus)
+      file = runner
+      summaries = file.call
+      # Of every test that ran, including those that skipped themselves
+      @names = file.names
+      @not_run = focus ? focus - file.seen : []
+      return summaries unless focus
+
+      summaries.select { |id, _| focus.include?(id) || id == Execution::OUTSIDE }
+    end
+
+    def runner(only: nil)
+      Runner.new(@path, runs: @runs, only: only, jobs: @jobs, adapter: @adapter)
+    end
+
+    def diagnose(summary)
+      diagnosis = Diagnosis.new(name: summary.name, id: summary.id, file_runs: summary, isolated_runs: nil,
+                                culprits: nil)
+      # Errors outside of tests cannot run on their own
+      return diagnosis if summary.failures.zero? || summary.id == Execution::OUTSIDE
+
+      diagnosis = diagnosis.with(isolated_runs: isolated(summary))
+      seeds = bisect_seeds(diagnosis)
+      seeds.empty? ? diagnosis : diagnosis.with(culprits: bisect(summary, seeds))
+    end
+
+    # Failing seeds worth a bisect. A test that fails on its own may also fail after other tests,
+    # with an error it never had alone: its seeds are bisected too
+    def bisect_seeds(diagnosis)
+      errors = diagnosis.file_runs.errors
+      case diagnosis.verdict
+      when :order_dependent then seeds_of(errors)
+      when :fails_on_its_own then seeds_of(errors.except(*diagnosis.isolated_runs.errors.keys))
+      else []
+      end
+    end
+
+    # The first seed of every error first, as a single odd failure may be the first one
+    def seeds_of(errors)
+      firsts = errors.values.map(&:first)
+      (firsts + errors.values.flatten).uniq.first(SEEDS)
+    end
+
+    # When no run alone reached the test, nil: the test failed, and whether it fails on its own
+    # is not known. A test that skipped itself every time it ran alone, and fails only after
+    # other tests, is order dependent
+    def isolated(summary)
+      alone = runner(only: summary.id)
+      alone.call
+      alone.all[summary.id]
+    rescue Execution::NoResults
+      nil
+    end
+
+    # Tries the seeds one by one until bisect finds culprits, or finds the test failing alone
+    def bisect(summary, seeds)
+      culprits = nil
+      seeds.each do |seed|
+        culprits = @adapter.bisect(@path, summary.id, seed: seed)
+        break if culprits.found? || culprits.alone
+      end
+      culprits.with(names: culprits.ids.map { |id| label(id, culprits.group_wide.include?(id)) })
+    end
+
+    # Tests with the same name are told apart by their ids
+    def label(id, group_wide)
+      name = @names.fetch(id, id)
+      name = "#{name} (#{id})" if @names.count { |_, other| other == name } > 1
+      group_wide ? "#{name} or any other test of its group (a hook of the group?)" : name
+    end
+  end
+end

data/lib/deflake/json_report.rb

@@ -0,0 +1,41 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Deflake
+  # Prints diagnoses as JSON for CI and other tools. The format may change before 1.0
+  class JsonReport
+    def initialize(out)
+      @out = out
+    end
+
+    # not_run: ids of tests on the line that never started, each one an object with the verdict "not_run"
+    def print(diagnoses, not_run: [], **)
+      objects = diagnoses.map { |diagnosis| as_json(diagnosis) } + not_run.map { |id| never_ran(id) }
+      # Names and paths in another encoding, as under LANG=C, would stop the generator
+      @out.puts JSON.pretty_generate(ResultFile.clean(objects))
+    end
+
+    private
+
+    def as_json(diagnosis)
+      runs = diagnosis.file_runs
+      { name: diagnosis.name, id: diagnosis.id, verdict: diagnosis.verdict,
+        runs: counts(runs), isolated_runs: diagnosis.isolated_runs && counts(diagnosis.isolated_runs),
+        errors: runs.errors.map { |message, seeds| { message: message, seeds: seeds } },
+        leaks: runs.leaks.map { |message, count| { message: message, runs: count } },
+        culprits: diagnosis.culprits&.to_h }
+    end
+
+    def never_ran(id)
+      { name: id, id: id, verdict: :not_run, runs: counts(Summary.new([], id: id)), isolated_runs: nil,
+        errors: [], leaks: [], culprits: nil }
+    end
+
+    def counts(summary)
+      # Not rounded: one failure in thousands of runs is not 0
+      { total: summary.total, failures: summary.failures, failure_rate: summary.failure_rate,
+        skipped: summary.skipped }
+    end
+  end
+end

data/lib/deflake/leaks/cache.rb

@@ -0,0 +1,29 @@
+# frozen_string_literal: true
+
+module Deflake
+  module Leaks
+    # Keys left in Rails.cache, when it is a MemoryStore
+    class Cache
+      # How many of the new keys a message lists
+      SHOWN = 3
+
+      def snapshot
+        return unless defined?(::Rails) && ::Rails.respond_to?(:cache)
+        return unless defined?(::ActiveSupport::Cache::MemoryStore)
+
+        store = ::Rails.cache
+        store.instance_variable_get(:@data).keys if store.is_a?(::ActiveSupport::Cache::MemoryStore)
+      end
+
+      def diff(before, after)
+        added = after - before
+        return [] if added.empty?
+
+        # Long numbers, such as times and versions, change from run to run
+        listed = added.first(SHOWN).map { |key| key.inspect.gsub(/\d{6,}/, "N") }.join(", ")
+        listed += ", ..." if added.size > SHOWN
+        ["Rails.cache has #{added.size} new #{added.size == 1 ? "key" : "keys"}: #{listed}"]
+      end
+    end
+  end
+end

data/lib/deflake/leaks/clock.rb

@@ -0,0 +1,60 @@
+# frozen_string_literal: true
+
+module Deflake
+  module Leaks
+    # Time.now frozen or moved away from the real clock: travel_to, freeze_time or Timecop left on
+    class Clock
+      # Seconds Time.now may be off without being a leak: the real clock moves while it is read
+      TOLERANCE = 1
+      # How long to watch Time.now to tell whether it moves
+      TICK = 0.0001
+      # The wait for the clock to move must end whatever happens
+      CHECKS = 100_000
+      # Taken when deflake loads, before the tests: Timecop's mock_process_clock replaces
+      # Process.clock_gettime, and the real clock would move with the mocked one
+      CLOCK = Process.method(:clock_gettime)
+
+      # [offset from the real clock in seconds, whether Time.now stands still]
+      def snapshot
+        first = now
+        start = real
+        CHECKS.times { break if real - start >= TICK }
+        [first.to_f - start, now == first]
+      end
+
+      # Where the clock ends up counts: a test that brings back a clock another test moved
+      # leaves nothing behind, and one that moves it says how far from the real clock it is
+      def diff(before, after)
+        (offset_before, frozen_before), (offset_after, frozen_after) = before, after
+        return [] if frozen_before
+        if frozen_after
+          # Without the offset, which grows with every second the clock stands still
+          return ["Time.now is frozen (freeze_time or travel_to without travel_back?)"]
+        end
+        return [] if (offset_after - offset_before).abs < TOLERANCE || offset_after.abs < TOLERANCE
+
+        ["Time.now is#{shift(offset_after)} (travel_to without travel_back?)"]
+      end
+
+      private
+
+      def now = ::Time.now
+      def real = CLOCK.call(Process::CLOCK_REALTIME)
+
+      def shift(seconds)
+        return "" if seconds.abs < TOLERANCE
+
+        " #{duration(seconds.abs)} #{seconds.positive? ? "ahead of" : "behind"} the real clock"
+      end
+
+      # In whole seconds, so a day measured a moment early is still a day
+      def duration(seconds)
+        seconds = seconds.round
+        [[86_400, "day"], [3600, "hour"], [60, "minute"], [1, "second"]].each do |size, unit|
+          count = seconds.fdiv(size).round
+          return "#{count} #{unit}#{"s" unless count == 1}" if count.positive? && seconds >= size
+        end
+      end
+    end
+  end
+end