deflake 0.1.0

data/lib/deflake/tail.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+module Deflake
+  # The end of a test process's output, where test frameworks say what went wrong, cleaned of
+  # what changes from run to run, so one error stays one error
+  module Tail
+    # How many lines an error shows
+    LINES = 15
+    # How much of the end of the output is read: a log of hundreds of megabytes stays on disk
+    KEEP = 256 * 1024
+    # Whole lines the test frameworks print that change from run to run (the seed, the timing,
+    # the counts, the progress dots)
+    NOISE = Regexp.union(/\A(?:Run options:|Randomized with seed \d+\z|Finished in |# Running:\z|[.FESP*]+\z)/,
+                         /\A\d+ runs?, \d+ assertions?/, /\A\d+ examples?, \d+ failures?/)
+    # Progress dots stuck to the start of a line the tests printed, but not those of "./path"
+    PROGRESS = %r{\A[.FESP*]*\.(?![./])}
+    # A seed inside a line, such as "Started with run options --seed 41234"
+    SEED = /\b(seed[ =:]+)\d+/i
+    # Frames of a backtrace, which Ruby prints after the message
+    FRAME = /\A\s+from /
+    # Frames kept of each backtrace
+    FRAMES = 3
+
+    module_function
+
+    # The message, and the end of the output when there is one. Long backtraces are cut short,
+    # so the message before them stays in
+    def with(message, output)
+      tail = lines(output)
+      tail.empty? ? message : "#{message}, its output ended with:\n#{tail.join("\n")}"
+    end
+
+    def read(log)
+      File.open(log, "rb") do |file|
+        cut = file.size > KEEP
+        file.seek(-KEEP, IO::SEEK_END) if cut
+        text = file.read
+        cut ? text.sub(/\A[^\n]*\n/, "") : text
+      end
+    end
+
+    def lines(output)
+      lines = without_pending(output.lines.map(&:rstrip)).map { |line| line.sub(PROGRESS, "").gsub(SEED, '\1N') }
+      short_frames(lines.reject { |line| line.empty? || line.strip.match?(NOISE) }).last(LINES)
+    end
+
+    # RSpec lists pending examples in the order they ran, which changes with the seed
+    def without_pending(lines)
+      skipping = false
+      lines.reject do |line|
+        skipping = true if line.start_with?("Pending:")
+        skipping = false if skipping && !line.empty? && !line.start_with?(" ", "Pending:")
+        skipping
+      end
+    end
+
+    def short_frames(lines)
+      run = 0
+      lines.filter_map do |line|
+        run = line.match?(FRAME) ? run + 1 : 0
+        next line if run <= FRAMES
+
+        "\t ... more frames" if run == FRAMES + 1
+      end
+    end
+  end
+end

data/lib/deflake/text_report.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+module Deflake
+  # Prints diagnoses in a form meant for people
+  class TextReport
+    VERDICTS = {
+      failed: "failed, could not run it alone",
+      fails_on_its_own: "fails on its own",
+      order_dependent: "fails only after other tests",
+      random: "fails at random, not after particular tests"
+    }.freeze
+
+    def initialize(out)
+      @out = out
+    end
+
+    # shared: names more than one test of the file has, even tests left out of the report
+    def print(diagnoses, shared: [], **)
+      # Tests with the same name, such as RSpec examples from shared groups, get their ids too
+      @shared_names = diagnoses.map(&:name).tally.select { |_, count| count > 1 }.keys | shared
+      diagnoses.each { |diagnosis| print_diagnosis(diagnosis) }
+    end
+
+    private
+
+    def print_diagnosis(diagnosis)
+      @out.puts "#{heading(diagnosis)}: #{outcome(diagnosis)}"
+      print_leaks(diagnosis.file_runs)
+      return if diagnosis.verdict == :stable
+
+      print_verdict(diagnosis)
+      print_culprits(diagnosis) if diagnosis.culprits
+      print_errors(diagnosis.file_runs)
+      print_errors(diagnosis.isolated_runs, "alone, ") if alone_errors?(diagnosis)
+    end
+
+    def heading(diagnosis)
+      @shared_names.include?(diagnosis.name) ? "#{diagnosis.name} (#{diagnosis.id})" : diagnosis.name
+    end
+
+    def print_verdict(diagnosis)
+      verdict = VERDICTS.fetch(diagnosis.verdict)
+      return @out.puts("  verdict: #{verdict}") unless diagnosis.isolated_runs
+
+      @out.puts "  verdict: #{verdict} (alone: #{failed(diagnosis.isolated_runs)})"
+    end
+
+    # Errors outside of tests that only leaked failed in no run, which says nothing
+    def outcome(diagnosis)
+      runs = diagnosis.file_runs
+      return "none in #{runs.total} runs, but leaks" if diagnosis.id == Execution::OUTSIDE && runs.failures.zero?
+
+      failed(runs)
+    end
+
+    # Runs where the test skipped itself are not counted, but said
+    def failed(summary)
+      return "skipped in all #{summary.skipped} runs" if summary.total.zero?
+
+      skipped = ", skipped in #{summary.skipped} more" if summary.skipped.positive?
+      "failed #{summary.failures} of #{summary.total} runs (#{percent(summary.failure_rate)})#{skipped}"
+    end
+
+    # A rare failure is not 0%, and a failure in all but a few runs is not 100%
+    def percent(rate)
+      return "<1%" if rate.positive? && rate < 0.005
+      return ">99%" if rate < 1 && rate >= 0.995
+
+      "#{(rate * 100).round}%"
+    end
+
+    def print_culprits(diagnosis)
+      culprits = diagnosis.culprits
+      @out.puts "  #{culprit_line(culprits, diagnosis.verdict)}", "  reproduce: #{culprits.command}"
+    end
+
+    def culprit_line(culprits, verdict)
+      if culprits.alone then "fails with no other test before it with this seed"
+      elsif culprits.found? then "culprit: #{culprits.names.join(", ")}"
+      elsif culprits.hidden
+        "culprit: not found, tests in a class without a name ran too, and deflake cannot run them by name"
+      elsif verdict == :random then "culprit: none, the failure did not happen again with the same seed and order"
+      else "culprit: not found, the failure did not happen again with the same seed"
+      end
+    end
+
+    def print_leaks(summary)
+      summary.leaks.each do |leak, count|
+        @out.puts "  leaks: #{leak} (#{count} of #{summary.total + summary.skipped} runs)"
+      end
+    end
+
+    # Errors of the runs alone that the runs of the file did not have
+    def alone_errors?(diagnosis)
+      alone = diagnosis.isolated_runs
+      return false unless alone&.failures&.positive?
+
+      !(alone.errors.keys - diagnosis.file_runs.errors.keys).empty?
+    end
+
+    def print_errors(summary, prefix = "")
+      errors = summary.errors
+      errors = errors.except(*@printed) unless prefix.empty?
+      errors.each do |error, seeds|
+        @out.puts "  #{prefix}seeds #{seeds.join(", ")}:", error.to_s.gsub(/^(?=.)/, "    ")
+      end
+      @printed = errors.keys
+    end
+  end
+end

data/lib/deflake/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Deflake
+  VERSION = "0.1.0"
+end

data/lib/deflake.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require_relative "deflake/version"
+
+module Deflake
+  class Error < StandardError; end
+
+  # deflake's own code, added with -I to every test process it starts
+  LIB = __dir__
+end
+
+require_relative "deflake/run"
+require_relative "deflake/summary"
+require_relative "deflake/result_file"
+require_relative "deflake/tail"
+require_relative "deflake/child"
+require_relative "deflake/execution"
+require_relative "deflake/culprits"
+require_relative "deflake/bisect"
+require_relative "deflake/adapters"
+require_relative "deflake/runner"
+require_relative "deflake/diagnosis"
+require_relative "deflake/investigation"
+require_relative "deflake/text_report"
+require_relative "deflake/json_report"
+require_relative "deflake/cli"

metadata

@@ -0,0 +1,88 @@
+--- !ruby/object:Gem::Specification
+name: deflake
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Roman Belyaev
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies: []
+description: |
+  deflake runs a suspicious test many times, measures how often it fails,
+  tells apart failures that happen on their own from failures caused by test
+  order, finds the tests that cause them, and detects global state that tests
+  leave behind: ENV, the clock, globals, Rails.cache, Current attributes,
+  database rows and enqueued jobs.
+email:
+- slyrs@yandex.ru
+executables:
+- deflake
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE.txt
+- README.md
+- exe/deflake
+- lib/deflake.rb
+- lib/deflake/adapters.rb
+- lib/deflake/adapters/minitest.rb
+- lib/deflake/adapters/rspec.rb
+- lib/deflake/bisect.rb
+- lib/deflake/child.rb
+- lib/deflake/cli.rb
+- lib/deflake/culprits.rb
+- lib/deflake/diagnosis.rb
+- lib/deflake/execution.rb
+- lib/deflake/investigation.rb
+- lib/deflake/json_report.rb
+- lib/deflake/leaks.rb
+- lib/deflake/leaks/cache.rb
+- lib/deflake/leaks/clock.rb
+- lib/deflake/leaks/current.rb
+- lib/deflake/leaks/env.rb
+- lib/deflake/leaks/globals.rb
+- lib/deflake/leaks/jobs.rb
+- lib/deflake/leaks/rows.rb
+- lib/deflake/minitest.rb
+- lib/deflake/minitest_reporter.rb
+- lib/deflake/minitest_tests.rb
+- lib/deflake/outcome.rb
+- lib/deflake/result_file.rb
+- lib/deflake/rspec.rb
+- lib/deflake/rspec_formatter.rb
+- lib/deflake/run.rb
+- lib/deflake/runner.rb
+- lib/deflake/summary.rb
+- lib/deflake/tail.rb
+- lib/deflake/text_report.rb
+- lib/deflake/version.rb
+homepage: https://github.com/SlyDeath/ruby-deflake
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/SlyDeath/ruby-deflake
+  source_code_uri: https://github.com/SlyDeath/ruby-deflake
+  changelog_uri: https://github.com/SlyDeath/ruby-deflake/blob/main/CHANGELOG.md
+  bug_tracker_uri: https://github.com/SlyDeath/ruby-deflake/issues
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.3'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.20
+specification_version: 4
+summary: Find flaky tests in Minitest and RSpec and explain why they fail.
+test_files: []