rspec-flake-classifier 0.1.0

data/lib/rspec/flake/classifier/rerun/bisect_dependency_search.rb

@@ -0,0 +1,81 @@
+# frozen_string_literal: true
+
+require_relative "result"
+
+module RSpec
+  module FlakeClassifier
+    module Rerun
+      class BisectDependencySearch
+        BisectRunResult = Struct.new(:all_example_ids, :failed_example_ids, keyword_init: true)
+
+        def initialize(runner:)
+          @runner = runner
+        end
+
+        def find(example_id, prior_examples, seed: nil)
+          return [] if prior_examples.empty?
+
+          require_rspec_bisect!
+          result = minimizer(example_id, prior_examples, seed).find_minimal_repro
+          Array(result) - [example_id]
+        rescue LoadError, NameError, StandardError
+          nil
+        end
+
+        private
+
+        attr_reader :runner
+
+        def require_rspec_bisect!
+          require "rspec/core/bisect/example_minimizer"
+        end
+
+        def minimizer(example_id, prior_examples, seed)
+          RSpec::Core::Bisect::ExampleMinimizer.new(
+            ShellCommand.new,
+            RunnerAdapter.new(runner, example_id, prior_examples, seed),
+            NullNotifier.new
+          )
+        end
+
+        class ShellCommand
+          def original_cli_args
+            []
+          end
+
+          def repro_command_from(locations)
+            locations.join(" ")
+          end
+        end
+
+        class RunnerAdapter
+          def initialize(runner, example_id, prior_examples, seed)
+            @runner = runner
+            @example_id = example_id
+            @prior_examples = prior_examples
+            @seed = seed
+          end
+
+          def original_results
+            BisectRunResult.new(
+              all_example_ids: @prior_examples + [@example_id],
+              failed_example_ids: [@example_id]
+            )
+          end
+
+          def run(example_ids)
+            result = @runner.run(example_ids, seed: @seed)
+            BisectRunResult.new(
+              all_example_ids: Array(example_ids),
+              failed_example_ids: result.failed? ? [@example_id] : []
+            )
+          end
+        end
+
+        class NullNotifier
+          def publish(*); end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/rerun/isolated_runner.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require "open3"
+require "rbconfig"
+require_relative "result"
+
+module RSpec
+  module FlakeClassifier
+    module Rerun
+      class IsolatedRunner
+        DISABLE_ENV = "RSPEC_FLAKE_CLASSIFIER_CHILD"
+
+        attr_reader :command
+
+        def initialize(command: nil)
+          @command = Array(command || default_command)
+        end
+
+        def run(example_ids, seed: nil, env: {}, extra_args: [])
+          ids = Array(example_ids)
+          args = command + ids
+          args += ["--seed", seed.to_s] if seed
+          args += Array(extra_args)
+          child_env = { DISABLE_ENV => "1" }.merge(stringify_env(env))
+          started_at = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+          stdout, stderr, process_status = Open3.capture3(child_env, *args)
+          duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started_at
+
+          Result.new(
+            example_ids: ids,
+            seed: seed,
+            status: process_status.success? ? "passed" : "failed",
+            exit_status: process_status.exitstatus,
+            stdout: stdout,
+            stderr: stderr,
+            duration: duration,
+            command: args
+          )
+        rescue SystemCallError => e
+          Result.new(
+            example_ids: ids,
+            seed: seed,
+            status: "failed",
+            exit_status: nil,
+            stdout: "",
+            stderr: e.message,
+            duration: 0.0,
+            command: args
+          )
+        end
+
+        private
+
+        def default_command
+          return ENV.fetch("RSPEC_FLAKE_RSPEC_COMMAND").split(" ") if ENV["RSPEC_FLAKE_RSPEC_COMMAND"]
+          return ["bundle", "exec", "rspec"] if File.file?("Gemfile")
+
+          [RbConfig.ruby, "-S", "rspec"]
+        end
+
+        def stringify_env(env)
+          env.each_with_object({}) do |(key, value), result|
+            result[key.to_s] = value.to_s
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/rerun/protocol.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require_relative "isolated_runner"
+require_relative "bisect_dependency_search"
+
+module RSpec
+  module FlakeClassifier
+    module Rerun
+      class Protocol
+        attr_reader :runner, :same_order_runs
+
+        def initialize(runner: IsolatedRunner.new, same_order_runs: 3)
+          @runner = runner
+          @same_order_runs = same_order_runs
+        end
+
+        def investigate(example_id, seed: nil, prior_examples: [])
+          sequence = Array(prior_examples) + [example_id]
+          runs = run_same_order(sequence, seed)
+          return true_failure(example_id, runs) if runs.all?(&:failed?)
+
+          single = runner.run(example_id, seed: seed, extra_args: ["--order", "defined"])
+          runs << single
+
+          if !Array(prior_examples).empty? && runs.any?(&:failed?) && single.passed?
+            dependencies = find_dependency(example_id, prior_examples, seed)
+            return flaky(example_id, "od", runs, dependencies, "victim")
+          end
+
+          flaky(example_id, "nod", runs)
+        end
+
+        private
+
+        def run_same_order(example_ids, seed)
+          Array.new(same_order_runs) { runner.run(example_ids, seed: seed) }
+        end
+
+        def true_failure(example_id, runs)
+          Investigation.new(example_id: example_id, status: "true_failure", order_type: nil, runs: runs)
+        end
+
+        def flaky(example_id, order_type, runs, dependency_examples = [], dependency_role = nil)
+          Investigation.new(
+            example_id: example_id,
+            status: "flaky",
+            order_type: order_type,
+            runs: runs,
+            dependency_examples: dependency_examples,
+            dependency_role: dependency_role
+          )
+        end
+
+        def find_dependency(example_id, prior_examples, seed)
+          candidates = Array(prior_examples)
+          return [] if candidates.empty?
+
+          bisect_result = BisectDependencySearch.new(runner: runner).find(example_id, candidates, seed: seed)
+          return bisect_result if bisect_result
+
+          found = []
+          search = candidates
+
+          until search.empty?
+            half_size = [(search.length / 2.0).ceil, 1].max
+            left = search.first(half_size)
+            right = search.drop(half_size)
+            run = runner.run(left + [example_id], seed: seed)
+
+            if run.failed?
+              found = left
+              search = left.length == search.length ? [] : left
+            else
+              search = right
+            end
+          end
+
+          found
+        end
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/rerun/result.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module RSpec
+  module FlakeClassifier
+    module Rerun
+      Result = Struct.new(
+        :example_ids,
+        :seed,
+        :status,
+        :exit_status,
+        :stdout,
+        :stderr,
+        :duration,
+        :command,
+        keyword_init: true
+      ) do
+        def passed?
+          status.to_s == "passed"
+        end
+
+        def failed?
+          status.to_s == "failed"
+        end
+
+        def to_h
+          {
+            "example_ids" => Array(example_ids),
+            "seed" => seed,
+            "status" => status,
+            "exit_status" => exit_status,
+            "duration" => duration,
+            "command" => Array(command)
+          }
+        end
+      end
+
+      class Investigation
+        attr_reader :example_id, :status, :order_type, :runs, :dependency_examples,
+                    :dependency_role
+
+        def initialize(example_id:, status:, order_type:, runs:, dependency_examples: [], dependency_role: nil)
+          @example_id = example_id
+          @status = status.to_s
+          @order_type = order_type&.to_s
+          @runs = runs
+          @dependency_examples = dependency_examples
+          @dependency_role = dependency_role&.to_s
+        end
+
+        def flaky?
+          status == "flaky"
+        end
+
+        def true_failure?
+          status == "true_failure"
+        end
+
+        def labels
+          labels = []
+          labels << "flaky" if flaky?
+          labels << "order-dependent" if order_type == "od"
+          labels << "non-order-dependent" if order_type == "nod"
+          labels << dependency_role if dependency_role
+          labels
+        end
+
+        def to_h
+          {
+            "example_id" => example_id,
+            "status" => status,
+            "order_type" => order_type,
+            "dependency_role" => dependency_role,
+            "labels" => labels,
+            "dependency_examples" => dependency_examples,
+            "polluter_candidates" => dependency_role == "victim" ? dependency_examples : [],
+            "runs" => runs.map(&:to_h)
+          }
+        end
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/runtime_controls.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module RSpec
+  module FlakeClassifier
+    module RuntimeControls
+      module_function
+
+      def activate!
+        freeze_time if ENV["RSPEC_FLAKE_TIME_NOW"]
+        block_network if ENV["RSPEC_FLAKE_BLOCK_NETWORK"] == "1"
+      end
+
+      def freeze_time
+        timestamp = Integer(ENV.fetch("RSPEC_FLAKE_TIME_NOW"))
+        return if Time.singleton_class.method_defined?(:flake_classifier_original_now)
+
+        Time.singleton_class.class_eval do
+          alias_method :flake_classifier_original_now, :now
+
+          define_method(:now) do
+            at(timestamp).getlocal
+          end
+        end
+      rescue ArgumentError
+        nil
+      end
+
+      def block_network
+        block_webmock
+        require "socket"
+        patch_tcp_socket
+      rescue LoadError
+        nil
+      end
+
+      def block_webmock
+        require "webmock"
+        return unless defined?(::WebMock) && ::WebMock.respond_to?(:disable_net_connect!)
+
+        ::WebMock.disable_net_connect!
+      rescue LoadError
+        nil
+      end
+
+      def patch_tcp_socket
+        return if TCPSocket.singleton_class.method_defined?(:flake_classifier_original_open)
+
+        TCPSocket.singleton_class.class_eval do
+          alias_method :flake_classifier_original_open, :open
+          alias_method :flake_classifier_original_new, :new
+
+          define_method(:open) do |*|
+            raise IOError, "network disabled by rspec-flake-classifier"
+          end
+
+          define_method(:new) do |*|
+            raise IOError, "network disabled by rspec-flake-classifier"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/sensitivity.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+module RSpec
+  module FlakeClassifier
+    class Sensitivity
+      Result = Struct.new(:example_id, :factors, :runs, keyword_init: true) do
+        def to_h
+          {
+            "example_id" => example_id,
+            "factors" => factors,
+            "runs" => runs.map(&:to_h)
+          }
+        end
+      end
+
+      def initialize(runner:)
+        @runner = runner
+      end
+
+      def analyze(example_id, factors: %i[time randomness network], seed: nil)
+        runs = []
+        factor_results = {}
+
+        Array(factors).each do |factor|
+          case factor.to_sym
+          when :time
+            factor_results["time"] = run_time_variants(example_id, seed, runs)
+          when :randomness
+            factor_results["randomness"] = run_randomness_variants(example_id, seed, runs)
+          when :network
+            factor_results["network"] = run_network_variant(example_id, seed, runs)
+          end
+        end
+
+        Result.new(example_id: example_id, factors: factor_results, runs: runs)
+      end
+
+      private
+
+      attr_reader :runner
+
+      def run_time_variants(example_id, seed, runs)
+        variants = [
+          ["UTC", "1704067200"],
+          ["Pacific/Kiritimati", "1704067200"],
+          ["America/Los_Angeles", "1704153600"]
+        ]
+        statuses = variants.map do |tz, timestamp|
+          run = runner.run(example_id, seed: seed, env: { "TZ" => tz, "RSPEC_FLAKE_TIME_NOW" => timestamp })
+          runs << run
+          ["#{tz}@#{timestamp}", run.status]
+        end
+        factor_result(statuses)
+      end
+
+      def run_randomness_variants(example_id, seed, runs)
+        seeds = [seed || 12_345, 54_321].uniq
+        statuses = seeds.map do |variant_seed|
+          run = runner.run(example_id, seed: variant_seed)
+          runs << run
+          [variant_seed.to_s, run.status]
+        end
+        factor_result(statuses)
+      end
+
+      def run_network_variant(example_id, seed, runs)
+        allowed = runner.run(example_id, seed: seed)
+        blocked = runner.run(example_id, seed: seed, env: { "RSPEC_FLAKE_BLOCK_NETWORK" => "1" })
+        runs.concat([allowed, blocked])
+        factor_result([["allowed", allowed.status], ["blocked", blocked.status]])
+      end
+
+      def factor_result(statuses)
+        unique_statuses = statuses.map(&:last).uniq
+        {
+          "sensitive" => unique_statuses.length > 1,
+          "observations" => statuses.to_h
+        }
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/signature.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+require "digest"
+
+module RSpec
+  module FlakeClassifier
+    class Signature
+      PLACEHOLDERS = {
+        object_address: "0xADDR",
+        tmp_path: "<TMP>",
+        line: ":LINE",
+        seed: "SEED",
+        timestamp: "<TIME>",
+        cwd: "<PWD>"
+      }.freeze
+
+      attr_reader :message, :backtrace, :normalized, :digest
+
+      def self.from_exception(exception)
+        new(
+          message: exception.message,
+          backtrace: Array(exception.backtrace)
+        )
+      end
+
+      def initialize(message:, backtrace: [])
+        @message = message.to_s
+        @backtrace = Array(backtrace).map(&:to_s)
+        @normalized = normalize([@message, *@backtrace].join("\n"))
+        @digest = Digest::SHA256.hexdigest(@normalized)
+      end
+
+      def to_h
+        {
+          "digest" => digest,
+          "message" => message,
+          "normalized" => normalized,
+          "backtrace" => backtrace
+        }
+      end
+
+      private
+
+      def normalize(text)
+        normalized = text.dup
+        normalized = normalized.gsub(Dir.pwd, PLACEHOLDERS.fetch(:cwd)) if Dir.pwd && !Dir.pwd.empty?
+        normalized = normalized.gsub(%r{#<([^>]+):0x[0-9a-f]+>}i, '#<\1:0xADDR>')
+        normalized = normalized.gsub(/0x[0-9a-f]+/i, PLACEHOLDERS.fetch(:object_address))
+        normalized = normalized.gsub(%r{(?:/private)?/var/folders/[^\s:]+}, PLACEHOLDERS.fetch(:tmp_path))
+        normalized = normalized.gsub(%r{(?:/private)?/tmp/[^\s:]+}, PLACEHOLDERS.fetch(:tmp_path))
+        normalized = normalized.gsub(%r{:[0-9]+(?::in `[^`]+')?}, PLACEHOLDERS.fetch(:line))
+        normalized = normalized.gsub(/\bseed(?:ed)?(?:\s+with)?\s+[0-9]+\b/i, "seed #{PLACEHOLDERS.fetch(:seed)}")
+        normalized = normalized.gsub(/\b[0-9]{4}-[0-9]{2}-[0-9]{2}[T\s][0-9:.+-Z]+\b/, PLACEHOLDERS.fetch(:timestamp))
+        normalized = normalized.gsub(/\b[0-9]{4}-[0-9]{2}-[0-9]{2}\b/, PLACEHOLDERS.fetch(:timestamp))
+        normalized.gsub(/\b[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]+)?\b/, PLACEHOLDERS.fetch(:timestamp))
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/store/jsonl_store.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "time"
+
+module RSpec
+  module FlakeClassifier
+    module Store
+      class JSONLStore
+        FILENAME = "failures.jsonl"
+
+        attr_reader :path
+
+        def initialize(path)
+          @path = path.to_s
+        end
+
+        def record(signature, classification: nil, metadata: {})
+          current = find(signature.digest)
+          now = Time.now.utc.iso8601
+          entry = build_entry(signature, current, classification, metadata, now)
+
+          FileUtils.mkdir_p(path)
+          File.open(file_path, "a") { |file| file.puts(JSON.generate(entry)) }
+          entry
+        end
+
+        def update_classification(digest, classification:, metadata: {})
+          current = find(digest)
+          return nil unless current
+
+          classification_hash = classification_to_hash(classification)
+          labels = labels_from(classification_hash)
+          labels = Array(current.fetch("labels", [])) if labels.empty?
+          entry = current.merge(
+            "classification" => classification_hash,
+            "labels" => labels,
+            "metadata" => current.fetch("metadata", {}).merge(stringify_keys(metadata))
+          )
+          FileUtils.mkdir_p(path)
+          File.open(file_path, "a") { |file| file.puts(JSON.generate(entry)) }
+          entry
+        end
+
+        def find(digest)
+          all[digest]
+        end
+
+        def known_flake?(digest)
+          entry = find(digest)
+          return false unless entry
+
+          labels = Array(entry["labels"]).map(&:to_s)
+          entry.dig("classification", "status") == "flaky" || labels.any? { |label| label.to_s.include?("flaky") }
+        end
+
+        def all
+          return {} unless File.file?(file_path)
+
+          File.foreach(file_path).each_with_object({}) do |line, entries|
+            next if line.strip.empty?
+
+            entry = JSON.parse(line)
+            entries[entry.fetch("digest")] = entry
+          rescue JSON::ParserError
+            next
+          end
+        end
+
+        def entries
+          all.values.sort_by { |entry| entry.fetch("last_seen", "") }.reverse
+        end
+
+        private
+
+        def file_path
+          File.join(path, FILENAME)
+        end
+
+        def build_entry(signature, current, classification, metadata, now)
+          classification_hash = classification_to_hash(classification)
+          labels = labels_from(classification_hash)
+          labels = Array(current&.fetch("labels", [])) if labels.empty?
+
+          {
+            "digest" => signature.digest,
+            "normalized" => signature.normalized,
+            "first_seen" => current&.fetch("first_seen", nil) || now,
+            "last_seen" => now,
+            "occurrences" => current&.fetch("occurrences", 0).to_i + 1,
+            "classification" => classification_hash,
+            "labels" => labels,
+            "example_ids" => merge_values(current, "example_ids", metadata[:example_id]),
+            "commits" => merge_values(current, "commits", metadata[:commit]),
+            "metadata" => stringify_keys(metadata)
+          }
+        end
+
+        def classification_to_hash(classification)
+          return nil unless classification
+          return classification.to_h if classification.respond_to?(:to_h)
+
+          classification
+        end
+
+        def labels_from(classification_hash)
+          Array(classification_hash&.fetch("labels", nil)).map do |label|
+            if label.respond_to?(:fetch)
+              label.fetch("category", label.fetch(:category, label.to_s))
+            else
+              label.to_s
+            end
+          end
+        end
+
+        def merge_values(current, key, value)
+          values = Array(current&.fetch(key, []))
+          values << value if value
+          values.map(&:to_s).uniq
+        end
+
+        def stringify_keys(hash)
+          hash.each_with_object({}) do |(key, value), result|
+            result[key.to_s] = value
+          end
+        end
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/version.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module RSpec
+  module FlakeClassifier
+    VERSION = "0.1.0"
+  end
+end
+
+module Rspec
+  module Flake
+    Classifier = ::RSpec::FlakeClassifier unless const_defined?(:Classifier, false)
+  end
+end