rspec-flake-classifier 0.1.0

data/lib/rspec/flake/classifier.rb

@@ -0,0 +1,285 @@
+# frozen_string_literal: true
+
+require_relative "classifier/classify/classifier"
+require_relative "classifier/configuration"
+require_relative "classifier/coverage_snapshot"
+require_relative "classifier/deflaker"
+require_relative "classifier/example_history"
+require_relative "classifier/evaluation"
+require_relative "classifier/features"
+require_relative "classifier/integrations"
+require_relative "classifier/predictor"
+require_relative "classifier/probe_evidence"
+require_relative "classifier/rerun/isolated_runner"
+require_relative "classifier/rerun/protocol"
+require_relative "classifier/runtime_controls"
+require_relative "classifier/sensitivity"
+require_relative "classifier/signature"
+require_relative "classifier/store/jsonl_store"
+require_relative "classifier/version"
+
+module RSpec
+  module FlakeClassifier
+    class Error < StandardError; end
+
+    RuntimeControls.activate!
+
+    class << self
+      def configuration
+        @configuration ||= Configuration.new
+      end
+
+      def configure(rspec_config = nil)
+        yield(configuration) if block_given?
+        RSpecIntegration.install(rspec_config, configuration) if rspec_config
+        configuration
+      end
+
+      def signature_for(exception)
+        Signature.from_exception(exception)
+      end
+
+      def store
+        Store::JSONLStore.new(configuration.store_path)
+      end
+
+      def classify(input = nil, **attributes)
+        Classify::Classifier.new.classify(input, **attributes)
+      end
+
+      def investigate(example_id, seed: nil, prior_examples: [], runner: nil)
+        runner ||= Rerun::IsolatedRunner.new(command: configuration.rspec_command)
+        Rerun::Protocol.new(
+          runner: runner,
+          same_order_runs: configuration.auto_rerun_attempts
+        ).investigate(example_id, seed: seed, prior_examples: prior_examples)
+      end
+
+      def sensitivity(example_id, factors: configuration.sensitivity_factors, seed: nil, runner: nil)
+        runner ||= Rerun::IsolatedRunner.new(command: configuration.rspec_command)
+        Sensitivity.new(runner: runner).analyze(example_id, factors: factors, seed: seed)
+      end
+    end
+
+    module RSpecIntegration
+      module_function
+
+      def install(rspec_config, configuration)
+        return if ENV[Rerun::IsolatedRunner::DISABLE_ENV] == "1"
+        return if rspec_config.instance_variable_defined?(:@flake_classifier_installed)
+
+        rspec_config.instance_variable_set(:@flake_classifier_installed, true)
+        reset!
+        install_around_hook(rspec_config) do |example|
+          prior_examples = RSpecIntegration.prior_examples
+          coverage_before = CoverageSnapshot.capture
+          example.run
+          coverage = RSpecIntegration.coverage_for(example, configuration, coverage_before)
+          if example.exception
+            RSpecIntegration.handle_failure(
+              example,
+              configuration,
+              prior_examples: prior_examples,
+              coverage: coverage
+            )
+          end
+        ensure
+          RSpecIntegration.record_example(example)
+        end
+      end
+
+      def install_around_hook(rspec_config, &hook)
+        if rspec_config.respond_to?(:hooks) && rspec_config.hooks.respond_to?(:register)
+          rspec_config.hooks.register(:append, :around, :each, &hook)
+        else
+          rspec_config.around(:each, &hook)
+        end
+      end
+
+      def handle_failure(example, configuration, prior_examples: [], coverage: {})
+        signature = Signature.from_exception(example.exception)
+        store = Store::JSONLStore.new(configuration.store_path)
+        known_flake = store.known_flake?(signature.digest)
+        probes = ProbeEvidence.new(provider: configuration.probe_provider).collect(example)
+        deflaker = deflaker_result(configuration, coverage)
+        sensitivity = nil
+        metadata = failure_metadata(example, prior_examples, coverage, probes, deflaker, sensitivity)
+
+        if known_flake && configuration.skip_known_flakes
+          store.record(signature, classification: known_classification, metadata: metadata)
+          clear_exception(example)
+          return
+        end
+
+        sensitivity = sensitivity_result(example, configuration)
+        metadata = failure_metadata(example, prior_examples, coverage, probes, deflaker, sensitivity)
+
+        investigation = nil
+        if configuration.auto_rerun
+          investigation = rerun_example(example, configuration, prior_examples)
+          clear_exception(example) if investigation.flaky?
+        end
+
+        classification = classify_failure(
+          example,
+          investigation,
+          probes: probes,
+          deflaker: deflaker,
+          sensitivity: sensitivity
+        )
+        Integrations.tag_ci_execution(example: example, signature: signature, classification: classification)
+        store.record(signature, classification: classification, metadata: metadata)
+        example.metadata[:flake_classifier] = {
+          signature: signature.digest,
+          known_flake: known_flake,
+          coverage: coverage,
+          deflaker: deflaker&.to_h,
+          sensitivity: sensitivity&.to_h,
+          probes: probes,
+          investigation: investigation&.to_h,
+          classification: classification.to_h
+        }
+      end
+
+      def rerun_example(example, configuration, prior_examples)
+        RSpec::FlakeClassifier.investigate(
+          example.id,
+          seed: current_seed,
+          prior_examples: prior_examples,
+          runner: Rerun::IsolatedRunner.new(command: configuration.rspec_command)
+        )
+      end
+
+      def classify_failure(example, investigation, probes:, deflaker:, sensitivity:)
+        Classify::Classifier.new.classify(
+          message: example.exception.message,
+          backtrace: example.exception.backtrace,
+          order_dependent: investigation&.order_type == "od",
+          duration_samples: investigation&.runs&.map(&:duration),
+          resources: probes.fetch(:resources, []),
+          files: probes.fetch(:files, []),
+          sockets: probes.fetch(:sockets, []),
+          source: probes[:source],
+          metadata: {
+            deflaker: deflaker&.to_h,
+            sensitivity: sensitivity&.to_h
+          }.compact
+        )
+      end
+
+      def coverage_for(example, configuration, coverage_before)
+        provided = provider_value(configuration.coverage_provider, example)
+        return provided if provided
+
+        integration_coverage = Integrations.coverage_for(example)
+        return integration_coverage if integration_coverage
+
+        metadata_coverage = example.metadata[:flake_classifier_coverage] || example.metadata[:covered_lines]
+        return metadata_coverage if metadata_coverage
+
+        coverage_before.delta
+      end
+
+      def deflaker_result(configuration, coverage)
+        return nil unless configuration.deflaker
+        return nil if coverage.nil? || coverage.empty?
+
+        Deflaker.new.suspect?(
+          coverage: coverage,
+          changed_lines: changed_lines(configuration),
+          base: deflaker_base(configuration)
+        )
+      end
+
+      def sensitivity_result(example, configuration)
+        return nil unless configuration.run_sensitivity
+
+        RSpec::FlakeClassifier.sensitivity(
+          example.id,
+          factors: configuration.sensitivity_factors,
+          seed: current_seed,
+          runner: Rerun::IsolatedRunner.new(command: configuration.rspec_command)
+        )
+      end
+
+      def changed_lines(configuration)
+        provider_value(configuration.changed_lines_provider)
+      end
+
+      def deflaker_base(configuration)
+        return "HEAD" unless configuration.deflaker.is_a?(Hash)
+
+        configuration.deflaker.fetch(:base, configuration.deflaker.fetch("base", "HEAD"))
+      end
+
+      def provider_value(provider, *args)
+        return nil unless provider
+
+        return provider.call if provider.respond_to?(:arity) && provider.arity.zero?
+
+        provider.call(*args)
+      rescue StandardError
+        nil
+      end
+
+      def failure_metadata(example, prior_examples, coverage, probes, deflaker, sensitivity)
+        {
+          example_id: example.id,
+          file_path: example.metadata[:file_path],
+          line_number: example.metadata[:line_number],
+          prior_examples: prior_examples,
+          coverage: coverage,
+          probes: probes,
+          deflaker: deflaker&.to_h,
+          sensitivity: sensitivity&.to_h
+        }.compact
+      end
+
+      def known_classification
+        Classify::Result.new(
+          status: "flaky",
+          labels: [
+            Classify::Label.new(
+              category: "known_flaky",
+              confidence: 1.0,
+              evidence: ["signature matched the flake store"],
+              action: "Inspect the stored classification and linked commits."
+            )
+          ]
+        )
+      end
+
+      def clear_exception(example)
+        example.instance_variable_set(:@exception, nil)
+      end
+
+      def prior_examples
+        history.prior_examples
+      end
+
+      def record_example(example)
+        history.record(example)
+      end
+
+      def reset!
+        @history = ExampleHistory.new
+      end
+
+      def history
+        @history ||= ExampleHistory.new
+      end
+
+      def current_seed
+        return unless defined?(::RSpec) && ::RSpec.respond_to?(:configuration)
+
+        ::RSpec.configuration.seed
+      end
+    end
+  end
+end
+
+module Rspec
+  module Flake
+    Classifier = ::RSpec::FlakeClassifier unless const_defined?(:Classifier, false)
+  end
+end

data/sig/rspec/flake/classifier.rbs

@@ -0,0 +1,176 @@
+module RSpec
+  module FlakeClassifier
+    VERSION: String
+
+    def self.configuration: () -> Configuration
+    def self.configure: (?untyped rspec_config) { (Configuration) -> void } -> Configuration
+    def self.signature_for: (Exception exception) -> Signature
+    def self.store: () -> Store::JSONLStore
+    def self.classify: (?untyped input, **untyped attributes) -> Classify::Result
+    def self.investigate: (String example_id, ?seed: Integer?, ?prior_examples: Array[String], ?runner: untyped) -> Rerun::Investigation
+    def self.sensitivity: (String example_id, ?factors: Array[Symbol], ?seed: Integer?, ?runner: untyped) -> Sensitivity::Result
+
+    class Configuration
+      attr_accessor store: String
+      attr_accessor auto_rerun: bool | Hash[Symbol, untyped]
+      attr_accessor deflaker: bool
+      attr_accessor rspec_command: Array[String]?
+      attr_accessor same_order_runs: Integer
+      attr_accessor skip_known_flakes: bool
+      attr_accessor coverage_provider: untyped
+      attr_accessor changed_lines_provider: untyped
+      attr_accessor probe_provider: untyped
+      attr_accessor run_sensitivity: bool
+      attr_accessor sensitivity_factors: Array[Symbol]
+
+      def initialize: () -> void
+      def auto_rerun_attempts: () -> Integer
+      def store_path: () -> String
+    end
+
+    class Signature
+      attr_reader message: String
+      attr_reader backtrace: Array[String]
+      attr_reader normalized: String
+      attr_reader digest: String
+
+      def self.from_exception: (Exception exception) -> Signature
+      def initialize: (message: String, ?backtrace: Array[String]) -> void
+      def to_h: () -> Hash[String, untyped]
+    end
+
+    class Deflaker
+      def suspect?: (coverage: untyped, ?changed_lines: untyped, ?base: String) -> Deflaker::Result
+      def git_changed_lines: (?String base) -> Hash[String, Array[Integer]]
+      def parse_diff: (String diff) -> Hash[String, Array[Integer]]
+
+      Result: singleton(::Struct)
+    end
+
+    class Sensitivity
+      def initialize: (runner: untyped) -> void
+      def analyze: (String example_id, ?factors: Array[Symbol], ?seed: Integer?) -> Sensitivity::Result
+
+      Result: singleton(::Struct)
+    end
+
+    class CoverageSnapshot
+      def self.capture: () -> CoverageSnapshot
+      def self.null: () -> CoverageSnapshot
+      def initialize: (Hash[String, untyped] data) -> void
+      def delta: () -> Hash[String, Array[Integer]]
+    end
+
+    class ProbeEvidence
+      def initialize: (?provider: untyped) -> void
+      def collect: (untyped example) -> Hash[Symbol, untyped]
+    end
+
+    class ExampleHistory
+      def initialize: () -> void
+      def prior_examples: () -> Array[String]
+      def record: (untyped example) -> void
+      def clear: () -> void
+    end
+
+    class Features
+      def extract: (?file: String?, ?source: String?, ?duration: Float?, ?metadata: Hash[Symbol, untyped]) -> Hash[String, untyped]
+    end
+
+    class Predictor
+      attr_reader weights: Hash[String, Float]
+
+      def self.train: (Array[Hash[String | Symbol, untyped]] examples) -> Predictor
+      def self.load: (String path) -> Predictor
+      def initialize: (?weights: Hash[String | Symbol, untyped]) -> void
+      def score: (Hash[String | Symbol, untyped] features) -> Float
+      def rank: (Array[Hash[String | Symbol, untyped]] feature_sets) -> Array[Hash[String, untyped]]
+      def to_h: () -> Hash[String, Hash[String, Float]]
+    end
+
+    class Evaluation
+      def classification: (predictions: Array[Hash[String | Symbol, untyped]], ground_truth: Array[Hash[String | Symbol, untyped]]) -> Hash[String, untyped]
+      def deflaker: (records: Array[Hash[String | Symbol, untyped]]) -> Hash[String, untyped]
+      def signatures: (records: Array[Hash[String | Symbol, untyped]]) -> Hash[String, untyped]
+      def idflakies: (records: Array[Hash[String | Symbol, untyped]]) -> Hash[String, untyped]
+    end
+
+    module Integrations
+      def self.coverage_for: (untyped example) -> untyped
+      def self.probe_for: (untyped example) -> untyped
+      def self.tag_ci_execution: (example: untyped, signature: Signature, classification: untyped) -> void
+    end
+
+    class Formatter
+      def initialize: (untyped output) -> void
+      def example_failed: (untyped notification) -> void
+      def dump_summary: (untyped notification) -> void
+    end
+
+    class JUnitFormatter
+      def initialize: (untyped output) -> void
+      def example_passed: (untyped notification) -> void
+      def example_failed: (untyped notification) -> void
+      def dump_summary: (untyped notification) -> void
+    end
+
+    module Store
+      class JSONLStore
+        attr_reader path: String
+
+        def initialize: (String path) -> void
+        def record: (Signature signature, ?classification: untyped, ?metadata: Hash[Symbol, untyped]) -> Hash[String, untyped]
+        def update_classification: (String digest, classification: untyped, ?metadata: Hash[Symbol, untyped]) -> Hash[String, untyped]?
+        def find: (String digest) -> Hash[String, untyped]?
+        def known_flake?: (String digest) -> bool
+        def all: () -> Hash[String, Hash[String, untyped]]
+        def entries: () -> Array[Hash[String, untyped]]
+      end
+    end
+
+    module Classify
+      Label: singleton(::Struct)
+
+      class Result
+        attr_reader status: String
+        attr_reader labels: Array[untyped]
+        attr_reader evidence: Array[String]
+
+        def initialize: (status: String, labels: Array[untyped], ?evidence: Array[String]) -> void
+        def primary_label: () -> untyped
+        def flaky?: () -> bool
+        def to_h: () -> Hash[String, untyped]
+      end
+    end
+
+    module Rerun
+      Result: singleton(::Struct)
+
+      class BisectDependencySearch
+        def initialize: (runner: untyped) -> void
+        def find: (String example_id, Array[String] prior_examples, ?seed: Integer?) -> Array[String]?
+      end
+
+      class Investigation
+        attr_reader example_id: String
+        attr_reader status: String
+        attr_reader order_type: String?
+        attr_reader runs: Array[untyped]
+        attr_reader dependency_examples: Array[String]
+        attr_reader dependency_role: String?
+
+        def flaky?: () -> bool
+        def true_failure?: () -> bool
+        def labels: () -> Array[String]
+        def to_h: () -> Hash[String, untyped]
+      end
+    end
+  end
+end
+
+module Rspec
+  module Flake
+    module Classifier
+    end
+  end
+end

metadata

@@ -0,0 +1,135 @@
+--- !ruby/object:Gem::Specification
+name: rspec-flake-classifier
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Yudai Takada
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rspec-core
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.10'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.10'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-covers
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: rspec-hermetic
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: 0.1.0
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '1.0'
+description: RSpec flake classifier records normalized failure signatures, reruns
+  failing examples in isolated processes, and assigns flaky-test cause labels.
+email:
+- t.yudai92@gmail.com
+executables:
+- rspec-flake
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- Rakefile
+- exe/rspec-flake
+- lib/rspec/flake/classifier.rb
+- lib/rspec/flake/classifier/classify/classifier.rb
+- lib/rspec/flake/classifier/classify/context.rb
+- lib/rspec/flake/classifier/classify/result.rb
+- lib/rspec/flake/classifier/cli.rb
+- lib/rspec/flake/classifier/configuration.rb
+- lib/rspec/flake/classifier/coverage_snapshot.rb
+- lib/rspec/flake/classifier/deflaker.rb
+- lib/rspec/flake/classifier/evaluation.rb
+- lib/rspec/flake/classifier/example_history.rb
+- lib/rspec/flake/classifier/features.rb
+- lib/rspec/flake/classifier/formatter.rb
+- lib/rspec/flake/classifier/integrations.rb
+- lib/rspec/flake/classifier/predictor.rb
+- lib/rspec/flake/classifier/probe_evidence.rb
+- lib/rspec/flake/classifier/rerun/bisect_dependency_search.rb
+- lib/rspec/flake/classifier/rerun/isolated_runner.rb
+- lib/rspec/flake/classifier/rerun/protocol.rb
+- lib/rspec/flake/classifier/rerun/result.rb
+- lib/rspec/flake/classifier/runtime_controls.rb
+- lib/rspec/flake/classifier/sensitivity.rb
+- lib/rspec/flake/classifier/signature.rb
+- lib/rspec/flake/classifier/store/jsonl_store.rb
+- lib/rspec/flake/classifier/version.rb
+- sig/rspec/flake/classifier.rbs
+homepage: https://github.com/ydah/rspec-flake-classifier
+licenses:
+- MIT
+metadata:
+  allowed_push_host: https://rubygems.org
+  homepage_uri: https://github.com/ydah/rspec-flake-classifier
+  source_code_uri: https://github.com/ydah/rspec-flake-classifier
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.6
+specification_version: 4
+summary: Detect, record, and classify flaky RSpec failures.
+test_files: []