rspec-flake-classifier 0.1.0

data/lib/rspec/flake/classifier/features.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+module RSpec
+  module FlakeClassifier
+    class Features
+      RESOURCE_PATTERNS = {
+        "network" => /Net::HTTP|HTTP\.|Faraday|Socket|TCPSocket|UDPSocket|WebMock|VCR/,
+        "filesystem" => /\bFile\b|\bDir\b|Tempfile|Pathname|tmpdir|fixture/,
+        "time" => /\bTime\b|\bDate\b|Timecop|travel_to|freeze_time|timezone|TZ/,
+        "randomness" => /Random|SecureRandom|srand|shuffle|sample|Faker/,
+        "concurrency" => /Thread|Fiber|Mutex|Queue|Concurrent::|fork/,
+        "async_wait" => /sleep|wait|eventually|Capybara|Selenium/
+      }.freeze
+
+      def extract(file: nil, source: nil, duration: nil, metadata: {})
+        source ||= file && File.file?(file) ? File.read(file) : ""
+        features = {
+          "file" => file,
+          "duration" => duration&.to_f,
+          "line_count" => source.lines.count,
+          "expectation_count" => source.scan(/\b(expect|should|is_expected)\b/).length,
+          "example_count" => source.scan(/\b(it|specify|example)\s+["'{(]/).length,
+          "metadata" => stringify_keys(metadata)
+        }
+
+        RESOURCE_PATTERNS.each do |name, pattern|
+          features["uses_#{name}"] = source.match?(pattern)
+        end
+
+        features
+      end
+
+      private
+
+      def stringify_keys(hash)
+        hash.each_with_object({}) do |(key, value), result|
+          result[key.to_s] = value
+        end
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/formatter.rb

@@ -0,0 +1,194 @@
+# frozen_string_literal: true
+
+require "json"
+require "cgi"
+require_relative "../classifier"
+
+module RSpec
+  module FlakeClassifier
+    class Formatter
+      ::RSpec::Core::Formatters.register self, :example_failed, :dump_summary if defined?(::RSpec::Core)
+
+      def initialize(output)
+        @output = output
+        @failures = []
+      end
+
+      def example_failed(notification)
+        example = notification.example
+        exception = notification_exception(notification, example)
+        signature = Signature.from_exception(exception)
+        classification = classify(example, exception)
+        @failures << failure_hash(example, exception, signature, classification)
+      end
+
+      def dump_summary(_notification)
+        output.puts(JSON.pretty_generate("failures" => failures))
+      end
+
+      private
+
+      attr_reader :output, :failures
+
+      def classify(example, exception)
+        stored = example.metadata[:flake_classifier]&.fetch(:classification, nil)
+        return stored if stored
+
+        Classify::Classifier.new.classify(message: exception.message, backtrace: exception.backtrace).to_h
+      end
+
+      def failure_hash(example, exception, signature, classification)
+        labels = Array(classification["labels"]).map { |label| label_category(label) }
+        {
+          "id" => example.id,
+          "description" => example.full_description,
+          "file_path" => example.metadata[:file_path],
+          "line_number" => example.metadata[:line_number],
+          "message" => exception.message,
+          "signature" => signature.digest,
+          "flaky" => flaky?(classification),
+          "labels" => labels,
+          "ci" => ci_metadata(example, signature.digest, labels, classification),
+          "classification" => classification
+        }
+      end
+
+      def flaky?(classification)
+        classification["status"] == "flaky" || !Array(classification["labels"]).empty?
+      end
+
+      def notification_exception(notification, example)
+        return notification.exception if notification.respond_to?(:exception)
+
+        example.exception
+      end
+
+      def label_category(label)
+        return label.fetch("category", label.fetch(:category, label.to_s)) if label.respond_to?(:fetch)
+
+        label.to_s
+      end
+
+      def ci_metadata(example, signature, labels, classification)
+        {
+          "buildkite" => {
+            "flaky" => flaky?(classification),
+            "flake_signature" => signature,
+            "flaky_labels" => labels
+          },
+          "circleci" => {
+            "result" => "failure",
+            "source" => example.metadata[:file_path].to_s,
+            "run_time" => run_time(example),
+            "flaky" => flaky?(classification),
+            "flaky_labels" => labels.join(",")
+          }
+        }
+      rescue StandardError
+        {
+          "buildkite" => { "flaky" => flaky?(classification), "flake_signature" => signature, "flaky_labels" => labels },
+          "circleci" => { "result" => "failure", "source" => example.metadata[:file_path].to_s, "flaky" => flaky?(classification), "flaky_labels" => labels.join(",") }
+        }
+      end
+
+      def run_time(example)
+        return nil unless example.respond_to?(:execution_result)
+        return nil unless example.execution_result.respond_to?(:run_time)
+
+        example.execution_result.run_time.to_f
+      end
+    end
+
+    class JUnitFormatter
+      ::RSpec::Core::Formatters.register self, :example_failed, :example_passed, :dump_summary if defined?(::RSpec::Core)
+
+      def initialize(output)
+        @output = output
+        @testcases = []
+      end
+
+      def example_passed(notification)
+        @testcases << testcase(notification.example)
+      end
+
+      def example_failed(notification)
+        example = notification.example
+        exception = notification.respond_to?(:exception) ? notification.exception : example.exception
+        signature = Signature.from_exception(exception)
+        classification = classify(example, exception)
+        @testcases << testcase(
+          example,
+          failure: exception.message,
+          signature: signature.digest,
+          classification: classification
+        )
+      end
+
+      def dump_summary(_notification)
+        output.puts(%(<?xml version="1.0" encoding="UTF-8"?>))
+        output.puts(%(<testsuite tests="#{testcases.length}">))
+        testcases.each { |attributes| output.puts(render_case(attributes)) }
+        output.puts("</testsuite>")
+      end
+
+      private
+
+      attr_reader :output, :testcases
+
+      def classify(example, exception)
+        stored = example.metadata[:flake_classifier]&.fetch(:classification, nil)
+        return stored if stored
+
+        Classify::Classifier.new.classify(message: exception.message, backtrace: exception.backtrace).to_h
+      end
+
+      def testcase(example, failure: nil, signature: nil, classification: nil)
+        labels = Array(classification&.fetch("labels", nil)).map { |label| label_category(label) }
+        {
+          "classname" => example.metadata[:file_path].to_s,
+          "name" => example.full_description,
+          "file" => example.metadata[:file_path].to_s,
+          "line" => example.metadata[:line_number].to_s,
+          "flaky" => (!labels.empty?).to_s,
+          "flaky_labels" => labels.join(","),
+          "flake_signature" => signature.to_s,
+          "properties" => {
+            "buildkite.flaky" => (!labels.empty?).to_s,
+            "buildkite.flaky_labels" => labels.join(","),
+            "buildkite.flake_signature" => signature.to_s,
+            "circleci.flaky" => (!labels.empty?).to_s,
+            "circleci.flaky_labels" => labels.join(","),
+            "circleci.source" => example.metadata[:file_path].to_s
+          },
+          "failure" => failure
+        }
+      end
+
+      def render_case(attributes)
+        failure = attributes.delete("failure")
+        properties = attributes.delete("properties") || {}
+        xml_attributes = attributes.map do |key, value|
+          %(#{key}="#{CGI.escapeHTML(value.to_s)}")
+        end.join(" ")
+        return %(  <testcase #{xml_attributes}>#{render_properties(properties)}</testcase>) unless failure
+
+        %(  <testcase #{xml_attributes}>#{render_properties(properties)}<failure>#{CGI.escapeHTML(failure)}</failure></testcase>)
+      end
+
+      def label_category(label)
+        return label.fetch("category", label.fetch(:category, label.to_s)) if label.respond_to?(:fetch)
+
+        label.to_s
+      end
+
+      def render_properties(properties)
+        return "" if properties.empty?
+
+        values = properties.map do |key, value|
+          %(    <property name="#{CGI.escapeHTML(key.to_s)}" value="#{CGI.escapeHTML(value.to_s)}"/>)
+        end.join
+        "<properties>#{values}</properties>"
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/integrations.rb

@@ -0,0 +1,247 @@
+# frozen_string_literal: true
+
+module RSpec
+  module FlakeClassifier
+    module Integrations
+      module_function
+
+      def coverage_for(example)
+        RSpecCovers.coverage_for(example)
+      end
+
+      def probe_for(example)
+        RSpecHermetic.probe_for(example)
+      end
+
+      def tag_ci_execution(example:, signature:, classification:)
+        BuildkiteCollector.tag_execution(example: example, signature: signature, classification: classification)
+      end
+
+      def constant(name)
+        name.to_s.split("::").reduce(Object) do |namespace, const_name|
+          return nil unless namespace.const_defined?(const_name, false)
+
+          namespace.const_get(const_name, false)
+        end
+      rescue NameError
+        nil
+      end
+
+      def try_require(feature)
+        require feature
+        true
+      rescue LoadError
+        false
+      end
+
+      def call_first(receivers, methods, *args)
+        Array(receivers).compact.each do |receiver|
+          Array(methods).each do |method_name|
+            next unless receiver.respond_to?(method_name)
+
+            value = receiver.public_send(method_name, *args)
+            return value if value
+          rescue ArgumentError
+            next
+          end
+        end
+        nil
+      end
+
+      module RSpecCovers
+        module_function
+
+        def coverage_for(example)
+          Integrations.try_require("rspec/covers")
+          result = reporter_result(example)
+          return line_hash(result.executed_locations) if result&.respond_to?(:executed_locations)
+
+          receivers = [
+            Integrations.constant("RSpec::Covers"),
+            Integrations.constant("RSpec::Covers::Probe"),
+            Integrations.constant("RSpec::Covers::Store"),
+            Integrations.constant("RSpec::Covers::Coverage"),
+            Integrations.constant("Rspec::Covers")
+          ]
+          Integrations.call_first(
+            receivers,
+            %i[coverage_for covered_lines_for example_coverage result_for for_example],
+            example
+          )
+        end
+
+        def reporter_result(example)
+          covers = Integrations.constant("RSpec::Covers")
+          return nil unless covers&.respond_to?(:reporter)
+          return nil unless covers.reporter.respond_to?(:results)
+
+          covers.reporter.results.reverse.find do |result|
+            result.respond_to?(:id) && result.id == example.id ||
+              result.respond_to?(:example) && result.example.equal?(example)
+          end
+        end
+
+        def line_hash(locations)
+          Array(locations).each_with_object(Hash.new { |hash, key| hash[key] = [] }) do |location, result|
+            file = location.respond_to?(:file) ? location.file : location[:file]
+            line = location.respond_to?(:line) ? location.line : location[:line]
+            result[relative_path(file)] << line.to_i if file && line
+          end.transform_values { |lines| lines.uniq.sort }
+        end
+
+        def relative_path(file)
+          path = file.to_s
+          return path unless path.start_with?(Dir.pwd)
+
+          path.delete_prefix("#{Dir.pwd}/")
+        end
+      end
+
+      module RSpecHermetic
+        module_function
+
+        def probe_for(example)
+          Integrations.try_require("rspec/hermetic")
+          record = runner_record(example)
+          return normalize_record(record) if record
+
+          receivers = [
+            Integrations.constant("RSpec::Hermetic"),
+            Integrations.constant("RSpec::Hermetic::Forensic"),
+            Integrations.constant("RSpec::Hermetic::Probe"),
+            Integrations.constant("Rspec::Hermetic")
+          ]
+          value = Integrations.call_first(
+            receivers,
+            %i[evidence_for forensic_for probe_for result_for for_example],
+            example
+          )
+          return normalize_hash(value) if value
+
+          collect_individual(receivers, example)
+        end
+
+        def collect_individual(receivers, example)
+          {
+            resources: Integrations.call_first(receivers, %i[resources_for resource_leaks_for leaks_for], example),
+            files: Integrations.call_first(receivers, %i[files_for filesystem_for], example),
+            sockets: Integrations.call_first(receivers, %i[sockets_for network_for], example)
+          }.compact
+        end
+
+        def runner_record(example)
+          hermetic = Integrations.constant("RSpec::Hermetic")
+          return nil unless hermetic&.respond_to?(:runner)
+          return nil unless hermetic.runner&.respond_to?(:records)
+
+          hermetic.runner.records.reverse.find do |record|
+            candidate = record[:example] || record["example"]
+            candidate.equal?(example) || candidate&.respond_to?(:id) && candidate.id == example.id
+          end
+        end
+
+        def normalize_record(record)
+          changes = Array((record[:changes] || record["changes"])&.changes)
+          evidence = changes.map { |change| change_summary(change) }
+          {
+            resources: evidence_for(changes, /resource|runtime|global|constant|env|random|time|rails/i),
+            files: evidence_for(changes, /file|filesystem|tmp|dir/i),
+            sockets: evidence_for(changes, /socket|network|tcp|udp|http/i),
+            source: evidence.join("\n")
+          }.compact
+        end
+
+        def evidence_for(changes, pattern)
+          changes.select { |change| change.probe.to_s.match?(pattern) || change.key.to_s.match?(pattern) }
+                 .map { |change| change_summary(change) }
+        end
+
+        def change_summary(change)
+          "#{change.probe}:#{change.key} #{stable_value(change.before)} -> #{stable_value(change.after)}"
+        end
+
+        def stable_value(value)
+          return "<missing>" if missing_value?(value)
+
+          value.inspect
+        end
+
+        def missing_value?(value)
+          change = Integrations.constant("RSpec::Hermetic::Change")
+          change && change.const_defined?(:MISSING) && value.equal?(change.const_get(:MISSING))
+        end
+
+        def normalize_hash(value)
+          return value.to_h if value.respond_to?(:to_h)
+
+          {
+            resources: value.respond_to?(:resources) ? value.resources : nil,
+            files: value.respond_to?(:files) ? value.files : nil,
+            sockets: value.respond_to?(:sockets) ? value.sockets : nil,
+            source: value.respond_to?(:source) ? value.source : nil
+          }.compact
+        end
+      end
+
+      module BuildkiteCollector
+        module_function
+
+        def tag_execution(example:, signature:, classification:)
+          labels = labels_for(classification)
+          tags = {
+            "rspec_flake_classifier.flaky" => flaky?(classification, labels).to_s,
+            "rspec_flake_classifier.labels" => labels.join(","),
+            "rspec_flake_classifier.signature" => signature.digest.to_s
+          }
+          return if tag_current_execution(tags)
+
+          tag_recorded_trace(example, tags)
+        rescue StandardError
+          nil
+        end
+
+        def tag_current_execution(tags)
+          collector = Integrations.constant("Buildkite::TestCollector")
+          return false unless collector&.respond_to?(:tag_execution)
+
+          tags.each { |key, value| collector.tag_execution(key, value) }
+          true
+        rescue StandardError
+          false
+        end
+
+        def tag_recorded_trace(example, tags)
+          trace = buildkite_trace(example)
+          return unless trace&.respond_to?(:tags)
+          return unless trace.tags.respond_to?(:[]=)
+
+          tags.each { |key, value| trace.tags[key] = value }
+        end
+
+        def buildkite_trace(example)
+          collector = Integrations.constant("Buildkite::TestCollector")
+          uploader = collector&.respond_to?(:uploader) ? collector.uploader : nil
+          return nil unless uploader&.respond_to?(:traces)
+
+          uploader.traces[example.id]
+        end
+
+        def labels_for(classification)
+          hash = classification.respond_to?(:to_h) ? classification.to_h : classification
+          Array(hash&.fetch("labels", nil)).map do |label|
+            if label.respond_to?(:fetch)
+              label.fetch("category", label.fetch(:category, label.to_s)).to_s
+            else
+              label.to_s
+            end
+          end
+        end
+
+        def flaky?(classification, labels)
+          hash = classification.respond_to?(:to_h) ? classification.to_h : classification
+          hash&.fetch("status", nil) == "flaky" || labels.any?
+        end
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/predictor.rb

@@ -0,0 +1,144 @@
+# frozen_string_literal: true
+
+require "json"
+
+module RSpec
+  module FlakeClassifier
+    class Predictor
+      WEIGHTS = {
+        "duration" => 0.15,
+        "uses_network" => 0.2,
+        "uses_filesystem" => 0.08,
+        "uses_time" => 0.15,
+        "uses_randomness" => 0.15,
+        "uses_concurrency" => 0.17,
+        "uses_async_wait" => 0.2
+      }.freeze
+
+      attr_reader :weights
+
+      def self.train(examples)
+        new(weights: Trainer.new(Array(examples)).weights)
+      end
+
+      def self.load(path)
+        payload = JSON.parse(File.read(path))
+        new(weights: payload.fetch("weights", payload))
+      end
+
+      def initialize(weights: WEIGHTS)
+        @weights = normalize_weights(weights)
+      end
+
+      def score(features)
+        score = weights.sum do |key, weight|
+          feature_score(features, key) * weight
+        end
+        [[score, 0.0].max, 1.0].min.round(4)
+      end
+
+      def rank(feature_sets)
+        Array(feature_sets).map do |features|
+          features.merge("flake_score" => score(features), "priority" => priority(score(features)))
+        end.sort_by { |features| -features.fetch("flake_score") }
+      end
+
+      def to_h
+        { "weights" => weights }
+      end
+
+      private
+
+      def feature_score(features, key)
+        value = features[key] || features[key.to_sym]
+        return duration_score(value) if key == "duration"
+
+        value ? 1.0 : 0.0
+      end
+
+      def duration_score(value)
+        duration = Float(value || 0.0)
+        return 0.0 unless duration.positive?
+
+        [duration / 10.0, 1.0].min
+      rescue ArgumentError, TypeError
+        0.0
+      end
+
+      def priority(score)
+        return "high" if score >= 0.45
+        return "medium" if score >= 0.2
+
+        "low"
+      end
+
+      def normalize_weights(input)
+        weights = WEIGHTS.merge(stringify_keys(input)).transform_values { |value| Float(value) }
+        total = weights.values.sum
+        return WEIGHTS if total <= 0.0
+
+        weights.transform_values { |value| (value / total).round(6) }
+      rescue ArgumentError, TypeError
+        WEIGHTS
+      end
+
+      def stringify_keys(hash)
+        hash.each_with_object({}) { |(key, value), result| result[key.to_s] = value }
+      end
+
+      class Trainer
+        def initialize(examples)
+          @examples = examples
+        end
+
+        def weights
+          positives = examples.select { |example| flaky?(example) }
+          negatives = examples.reject { |example| flaky?(example) }
+          return WEIGHTS if positives.empty? || negatives.empty?
+
+          raw = WEIGHTS.keys.to_h do |key|
+            [key, [average(positives, key) - average(negatives, key), 0.0].max]
+          end
+          normalize(raw)
+        end
+
+        private
+
+        attr_reader :examples
+
+        def flaky?(example)
+          value = example["flaky"] || example[:flaky] || example["label"] || example[:label] ||
+                  example["status"] || example[:status]
+          value == true || value.to_s == "flaky"
+        end
+
+        def average(records, key)
+          records.sum { |record| feature_score(record, key) } / records.length.to_f
+        end
+
+        def feature_score(record, key)
+          value = record[key] || record[key.to_sym]
+          return duration_score(value) if key == "duration"
+
+          value ? 1.0 : 0.0
+        end
+
+        def duration_score(value)
+          duration = Float(value || 0.0)
+          return 0.0 unless duration.positive?
+
+          [duration / 10.0, 1.0].min
+        rescue ArgumentError, TypeError
+          0.0
+        end
+
+        def normalize(raw)
+          total = raw.values.sum
+          return WEIGHTS if total <= 0.0
+
+          raw.transform_values { |value| (value / total).round(6) }
+        end
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/probe_evidence.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require_relative "integrations"
+
+module RSpec
+  module FlakeClassifier
+    class ProbeEvidence
+      EMPTY = {
+        resources: [],
+        files: [],
+        sockets: [],
+        source: nil
+      }.freeze
+
+      def initialize(provider: nil)
+        @provider = provider
+      end
+
+      def collect(example)
+        data = from_integration(example).merge(from_provider(example)) do |_key, integration_value, provider_value|
+          merge_value(integration_value, provider_value)
+        end.merge(from_metadata(example)) do |_key, provider_value, metadata_value|
+          merge_value(provider_value, metadata_value)
+        end
+        EMPTY.merge(data)
+      end
+
+      private
+
+      attr_reader :provider
+
+      def from_integration(example)
+        normalize(Integrations.probe_for(example))
+      end
+
+      def from_provider(example)
+        return {} unless provider
+
+        normalize(provider.call(example))
+      rescue StandardError => e
+        { resources: ["probe provider failed: #{e.class}: #{e.message}"] }
+      end
+
+      def from_metadata(example)
+        metadata = example.metadata
+        normalize(
+          resources: metadata[:flake_classifier_resources] || metadata[:hermetic_resources],
+          files: metadata[:flake_classifier_files] || metadata[:hermetic_files],
+          sockets: metadata[:flake_classifier_sockets] || metadata[:hermetic_sockets],
+          source: metadata[:flake_classifier_source]
+        )
+      end
+
+      def normalize(data)
+        hash = data || {}
+        {
+          resources: array_value(hash, :resources),
+          files: array_value(hash, :files),
+          sockets: array_value(hash, :sockets),
+          source: hash[:source] || hash["source"]
+        }.compact
+      end
+
+      def array_value(hash, key)
+        Array(hash[key] || hash[key.to_s]).compact
+      end
+
+      def merge_value(left, right)
+        return right if left.nil?
+        return left if right.nil?
+        return [*Array(left), *Array(right)].uniq unless left.is_a?(String) || right.is_a?(String)
+
+        [left, right].compact.join("\n")
+      end
+    end
+  end
+end