rspec-flake-classifier 0.1.0

data/lib/rspec/flake/classifier/cli.rb

@@ -0,0 +1,298 @@
+# frozen_string_literal: true
+
+require "json"
+require "optparse"
+require_relative "../classifier"
+
+module RSpec
+  module FlakeClassifier
+    class CLI
+      def initialize(argv, stdout: $stdout, stderr: $stderr)
+        @argv = argv.dup
+        @stdout = stdout
+        @stderr = stderr
+        @options = { store: Configuration::DEFAULT_STORE, format: "text" }
+      end
+
+      def run
+        command = argv.shift
+        return usage(1) unless command
+
+        case command
+        when "investigate" then investigate
+        when "classify" then classify
+        when "features" then features
+        when "predict" then predict
+        when "train" then train
+        when "evaluate" then evaluate
+        when "sensitivity" then sensitivity
+        when "report" then report
+        else
+          stderr.puts("Unknown command: #{command}")
+          usage(1)
+        end
+      rescue OptionParser::ParseError, Error => e
+        stderr.puts(e.message)
+        1
+      end
+
+      private
+
+      attr_reader :argv, :stdout, :stderr, :options
+
+      def investigate
+        parser = OptionParser.new do |parser|
+          parser.banner = "Usage: rspec-flake investigate EXAMPLE_ID [options]"
+          parser.on("--seed SEED", Integer) { |value| options[:seed] = value }
+          parser.on("--attempts N", Integer) { |value| options[:attempts] = value }
+          parser.on("--prior EXAMPLE_ID") { |value| (options[:prior_examples] ||= []) << value }
+          parser.on("--prior-file PATH") { |value| options[:prior_file] = value }
+          parser.on("--store PATH") { |value| options[:store] = value }
+          parser.on("--json") { options[:format] = "json" }
+        end
+        parser.parse!(argv)
+        example_id = argv.shift || raise(Error, "EXAMPLE_ID is required")
+
+        configuration.store = options[:store]
+        configuration.same_order_runs = options[:attempts] if options[:attempts]
+        result = RSpec::FlakeClassifier.investigate(
+          example_id,
+          seed: options[:seed],
+          prior_examples: prior_examples
+        )
+        write_result(result.to_h)
+        0
+      end
+
+      def classify
+        parser = OptionParser.new do |parser|
+          parser.banner = "Usage: rspec-flake classify [MESSAGE] [options]"
+          parser.on("--from-store") { options[:from_store] = true }
+          parser.on("--store PATH") { |value| options[:store] = value }
+          parser.on("--json") { options[:format] = "json" }
+        end
+        parser.parse!(argv)
+
+        if options[:from_store]
+          classify_store
+        else
+          message = argv.join(" ")
+          raise Error, "MESSAGE is required unless --from-store is used" if message.empty?
+
+          write_result(RSpec::FlakeClassifier.classify(message: message).to_h)
+        end
+        0
+      end
+
+      def classify_store
+        store = Store::JSONLStore.new(options[:store])
+        entries = store.entries.map do |entry|
+          result = RSpec::FlakeClassifier.classify(message: entry.fetch("normalized", ""))
+          store.update_classification(
+            entry.fetch("digest"),
+            classification: result,
+            metadata: { classified_by: "rspec-flake classify" }
+          )
+        end
+        write_result("entries" => entries)
+      end
+
+      def report
+        parser = OptionParser.new do |parser|
+          parser.banner = "Usage: rspec-flake report [options]"
+          parser.on("--store PATH") { |value| options[:store] = value }
+          parser.on("--json") { options[:format] = "json" }
+        end
+        parser.parse!(argv)
+        entries = Store::JSONLStore.new(options[:store]).entries
+        write_result(report_hash(entries))
+        0
+      end
+
+      def features
+        parser = OptionParser.new do |parser|
+          parser.banner = "Usage: rspec-flake features FILE [options]"
+          parser.on("--duration SECONDS", Float) { |value| options[:duration] = value }
+          parser.on("--json") { options[:format] = "json" }
+        end
+        parser.parse!(argv)
+        file = argv.shift || raise(Error, "FILE is required")
+        write_result(RSpec::FlakeClassifier::Features.new.extract(file: file, duration: options[:duration]))
+        0
+      end
+
+      def sensitivity
+        parser = OptionParser.new do |parser|
+          parser.banner = "Usage: rspec-flake sensitivity EXAMPLE_ID [options]"
+          parser.on("--seed SEED", Integer) { |value| options[:seed] = value }
+          parser.on("--factor FACTOR") { |value| (options[:factors] ||= []) << value.to_sym }
+          parser.on("--json") { options[:format] = "json" }
+        end
+        parser.parse!(argv)
+        example_id = argv.shift || raise(Error, "EXAMPLE_ID is required")
+        factors = options[:factors] || configuration.sensitivity_factors
+        write_result(RSpec::FlakeClassifier.sensitivity(example_id, factors: factors, seed: options[:seed]).to_h)
+        0
+      end
+
+      def predict
+        parser = OptionParser.new do |parser|
+          parser.banner = "Usage: rspec-flake predict FILE... [options]"
+          parser.on("--weights PATH") { |value| options[:weights] = value }
+          parser.on("--json") { options[:format] = "json" }
+        end
+        parser.parse!(argv)
+        raise Error, "at least one FILE is required" if argv.empty?
+
+        extractor = RSpec::FlakeClassifier::Features.new
+        features = argv.map { |file| extractor.extract(file: file) }
+        write_result("predictions" => predictor.rank(features))
+        0
+      end
+
+      def train
+        parser = OptionParser.new do |parser|
+          parser.banner = "Usage: rspec-flake train FEATURES_JSON_OR_JSONL [options]"
+          parser.on("--out PATH") { |value| options[:out] = value }
+          parser.on("--json") { options[:format] = "json" }
+        end
+        parser.parse!(argv)
+        file = argv.shift || raise(Error, "FEATURES_JSON_OR_JSONL is required")
+        model = RSpec::FlakeClassifier::Predictor.train(read_records(file))
+        File.write(options[:out], JSON.pretty_generate(model.to_h)) if options[:out]
+        write_result(model.to_h)
+        0
+      end
+
+      def evaluate
+        parser = OptionParser.new do |parser|
+          parser.banner = "Usage: rspec-flake evaluate [options]"
+          parser.on("--predictions PATH") { |value| options[:predictions] = value }
+          parser.on("--ground-truth PATH") { |value| options[:ground_truth] = value }
+          parser.on("--deflaker PATH") { |value| options[:deflaker_records] = value }
+          parser.on("--signatures PATH") { |value| options[:signature_records] = value }
+          parser.on("--idflakies PATH") { |value| options[:idflakies_records] = value }
+          parser.on("--json") { options[:format] = "json" }
+        end
+        parser.parse!(argv)
+        result = evaluation_result
+        raise Error, "no evaluation inputs provided" if result.empty?
+
+        write_result(result)
+        0
+      end
+
+      def report_hash(entries)
+        labels = Hash.new(0)
+        entries.each do |entry|
+          Array(entry["labels"]).each { |label| labels[label] += entry.fetch("occurrences", 1).to_i }
+        end
+
+        {
+          "total_signatures" => entries.length,
+          "total_occurrences" => entries.sum { |entry| entry.fetch("occurrences", 1).to_i },
+          "labels" => labels.sort.to_h,
+          "entries" => entries
+        }
+      end
+
+      def write_result(hash)
+        if options[:format] == "json"
+          stdout.puts(JSON.pretty_generate(hash))
+        else
+          stdout.puts(render_text(hash))
+        end
+      end
+
+      def render_text(hash)
+        return render_entries(hash.fetch("entries")) if hash.key?("entries")
+
+        hash.map { |key, value| "#{key}: #{value.inspect}" }.join("\n")
+      end
+
+      def render_entries(entries)
+        return "No flake signatures recorded." if entries.empty?
+
+        entries.map do |entry|
+          labels = Array(entry["labels"]).join(", ")
+          "#{entry["digest"]} occurrences=#{entry["occurrences"]} labels=#{labels}"
+        end.join("\n")
+      end
+
+      def usage(code)
+        stdout.puts(<<~TEXT)
+          Usage:
+            rspec-flake investigate EXAMPLE_ID [--seed SEED] [--attempts N] [--json]
+            rspec-flake classify MESSAGE [--json]
+            rspec-flake classify --from-store [--store PATH] [--json]
+            rspec-flake features FILE [--duration SECONDS] [--json]
+            rspec-flake predict FILE... [--weights PATH] [--json]
+            rspec-flake train FEATURES_JSON_OR_JSONL [--out PATH] [--json]
+            rspec-flake evaluate [--predictions PATH --ground-truth PATH] [--deflaker PATH] [--signatures PATH] [--idflakies PATH] [--json]
+            rspec-flake sensitivity EXAMPLE_ID [--seed SEED] [--factor FACTOR] [--json]
+            rspec-flake report [--store PATH] [--json]
+        TEXT
+        code
+      end
+
+      def prior_examples
+        Array(options[:prior_examples]) + prior_examples_from_file
+      end
+
+      def prior_examples_from_file
+        return [] unless options[:prior_file]
+
+        File.readlines(options[:prior_file], chomp: true).map(&:strip).reject(&:empty?)
+      end
+
+      def configuration
+        RSpec::FlakeClassifier.configuration
+      end
+
+      def predictor
+        return RSpec::FlakeClassifier::Predictor.load(options[:weights]) if options[:weights]
+
+        RSpec::FlakeClassifier::Predictor.new
+      end
+
+      def evaluation_result
+        evaluator = RSpec::FlakeClassifier::Evaluation.new
+        result = {}
+        if options[:predictions] && options[:ground_truth]
+          result["classification"] = evaluator.classification(
+            predictions: read_records(options[:predictions]),
+            ground_truth: read_records(options[:ground_truth])
+          )
+        end
+        result["deflaker"] = evaluator.deflaker(records: read_records(options[:deflaker_records])) if options[:deflaker_records]
+        result["signatures"] = evaluator.signatures(records: read_records(options[:signature_records])) if options[:signature_records]
+        result["idflakies"] = evaluator.idflakies(records: read_records(options[:idflakies_records])) if options[:idflakies_records]
+        result
+      end
+
+      def read_records(path)
+        content = File.read(path)
+        stripped = content.strip
+        return [] if stripped.empty?
+
+        parsed = json_records(stripped)
+        return parsed if parsed
+
+        stripped.each_line.map { |line| JSON.parse(line) }
+      end
+
+      def json_records(content)
+        parsed = JSON.parse(content)
+        return parsed if parsed.is_a?(Array)
+        return parsed.fetch("entries") if parsed.key?("entries")
+        return parsed.fetch("predictions") if parsed.key?("predictions")
+        return parsed.fetch("records") if parsed.key?("records")
+        return parsed.fetch("features") if parsed.key?("features")
+
+        [parsed]
+      rescue JSON::ParserError
+        nil
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/configuration.rb

@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+
+module RSpec
+  module FlakeClassifier
+    class Configuration
+      DEFAULT_STORE = ".rspec_flake_store"
+
+      attr_accessor :store, :auto_rerun, :deflaker, :rspec_command,
+                    :same_order_runs, :skip_known_flakes, :coverage_provider,
+                    :changed_lines_provider, :probe_provider, :run_sensitivity,
+                    :sensitivity_factors
+
+      def initialize
+        @store = DEFAULT_STORE
+        @auto_rerun = false
+        @deflaker = false
+        @rspec_command = nil
+        @same_order_runs = 3
+        @skip_known_flakes = false
+        @coverage_provider = nil
+        @changed_lines_provider = nil
+        @probe_provider = nil
+        @run_sensitivity = false
+        @sensitivity_factors = %i[time randomness network]
+      end
+
+      def auto_rerun_attempts
+        return same_order_runs unless auto_rerun.is_a?(Hash)
+
+        Integer(auto_rerun.fetch(:failures, same_order_runs))
+      rescue ArgumentError, TypeError
+        same_order_runs
+      end
+
+      def store_path
+        store.to_s.empty? ? DEFAULT_STORE : store.to_s
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/coverage_snapshot.rb

@@ -0,0 +1,89 @@
+# frozen_string_literal: true
+
+require "coverage"
+
+module RSpec
+  module FlakeClassifier
+    class CoverageSnapshot
+      def self.capture
+        return null unless Coverage.respond_to?(:running?) && Coverage.running?
+        return null unless Coverage.respond_to?(:peek_result)
+
+        new(Coverage.peek_result)
+      rescue RuntimeError
+        null
+      end
+
+      def self.null
+        new({})
+      end
+
+      def initialize(data)
+        @data = normalize(data)
+      end
+
+      def delta
+        after = self.class.capture_snapshot
+        after.each_with_object({}) do |(file, counts), result|
+          changed = line_delta(Array(@data[file]), counts)
+          result[relative_path(file)] = changed unless changed.empty?
+        end
+      end
+
+      def self.capture_snapshot
+        return {} unless Coverage.respond_to?(:running?) && Coverage.running?
+        return {} unless Coverage.respond_to?(:peek_result)
+
+        normalize_static(Coverage.peek_result)
+      rescue RuntimeError
+        {}
+      end
+
+      def self.normalize_static(data)
+        data.each_with_object({}) do |(file, value), result|
+          result[file.to_s] = line_counts(value)
+        end
+      end
+
+      def self.line_counts(value)
+        return Array(value) if value.is_a?(Array)
+        return Array(value[:lines]) if value.is_a?(Hash) && value.key?(:lines)
+        return Array(value["lines"]) if value.is_a?(Hash) && value.key?("lines")
+
+        []
+      end
+
+      private
+
+      def normalize(data)
+        self.class.normalize_static(data)
+      end
+
+      def line_delta(before, after)
+        after.each_with_index.each_with_object([]) do |(after_count, index), lines|
+          next unless count_positive?(after_count)
+
+          before_count = before[index]
+          lines << index + 1 if count_value(after_count) > count_value(before_count)
+        end
+      end
+
+      def count_positive?(count)
+        count_value(count).positive?
+      end
+
+      def count_value(count)
+        Integer(count || 0)
+      rescue ArgumentError, TypeError
+        0
+      end
+
+      def relative_path(file)
+        path = file.to_s
+        return path unless path.start_with?(Dir.pwd)
+
+        path.delete_prefix("#{Dir.pwd}/")
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/deflaker.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+require "open3"
+
+module RSpec
+  module FlakeClassifier
+    class Deflaker
+      Result = Struct.new(:suspected, :reason, :covered_lines, :changed_lines, keyword_init: true) do
+        def suspected?
+          suspected
+        end
+
+        def to_h
+          {
+            "suspected" => suspected,
+            "reason" => reason,
+            "covered_lines" => covered_lines,
+            "changed_lines" => changed_lines
+          }
+        end
+      end
+
+      def suspect?(coverage:, changed_lines: nil, base: "HEAD")
+        covered = normalize_lines(coverage)
+        changed = normalize_lines(changed_lines || git_changed_lines(base))
+        intersection = intersect(covered, changed)
+
+        Result.new(
+          suspected: !covered.empty? && intersection.empty?,
+          reason: intersection.empty? ? "covered code did not intersect changed lines" : "covered changed lines",
+          covered_lines: covered,
+          changed_lines: changed
+        )
+      end
+
+      def git_changed_lines(base = "HEAD")
+        stdout, _stderr, status = Open3.capture3("git", "diff", "--unified=0", base)
+        return {} unless status.success?
+
+        parse_diff(stdout)
+      end
+
+      def parse_diff(diff)
+        changed = Hash.new { |hash, key| hash[key] = [] }
+        current_file = nil
+
+        diff.each_line do |line|
+          if (match = line.match(%r{\A\+\+\+ b/(.+)}))
+            current_file = match[1]
+            next
+          end
+
+          next unless current_file
+
+          match = line.match(/\A@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/)
+          next unless match
+
+          start = match[1].to_i
+          length = (match[2] || "1").to_i
+          next if length.zero?
+
+          changed[current_file].concat((start...(start + length)).to_a)
+        end
+
+        changed.transform_values { |lines| lines.uniq.sort }
+      end
+
+      private
+
+      def normalize_lines(lines)
+        case lines
+        when Hash
+          lines.each_with_object({}) do |(file, file_lines), result|
+            normalized = Array(file_lines).map(&:to_i).reject(&:zero?).uniq.sort
+            result[file.to_s] = normalized unless normalized.empty?
+          end
+        when Array
+          lines.each_with_object(Hash.new { |hash, key| hash[key] = [] }) do |entry, result|
+            file, line = normalize_entry(entry)
+            result[file] << line if file && line.positive?
+          end.transform_values { |file_lines| file_lines.uniq.sort }
+        else
+          {}
+        end
+      end
+
+      def normalize_entry(entry)
+        return [entry[:file].to_s, entry[:line].to_i] if entry.is_a?(Hash) && entry.key?(:file)
+        return [entry["file"].to_s, entry["line"].to_i] if entry.is_a?(Hash) && entry.key?("file")
+
+        Array(entry).then { |file, line| [file.to_s, line.to_i] }
+      end
+
+      def intersect(left, right)
+        left.each_with_object({}) do |(file, lines), result|
+          common = lines & Array(right[file])
+          result[file] = common unless common.empty?
+        end
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/evaluation.rb

@@ -0,0 +1,127 @@
+# frozen_string_literal: true
+
+module RSpec
+  module FlakeClassifier
+    class Evaluation
+      def classification(predictions:, ground_truth:)
+        expected = index_by_id(ground_truth)
+        rows = Array(predictions).filter_map do |prediction|
+          id = record_id(prediction)
+          truth = Array(expected[id])
+          next if id.nil? || truth.empty?
+
+          predicted = labels_for(prediction)
+          {
+            "id" => id,
+            "truth" => truth,
+            "predicted" => predicted,
+            "top1" => truth.include?(predicted.first),
+            "top2" => !(truth & predicted.first(2)).empty?
+          }
+        end
+        metric_hash(rows)
+      end
+
+      def deflaker(records:)
+        flaky = Array(records).select { |record| flaky?(record) }
+        captured = flaky.select { |record| deflaker_suspected?(record) }
+        {
+          "flaky_count" => flaky.length,
+          "captured_count" => captured.length,
+          "recall" => ratio(captured.length, flaky.length)
+        }
+      end
+
+      def signatures(records:)
+        flaky = Array(records).select { |record| flaky?(record) }
+        signed = flaky.select { |record| signature_for(record) }
+        collision_groups = signed.group_by { |record| signature_for(record) }.values.count do |group|
+          group.flat_map { |record| labels_for(record) }.uniq.length > 1
+        end
+        {
+          "flaky_count" => flaky.length,
+          "signed_count" => signed.length,
+          "capture_rate" => ratio(signed.length, flaky.length),
+          "collision_groups" => collision_groups
+        }
+      end
+
+      def idflakies(records:)
+        counts = Hash.new(0)
+        Array(records).each do |record|
+          order_type = record["order_type"] || record[:order_type] || record.dig("investigation", "order_type")
+          counts[order_type || "unknown"] += 1
+        end
+        total = counts.values.sum
+        {
+          "total" => total,
+          "counts" => counts.sort.to_h,
+          "od_ratio" => ratio(counts["od"], total),
+          "nod_ratio" => ratio(counts["nod"], total)
+        }
+      end
+
+      private
+
+      def index_by_id(records)
+        Array(records).each_with_object({}) do |record, result|
+          id = record_id(record)
+          result[id] = labels_for(record) if id
+        end
+      end
+
+      def metric_hash(rows)
+        {
+          "total" => rows.length,
+          "top1_accuracy" => ratio(rows.count { |row| row.fetch("top1") }, rows.length),
+          "top2_accuracy" => ratio(rows.count { |row| row.fetch("top2") }, rows.length),
+          "rows" => rows
+        }
+      end
+
+      def record_id(record)
+        record["id"] || record[:id] || record["example_id"] || record[:example_id] ||
+          record["file"] || record[:file]
+      end
+
+      def labels_for(record)
+        labels = record["labels"] || record[:labels] ||
+                 record.dig("classification", "labels") ||
+                 record["categories"] || record[:categories] ||
+                 record["truth"] || record[:truth] ||
+                 record["label"] || record[:label]
+        Array(labels).map { |label| label_category(label) }.reject(&:empty?)
+      end
+
+      def label_category(label)
+        return label.fetch("category", label.fetch(:category, label.to_s)) if label.respond_to?(:fetch)
+
+        label.to_s
+      end
+
+      def flaky?(record)
+        return true if record["flaky"] == true || record[:flaky] == true
+        return true if record["status"].to_s == "flaky" || record[:status].to_s == "flaky"
+        return true if record.dig("classification", "status").to_s == "flaky"
+
+        labels_for(record).any? { |label| label.include?("flaky") || label == "network" || label == "time" }
+      end
+
+      def deflaker_suspected?(record)
+        labels_for(record).include?("suspected_flaky_deflaker") ||
+          record.dig("metadata", "deflaker", "suspected") == true ||
+          record.dig("deflaker", "suspected") == true
+      end
+
+      def signature_for(record)
+        record["signature"] || record[:signature] || record["digest"] || record[:digest]
+      end
+
+      def ratio(numerator, denominator)
+        return 0.0 if denominator.to_i.zero?
+
+        (numerator.to_f / denominator).round(4)
+      end
+    end
+  end
+end

data/lib/rspec/flake/classifier/example_history.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module RSpec
+  module FlakeClassifier
+    class ExampleHistory
+      def initialize
+        @ids = []
+      end
+
+      def prior_examples
+        @ids.dup
+      end
+
+      def record(example)
+        id = example.respond_to?(:id) ? example.id : nil
+        @ids << id if id && !@ids.include?(id)
+      end
+
+      def clear
+        @ids.clear
+      end
+    end
+  end
+end