sixth_sense 0.1.0

data/lib/sixth_sense/cli.rb

@@ -0,0 +1,229 @@
+# frozen_string_literal: true
+
+require "optparse"
+require "json"
+
+require_relative "analysis_runner"
+require_relative "changed_files"
+require_relative "config"
+require_relative "engines/mutant"
+require_relative "guardrail/baseline"
+require_relative "guardrail/evaluator"
+require_relative "mutation_cache"
+require_relative "reporters/console"
+require_relative "reporters/json"
+require_relative "reporters/markdown"
+require_relative "reporters/sarif"
+require_relative "reporters/html"
+
+module SixthSense
+  class CLI
+    def run(argv)
+      command = argv.shift || "analyze"
+      case command
+      when "analyze" then analyze(argv)
+      when "guard" then guard(argv)
+      when "baseline" then baseline(argv)
+      when "explain" then explain(argv)
+      when "calibrate" then calibrate(argv)
+      when "mutation-plan" then mutation_plan(argv)
+      when "mutation-run" then mutation_run(argv)
+      else
+        warn "Unknown command: #{command}"
+        2
+      end
+    end
+
+    private
+
+    def analyze(argv)
+      options = parse_analysis_options(argv)
+      reports = run_analysis(argv, options)
+      write_report(reports, options)
+      0
+    end
+
+    def guard(argv)
+      options = parse_analysis_options(argv)
+      options[:level] ||= Config.load(options[:config_path]).fetch(:guardrail, :require_level, default: 0).to_i
+      guard_config = Config.load(options[:config_path])
+      diff_ref = options[:diff] || (guard_config.fetch(:guardrail, :diff_only, default: false) && guard_config.fetch(:guardrail, :diff_ref, default: "origin/main"))
+      paths = diff_ref ? ChangedFiles.test_files(diff_ref) : argv
+      reports = run_analysis(paths, options)
+      write_report(reports, options)
+
+      config = Config.load(options[:config_path])
+      evaluation = Guardrail::Evaluator.new(config: config).evaluate(reports, level: options[:level])
+      print_guard_violations(evaluation)
+      evaluation.passed? ? 0 : 1
+    end
+
+    def baseline(argv)
+      options = parse_analysis_options(argv)
+      reports = run_analysis(argv, options)
+      baseline = Guardrail::Baseline.load
+      baseline.update(reports, level: options[:level], force: options[:force])
+      baseline.write
+      puts "Updated #{Guardrail::Baseline::DEFAULT_PATH}"
+      0
+    end
+
+    def explain(argv)
+      analyzer_id = argv.shift
+      analyzer = SixthSense.analyzers.find { |item| item.analyzer_id == analyzer_id }
+      unless analyzer
+        warn "Unknown analyzer: #{analyzer_id}"
+        return 2
+      end
+
+      puts "#{analyzer.analyzer_id} axis=#{analyzer.axis} level=#{analyzer.level}"
+      analyzer.references.each do |reference|
+        doi = reference.doi ? " doi:#{reference.doi}" : ""
+        puts "- #{reference.authors} (#{reference.year}). #{reference.title}. #{reference.venue}.#{doi}"
+      end
+      0
+    end
+
+    def mutation_plan(argv)
+      options = parse_analysis_options(argv)
+      config = Config.load(options[:config_path])
+      apply_mutation_mode(config, options)
+      runner = AnalysisRunner.new(config: config)
+      test_files = runner.discover(argv).filter_map do |path|
+        adapter_class = FrameworkAdapter.for_path(path)
+        adapter_class&.new&.parse(path)
+      end
+      engine = Engines::Mutant.new(config: config.fetch(:mutation, default: {}))
+      available = engine.availability
+      warn "mutation engine unavailable: #{available.reason}" unless available.available?
+      plan = engine.plan(sut_units: test_files.flat_map(&:sut_candidates), test_files: test_files)
+      puts JSON.pretty_generate(
+        {
+          availability: { available: available.available?, reason: available.reason },
+          plan: plan.to_h
+        }
+      )
+      available.available? ? 0 : 1
+    end
+
+    def mutation_run(argv)
+      options = parse_analysis_options(argv)
+      config = Config.load(options[:config_path])
+      apply_mutation_mode(config, options)
+      runner = AnalysisRunner.new(config: config)
+      test_files = runner.discover(argv).filter_map do |path|
+        adapter_class = FrameworkAdapter.for_path(path)
+        adapter_class&.new&.parse(path)
+      end
+      engine = Engines::Mutant.new(config: config.fetch(:mutation, default: {}))
+      plan = engine.plan(sut_units: test_files.flat_map(&:sut_candidates), test_files: test_files)
+      result = engine.run(plan, cache: config.fetch(:mutation, :cache_dir, default: MutationCache::DEFAULT_ROOT))
+      puts JSON.pretty_generate(result.to_h)
+      result.success? ? 0 : 1
+    end
+
+    def calibrate(argv)
+      options = parse_analysis_options(argv)
+      level1 = run_analysis(argv, options.merge(level: 1, format: options[:format]))
+      level2 = run_analysis(argv, options.merge(level: 2, format: options[:format]))
+      pairs = level1.zip(level2).filter_map do |left, right|
+        coverage = left.axis_scores.find { |score| score.axis == :adequacy }
+        mutation = right.axis_scores.find { |score| score.axis == :adequacy }
+        [coverage.value, mutation.value] if coverage&.measured? && mutation&.measured?
+      end
+      pairs = pairs.first(options[:sample]) if options[:sample]
+      rho = spearman(pairs)
+      recommendation = rho.nil? || rho < 0.5 ? "require_level: 2" : "require_level: 1"
+      puts JSON.pretty_generate({ sample_size: pairs.length, spearman_rho: rho, recommendation: recommendation })
+      0
+    end
+
+    def parse_analysis_options(argv)
+      options = {
+        format: "console",
+        level: 0,
+        config_path: ".sixth_sense.yml"
+      }
+      parser = OptionParser.new do |opts|
+        opts.on("--level LEVEL", Integer) { |value| options[:level] = value }
+        opts.on("--format FORMAT") { |value| options[:format] = value }
+        opts.on("-o", "--output PATH") { |value| options[:output] = value }
+        opts.on("--config PATH") { |value| options[:config_path] = value }
+        opts.on("--diff REF") { |value| options[:diff] = value }
+        opts.on("--all") { options[:all] = true }
+        opts.on("--force") { options[:force] = true }
+        opts.on("--sample N", Integer) { |value| options[:sample] = value }
+        opts.on("--mutation-mode MODE") { |value| options[:mutation_mode] = value }
+        opts.on("--score-engine ENGINE") { |value| options[:score_engine] = value }
+      end
+      parser.parse!(argv)
+      options
+    end
+
+    def run_analysis(paths, options)
+      config = Config.load(options[:config_path])
+      selected_paths = options[:all] ? [] : paths
+      AnalysisRunner.new(config: config).analyze(paths: selected_paths, level: options[:level])
+    end
+
+    def apply_mutation_mode(config, options)
+      config.data["mutation"] ||= {}
+      config.data["mutation"]["mode"] = options[:mutation_mode] if options[:mutation_mode]
+      config.data["mutation"]["score_engine"] = options[:score_engine] if options[:score_engine]
+    end
+
+    def write_report(reports, options)
+      rendered = reporter_for(options[:format]).render(reports)
+      if options[:output]
+        File.write(options[:output], rendered)
+      else
+        puts rendered
+      end
+    end
+
+    def reporter_for(format)
+      case format
+      when "console" then Reporters::Console.new
+      when "json" then Reporters::Json.new
+      when "markdown" then Reporters::Markdown.new
+      when "sarif" then Reporters::Sarif.new
+      when "html" then Reporters::Html.new
+      else
+        raise Error, "unknown format: #{format}"
+      end
+    end
+
+    def print_guard_violations(evaluation)
+      evaluation.violations.each do |violation|
+        warn "guard violation: #{violation}"
+      end
+      evaluation.baseline_violations.each do |violation|
+        warn "ratchet violation: #{violation}"
+      end
+      evaluation.warnings.each do |warning|
+        warn "guard warning: #{warning}"
+      end
+    end
+
+    def spearman(pairs)
+      return nil if pairs.length < 2
+
+      left_ranks = ranks(pairs.map(&:first))
+      right_ranks = ranks(pairs.map(&:last))
+      mean_left = left_ranks.sum / left_ranks.length.to_f
+      mean_right = right_ranks.sum / right_ranks.length.to_f
+      numerator = left_ranks.zip(right_ranks).sum { |left, right| (left - mean_left) * (right - mean_right) }
+      denominator = Math.sqrt(left_ranks.sum { |left| (left - mean_left)**2 } * right_ranks.sum { |right| (right - mean_right)**2 })
+      return nil if denominator.zero?
+
+      (numerator / denominator).round(4)
+    end
+
+    def ranks(values)
+      sorted = values.each_with_index.sort_by(&:first)
+      ranks = Array.new(values.length)
+      sorted.each_with_index { |(_value, original_index), rank| ranks[original_index] = rank + 1 }
+      ranks
+    end
+  end
+end

data/lib/sixth_sense/config.rb

@@ -0,0 +1,91 @@
+# frozen_string_literal: true
+
+require "yaml"
+
+module SixthSense
+  class Config
+    DEFAULT = {
+      "guardrail" => {
+        "fail_under" => {},
+        "require_level" => 0,
+        "ratchet" => false,
+        "tolerance" => 1.0,
+        "diff_only" => false,
+        "diff_ref" => "origin/main"
+      },
+      "quality" => {
+        "assertion_roulette_threshold" => 3,
+        "assertion_density_min" => 0.05,
+        "smell_cap" => 5
+      },
+      "adequacy" => {
+        "coverage_transform" => "raw",
+        "coverage_gamma" => 0.5
+      },
+      "coverage" => {
+        "enabled" => true,
+        "load_path" => "lib",
+        "checked_mode" => "windowed",
+        "checked_window" => 20
+      },
+      "mutation" => {
+        "engine" => "mutant",
+        "mode" => "score",
+        "score_engine" => "auto",
+        "version_requirement" => ">= 0",
+        "timeout" => 10,
+        "timeout_policy" => "killed",
+        "jobs" => "auto",
+        "cache_dir" => ".sixth_sense_cache/kill_matrix",
+        "neutral_runs" => 2,
+        "max_mutants" => 500,
+        "equivalent_threshold" => 0.2,
+        "operators" => {
+          "exclude" => []
+        }
+      }
+    }.freeze
+
+    attr_reader :data
+
+    def self.load(path = ".sixth_sense.yml")
+      file_data = File.file?(path) ? YAML.load_file(path) : {}
+      new(deep_merge(DEFAULT, file_data || {}))
+    end
+
+    def self.deep_merge(left, right)
+      merged = deep_dup(left)
+      right.each do |key, value|
+        merged[key] = if merged[key].is_a?(Hash) && value.is_a?(Hash)
+                        deep_merge(merged[key], value)
+                      else
+                        deep_dup(value)
+                      end
+      end
+      merged
+    end
+
+    def self.deep_dup(value)
+      case value
+      when Hash
+        value.each_with_object({}) { |(key, nested), copy| copy[key] = deep_dup(nested) }
+      when Array
+        value.map { |nested| deep_dup(nested) }
+      else
+        value
+      end
+    end
+
+    def initialize(data = DEFAULT)
+      @data = data
+    end
+
+    def fetch(*keys, default: nil)
+      keys.reduce(data) do |current, key|
+        break default unless current.respond_to?(:fetch)
+
+        current.fetch(key.to_s) { current.fetch(key.to_sym, default) }
+      end
+    end
+  end
+end

data/lib/sixth_sense/engines/mutant.rb

@@ -0,0 +1,258 @@
+# frozen_string_literal: true
+
+require "rubygems"
+require "json"
+require "open3"
+require "rbconfig"
+
+require_relative "../framework_adapter"
+require_relative "../mutation_engine"
+require_relative "../mutation_cache"
+require_relative "../mutation_matrix_producer"
+require_relative "../mutation_score_cache_writer"
+
+module SixthSense
+  module Engines
+    class Mutant < MutationEngine
+      SUPPORTED_ENGINE = "ruby"
+
+      def initialize(config: {})
+        @config = config
+      end
+
+      def availability
+        return Availability.new(available: true, reason: "sixth_sense matrix producer is available") if matrix_mode?
+        return Availability.new(available: true, reason: "sixth_sense built-in score producer is available") if builtin_score_engine?
+
+        external = external_availability
+        return external if external.available? || external_score_engine?
+
+        Availability.new(available: true, reason: "sixth_sense built-in score producer fallback is available")
+      end
+
+      def external_availability
+        return Availability.new(available: false, reason: "mutant requires CRuby") unless RUBY_ENGINE == SUPPORTED_ENGINE
+        mutant_spec = gem_spec("mutant")
+        return Availability.new(available: false, reason: "mutant gem is not installed") unless mutant_spec
+        return Availability.new(available: false, reason: "mutant-rspec gem is not installed") unless gem_installed?("mutant-rspec")
+        unless supported_mutant_version?(mutant_spec.version)
+          return Availability.new(available: false, reason: "mutant #{mutant_spec.version} does not satisfy #{@config.fetch("version_requirement", ">= 0")}")
+        end
+
+        Availability.new(available: true, reason: "mutant and mutant-rspec are available")
+      end
+
+      def plan(sut_units:, test_files:)
+        configured = test_files.flat_map { |test_file| Array(test_file.metadata&.fetch(:mutation_subjects, [])) }
+        subjects = (configured + sut_units.flat_map { |unit| expressions_for(unit) }).uniq
+        MutationPlan.new(
+          engine: "mutant",
+          mode: @config.fetch("mode", "score"),
+          subjects: subjects,
+          test_files: test_files.map(&:path),
+          source_units: sut_units.map(&:to_h)
+        )
+      end
+
+      def run(plan, cache:)
+        return failed_result(plan, ["sixth_sense", "mutation-run"], "mutation plan has no subjects") if plan.subjects.empty?
+        return run_matrix(plan, cache: cache) if matrix_mode?(plan.mode)
+        return run_builtin_score(plan, cache: cache) if builtin_score_engine? || requires_builtin_score?(plan)
+
+        available = external_availability
+        unless available.available?
+          return failed_result(plan, command_for(plan, cache: cache), "mutation engine unavailable: #{available.reason}")
+        end
+        begin
+          neutral_run!(plan)
+        rescue Error => error
+          return failed_result(plan, command_for(plan, cache: cache), error.message)
+        end
+        command = command_for(plan, cache: cache)
+        stdout, stderr, status = Open3.capture3(*command)
+        MutationScoreCacheWriter.new(cache: cache).write(plan, stdout: stdout, stderr: stderr) if status.success? && cache
+        MutationRunResult.new(
+          plan: plan,
+          command: command,
+          success: status.success?,
+          stdout: stdout,
+          stderr: stderr,
+          status: status.exitstatus
+        )
+      end
+
+      private
+
+      def matrix_mode?(mode = @config.fetch("mode", "score"))
+        mode.to_s == "matrix"
+      end
+
+      def builtin_score_engine?
+        @config.fetch("score_engine", "auto").to_s == "builtin"
+      end
+
+      def external_score_engine?
+        @config.fetch("score_engine", "auto").to_s == "mutant"
+      end
+
+      def requires_builtin_score?(plan)
+        return false if external_score_engine?
+
+        !external_availability.available? || plan_frameworks(plan).any? { |framework| framework != :rspec }
+      end
+
+      def plan_frameworks(plan)
+        plan.test_files.filter_map do |path|
+          FrameworkAdapter.for_path(path)&.new&.parse(path)&.framework
+        end
+      end
+
+      def run_matrix(plan, cache:)
+        producer = MutationMatrixProducer.new(config: @config)
+        begin
+          producer.neutral_run!(plan.test_files)
+        rescue Error => error
+          write_unstable_cache(plan, cache, error.message) if cache
+          return failed_result(plan, ["sixth_sense", "mutation-run", "--mutation-mode", "matrix"], error.message)
+        end
+        result = producer.run(plan)
+        result.write(cache)
+        MutationRunResult.new(
+          plan: plan,
+          command: ["sixth_sense", "mutation-run", "--mutation-mode", "matrix"],
+          success: true,
+          stdout: JSON.generate(result.payload),
+          stderr: "",
+          status: 0
+        )
+      end
+
+      def run_builtin_score(plan, cache:)
+        producer = MutationMatrixProducer.new(config: @config)
+        begin
+          producer.neutral_run!(plan.test_files)
+        rescue Error => error
+          write_unstable_cache(plan, cache, error.message) if cache
+          return failed_result(plan, ["sixth_sense", "mutation-run", "--mutation-mode", "score"], error.message)
+        end
+
+        result = producer.run(plan)
+        score = score_for(result.payload)
+        MutationScoreCacheWriter.new(cache: cache).write_score(plan, score) if cache
+        stdout = JSON.generate(
+          {
+            "engine" => "sixth_sense/matrix",
+            "mode" => "score",
+            "mutation_score" => score,
+            "source_mode" => "built_in_matrix"
+          }
+        )
+        MutationRunResult.new(
+          plan: plan,
+          command: ["sixth_sense", "mutation-run", "--mutation-mode", "score", "--score-engine", "builtin"],
+          success: true,
+          stdout: stdout,
+          stderr: "",
+          status: 0
+        )
+      end
+
+      def score_for(payload)
+        mutants = payload.fetch("subjects", []).flat_map { |subject| subject.fetch("mutants", []) }
+        countable = mutants.reject { |mutant| mutant.fetch("status", "alive") == "skipped" }
+        return 0.0 if countable.empty?
+
+        killed = countable.count do |mutant|
+          status = mutant.fetch("status", "alive")
+          status == "killed" || (status == "timeout" && @config.fetch("timeout_policy", "killed").to_s == "killed")
+        end
+        killed.to_f / countable.length
+      end
+
+      def failed_result(plan, command, message)
+        MutationRunResult.new(
+          plan: plan,
+          command: command,
+          success: false,
+          stdout: JSON.generate({ error: message }),
+          stderr: message,
+          status: 1
+        )
+      end
+
+      def gem_installed?(name)
+        Gem::Specification.find_all_by_name(name).any?
+      end
+
+      def gem_spec(name)
+        Gem::Specification.find_all_by_name(name).first
+      end
+
+      def supported_mutant_version?(version)
+        Gem::Requirement.new(@config.fetch("version_requirement", ">= 0")).satisfied_by?(version)
+      rescue Gem::Requirement::BadRequirementError
+        false
+      end
+
+      def expressions_for(unit)
+        return unit.constants if unit.methods.empty?
+
+        unit.constants.flat_map do |constant|
+          [constant] + unit.methods.map { |method_name| "#{constant}##{method_name}" }
+        end
+      end
+
+      def command_for(plan, cache:)
+        command = ["mutant", "run", "--integration", "rspec", "--usage", @config.fetch("usage", "opensource")]
+        command.concat(["-I", @config.fetch("load_path", "lib")])
+        timeout = @config.fetch("timeout", nil)
+        command.concat(["--mutation-timeout", timeout.to_s]) if timeout
+        jobs = @config.fetch("jobs", nil)
+        command.concat(["--jobs", jobs.to_s]) if jobs && jobs != "auto"
+        command.concat(["--env", "SIXTH_SENSE_CACHE=#{cache}"]) if cache
+        plan.test_files.each do |test_file|
+          command.concat(["--integration-argument", test_file])
+        end
+        plan.subjects.each do |subject|
+          command.concat(["--start-subject", subject])
+        end
+        command
+      end
+
+      def write_unstable_cache(plan, cache, message)
+        mutation_cache = MutationCache.new(root: cache)
+        plan.test_files.each do |test_file|
+          mutation_cache.write(
+            Model::TestFile.new(path: test_file, framework: :rspec, test_cases: [], sut_candidates: []),
+            {
+              "version" => 1,
+              "engine" => "sixth_sense/matrix",
+              "mode" => "matrix",
+              "metadata" => {
+                "warnings" => [
+                  {
+                    "type" => "unstable_baseline",
+                    "message" => message
+                  }
+                ]
+              },
+              "subjects" => []
+            }
+          )
+        end
+      end
+
+      def neutral_run!(plan)
+        runs = @config.fetch("neutral_runs", 2).to_i
+        statuses = runs.times.map do
+          command = [RbConfig.ruby, "-I", @config.fetch("load_path", "lib"), "-e", "require 'rspec/core'; require 'rspec/expectations'; exit RSpec::Core::Runner.run(ARGV)", "--", "--options", File::NULL, *plan.test_files]
+          _stdout, stderr, status = Open3.capture3(*command)
+          raise Error, "neutral run failed: #{stderr}" unless status.success?
+
+          status.exitstatus
+        end
+        raise Error, "neutral run was unstable" unless statuses.uniq.length == 1
+      end
+    end
+  end
+end

data/lib/sixth_sense/framework_adapter.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module SixthSense
+  class FrameworkAdapter
+    class << self
+      def register(name, adapter_class)
+        registry[name.to_sym] = adapter_class
+      end
+
+      def registry
+        @registry ||= {}
+      end
+
+      def for_path(path)
+        registry.values.find { |adapter_class| adapter_class.new.handles?(path) }
+      end
+    end
+
+    def handles?(_path)
+      raise NotImplementedError
+    end
+
+    def parse(_path)
+      raise NotImplementedError
+    end
+
+    def run_with_coverage(test_files:, isolation:)
+      raise NotImplementedError
+    end
+
+    def mutation_session(test_files:)
+      raise NotImplementedError
+    end
+
+    def mutation_test_command(test_case:, config:)
+      nil
+    end
+
+    def coverage_test_command(test_case:, config:)
+      nil
+    end
+
+    def checked_coverage_test_command(test_case:, config:)
+      nil
+    end
+
+    def map_to_sut(_test_file)
+      raise NotImplementedError
+    end
+
+    def assertion_patterns
+      []
+    end
+  end
+end