sixth_sense 0.1.0

data/lib/sixth_sense/mutation_matrix_producer.rb

@@ -0,0 +1,308 @@
+# frozen_string_literal: true
+
+require "json"
+require "open3"
+require "rbconfig"
+
+require_relative "adapters/minitest"
+require_relative "adapters/rspec"
+require_relative "adapters/test_unit"
+require_relative "framework_adapter"
+require_relative "mutation_cache"
+require_relative "mutation_matrix_mutant_generator"
+
+module SixthSense
+  class MutationMatrixResult
+    attr_reader :payload
+    def initialize(payload)
+      @payload = payload
+    end
+
+    def write(cache_root)
+      cache = MutationCache.new(root: cache_root)
+      test_files.each do |test_file|
+        cache.write(Model::TestFile.new(path: test_file, framework: :rspec, test_cases: [], sut_candidates: []), payload_for(test_file))
+      end
+    end
+
+    private
+
+    def test_files
+      payload.fetch("test_files")
+    end
+
+    def payload_for(test_file)
+      subjects = payload.fetch("subjects").map do |subject|
+        subject.merge(
+          "mutants" => subject.fetch("mutants").map do |mutant|
+            killed_by = Array(mutant["killed_by"]).select { |id| id.start_with?(test_file) }
+            executed = Array(mutant["executed"]).select { |id| id.start_with?(test_file) }
+            mutant.merge("killed_by" => killed_by, "executed" => executed)
+          end
+        )
+      end
+      payload.merge("subjects" => subjects)
+    end
+  end
+
+  class MutationMatrixProducer
+    def initialize(config: {})
+      @config = config
+    end
+
+    # Paper basis: Luo et al. 2014 (doi:10.1145/2635868.2635920) motivates
+    # isolating flaky/unstable baseline behavior before mutation analysis.
+    def neutral_run!(test_files)
+      runs = neutral_runs.times.map do
+        test_cases(test_files).map do |entry|
+          [entry.fetch(:test_case).id, run_test_case(entry)]
+        end
+      end
+      failed = runs.flatten(1).find { |_id, status| status != :passed }
+      raise Error, "neutral run failed for #{failed.fetch(0)}: #{failed.fetch(1)}" if failed
+      raise Error, "neutral run was unstable" unless runs.uniq.length == 1
+    end
+
+    # Paper basis: Jia/Harman 2011 (doi:10.1109/TSE.2010.62) defines mutation
+    # testing over generated mutants; this produces the per-test kill matrix
+    # consumed by mutation adequacy and redundancy analyzers.
+    def run(plan)
+      selection = MutationMatrixMutantGenerator.new(config: @config).generate(plan)
+      mutants = selection.fetch(:mutants)
+      sampling = selection.fetch(:sampling)
+      subjects = plan.source_units.map do |unit|
+        source_mutants = mutants.select { |mutant| File.expand_path(mutant.path) == File.expand_path(unit.fetch(:path) { unit["path"] }) }
+        {
+          "expression" => Array(unit[:constants] || unit["constants"]).first.to_s,
+          "source_digest" => digest_for(unit.fetch(:path) { unit["path"] }),
+          "mutants" => source_mutants.map { |mutant| execute_mutant(mutant, plan.test_files) }
+        }
+      end
+      MutationMatrixResult.new(
+        {
+          "version" => 1,
+          "engine" => "sixth_sense/matrix",
+          "mode" => "matrix",
+          "test_files" => plan.test_files,
+          "metadata" => {
+            "timeout_policy" => timeout_policy,
+            "sampling" => sampling,
+            "warnings" => sampling.fetch("sampled") ? ["mutation sample truncated to #{sampling.fetch("selected_count")} of #{sampling.fetch("generated_count")} generated mutants"] : []
+          },
+          "subjects" => subjects
+        }
+      )
+    end
+
+    private
+
+    # Paper basis: Koochakzadeh/Garousi 2010 (doi:10.1155/2010/932686) uses
+    # killed-mutant sets per test for redundancy; every test is run to record
+    # killed_by instead of using fail-fast mutation scoring.
+    def execute_mutant(mutant, test_files)
+      return skipped_mutant(mutant) if mutant.status == "skipped"
+
+      killed_by = []
+      executed = []
+      timed_out = []
+      with_mutation(mutant) do
+        test_cases(test_files).each do |test_case|
+          executed << test_case.fetch(:test_case).id
+          case run_test_case(test_case)
+          when :passed
+            next
+          when :timeout
+            timed_out << test_case.fetch(:test_case).id
+            killed_by << test_case.fetch(:test_case).id if timeout_policy == "killed"
+          else
+            killed_by << test_case.fetch(:test_case).id
+          end
+        end
+      end
+      {
+        "id" => mutant.id,
+        "operator" => mutant.operator,
+        "location" => { "file" => mutant.path, "line" => mutant.line },
+        "diff" => "-#{mutant.original}+#{mutant.replacement}",
+        "status" => mutant_status(killed_by, timed_out),
+        "killed_by" => killed_by,
+        "executed" => executed,
+        "timed_out" => timed_out,
+        "metadata" => { "timeout_policy" => timeout_policy }
+      }
+    end
+
+    def with_mutation(mutant)
+      original = File.read(mutant.path)
+      File.write(mutant.path, mutated_source(original, mutant))
+      yield
+    ensure
+      File.write(mutant.path, original) if original
+    end
+
+    def mutated_source(original, mutant)
+      range = mutant.metadata["range"] || mutant.metadata[:range]
+      return line_mutated_source(original, mutant) unless range
+
+      start_offset = range["start"] || range[:start]
+      end_offset = range["end"] || range[:end]
+      original.byteslice(0, start_offset) + mutant.replacement + original.byteslice(end_offset, original.bytesize - end_offset).to_s
+    end
+
+    def line_mutated_source(original, mutant)
+      lines = original.lines
+      lines[mutant.line - 1] = mutant.replacement
+      lines.join
+    end
+
+    def test_cases(test_files)
+      @test_cases_by_paths ||= {}
+      key = test_files.join(File::PATH_SEPARATOR)
+      return @test_cases_by_paths[key] if @test_cases_by_paths.key?(key)
+
+      test_files.flat_map do |path|
+        adapter, test_file = parse_test_file(path)
+        test_file.test_cases.reject { |test_case| test_case.metadata[:pending] }.map do |test_case|
+          { adapter: adapter, framework: test_file.framework, test_case: test_case }
+        end
+      end.then { |cases| @test_cases_by_paths[key] = cases }
+    end
+
+    def parse_test_file(path)
+      adapter_class = FrameworkAdapter.for_path(path)
+      raise Error, "no framework adapter accepted #{path}" unless adapter_class
+
+      adapter = adapter_class.new
+      [adapter, adapter.parse(path)]
+    end
+
+    def run_test_case(entry)
+      run_command(test_command(entry))
+    end
+
+    def test_command(entry)
+      custom_command = entry.fetch(:adapter).mutation_test_command(test_case: entry.fetch(:test_case), config: @config)
+      return custom_command if custom_command
+
+      framework = entry.fetch(:framework)
+      test_case = entry.fetch(:test_case)
+      case framework
+      when :rspec
+        [ruby, *load_path_args, "-e", rspec_runner_source, "--", "--options", File::NULL, "#{test_case.location.path}:#{test_case.location.line}"]
+      when :minitest
+        [ruby, *load_path_args, "-e", minitest_runner_source, test_case.location.path, test_case.id.split(":", 2).last]
+      when :test_unit
+        [ruby, *load_path_args, "-e", test_unit_runner_source, test_case.location.path, test_case.id.split(":", 2).last]
+      else
+        raise NotImplementedError, "mutation matrix execution is not implemented for #{framework}"
+      end
+    end
+
+    def ruby
+      RbConfig.ruby
+    end
+
+    def load_path_args
+      ["-I", @config.fetch("load_path", "lib")]
+    end
+
+    def rspec_runner_source
+      "require 'rspec/core'; require 'rspec/expectations'; exit RSpec::Core::Runner.run(ARGV)"
+    end
+
+    def minitest_runner_source
+      <<~'RUBY'
+        require File.expand_path(ARGV[0])
+        Minitest.seed ||= 0
+        klass = Minitest::Runnable.runnables.select { |candidate| candidate < Minitest::Test }.find { |candidate| candidate.runnable_methods.include?(ARGV[1]) }
+        exit 1 unless klass
+        result = klass.new(ARGV[1]).run
+        Minitest::Runnable.runnables.clear
+        exit(result.passed? ? 0 : 1)
+      RUBY
+    end
+
+    def test_unit_runner_source
+      <<~'RUBY'
+        require "test/unit"
+        require "test/unit/testresult"
+        require "test/unit/worker-context"
+        Test::Unit::AutoRunner.need_auto_run = false
+        require File.expand_path(ARGV[0])
+        klass = ObjectSpace.each_object(Class).select { |candidate| candidate < Test::Unit::TestCase }.find { |candidate| candidate.public_instance_methods.map(&:to_s).include?(ARGV[1]) }
+        exit 1 unless klass
+        result = Test::Unit::TestResult.new
+        test = klass.new(ARGV[1])
+        test.instance_variable_set(:@passed_assertions, [])
+        test.run(result) { |_event, *_args| }
+        exit(result.passed? ? 0 : 1)
+      RUBY
+    end
+
+    def run_command(command)
+      Open3.popen3(*command) do |stdin, stdout, stderr, wait_thread|
+        stdin.close
+        stdout_reader = Thread.new { stdout.read }
+        stderr_reader = Thread.new { stderr.read }
+        return command_status(wait_thread, stdout_reader, stderr_reader) if wait_thread.join(timeout_seconds)
+
+        terminate(wait_thread)
+        stdout_reader.kill
+        stderr_reader.kill
+        :timeout
+      end
+    end
+
+    def command_status(wait_thread, stdout_reader, stderr_reader)
+      stdout_reader.value
+      stderr_reader.value
+      wait_thread.value.success? ? :passed : :failed
+    end
+
+    def terminate(wait_thread)
+      Process.kill("TERM", wait_thread.pid)
+      return if wait_thread.join(0.2)
+
+      Process.kill("KILL", wait_thread.pid)
+    rescue Errno::ESRCH
+      nil
+    end
+
+    def digest_for(path)
+      "sha256:#{Digest::SHA256.file(path).hexdigest}"
+    end
+
+    def timeout_seconds
+      @config.fetch("timeout", 10).to_f
+    end
+
+    def timeout_policy
+      @config.fetch("timeout_policy", "killed").to_s
+    end
+
+    def neutral_runs
+      [@config.fetch("neutral_runs", 2).to_i, 1].max
+    end
+
+    def skipped_mutant(mutant)
+      {
+        "id" => mutant.id,
+        "operator" => mutant.operator,
+        "location" => { "file" => mutant.path, "line" => mutant.line },
+        "diff" => "-#{mutant.original}+#{mutant.replacement}",
+        "status" => "skipped",
+        "killed_by" => [],
+        "executed" => [],
+        "timed_out" => [],
+        "metadata" => mutant.metadata.merge("timeout_policy" => timeout_policy)
+      }
+    end
+
+    def mutant_status(killed_by, timed_out)
+      return "timeout" if timed_out.any?
+      return "killed" if killed_by.any?
+
+      "alive"
+    end
+  end
+end

data/lib/sixth_sense/mutation_score_cache_writer.rb

@@ -0,0 +1,75 @@
+# frozen_string_literal: true
+
+require "digest"
+
+require_relative "model"
+require_relative "mutation_cache"
+
+module SixthSense
+  class MutationScoreCacheWriter
+    def initialize(cache:)
+      @cache = cache
+    end
+
+    def write(plan, stdout:, stderr:)
+      score = parse_mutation_score([stdout, stderr].join("\n"))
+      return false unless score
+
+      write_score(plan, score)
+    end
+
+    def write_score(plan, score)
+      mutation_cache = MutationCache.new(root: @cache)
+      plan.test_files.each do |test_file|
+        mutation_cache.write(test_file_model(test_file), payload(plan, score))
+      end
+      true
+    end
+
+    private
+
+    def payload(plan, score)
+      {
+        "version" => 1,
+        "engine" => "mutant",
+        "mode" => "score",
+        "summary" => { "mutation_score" => score },
+        "metadata" => { "warnings" => ["score mode cache contains adequacy data only"] },
+        "subjects" => plan.source_units.map { |unit| subject_payload(unit) }
+      }
+    end
+
+    def subject_payload(unit)
+      {
+        "expression" => Array(unit[:constants] || unit["constants"]).first.to_s,
+        "source_digest" => source_digest(unit),
+        "mutants" => []
+      }
+    end
+
+    def parse_mutation_score(output)
+      percentage = output[/mutation\s+score:\s*([0-9]+(?:\.[0-9]+)?)%/i, 1] ||
+                   output[/score:\s*([0-9]+(?:\.[0-9]+)?)%/i, 1]
+      return percentage.to_f / 100.0 if percentage
+
+      counts = output.match(/(?<killed>\d+)\s+killed.*?(?<total>\d+)\s+(?:total|mutations?)/im)
+      return unless counts
+
+      total = counts[:total].to_i
+      return if total.zero?
+
+      counts[:killed].to_i.to_f / total
+    end
+
+    def source_digest(unit)
+      path = unit[:path] || unit["path"]
+      return nil unless path && File.file?(path)
+
+      "sha256:#{Digest::SHA256.file(path).hexdigest}"
+    end
+
+    def test_file_model(path)
+      Model::TestFile.new(path: path, framework: :rspec, test_cases: [], sut_candidates: [])
+    end
+  end
+end

data/lib/sixth_sense/rake_task.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+require "rake"
+
+require_relative "cli"
+
+module SixthSense
+  class RakeTask
+    def initialize(name = :sixth_sense)
+      desc "Run SixthSense guardrails"
+      task name do
+        exit CLI.new.run(["guard"])
+      end
+    end
+  end
+end

data/lib/sixth_sense/reporters/console.rb

@@ -0,0 +1,31 @@
+# frozen_string_literal: true
+
+module SixthSense
+  module Reporters
+    class Console
+      def render(reports)
+        reports.map { |report| render_report(report) }.join("\n\n")
+      end
+
+      private
+
+      def render_report(report)
+        lines = []
+        lines << "#{report.test_file.path} composite=#{format_score(report.composite)} level=#{report.level}"
+        report.axis_scores.each do |score|
+          lines << "  #{score.axis}: #{format_score(score.value)} confidence=#{score.confidence} measured=#{score.measured?}"
+        end
+        report.findings.first(10).each do |finding|
+          lines << "  #{finding.severity} #{finding.analyzer_id}/#{finding.rule_id} #{finding.location}: #{finding.message}"
+        end
+        remaining = report.findings.length - 10
+        lines << "  ... #{remaining} more finding(s)" if remaining.positive?
+        lines.join("\n")
+      end
+
+      def format_score(value)
+        value.nil? ? "not_measured" : format("%.2f", value)
+      end
+    end
+  end
+end

data/lib/sixth_sense/reporters/html.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module SixthSense
+  module Reporters
+    class Html
+      def render(reports)
+        <<~HTML
+          <!doctype html>
+          <html lang="en">
+          <head>
+            <meta charset="utf-8">
+            <title>SixthSense Report</title>
+            <style>
+              body { font-family: system-ui, sans-serif; margin: 2rem; color: #1f2937; }
+              table { border-collapse: collapse; width: 100%; }
+              th, td { border: 1px solid #d1d5db; padding: .5rem; text-align: left; }
+              th { background: #f3f4f6; }
+              .finding { margin: .35rem 0; }
+            </style>
+          </head>
+          <body>
+            <h1>SixthSense Report</h1>
+            #{summary_table(reports)}
+            #{findings(reports)}
+          </body>
+          </html>
+        HTML
+      end
+
+      private
+
+      def summary_table(reports)
+        rows = reports.map do |report|
+          scores = report.axis_scores.to_h { |score| [score.axis, score] }
+          "<tr><td>#{escape(report.test_file.path)}</td><td>#{score(scores[:adequacy])}</td><td>#{score(scores[:redundancy])}</td><td>#{score(scores[:quality])}</td><td>#{number(report.composite)}</td></tr>"
+        end.join("\n")
+        "<table><thead><tr><th>File</th><th>Adequacy</th><th>Redundancy</th><th>Quality</th><th>Composite</th></tr></thead><tbody>#{rows}</tbody></table>"
+      end
+
+      def findings(reports)
+        body = reports.flat_map(&:findings).map do |finding|
+          "<li class=\"finding\"><strong>#{escape(finding.severity)}</strong> #{escape(finding.analyzer_id)}/#{escape(finding.rule_id)} #{escape(finding.location)}: #{escape(finding.message)}</li>"
+        end.join("\n")
+        "<h2>Findings</h2><ul>#{body}</ul>"
+      end
+
+      def score(axis_score)
+        return "not measured" unless axis_score&.measured?
+
+        "#{number(axis_score.value)} (#{escape(axis_score.confidence)})"
+      end
+
+      def number(value)
+        value.nil? ? "not measured" : format("%.2f", value)
+      end
+
+      def escape(value)
+        value.to_s.gsub("&", "&amp;").gsub("<", "&lt;").gsub(">", "&gt;")
+      end
+    end
+  end
+end

data/lib/sixth_sense/reporters/json.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SixthSense
+  module Reporters
+    class Json
+      def render(reports)
+        JSON.pretty_generate(
+          {
+            version: SixthSense::VERSION,
+            reports: reports.map(&:to_h)
+          }
+        )
+      end
+    end
+  end
+end

data/lib/sixth_sense/reporters/markdown.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+module SixthSense
+  module Reporters
+    class Markdown
+      def render(reports)
+        lines = ["| Test file | Adequacy | Redundancy | Quality | Composite |", "|---|---:|---:|---:|---:|"]
+        reports.each do |report|
+          axes = report.axis_scores.to_h { |score| [score.axis, score] }
+          lines << [
+            report.test_file.path,
+            score_cell(axes[:adequacy]),
+            score_cell(axes[:redundancy]),
+            score_cell(axes[:quality]),
+            numeric(report.composite)
+          ].join(" | ").prepend("| ").concat(" |")
+        end
+        lines.join("\n")
+      end
+
+      private
+
+      def score_cell(score)
+        return "not measured" unless score&.measured?
+
+        "#{numeric(score.value)} (#{score.confidence})"
+      end
+
+      def numeric(value)
+        value.nil? ? "not measured" : format("%.2f", value)
+      end
+    end
+  end
+end

data/lib/sixth_sense/reporters/sarif.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+require "json"
+
+module SixthSense
+  module Reporters
+    class Sarif
+      def render(reports)
+        JSON.pretty_generate(
+          {
+            version: "2.1.0",
+            runs: [
+              {
+                tool: {
+                  driver: {
+                    name: "sixth_sense",
+                    informationUri: "https://github.com/ydah/sixth_sense",
+                    rules: rules_for(reports)
+                  }
+                },
+                results: results_for(reports)
+              }
+            ]
+          }
+        )
+      end
+
+      private
+
+      def rules_for(reports)
+        reports.flat_map(&:findings).uniq { |finding| rule_id(finding) }.map do |finding|
+          {
+            id: rule_id(finding),
+            name: finding.rule_id,
+            shortDescription: { text: finding.message }
+          }
+        end
+      end
+
+      def results_for(reports)
+        reports.flat_map(&:findings).map do |finding|
+          {
+            ruleId: rule_id(finding),
+            level: sarif_level(finding.severity),
+            message: { text: finding.message },
+            locations: [location_for(finding)]
+          }
+        end
+      end
+
+      def location_for(finding)
+        location = finding.location
+        {
+          physicalLocation: {
+            artifactLocation: { uri: location&.path.to_s },
+            region: {
+              startLine: location&.line || 1,
+              startColumn: location&.column || 1
+            }
+          }
+        }
+      end
+
+      def rule_id(finding)
+        "#{finding.analyzer_id}:#{finding.rule_id}"
+      end
+
+      def sarif_level(severity)
+        case severity
+        when :error, "error" then "error"
+        when :warning, "warning" then "warning"
+        else "note"
+        end
+      end
+    end
+  end
+end