sixth_sense 0.1.0

data/lib/sixth_sense/guardrail/baseline.rb

@@ -0,0 +1,135 @@
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+require "time"
+
+module SixthSense
+  module Guardrail
+    class Baseline
+      DEFAULT_PATH = ".sixth_sense_baseline.json"
+
+      attr_reader :path, :data
+
+      def self.load(path = DEFAULT_PATH)
+        data = File.file?(path) ? JSON.parse(File.read(path)) : empty_data
+        new(path: path, data: data)
+      end
+
+      def self.empty_data
+        {
+          "version" => 1,
+          "generated_at" => nil,
+          "level" => 0,
+          "files" => {}
+        }
+      end
+
+      def initialize(path:, data:)
+        @path = path
+        @data = data
+      end
+
+      def violations(reports, current_level:, tolerance:)
+        return [] if current_level < data.fetch("level", 0).to_i
+
+        reports.flat_map do |report|
+          baseline_file = data.fetch("files", {})[report.test_file.path]
+          next [] unless baseline_file
+          next [] if baseline_file["digest"] == digest_for(report.test_file)
+
+          report.axis_scores.filter_map do |score|
+            next unless score.measured?
+
+            baseline_value = baseline_file[score.axis.to_s]
+            next unless baseline_value
+            next unless score.value < baseline_value.to_f - tolerance
+
+            {
+              path: report.test_file.path,
+              axis: score.axis,
+              current: score.value,
+              baseline: baseline_value.to_f,
+              tolerance: tolerance
+            }
+          end
+        end
+      end
+
+      def warnings(reports, current_level:, tolerance:)
+        return [] if current_level < data.fetch("level", 0).to_i
+
+        reports.flat_map do |report|
+          baseline_file = data.fetch("files", {})[report.test_file.path]
+          next [] unless baseline_file
+          next [] unless baseline_file["digest"] == digest_for(report.test_file)
+
+          report.axis_scores.filter_map do |score|
+            next unless score.measured?
+
+            baseline_value = baseline_file[score.axis.to_s]
+            next unless baseline_value
+            next unless (score.value - baseline_value.to_f).abs > tolerance
+
+            {
+              type: :flaky_measurement,
+              path: report.test_file.path,
+              axis: score.axis,
+              current: score.value,
+              baseline: baseline_value.to_f,
+              tolerance: tolerance
+            }
+          end
+        end
+      end
+
+      def update(reports, level:, force: false)
+        return self if !force && data.fetch("level", 0).to_i > level
+
+        next_data = self.class.empty_data
+        next_data["generated_at"] = Time.now.utc.iso8601
+        next_data["level"] = level
+        reports.each do |report|
+          existing = data.fetch("files", {})[report.test_file.path] || {}
+          next_data["files"][report.test_file.path] = updated_entry(report, existing, force: force)
+        end
+        @data = next_data
+      end
+
+      def write
+        File.write(path, JSON.pretty_generate(data) + "\n")
+      end
+
+      def digest_for(test_file)
+        digest = Digest::SHA256.new
+        ([test_file.path] + test_file.sut_candidates.map(&:path)).uniq.each do |path|
+          next unless File.file?(path)
+
+          digest.update(path)
+          digest.update(File.binread(path))
+        end
+        "sha256:#{digest.hexdigest}"
+      end
+
+      private
+
+      def updated_entry(report, existing, force:)
+        entry = {
+          "digest" => digest_for(report.test_file)
+        }
+        report.axis_scores.each do |score|
+          next unless score.measured?
+
+          old_value = existing[score.axis.to_s]
+          entry[score.axis.to_s] = if old_value && !force
+                                     [old_value.to_f, score.value].max.round(2)
+                                   else
+                                     score.value.round(2)
+                                   end
+        end
+        entry
+      end
+
+    end
+  end
+end

data/lib/sixth_sense/guardrail/evaluator.rb

@@ -0,0 +1,69 @@
+# frozen_string_literal: true
+
+require_relative "baseline"
+
+module SixthSense
+  module Guardrail
+    Evaluation = Struct.new(:passed, :violations, :baseline_violations, :warnings, keyword_init: true) do
+      def passed?
+        passed
+      end
+    end
+
+    class Evaluator
+      def initialize(config:, baseline: Baseline.load)
+        @config = config
+        @baseline = baseline
+      end
+
+      def evaluate(reports, level:)
+        threshold_violations = threshold_violations(reports, level: level)
+        tolerance = @config.fetch(:guardrail, :tolerance, default: 1.0).to_f
+        baseline_violations = ratchet? ? @baseline.violations(reports, current_level: level, tolerance: tolerance) : []
+        warnings = ratchet? ? @baseline.warnings(reports, current_level: level, tolerance: tolerance) : []
+
+        Evaluation.new(
+          passed: threshold_violations.empty? && baseline_violations.empty?,
+          violations: threshold_violations,
+          baseline_violations: baseline_violations,
+          warnings: warnings
+        )
+      end
+
+      private
+
+      def threshold_violations(reports, level:)
+        require_level = @config.fetch(:guardrail, :require_level, default: 0).to_i
+        level_violation = if level < require_level
+                            [{ type: :level, current: level, required: require_level }]
+                          else
+                            []
+                          end
+
+        fail_under = @config.fetch(:guardrail, :fail_under, default: {})
+        score_violations = reports.flat_map do |report|
+          report.axis_scores.filter_map do |score|
+            threshold = fail_under[score.axis.to_s] || fail_under[score.axis]
+            next unless threshold
+            next unless !score.measured? || score.value < threshold.to_f
+
+            {
+              type: :threshold,
+              path: report.test_file.path,
+              axis: score.axis,
+              current: score.value,
+              threshold: threshold.to_f,
+              measured: score.measured?
+            }
+          end
+        end
+
+        level_violation + score_violations
+      end
+
+      def ratchet?
+        @config.fetch(:guardrail, :ratchet, default: false)
+      end
+    end
+  end
+end

data/lib/sixth_sense/model.rb

@@ -0,0 +1,264 @@
+# frozen_string_literal: true
+
+require "set"
+
+require_relative "source_location"
+
+module SixthSense
+  module Model
+    TestFile = Struct.new(
+      :path,
+      :framework,
+      :test_cases,
+      :sut_candidates,
+      :metadata,
+      keyword_init: true
+    ) do
+      def assertions
+        test_cases.flat_map(&:assertions)
+      end
+
+      def executable_loc
+        test_cases.sum(&:executable_loc)
+      end
+
+      def to_h
+        {
+          path: path,
+          framework: framework,
+          test_cases: test_cases.map(&:to_h),
+          sut_candidates: sut_candidates.map(&:to_h),
+          metadata: metadata || {}
+        }
+      end
+    end
+
+    TestCase = Struct.new(
+      :id,
+      :description,
+      :location,
+      :assertions,
+      :ast,
+      :body,
+      :metadata,
+      keyword_init: true
+    ) do
+      def executable_loc
+        body.to_s.lines.count { |line| line.match?(/\S/) }
+      end
+
+      def to_h
+        {
+          id: id,
+          description: description,
+          location: location&.to_h,
+          assertions: assertions.map(&:to_h),
+          metadata: metadata
+        }
+      end
+    end
+
+    Assertion = Struct.new(
+      :location,
+      :matcher,
+      :subject_expr,
+      :message,
+      keyword_init: true
+    ) do
+      def signature
+        [matcher.to_s, subject_expr.to_s.gsub(/\s+/, " ").strip].join(":")
+      end
+
+      def to_h
+        {
+          location: location&.to_h,
+          matcher: matcher,
+          subject_expr: subject_expr,
+          message: message
+        }
+      end
+    end
+
+    CodeUnit = Struct.new(:path, :constants, :methods, keyword_init: true) do
+      def to_h
+        {
+          path: path,
+          constants: constants,
+          methods: methods
+        }
+      end
+    end
+
+    CoverageMap = Struct.new(
+      :per_test_lines,
+      :per_test_branches,
+      :total_lines,
+      :total_branches,
+      keyword_init: true
+    ) do
+      # Paper basis: Zhu/Hall/May 1997 (doi:10.1145/267580.267590)
+      # treats executed lines as coverage requirements for adequacy criteria.
+      def line_requirements_for(test_case_id)
+        normalize_requirements(per_test_lines.fetch(test_case_id, {}))
+      end
+
+      # Paper basis: Zhu/Hall/May 1997; branch requirements provide a stronger
+      # structural adequacy signal than line requirements when available.
+      def branch_requirements_for(test_case_id)
+        normalize_requirements(per_test_branches.fetch(test_case_id, {}))
+      end
+
+      # Paper basis: Zhu/Hall/May 1997; this is the raw satisfied-line
+      # requirement ratio used by adequacy/coverage.
+      def line_coverage_ratio
+        ratio_for(per_test_lines, total_lines)
+      end
+
+      # Paper basis: Zhu/Hall/May 1997; this is the raw satisfied-branch
+      # requirement ratio preferred by adequacy/coverage.
+      def branch_coverage_ratio
+        ratio_for(per_test_branches, total_branches)
+      end
+
+      private
+
+      def ratio_for(data, totals)
+        covered_requirements = data.values.flat_map do |requirements_by_path|
+          normalize_requirements(requirements_by_path).to_a
+        end.to_set
+        covered = covered_requirements.length
+        total = totals ? normalize_requirements(totals).length : covered
+        return nil if total.zero?
+
+        covered.to_f / total
+      end
+
+      def normalize_requirements(requirements_by_path)
+        requirements_by_path.flat_map do |path, requirements|
+          requirements.map { |requirement| [path, requirement] }
+        end.to_set
+      end
+    end
+
+    CheckedCoverageMap = Struct.new(:per_test_checked_lines, :executed_lines, keyword_init: true) do
+      # Paper basis: Schuler/Zeller 2011 (doi:10.1109/ICST.2011.32)
+      # scores the fraction of executed code that is checked by an oracle.
+      def checked_ratio
+        executed = normalize_requirements(executed_lines || {})
+        return nil if executed.empty?
+
+        checked = per_test_checked_lines.values.flat_map do |requirements_by_path|
+          normalize_requirements(requirements_by_path).to_a
+        end.to_set
+        (checked & executed).length.to_f / executed.length
+      end
+
+      # Paper basis: Schuler/Zeller 2011; unchecked executed lines are reported
+      # as oracle-quality gaps rather than ordinary coverage gaps.
+      def unchecked_lines
+        checked = per_test_checked_lines.values.flat_map do |requirements_by_path|
+          normalize_requirements(requirements_by_path).to_a
+        end.to_set
+        normalize_requirements(executed_lines || {}) - checked
+      end
+
+      private
+
+      def normalize_requirements(requirements_by_path)
+        requirements_by_path.flat_map do |path, requirements|
+          requirements.map { |requirement| [path, requirement] }
+        end.to_set
+      end
+    end
+
+    Mutant = Struct.new(
+      :id,
+      :operator,
+      :location,
+      :status,
+      :diff,
+      :killing_hint,
+      :metadata,
+      keyword_init: true
+    ) do
+      # Paper basis: Jia/Harman 2011 (doi:10.1109/TSE.2010.62) counts killed
+      # mutants as behaviorally distinguished by the test suite.
+      def killed?
+        return timeout_killed? if status == :timeout || status == "timeout"
+
+        status == :killed || status == "killed" || status == :timeout
+      end
+
+      # Paper basis: Jia/Harman 2011 describes equivalent mutants as a threat;
+      # equivalent/skipped mutants are excluded from the mutation-score denominator.
+      def excluded?
+        status == :excluded || status == "excluded" || status == :equivalent || status == :skipped || status == "skipped"
+      end
+
+      def to_h
+        {
+          id: id,
+          operator: operator,
+          location: location&.to_h,
+          status: status,
+          diff: diff,
+          killing_hint: killing_hint,
+          metadata: metadata || {}
+        }
+      end
+
+      private
+
+      def timeout_killed?
+        (metadata || {}).fetch("timeout_policy", "killed") == "killed"
+      end
+    end
+
+    KillMatrix = Struct.new(:mutants, :kills, :metadata, keyword_init: true) do
+      # Paper basis: Jia/Harman 2011 defines mutation score as killed mutants
+      # over non-excluded mutants; Just et al. 2014 validates its fault relation.
+      def mutation_score
+        override = (metadata || {}).dig("summary", "mutation_score")
+        return override.to_f if override
+
+        countable = mutants.reject(&:excluded?)
+        return nil if countable.empty?
+
+        countable.count(&:killed?).to_f / countable.length
+      end
+
+      def matrix?
+        (metadata || {}).fetch("mode", "matrix") == "matrix"
+      end
+
+      def surviving_mutants
+        mutants.reject(&:excluded?).reject(&:killed?)
+      end
+
+      def timeout_ratio
+        countable = mutants.reject(&:excluded?)
+        return 0.0 if countable.empty?
+
+        countable.count { |mutant| mutant.status == :timeout || mutant.status == "timeout" }.to_f / countable.length
+      end
+
+      def sampled?
+        !!(metadata || {}).dig("sampling", "sampled")
+      end
+
+      # Paper basis: Jia/Harman 2011 highlights equivalent mutants; the ratio is
+      # surfaced to warn about excessive denominator exclusions.
+      def equivalent_ratio
+        return 0.0 if mutants.empty?
+
+        mutants.count { |mutant| mutant.status == :equivalent || mutant.status == "equivalent" }.to_f / mutants.length
+      end
+
+      # Paper basis: Koochakzadeh/Garousi 2010 uses killed-mutant sets as
+      # requirements for mutation-based redundancy detection.
+      def kill_requirements_for(test_case_id)
+        kills.fetch(test_case_id, Set.new).to_set
+      end
+    end
+  end
+end

data/lib/sixth_sense/mutation_cache.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "set"
+
+require_relative "model"
+require_relative "source_location"
+
+module SixthSense
+  class MutationCache
+    DEFAULT_ROOT = ".sixth_sense_cache/kill_matrix"
+
+    def initialize(root: DEFAULT_ROOT)
+      @root = root
+    end
+
+    def load(test_file)
+      path = path_for(test_file.path)
+      return nil unless File.file?(path)
+
+      build_matrix(JSON.parse(File.read(path)), test_file)
+    end
+
+    def write(test_file, payload)
+      FileUtils.mkdir_p(@root)
+      File.write(path_for(test_file.path), JSON.pretty_generate(payload) + "\n")
+    end
+
+    def path_for(test_file_path)
+      File.join(@root, "#{test_file_path.gsub(%r{[^A-Za-z0-9]+}, "_").gsub(/\A_+|_+\z/, "")}.json")
+    end
+
+    private
+
+    def build_matrix(payload, test_file)
+      equivalent_ids = equivalent_annotations(test_file)
+      mutants = []
+      kills = Hash.new { |hash, key| hash[key] = Set.new }
+
+      payload.fetch("subjects", []).each do |subject|
+        subject.fetch("mutants", []).each do |entry|
+          status = equivalent_ids.include?(entry.fetch("id")) ? :equivalent : entry.fetch("status", "alive").to_sym
+          mutant = Model::Mutant.new(
+            id: entry.fetch("id"),
+            operator: entry["operator"],
+            location: location_for(entry["location"]),
+            status: status,
+            diff: entry["diff"],
+            killing_hint: entry["killing_hint"],
+            metadata: metadata_for(entry, payload)
+          )
+          mutants << mutant
+          Array(entry["killed_by"]).each { |test_case_id| kills[test_case_id] << mutant.id }
+        end
+      end
+
+      Model::KillMatrix.new(mutants: mutants, kills: kills, metadata: matrix_metadata(payload))
+    end
+
+    def location_for(payload)
+      return nil unless payload
+
+      SourceLocation.new(path: payload["file"], line: payload["line"], column: payload["column"] || 1)
+    end
+
+    def equivalent_annotations(test_file)
+      test_file.sut_candidates.flat_map do |unit|
+        next [] unless File.file?(unit.path)
+
+        File.read(unit.path).scan(/#\s*sixth_sense:equivalent\s+([A-Za-z0-9_.:-]+)/).flatten
+      end.to_set
+    end
+
+    def metadata_for(entry, payload)
+      (entry["metadata"] || {}).merge(
+        "timeout_policy" => payload.dig("metadata", "timeout_policy") || "killed",
+        "timed_out" => Array(entry["timed_out"])
+      )
+    end
+
+    def matrix_metadata(payload)
+      {
+        "engine" => payload["engine"],
+        "mode" => payload["mode"] || "matrix",
+        "summary" => payload["summary"] || {},
+        "sampling" => payload.dig("metadata", "sampling") || {},
+        "timeout_policy" => payload.dig("metadata", "timeout_policy") || "killed",
+        "warnings" => payload.dig("metadata", "warnings") || []
+      }
+    end
+  end
+end

data/lib/sixth_sense/mutation_engine.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module SixthSense
+  Availability = Struct.new(:available, :reason, keyword_init: true) do
+    def available?
+      available
+    end
+  end
+
+  MutationPlan = Struct.new(:engine, :mode, :subjects, :test_files, :source_units, keyword_init: true) do
+    def to_h
+      {
+        engine: engine,
+        mode: mode,
+        subjects: subjects,
+        test_files: test_files,
+        source_units: source_units
+      }
+    end
+  end
+
+  MutationRunResult = Struct.new(:plan, :command, :success, :stdout, :stderr, :status, keyword_init: true) do
+    def success?
+      success
+    end
+
+    def to_h
+      {
+        plan: plan.to_h,
+        command: command,
+        success: success,
+        stdout: stdout,
+        stderr: stderr,
+        status: status
+      }
+    end
+  end
+
+  class MutationEngine
+    def plan(sut_units:, test_files:)
+      raise NotImplementedError
+    end
+
+    def run(_plan, cache:)
+      raise NotImplementedError
+    end
+
+    def availability
+      raise NotImplementedError
+    end
+  end
+end