sixth_sense 0.1.0

data/lib/sixth_sense/result.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+module SixthSense
+  module Result
+    Reference = Struct.new(:authors, :title, :venue, :year, :doi, keyword_init: true) do
+      def to_h
+        {
+          authors: authors,
+          title: title,
+          venue: venue,
+          year: year,
+          doi: doi
+        }
+      end
+    end
+
+    Finding = Struct.new(
+      :analyzer_id,
+      :rule_id,
+      :severity,
+      :location,
+      :message,
+      :references,
+      :suggestion,
+      keyword_init: true
+    ) do
+      def to_h
+        {
+          analyzer_id: analyzer_id,
+          rule_id: rule_id,
+          severity: severity,
+          location: location&.to_h,
+          message: message,
+          references: references.map(&:to_h),
+          suggestion: suggestion
+        }
+      end
+    end
+
+    AxisScore = Struct.new(
+      :axis,
+      :value,
+      :confidence,
+      :findings,
+      :measured,
+      keyword_init: true
+    ) do
+      def measured?
+        measured != false && !value.nil?
+      end
+
+      def to_h
+        {
+          axis: axis,
+          value: value,
+          confidence: confidence,
+          measured: measured?,
+          findings: findings.map(&:to_h)
+        }
+      end
+    end
+
+    FileReport = Struct.new(
+      :test_file,
+      :axis_scores,
+      :composite,
+      :level,
+      :warnings,
+      keyword_init: true
+    ) do
+      def findings
+        axis_scores.flat_map(&:findings)
+      end
+
+      def to_h
+        {
+          test_file: test_file.to_h,
+          axis_scores: axis_scores.to_h { |score| [score.axis, score.to_h] },
+          composite: composite,
+          level: level,
+          warnings: warnings
+        }
+      end
+    end
+  end
+end

data/lib/sixth_sense/runners/checked_coverage_estimator.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+require "set"
+
+require_relative "../model"
+
+module SixthSense
+  module Runners
+    class CheckedCoverageEstimator
+      def estimate(test_file, coverage_map)
+        checked = {}
+        executed = Hash.new { |hash, key| hash[key] = Set.new }
+
+        test_file.test_cases.each do |test_case|
+          covered_by_path = group_requirements(coverage_map.line_requirements_for(test_case.id))
+          sut_paths = test_file.sut_candidates.map { |unit| File.expand_path(unit.path) }.to_set
+          covered_by_path.each do |path, lines|
+            executed[path].merge(lines) if sut_paths.include?(File.expand_path(path))
+          end
+          checked[test_case.id] = checked_lines_for(test_file, test_case, covered_by_path)
+        end
+
+        Model::CheckedCoverageMap.new(
+          per_test_checked_lines: checked,
+          executed_lines: executed.transform_values(&:to_a)
+        )
+      end
+
+      private
+
+      def checked_lines_for(test_file, test_case, covered_by_path)
+        tokens = assertion_tokens(test_case)
+        return {} if tokens.empty?
+
+        test_file.sut_candidates.each_with_object({}) do |unit, checked|
+          covered_path = covered_by_path.keys.find { |path| File.expand_path(path) == File.expand_path(unit.path) }
+          next unless covered_path
+          next unless token_matches_unit?(tokens, unit)
+
+          checked[covered_path] = covered_by_path.fetch(covered_path).to_a
+        end
+      end
+
+      def group_requirements(requirements)
+        requirements.each_with_object(Hash.new { |hash, key| hash[key] = Set.new }) do |(path, line), grouped|
+          grouped[path] << line
+        end
+      end
+
+      def assertion_tokens(test_case)
+        test_case.assertions.flat_map do |assertion|
+          assertion.subject_expr.to_s.scan(/[A-Za-z_]\w*[!?=]?/)
+        end.to_set
+      end
+
+      def token_matches_unit?(tokens, unit)
+        unit.constants.any? { |constant| tokens.include?(constant.split("::").last) } ||
+          unit.methods.any? { |method_name| tokens.include?(method_name.delete_suffix("=")) || tokens.include?(method_name) }
+      end
+    end
+  end
+end

data/lib/sixth_sense/runners/checked_coverage_runner.rb

@@ -0,0 +1,130 @@
+# frozen_string_literal: true
+
+require "json"
+require "open3"
+require "rbconfig"
+require "tempfile"
+
+require_relative "../framework_adapter"
+require_relative "../model"
+
+module SixthSense
+  module Runners
+    class CheckedCoverageRunner
+      def initialize(config: {})
+        @config = config
+      end
+
+      def run(test_files:)
+        test_files.to_h do |test_file|
+          [test_file.path, run_file(test_file)]
+        end
+      end
+
+      private
+
+      def run_file(test_file)
+        return fallback_empty unless checked_coverage_supported?(test_file)
+
+        per_test_checked = {}
+        executed = Hash.new { |hash, key| hash[key] = [] }
+        test_file.test_cases.each do |test_case|
+          next if test_case.metadata[:pending]
+
+          snapshot = process_snapshot(test_file, test_case)
+          per_test_checked[test_case.id] = snapshot.fetch("checked_lines", {})
+          snapshot.fetch("executed_lines", {}).each do |path, lines|
+            executed[path] |= lines
+          end
+        end
+
+        Model::CheckedCoverageMap.new(
+          per_test_checked_lines: per_test_checked,
+          executed_lines: executed
+        )
+      end
+
+      def fallback_empty
+        Model::CheckedCoverageMap.new(per_test_checked_lines: {}, executed_lines: {})
+      end
+
+      def process_snapshot(test_file, test_case)
+        Tempfile.create(["sixth_sense_checked", ".json"]) do |file|
+          file.close
+          env = {
+            "SIXTH_SENSE_CHECKED_OUTPUT" => file.path,
+            "SIXTH_SENSE_SUT_PATHS" => test_file.sut_candidates.map { |unit| File.expand_path(unit.path) }.join(File::PATH_SEPARATOR),
+            "SIXTH_SENSE_CHECKED_WINDOW" => checked_window.to_s
+          }
+          command = checked_coverage_command(test_file, test_case)
+          _stdout, stderr, status = Open3.capture3(env, *command)
+          raise Error, "checked coverage run failed for #{test_case.id}: #{stderr}" unless status.success?
+
+          JSON.parse(File.read(file.path))
+        end
+      end
+
+      def checked_coverage_command(test_file, test_case)
+        custom_command = adapter_for(test_file)&.checked_coverage_test_command(test_case: test_case, config: @config)
+        return custom_command if custom_command
+
+        case test_file.framework
+        when :rspec
+          [
+            RbConfig.ruby,
+            *load_paths.flat_map { |path| ["-I", path] },
+            "-rsixth_sense/runners/rspec_checked_coverage_probe",
+            "-e",
+            "SixthSense::Runners::RSpecCheckedCoverageProbe.run(ARGV[0], ARGV[1])",
+            test_case.location.path,
+            test_case.location.line.to_s
+          ]
+        when :minitest
+          [
+            RbConfig.ruby,
+            *load_paths.flat_map { |path| ["-I", path] },
+            "-rsixth_sense/runners/minitest_checked_coverage_probe",
+            "-e",
+            "SixthSense::Runners::MinitestCheckedCoverageProbe.run(ARGV[0], ARGV[1])",
+            test_case.location.path,
+            test_case.id.split(":", 2).last
+          ]
+        when :test_unit
+          [
+            RbConfig.ruby,
+            *load_paths.flat_map { |path| ["-I", path] },
+            "-rsixth_sense/runners/test_unit_checked_coverage_probe",
+            "-e",
+            "SixthSense::Runners::TestUnitCheckedCoverageProbe.run(ARGV[0], ARGV[1])",
+            test_case.location.path,
+            test_case.id.split(":", 2).last
+          ]
+        else
+          raise NotImplementedError, "checked coverage is not implemented for #{test_file.framework}"
+        end
+      end
+
+      def checked_coverage_supported?(test_file)
+        return true if %i[rspec minitest test_unit].include?(test_file.framework)
+
+        adapter = adapter_for(test_file)
+        adapter && adapter.method(:checked_coverage_test_command).owner != FrameworkAdapter
+      end
+
+      def load_paths
+        configured = Array(@config.fetch("load_path", "lib"))
+        (configured + [File.expand_path("../..", __dir__)]).uniq
+      end
+
+      def checked_window
+        return 0 if @config.fetch("checked_mode", "windowed").to_s == "strict"
+
+        [@config.fetch("checked_window", 20).to_i, 0].max
+      end
+
+      def adapter_for(test_file)
+        FrameworkAdapter.for_path(test_file.path)&.new
+      end
+    end
+  end
+end

data/lib/sixth_sense/runners/checked_coverage_trace.rb

@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+
+require "json"
+require "set"
+
+module SixthSense
+  module Runners
+    module CheckedCoverageTrace
+      module_function
+
+      # Paper basis: Schuler/Zeller 2011 (doi:10.1109/ICST.2011.32).
+      # This captures a TracePoint approximation of checked coverage rather
+      # than their full VM-level dynamic backward slicing.
+      def capture
+        executed = Hash.new { |hash, key| hash[key] = Set.new }
+        checked = Hash.new { |hash, key| hash[key] = Set.new }
+        trace = tracepoint(executed, checked)
+        trace.enable
+        yield
+      ensure
+        trace.disable if trace
+        write_snapshot(executed, checked) if executed && checked
+      end
+
+      # Paper basis: Schuler/Zeller 2011; assertion calls are treated as
+      # oracle boundaries, and SUT lines observed inside that boundary are checked.
+      def tracepoint(executed, checked)
+        TracePoint.new(:line, :call, :c_call, :return, :c_return) do |event|
+          case event.event
+          when :line
+            next unless sut_path?(event.path)
+
+            executed[event.path] << event.lineno
+            remember_line(event.path, event.lineno)
+            checked[event.path] << event.lineno if assertion_stack.positive?
+          when :call, :c_call
+            next unless assertion_method?(event.method_id)
+
+            mark_recent_lines(checked)
+            self.assertion_stack += 1
+          when :return, :c_return
+            next unless assertion_method?(event.method_id) && assertion_stack.positive?
+
+            self.assertion_stack -= 1
+            clear_recent_lines
+          end
+        end
+      end
+
+      def assertion_method?(method_id)
+        name = method_id.to_s
+        return true if %w[expect should should_not flunk].include?(name)
+        return true if name.start_with?("assert", "refute", "must_", "wont_")
+
+        false
+      end
+
+      def sut_path?(path)
+        sut_paths.include?(File.expand_path(path))
+      end
+
+      def sut_paths
+        @sut_paths ||= ENV.fetch("SIXTH_SENSE_SUT_PATHS", "").split(File::PATH_SEPARATOR).map { |path| File.expand_path(path) }.to_set
+      end
+
+      def assertion_stack
+        Thread.current[:sixth_sense_assertion_stack].to_i
+      end
+
+      def assertion_stack=(value)
+        Thread.current[:sixth_sense_assertion_stack] = value
+      end
+
+      def remember_line(path, line)
+        recent_lines << [path, line]
+        recent_lines.shift while recent_lines.length > recent_window
+      end
+
+      def mark_recent_lines(checked)
+        recent_lines.each do |path, line|
+          checked[path] << line
+        end
+      end
+
+      def clear_recent_lines
+        Thread.current[:sixth_sense_recent_sut_lines] = []
+      end
+
+      def recent_lines
+        Thread.current[:sixth_sense_recent_sut_lines] ||= []
+      end
+
+      def recent_window
+        ENV.fetch("SIXTH_SENSE_CHECKED_WINDOW", "20").to_i
+      end
+
+      def write_snapshot(executed, checked)
+        File.write(
+          ENV.fetch("SIXTH_SENSE_CHECKED_OUTPUT"),
+          JSON.generate(
+            {
+              executed_lines: executed.transform_values(&:to_a),
+              checked_lines: checked.transform_values(&:to_a)
+            }
+          )
+        )
+      end
+    end
+  end
+end

data/lib/sixth_sense/runners/coverage_runner.rb

@@ -0,0 +1,220 @@
+# frozen_string_literal: true
+
+require "coverage"
+require "json"
+require "open3"
+require "rbconfig"
+require "set"
+require "stringio"
+require "tempfile"
+
+require_relative "../framework_adapter"
+require_relative "../mutation_engine"
+require_relative "../model"
+require_relative "coverage_snapshot"
+
+module SixthSense
+  module Runners
+    class CoverageRunner
+      def initialize(config: {})
+        @config = config
+      end
+
+      def availability
+        return Availability.new(available: false, reason: "Ruby Coverage module is unavailable") unless defined?(Coverage)
+        return Availability.new(available: false, reason: "Coverage.start is unavailable") unless Coverage.respond_to?(:start)
+
+        Availability.new(available: true, reason: "Ruby Coverage module is available")
+      end
+
+      def run(test_files:, isolation: :process)
+        available = availability
+        unless available.available?
+          raise Error, "coverage runner unavailable: #{available.reason}"
+        end
+
+        test_files.to_h do |test_file|
+          [test_file.path, run_file(test_file, isolation: isolation)]
+        end
+      end
+
+      private
+
+      def run_file(test_file, isolation:)
+        per_test_lines = {}
+        per_test_branches = {}
+        all_lines = Hash.new { |hash, key| hash[key] = Set.new }
+        all_branches = Hash.new { |hash, key| hash[key] = Set.new }
+
+        test_file.test_cases.each do |test_case|
+          next if test_case.metadata[:pending]
+
+          snapshot = isolation == :none ? in_process_snapshot(test_file, test_case) : process_snapshot(test_file, test_case)
+          snapshot = filter_snapshot(snapshot, test_file)
+          per_test_lines[test_case.id] = snapshot.fetch("lines", {})
+          per_test_branches[test_case.id] = snapshot.fetch("branches", {})
+          merge_requirements(all_lines, per_test_lines[test_case.id])
+          merge_requirements(all_branches, per_test_branches[test_case.id])
+          merge_requirements(all_lines, snapshot.fetch("executable_lines", {}))
+          merge_requirements(all_branches, snapshot.fetch("executable_branches", {}))
+        end
+
+        Model::CoverageMap.new(
+          per_test_lines: deep_sets(per_test_lines),
+          per_test_branches: deep_sets(per_test_branches),
+          total_lines: all_lines.transform_values(&:to_a),
+          total_branches: all_branches.transform_values(&:to_a)
+        )
+      end
+
+      def empty_coverage_map
+        Model::CoverageMap.new(
+          per_test_lines: {},
+          per_test_branches: {},
+          total_lines: {},
+          total_branches: {}
+        )
+      end
+
+      def process_snapshot(test_file, test_case)
+        Tempfile.create(["sixth_sense_coverage", ".json"]) do |file|
+          file.close
+          command = coverage_command(test_file, test_case)
+          env = { "SIXTH_SENSE_COVERAGE_OUTPUT" => file.path }
+          _stdout, stderr, status = Open3.capture3(env, *command)
+          raise Error, "coverage run failed for #{test_case.id}: #{stderr}" unless status.success?
+
+          JSON.parse(File.read(file.path))
+        end
+      end
+
+      def coverage_command(test_file, test_case)
+        custom_command = adapter_for(test_file)&.coverage_test_command(test_case: test_case, config: @config)
+        return custom_command if custom_command
+
+        case test_file.framework
+        when :rspec
+          [
+            RbConfig.ruby,
+            *load_paths.flat_map { |path| ["-I", path] },
+            "-rsixth_sense/runners/rspec_coverage_probe",
+            "-e",
+            "SixthSense::Runners::RSpecCoverageProbe.run(ARGV[0], ARGV[1])",
+            test_case.location.path,
+            test_case.location.line.to_s
+          ]
+        when :minitest
+          [
+            RbConfig.ruby,
+            *load_paths.flat_map { |path| ["-I", path] },
+            "-rsixth_sense/runners/minitest_coverage_probe",
+            "-e",
+            "SixthSense::Runners::MinitestCoverageProbe.run(ARGV[0], ARGV[1])",
+            test_case.location.path,
+            test_case.id.split(":", 2).last
+          ]
+        when :test_unit
+          [
+            RbConfig.ruby,
+            *load_paths.flat_map { |path| ["-I", path] },
+            "-rsixth_sense/runners/test_unit_coverage_probe",
+            "-e",
+            "SixthSense::Runners::TestUnitCoverageProbe.run(ARGV[0], ARGV[1])",
+            test_case.location.path,
+            test_case.id.split(":", 2).last
+          ]
+        else
+          raise NotImplementedError, "coverage is not implemented for #{test_file.framework}"
+        end
+      end
+
+      def in_process_snapshot(test_file, test_case)
+        load_paths.each { |path| $LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path) }
+        Coverage.start(lines: true, branches: true)
+        passed = run_in_process(test_file, test_case)
+        result = Coverage.result
+        raise Error, "in-process coverage run failed for #{test_case.id}" unless passed
+
+        CoverageSnapshot.snapshot_for(result)
+      end
+
+      def run_in_process(test_file, test_case)
+        case test_file.framework
+        when :rspec then run_rspec_in_process(test_case)
+        when :minitest then run_minitest_in_process(test_case)
+        when :test_unit then run_test_unit_in_process(test_case)
+        else
+          raise NotImplementedError, "coverage is not implemented for #{test_file.framework}"
+        end
+      end
+
+      def run_rspec_in_process(test_case)
+        require "rspec/core"
+        require "rspec/expectations"
+
+        RSpec::Core::Runner.run(["--options", File::NULL, "#{test_case.location.path}:#{test_case.location.line}"], StringIO.new, StringIO.new).zero?
+      end
+
+      def run_minitest_in_process(test_case)
+        require File.expand_path(test_case.location.path)
+        Minitest.run(["--name", "/\\A#{Regexp.escape(test_case.id.split(":", 2).last)}\\z/"])
+      end
+
+      def run_test_unit_in_process(test_case)
+        require "test/unit"
+        require "test/unit/testresult"
+        require "test/unit/worker-context"
+        Test::Unit::AutoRunner.need_auto_run = false
+        require File.expand_path(test_case.location.path)
+        klass = test_unit_class_for(test_case.id.split(":", 2).last)
+        return false unless klass
+
+        result = Test::Unit::TestResult.new
+        test = klass.new(test_case.id.split(":", 2).last)
+        test.instance_variable_set(:@passed_assertions, [])
+        test.run(result) { |_event, *_args| }
+        result.passed?
+      end
+
+      def test_unit_classes
+        ObjectSpace.each_object(Class).select { |klass| klass < Test::Unit::TestCase }
+      end
+
+      def test_unit_class_for(test_name)
+        test_unit_classes.find { |candidate| candidate.public_instance_methods.map(&:to_s).include?(test_name) }
+      end
+
+      def filter_snapshot(snapshot, test_file)
+        allowed = test_file.sut_candidates.map { |unit| File.expand_path(unit.path) }.to_set
+        return snapshot if allowed.empty?
+
+        snapshot.transform_values do |requirements|
+          next requirements unless requirements.is_a?(Hash)
+
+          requirements.select { |path, _items| allowed.include?(File.expand_path(path)) }
+        end
+      end
+
+      def load_paths
+        configured = Array(@config.fetch("load_path", "lib"))
+        (configured + [File.expand_path("../..", __dir__)]).uniq
+      end
+
+      def adapter_for(test_file)
+        FrameworkAdapter.for_path(test_file.path)&.new
+      end
+
+      def merge_requirements(target, requirements)
+        requirements.each do |path, items|
+          target[path].merge(items)
+        end
+      end
+
+      def deep_sets(hash)
+        hash.transform_values do |requirements|
+          requirements.transform_values { |items| items.to_set }
+        end
+      end
+    end
+  end
+end

data/lib/sixth_sense/runners/coverage_snapshot.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+module SixthSense
+  module Runners
+    module CoverageSnapshot
+      module_function
+
+      def snapshot_for(result)
+        lines = {}
+        executable_lines = {}
+        branches = {}
+        executable_branches = {}
+        result.each do |path, payload|
+          line_counts = payload.is_a?(Hash) ? payload[:lines] || payload["lines"] : payload
+          branch_counts = payload[:branches] || payload["branches"] if payload.is_a?(Hash)
+          lines[path] = covered_lines(line_counts)
+          executable_lines[path] = executable_lines(line_counts)
+          branches[path] = covered_branches(branch_counts)
+          executable_branches[path] = executable_branches(branch_counts)
+        end
+        {
+          "lines" => lines,
+          "executable_lines" => executable_lines,
+          "branches" => branches,
+          "executable_branches" => executable_branches
+        }
+      end
+
+      def covered_lines(line_counts)
+        Array(line_counts).each_with_index.filter_map { |count, index| index + 1 if count.to_i.positive? }
+      end
+
+      def executable_lines(line_counts)
+        Array(line_counts).each_with_index.filter_map { |count, index| index + 1 unless count.nil? }
+      end
+
+      def covered_branches(branches)
+        return [] unless branches.respond_to?(:each)
+
+        branches.each_with_index.filter_map do |branch, index|
+          "#{branch_key(branch)}:#{index}" if branch_executed?(branch)
+        end
+      end
+
+      def executable_branches(branches)
+        return [] unless branches.respond_to?(:each)
+
+        branches.each_with_index.map { |branch, index| "#{branch_key(branch)}:#{index}" }
+      end
+
+      def branch_key(branch)
+        branch.respond_to?(:first) ? branch.first.inspect : branch.inspect
+      end
+
+      def branch_executed?(branch)
+        values = branch.respond_to?(:last) ? branch.last : branch
+        return values.to_i.positive? if values.is_a?(Integer)
+        return values.values.any? { |count| count.to_i.positive? } if values.respond_to?(:values)
+
+        false
+      end
+    end
+  end
+end