sixth_sense 0.1.0

data/lib/sixth_sense/runners/minitest_checked_coverage_probe.rb

@@ -0,0 +1,42 @@
+# frozen_string_literal: true
+
+require_relative "checked_coverage_trace"
+
+module SixthSense
+  module Runners
+    class MinitestCheckedCoverageProbe
+      class << self
+        def run(test_path, test_name)
+          status = CheckedCoverageTrace.capture do
+            require File.expand_path(test_path)
+            passed = run_test(test_name)
+            clear_minitest_runnables
+            passed ? 0 : 1
+          end
+          exit(status)
+        rescue LoadError => error
+          warn error.message
+          exit(127)
+        end
+
+        private
+
+        def run_test(test_name)
+          Minitest.seed ||= 0
+          klass = minitest_classes.find { |candidate| candidate.runnable_methods.include?(test_name) }
+          return false unless klass
+
+          klass.new(test_name).run.passed?
+        end
+
+        def minitest_classes
+          Minitest::Runnable.runnables.select { |candidate| candidate < Minitest::Test }
+        end
+
+        def clear_minitest_runnables
+          Minitest::Runnable.runnables.clear
+        end
+      end
+    end
+  end
+end

data/lib/sixth_sense/runners/minitest_coverage_probe.rb

@@ -0,0 +1,49 @@
+# frozen_string_literal: true
+
+require "coverage"
+require "json"
+
+require_relative "coverage_snapshot"
+
+module SixthSense
+  module Runners
+    class MinitestCoverageProbe
+      class << self
+        def run(test_path, test_name)
+          Coverage.start(lines: true, branches: true)
+          require File.expand_path(test_path)
+          passed = run_test(test_name)
+          clear_minitest_runnables
+          result = Coverage.result
+          write_snapshot(result)
+          exit(passed ? 0 : 1)
+        rescue LoadError => error
+          warn error.message
+          exit(127)
+        end
+
+        private
+
+        def run_test(test_name)
+          Minitest.seed ||= 0
+          klass = minitest_classes.find { |candidate| candidate.runnable_methods.include?(test_name) }
+          return false unless klass
+
+          klass.new(test_name).run.passed?
+        end
+
+        def minitest_classes
+          Minitest::Runnable.runnables.select { |candidate| candidate < Minitest::Test }
+        end
+
+        def clear_minitest_runnables
+          Minitest::Runnable.runnables.clear
+        end
+
+        def write_snapshot(result)
+          File.write(ENV.fetch("SIXTH_SENSE_COVERAGE_OUTPUT"), JSON.generate(CoverageSnapshot.snapshot_for(result)))
+        end
+      end
+    end
+  end
+end

data/lib/sixth_sense/runners/rspec_checked_coverage_probe.rb

@@ -0,0 +1,26 @@
+# frozen_string_literal: true
+
+require "stringio"
+
+require_relative "checked_coverage_trace"
+
+module SixthSense
+  module Runners
+    class RSpecCheckedCoverageProbe
+      class << self
+        def run(spec_path, line)
+          require "rspec/core"
+          require "rspec/expectations"
+
+          status = CheckedCoverageTrace.capture do
+            RSpec::Core::Runner.run(["--options", File::NULL, "#{spec_path}:#{line}"], StringIO.new, StringIO.new)
+          end
+          exit(status)
+        rescue LoadError => error
+          warn error.message
+          exit(127)
+        end
+      end
+    end
+  end
+end

data/lib/sixth_sense/runners/rspec_coverage_probe.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require "coverage"
+require "json"
+require "stringio"
+
+module SixthSense
+  module Runners
+    class RSpecCoverageProbe
+      class << self
+        def run(spec_path, line)
+          require "rspec/core"
+          require "rspec/expectations"
+
+          Coverage.start(lines: true, branches: true)
+          status = RSpec::Core::Runner.run(["--options", File::NULL, "#{spec_path}:#{line}"], StringIO.new, StringIO.new)
+          result = Coverage.result
+          write_snapshot(result)
+          exit(status)
+        rescue LoadError => error
+          warn error.message
+          exit(127)
+        end
+
+        private
+
+        def write_snapshot(result)
+          output = ENV.fetch("SIXTH_SENSE_COVERAGE_OUTPUT")
+          File.write(output, JSON.generate(snapshot_for(result)))
+        end
+
+        def snapshot_for(result)
+          lines = {}
+          executable_lines = {}
+          branches = {}
+          executable_branches = {}
+          result.each do |path, payload|
+            if payload.is_a?(Hash)
+              lines[path] = covered_lines(payload[:lines] || payload["lines"])
+              executable_lines[path] = executable_lines(payload[:lines] || payload["lines"])
+              branches[path] = covered_branches(payload[:branches] || payload["branches"])
+              executable_branches[path] = executable_branches(payload[:branches] || payload["branches"])
+            else
+              lines[path] = covered_lines(payload)
+              executable_lines[path] = executable_lines(payload)
+            end
+          end
+          {
+            lines: lines,
+            executable_lines: executable_lines,
+            branches: branches,
+            executable_branches: executable_branches
+          }
+        end
+
+        def covered_lines(line_counts)
+          Array(line_counts).each_with_index.filter_map do |count, index|
+            index + 1 if count.to_i.positive?
+          end
+        end
+
+        def executable_lines(line_counts)
+          Array(line_counts).each_with_index.filter_map do |count, index|
+            index + 1 unless count.nil?
+          end
+        end
+
+        def covered_branches(branches)
+          return [] unless branches.respond_to?(:each)
+
+          branches.each_with_index.filter_map do |branch, index|
+            "#{branch_key(branch)}:#{index}" if branch_executed?(branch)
+          end
+        end
+
+        def executable_branches(branches)
+          return [] unless branches.respond_to?(:each)
+
+          branches.each_with_index.map do |branch, index|
+            "#{branch_key(branch)}:#{index}"
+          end
+        end
+
+        def branch_key(branch)
+          branch.respond_to?(:first) ? branch.first.inspect : branch.inspect
+        end
+
+        def branch_executed?(branch)
+          values = branch.respond_to?(:last) ? branch.last : branch
+          return values.to_i.positive? if values.is_a?(Integer)
+          return values.values.any? { |count| count.to_i.positive? } if values.respond_to?(:values)
+
+          false
+        end
+      end
+    end
+  end
+end

data/lib/sixth_sense/runners/test_unit_checked_coverage_probe.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "checked_coverage_trace"
+
+module SixthSense
+  module Runners
+    class TestUnitCheckedCoverageProbe
+      class << self
+        def run(test_path, test_name)
+          status = CheckedCoverageTrace.capture do
+            require "test/unit"
+            require "test/unit/testresult"
+            require "test/unit/worker-context"
+            Test::Unit::AutoRunner.need_auto_run = false
+            require File.expand_path(test_path)
+            run_test(test_case_classes, test_name) ? 0 : 1
+          end
+          exit(status)
+        rescue LoadError => error
+          warn error.message
+          exit(127)
+        end
+
+        private
+
+        def run_test(classes, test_name)
+          klass = classes.find { |candidate| candidate.public_instance_methods.map(&:to_s).include?(test_name) }
+          return false unless klass
+
+          result = Test::Unit::TestResult.new
+          test = klass.new(test_name)
+          test.instance_variable_set(:@passed_assertions, [])
+          test.run(result) { |_event, *_args| }
+          result.passed?
+        end
+
+        def test_case_classes
+          ObjectSpace.each_object(Class).select { |klass| klass < Test::Unit::TestCase }
+        end
+      end
+    end
+  end
+end

data/lib/sixth_sense/runners/test_unit_coverage_probe.rb

@@ -0,0 +1,51 @@
+# frozen_string_literal: true
+
+require "coverage"
+require "json"
+
+require_relative "coverage_snapshot"
+
+module SixthSense
+  module Runners
+    class TestUnitCoverageProbe
+      class << self
+        def run(test_path, test_name)
+          Coverage.start(lines: true, branches: true)
+          require "test/unit"
+          require "test/unit/testresult"
+          require "test/unit/worker-context"
+          Test::Unit::AutoRunner.need_auto_run = false
+          require File.expand_path(test_path)
+          passed = run_test(test_case_classes, test_name)
+          result = Coverage.result
+          write_snapshot(result)
+          exit(passed ? 0 : 1)
+        rescue LoadError => error
+          warn error.message
+          exit(127)
+        end
+
+        private
+
+        def run_test(classes, test_name)
+          klass = classes.find { |candidate| candidate.public_instance_methods.map(&:to_s).include?(test_name) }
+          return false unless klass
+
+          result = Test::Unit::TestResult.new
+          test = klass.new(test_name)
+          test.instance_variable_set(:@passed_assertions, [])
+          test.run(result) { |_event, *_args| }
+          result.passed?
+        end
+
+        def test_case_classes
+          ObjectSpace.each_object(Class).select { |klass| klass < Test::Unit::TestCase }
+        end
+
+        def write_snapshot(result)
+          File.write(ENV.fetch("SIXTH_SENSE_COVERAGE_OUTPUT"), JSON.generate(CoverageSnapshot.snapshot_for(result)))
+        end
+      end
+    end
+  end
+end

data/lib/sixth_sense/scoring/aggregator.rb

@@ -0,0 +1,117 @@
+# frozen_string_literal: true
+
+require_relative "../result"
+
+module SixthSense
+  module Scoring
+    class Aggregator
+      AXES = %i[adequacy redundancy quality].freeze
+
+      def initialize(level:)
+        @level = level
+      end
+
+      def analyze(test_file, context)
+        analyzer_results = applicable_analyzers.map { |analyzer_class| analyzer_class.new.analyze(test_file, context) }
+        axis_scores = AXES.map { |axis| axis_score(axis, analyzer_results) }
+        measured_values = axis_scores.select(&:measured?).map(&:value)
+
+        Result::FileReport.new(
+          test_file: test_file,
+          axis_scores: axis_scores,
+          composite: measured_values.empty? ? nil : measured_values.min.round(2),
+          level: @level,
+          warnings: context.warnings
+        )
+      end
+
+      private
+
+      def applicable_analyzers
+        SixthSense.analyzers.select do |analyzer_class|
+          analyzer_class.level && analyzer_class.level <= @level
+        end
+      end
+
+      def axis_score(axis, analyzer_results)
+        results = analyzer_results.select { |item| item.axis == axis }
+        findings = results.flat_map(&:findings)
+        measured = results.select(&:measured?)
+
+        case axis
+        when :adequacy
+          adequacy_score(measured, findings)
+        when :redundancy
+          prioritized_score(axis, measured, findings, %w[redundancy/mutation_based redundancy/coverage_based redundancy/clone])
+        when :quality
+          minimum_score(axis, measured, findings)
+        else
+          unmeasured_axis(axis, findings)
+        end
+      end
+
+      # Paper basis: Just et al. 2014 and Papadakis et al. 2018 motivate the
+      # mutation-heavy weighting; Inozemtseva/Holmes 2014 and Schuler/Zeller
+      # 2011 keep coverage and checked coverage as lower-weight signals.
+      def adequacy_score(results, findings)
+        mutation = results.find { |item| item.analyzer_id == "adequacy/mutation" }
+        coverage = results.find { |item| item.analyzer_id == "adequacy/coverage" }
+        checked = results.find { |item| item.analyzer_id == "adequacy/checked_coverage" }
+        weighted = []
+        weighted << [mutation.score, 0.60] if mutation
+        weighted << [coverage.score, 0.25] if coverage
+        weighted << [checked.score, 0.15] if checked
+        return unmeasured_axis(:adequacy, findings) if weighted.empty?
+
+        total_weight = weighted.sum(&:last)
+        value = weighted.sum { |score, weight| score * (weight / total_weight) }
+        confidence = mutation ? :high : :medium
+        Result::AxisScore.new(
+          axis: :adequacy,
+          value: value.round(2),
+          confidence: confidence,
+          findings: findings,
+          measured: true
+        )
+      end
+
+      # Paper basis: Koochakzadeh/Garousi 2010 makes mutation requirements the
+      # more precise redundancy signal, with coverage minimization as fallback.
+      def prioritized_score(axis, results, findings, analyzer_ids)
+        selected = analyzer_ids.filter_map { |id| results.find { |item| item.analyzer_id == id } }.first
+        return unmeasured_axis(axis, findings) unless selected
+
+        Result::AxisScore.new(
+          axis: axis,
+          value: selected.score.round(2),
+          confidence: selected.confidence,
+          findings: selected.findings,
+          measured: true
+        )
+      end
+
+      def minimum_score(axis, results, findings)
+        return unmeasured_axis(axis, findings) if results.empty?
+
+        selected = results.min_by(&:score)
+        Result::AxisScore.new(
+          axis: axis,
+          value: selected.score.round(2),
+          confidence: selected.confidence,
+          findings: findings,
+          measured: true
+        )
+      end
+
+      def unmeasured_axis(axis, findings)
+        Result::AxisScore.new(
+          axis: axis,
+          value: nil,
+          confidence: :low,
+          findings: findings,
+          measured: false
+        )
+      end
+    end
+  end
+end

data/lib/sixth_sense/source_location.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module SixthSense
+  SourceLocation = Struct.new(:path, :line, :column, keyword_init: true) do
+    def to_h
+      {
+        path: path,
+        line: line,
+        column: column
+      }
+    end
+
+    def to_s
+      return path.to_s unless line
+
+      [path, line, column].compact.join(":")
+    end
+  end
+end

data/lib/sixth_sense/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module SixthSense
+  VERSION = "0.1.0"
+end

data/lib/sixth_sense.rb

@@ -0,0 +1,74 @@
+# frozen_string_literal: true
+
+require_relative "sixth_sense/version"
+
+module SixthSense
+  class Error < StandardError; end
+
+  class Plugin
+    def adapter(name, adapter_class)
+      FrameworkAdapter.register(name, adapter_class)
+    end
+
+    def analyzer(analyzer_class)
+      SixthSense.register_analyzer(analyzer_class)
+    end
+  end
+
+  class << self
+    def analyzers
+      @analyzers ||= []
+    end
+
+    def register_analyzer(analyzer_class)
+      return if analyzers.include?(analyzer_class)
+
+      analyzers << analyzer_class
+    end
+
+    def plugin
+      registry = Plugin.new
+      yield registry if block_given?
+      registry
+    end
+  end
+end
+
+require_relative "sixth_sense/source_location"
+require_relative "sixth_sense/model"
+require_relative "sixth_sense/result"
+require_relative "sixth_sense/framework_adapter"
+require_relative "sixth_sense/analyzer"
+require_relative "sixth_sense/analysis_context"
+require_relative "sixth_sense/adapters/rspec"
+require_relative "sixth_sense/adapters/test_unit"
+require_relative "sixth_sense/adapters/minitest"
+require_relative "sixth_sense/analyzers/quality_test_smells"
+require_relative "sixth_sense/analyzers/quality_assertion_density"
+require_relative "sixth_sense/analyzers/quality_flakiness"
+require_relative "sixth_sense/analyzers/adequacy_coverage"
+require_relative "sixth_sense/analyzers/adequacy_mutation"
+require_relative "sixth_sense/analyzers/adequacy_checked_coverage"
+require_relative "sixth_sense/analyzers/redundancy_coverage"
+require_relative "sixth_sense/analyzers/redundancy_mutation"
+require_relative "sixth_sense/analyzers/redundancy_clone"
+require_relative "sixth_sense/scoring/aggregator"
+require_relative "sixth_sense/reporters/console"
+require_relative "sixth_sense/reporters/json"
+require_relative "sixth_sense/reporters/markdown"
+require_relative "sixth_sense/reporters/sarif"
+require_relative "sixth_sense/reporters/html"
+require_relative "sixth_sense/config"
+require_relative "sixth_sense/analysis_runner"
+require_relative "sixth_sense/mutation_engine"
+require_relative "sixth_sense/mutation_cache"
+require_relative "sixth_sense/mutation_matrix_mutant_generator"
+require_relative "sixth_sense/mutation_matrix_producer"
+require_relative "sixth_sense/mutation_score_cache_writer"
+require_relative "sixth_sense/runners/coverage_runner"
+require_relative "sixth_sense/runners/minitest_coverage_probe"
+require_relative "sixth_sense/runners/test_unit_coverage_probe"
+require_relative "sixth_sense/runners/minitest_checked_coverage_probe"
+require_relative "sixth_sense/runners/test_unit_checked_coverage_probe"
+require_relative "sixth_sense/runners/checked_coverage_runner"
+require_relative "sixth_sense/runners/checked_coverage_estimator"

metadata

@@ -0,0 +1,113 @@
+--- !ruby/object:Gem::Specification
+name: sixth_sense
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Yudai Takada
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: prism
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.24'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0.24'
+description: Evaluates adequacy, redundancy, and maintainability of Ruby test suites
+  using coverage, checked coverage, mutation data, and test-smell analysis.
+email:
+- t.yudai92@gmail.com
+executables:
+- sixth_sense
+extensions: []
+extra_rdoc_files: []
+files:
+- LICENSE.txt
+- README.md
+- exe/sixth_sense
+- lib/sixth_sense.rb
+- lib/sixth_sense/adapters/minitest.rb
+- lib/sixth_sense/adapters/rspec.rb
+- lib/sixth_sense/adapters/test_unit.rb
+- lib/sixth_sense/analysis_context.rb
+- lib/sixth_sense/analysis_runner.rb
+- lib/sixth_sense/analyzer.rb
+- lib/sixth_sense/analyzers/adequacy_checked_coverage.rb
+- lib/sixth_sense/analyzers/adequacy_coverage.rb
+- lib/sixth_sense/analyzers/adequacy_mutation.rb
+- lib/sixth_sense/analyzers/quality_assertion_density.rb
+- lib/sixth_sense/analyzers/quality_flakiness.rb
+- lib/sixth_sense/analyzers/quality_test_smells.rb
+- lib/sixth_sense/analyzers/redundancy_clone.rb
+- lib/sixth_sense/analyzers/redundancy_coverage.rb
+- lib/sixth_sense/analyzers/redundancy_mutation.rb
+- lib/sixth_sense/analyzers/redundancy_requirement.rb
+- lib/sixth_sense/changed_files.rb
+- lib/sixth_sense/cli.rb
+- lib/sixth_sense/config.rb
+- lib/sixth_sense/engines/mutant.rb
+- lib/sixth_sense/framework_adapter.rb
+- lib/sixth_sense/guardrail/baseline.rb
+- lib/sixth_sense/guardrail/evaluator.rb
+- lib/sixth_sense/model.rb
+- lib/sixth_sense/mutation_cache.rb
+- lib/sixth_sense/mutation_engine.rb
+- lib/sixth_sense/mutation_matrix_mutant_generator.rb
+- lib/sixth_sense/mutation_matrix_producer.rb
+- lib/sixth_sense/mutation_score_cache_writer.rb
+- lib/sixth_sense/rake_task.rb
+- lib/sixth_sense/reporters/console.rb
+- lib/sixth_sense/reporters/html.rb
+- lib/sixth_sense/reporters/json.rb
+- lib/sixth_sense/reporters/markdown.rb
+- lib/sixth_sense/reporters/sarif.rb
+- lib/sixth_sense/result.rb
+- lib/sixth_sense/runners/checked_coverage_estimator.rb
+- lib/sixth_sense/runners/checked_coverage_runner.rb
+- lib/sixth_sense/runners/checked_coverage_trace.rb
+- lib/sixth_sense/runners/coverage_runner.rb
+- lib/sixth_sense/runners/coverage_snapshot.rb
+- lib/sixth_sense/runners/minitest_checked_coverage_probe.rb
+- lib/sixth_sense/runners/minitest_coverage_probe.rb
+- lib/sixth_sense/runners/rspec_checked_coverage_probe.rb
+- lib/sixth_sense/runners/rspec_coverage_probe.rb
+- lib/sixth_sense/runners/test_unit_checked_coverage_probe.rb
+- lib/sixth_sense/runners/test_unit_coverage_probe.rb
+- lib/sixth_sense/scoring/aggregator.rb
+- lib/sixth_sense/source_location.rb
+- lib/sixth_sense/version.rb
+homepage: https://github.com/ydah/sixth_sense
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/ydah/sixth_sense
+  source_code_uri: https://github.com/ydah/sixth_sense/tree/main
+  rubygems_mfa_required: 'true'
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: 3.2.0
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.6
+specification_version: 4
+summary: Evidence-based test suite adequacy analysis for Ruby test files.
+test_files: []