mutato 0.1.0

data/lib/mutato/report.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+module Mutato
+  class Report
+    def initialize(outcomes, mode, console)
+      @outcomes = outcomes
+      @mode = mode
+      @console = console
+    end
+
+    def show(started, flaky)
+      @console.say(
+        "\nresults: #{counts}",
+        format("%<wall>.0fs wall%<mean>s", wall: Clock.since(started), mean:)
+      )
+      @console.say("#{flaky.size} flaky tests fail with and without a mutation") if flaky.any?
+      @console.say(*@mode.summary(@outcomes))
+    end
+
+    private
+
+    def counts
+      groups = @outcomes.group_by { |outcome| outcome.fetch(:outcome) }
+      groups.map { |name, list| "#{name} #{list.size}" }
+        .join(", ")
+    end
+
+    def mean
+      seconds = @outcomes.map { |outcome| outcome.fetch(:seconds) }
+        .select(&:positive?)
+      return "" if seconds.empty?
+
+      format(", mean %<seconds>.2fs per executed mutant", seconds: seconds.sum / seconds.size)
+    end
+  end
+end

data/lib/mutato/rspec_adapter/progress.rb

@@ -0,0 +1,20 @@
+# frozen_string_literal: true
+
+module Mutato
+  module RSpecAdapter
+    class Progress
+      attr_reader :ran, :current
+
+      def initialize
+        @ran = []
+        @current = nil
+      end
+
+      def example_started(notification)
+        @current = notification.example.id
+        @ran << @current
+      end
+    end
+    private_constant :Progress
+  end
+end

data/lib/mutato/rspec_adapter/result.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+module Mutato
+  module RSpecAdapter
+    Result = Data.define(:status, :outside, :failing, :ran) do
+      def outcome
+        return { outcome: :caught, detail: "error outside examples", ran: } if outside
+        return { outcome: :missed, ran: } if status.zero?
+
+        { outcome: :caught, failing:, ran: }
+      end
+    end
+    public_constant :Result
+  end
+end

data/lib/mutato/rspec_adapter.rb

@@ -0,0 +1,129 @@
+# frozen_string_literal: true
+
+require "coverage"
+require_relative "rspec_adapter/progress"
+require_relative "rspec_adapter/result"
+
+module Mutato
+  # RSpec is one per process, so this is a module, not a class.
+  module RSpecAdapter
+    module_function
+
+    def boot(args)
+      # Cached iseqs cannot carry coverage.
+      ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"] ||= "1"
+      start_coverage
+      # Here, not at load: the project's bundle provides rspec.
+      require "rspec/core"
+      setup(RSpec::Core::Runner.new(RSpec::Core::ConfigurationOptions.new(args)))
+    end
+
+    # Before spec_helper, so files load instrumented; lines only, as method counters slow calls.
+    def start_coverage
+      Coverage.start(lines: true) unless Coverage.running?
+    end
+
+    def setup(runner)
+      @runner = runner
+      runner.setup($stderr, $stdout)
+      RSpec.configuration.backtrace_exclusion_patterns << %r{/mutato/}
+    end
+
+    def locations
+      RSpec.world.all_examples.to_h { |example| [example.id, example.metadata.fetch(:file_path)] }
+    end
+
+    # A SyntaxError sets rspec_is_quitting, any other load error wants_to_quit.
+    def load_failed?
+      world = RSpec.world
+      world.rspec_is_quitting || world.wants_to_quit
+    end
+
+    def current
+      @progress&.current
+    end
+
+    def stop
+      RSpec.world.wants_to_quit = true
+    end
+
+    # Child only.
+    def baseline(prefixes)
+      listener = CoverageListener.new(prefixes)
+      RSpec.configuration.reporter.register_listener(listener, *CoverageListener::EVENTS)
+      quiet(fail_fast: false)
+      Coverage.result(stop: false, clear: true)
+      Measure.new(status: run_all, **listener.tally.to_h)
+    end
+
+    # Child only. Stops at the first failure.
+    def run(ids)
+      prepare(ids)
+      Result.new(
+        status: run_all,
+        outside: RSpec.world.non_example_failure,
+        failing: first_failure,
+        ran: @progress.ran
+      )
+    end
+
+    def prepare(ids)
+      quiet(fail_fast: 1)
+      order(ids.each_with_index.to_h)
+      filter(ids.to_set)
+      watch
+    end
+
+    def watch
+      @progress = Progress.new
+      RSpec.configuration.reporter.register_listener(@progress, :example_started)
+    end
+
+    def run_all
+      @runner.run_specs(RSpec.world.ordered_example_groups)
+    end
+
+    def first_failure
+      RSpec.configuration.reporter.failed_examples.first&.id
+    end
+
+    # No status file, a fresh clock, and fail_fast forced: --fail-fast in .rspec is forced too.
+    def quiet(fail_fast:)
+      configuration = RSpec.configuration
+      configuration.example_status_persistence_file_path = nil
+      configuration.start_time = Time.now
+      configuration.force(fail_fast:)
+    end
+
+    def order(rank)
+      RSpec.configuration.register_ordering(:global) { |items| ordered(items, rank) }
+    end
+
+    def ordered(items, rank)
+      items.sort_by { |item| rank_of(item, rank) }
+    end
+
+    def rank_of(item, rank)
+      return rank.fetch(item.id, Float::INFINITY) if item.is_a?(RSpec::Core::Example)
+
+      ranks = item.descendant_filtered_examples.map { |example| rank.fetch(example.id, Float::INFINITY) }
+      ranks.min || Float::INFINITY
+    end
+
+    def filter(wanted)
+      filtered_examples.each_value { |examples| keep(examples, wanted) }
+    end
+
+    # Filtered examples are computed on demand, group by group.
+    def filtered_examples
+      world = RSpec.world
+      filtered = world.filtered_examples
+      world.example_groups.flat_map(&:descendants).each { |group| filtered[group] }
+      filtered
+    end
+
+    def keep(examples, wanted)
+      examples.select! { |example| wanted.include?(example.id) }
+    end
+  end
+end

data/lib/mutato/run.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Mutato
+  # Without a mutation, it confirms a kill.
+  class Run
+    def initialize(suite, mutation)
+      @suite = suite
+      @mutation = mutation
+      @installed = nil
+    end
+
+    def go(ids, budget, log:)
+      Mutato.config.run_hooks(:before_mutant)
+      result = Child.run(timeout: budget + [budget, 5.0].max, log:) { in_child(ids, budget) }
+      Output.cap(log)
+      result
+    end
+
+    private
+
+    def in_child(ids, budget)
+      install
+      checked(Watchdog.new(@suite.adapter, budget).run(ids))
+    rescue Installer::Unviable => error
+      { outcome: :unviable, detail: error.message }
+    end
+
+    def install
+      return unless @mutation
+
+      @installed = Installer.install(
+        @mutation.subject,
+        @mutation.mutated_source(File.read(@mutation.file))
+      )
+    end
+
+    def checked(verdict)
+      return verdict if verdict.fetch(:outcome) == :timeout || !@mutation
+      return verdict if Installer.current(@mutation.subject) == @installed
+
+      { outcome: :overwritten, detail: "the tests redefined the method (code reloader?)" }
+    end
+  end
+end

data/lib/mutato/selection.rb

@@ -0,0 +1,100 @@
+# frozen_string_literal: true
+
+module Mutato
+  class Selection
+    attr_reader :paths, :options
+
+    def self.by_id(todo, ids)
+      index = todo.to_h { |mutation| [mutation.id, mutation] }
+      ids.map { |id| index.fetch(id) }
+    end
+
+    # Under the current directory, the first directory of the path; elsewhere, the path itself.
+    def self.root_of(path)
+      here = Dir.pwd
+      inside = path.delete_prefix("#{here}/")
+      inside == path ? path : File.join(here, inside.split("/").first)
+    end
+
+    def initialize(paths, options)
+      @paths = paths
+      @options = options
+      @generators = nil
+      @diff = nil
+    end
+
+    def files
+      @paths.flat_map { |path| File.directory?(path) ? Dir["#{path}/**/*.rb"].sort : [path] }
+    end
+
+    def generators
+      @generators ||= files.map { |file| Generator.read(file) }
+    end
+
+    def subjects
+      generators.flat_map(&:subjects)
+    end
+
+    def mutations
+      limit = @options.limit
+      limit ? narrowed.first(limit) : narrowed
+    end
+
+    def controls
+      subjects.reject(&:skip).map { |subject| Mutation.control(subject) }
+    end
+
+    def unknown(todo)
+      @options.only.to_a - todo.map(&:id)
+    end
+
+    # The same random subset every run.
+    def chosen(todo)
+      only = @options.only
+      sample = @options.sample
+      todo = Selection.by_id(todo, only) if only
+      sample ? todo.sample(sample, random: Random.new(1)) : todo
+    end
+
+    def prefixes
+      roots = @paths.map { |path| Selection.root_of(File.expand_path(path)) }
+      roots.uniq
+    end
+
+    def summary(todo)
+      skipped, kept = subjects.partition(&:skip)
+      [
+        "#{todo.size} mutants over #{files.size} files, #{kept.size} methods",
+        *without_methods,
+        *dropped,
+        *skipped.map(&:skipped_line)
+      ]
+    end
+
+    private
+
+    def narrowed
+      all = generators.flat_map(&:mutations).select { |mutation| @options.genre?(mutation.genre) }
+      @options.diff ? all.select { |mutation| diff.touches?(mutation.file, mutation.span) } : all
+    end
+
+    def diff
+      @diff ||= Diff.read(@options.diff)
+    end
+
+    def without_methods
+      empty = generators.select { |generator| generator.subjects.empty? }
+      return if empty.none?
+
+      ["#{empty.size} files without methods (code in blocks or at class level is not mutated)"]
+    end
+
+    def dropped
+      dropped = generators.flat_map(&:dropped)
+      return if dropped.empty?
+
+      examples = dropped.first(3).map(&:label).join("; ")
+      ["#{dropped.size} mutants dropped as unparsable, e.g. #{examples}"]
+    end
+  end
+end

data/lib/mutato/session.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Mutato
+  class Session
+    def initialize(mode, selection, console)
+      @mode = mode
+      @selection = selection
+      @console = console
+    end
+
+    def call
+      todo = chosen
+      @console.say(*@selection.summary(todo))
+      nothing if todo.empty?
+      try(todo, Boot.new(@selection, @console).suite)
+    end
+
+    private
+
+    def chosen
+      todo = @mode.todo(@selection)
+      unknown = @selection.unknown(todo)
+      @console.die("no such mutant: #{unknown.join(", ")}") if unknown.any?
+      @selection.chosen(todo)
+    end
+
+    def nothing
+      @console.die(
+        "nothing to mutate under #{@selection.paths.join(" ")}",
+        code: @selection.options.diff ? 0 : 3
+      )
+    end
+
+    def try(todo, suite)
+      started = Clock.now
+      campaign = Campaign.new(suite, @console)
+      outcomes = campaign.try(todo) { |partial| conclude(partial, campaign, started) }
+      finish(conclude(outcomes, campaign, started))
+    end
+
+    def conclude(outcomes, campaign, started)
+      flaky = campaign.experience.flaky
+      Report.new(outcomes, @mode, @console).show(started, flaky)
+      campaign.suite.output.write(outcomes, flaky)
+      outcomes
+    end
+
+    def finish(outcomes)
+      @mode.annotate(outcomes, @console, @selection.options.format)
+      @mode.exit_code(outcomes)
+    end
+  end
+end

data/lib/mutato/skips.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require "prism"
+
+module Mutato
+  module Skips
+    ARID_DEFS = %i[to_s inspect].freeze
+    STUB = /NotImplementedError|NoMethodError|abstract|overrid|subclass/i
+    JUMPS = [Prism::ReturnNode, Prism::NextNode].freeze
+    private_constant :ARID_DEFS, :STUB, :JUMPS
+
+    module_function
+
+    def reason(subject, scope)
+      node = subject.node
+      return "explicit receiver" if Nodes.foreign_receiver?(node) || scope.foreign
+      return "defined inside a block" if scope.in_block
+      return "arid method name" if !subject.singleton && ARID_DEFS.include?(subject.name)
+
+      helper_reason(node) || Mutato.config.skip_reason(subject.qualified)
+    end
+
+    def helper_reason(node)
+      return "logging helper" if Arid.helper_name?(node.name)
+
+      statements = Nodes.statements_of(node.body)
+      body_reason(statements) if statements
+    end
+
+    def body_reason(statements)
+      return "template stub" if (statements in [only]) && stub?(only)
+
+      "logging only" if logging_only?(statements)
+    end
+
+    # `return unless x` in a method that otherwise only logs decides whether to log.
+    def logging_only?(statements)
+      arid = statements.map { |statement| Arid.arid_node?(statement) }
+      arid.any? && statements.zip(arid).all? { |statement, logs| logs || guard?(statement) }
+    end
+
+    def stub?(statement)
+      statement.is_a?(Prism::CallNode) && statement.name == :raise && statement.slice.match?(STUB)
+    end
+
+    def guard?(statement)
+      conditional = Nodes.one_of?(statement, Nodes::CONDITIONALS)
+      return false unless conditional && Nodes.else_less?(statement)
+
+      only = Nodes.single_statement(statement)
+      Nodes.one_of?(only, JUMPS) && only.arguments.nil?
+    end
+  end
+end

data/lib/mutato/soft_timeout.rb

@@ -0,0 +1,8 @@
+# frozen_string_literal: true
+
+module Mutato
+  # Not a StandardError: a worker loop's `rescue => e` would swallow it.
+  class SoftTimeout < Exception
+  end
+  public_constant :SoftTimeout
+end

data/lib/mutato/subject.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Mutato
+  Subject = Data.define(:file, :name, :chain, :node, :singleton, :receiver, :skip, :block_lines) do
+    # The constant that owns the method, `App::Calc`; empty at the top level.
+    def owner_name
+      receiver || chain.filter_map(&:last).join("::")
+    end
+
+    def qualified
+      "#{owner_name}#{singleton ? "." : "#"}#{name}"
+    end
+
+    alias_method :to_s, :qualified
+
+    def skipped_line
+      "skipped #{qualified}: #{skip}  #{file}:#{node.location.start_line}"
+    end
+  end
+  public_constant :Subject
+end

data/lib/mutato/subjects/scope.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+require "prism"
+
+module Mutato
+  class Subjects
+    # foreign: inside `class << obj` for an obj other than self.
+    Scope = Data.define(:chain, :in_block, :foreign) do
+      def self.top
+        new(chain: [], in_block: false, foreign: false)
+      end
+
+      def into(node)
+        case node
+        when Prism::ModuleNode, Prism::ClassNode then nest(
+          KINDS.fetch(node.class),
+          node.constant_path.slice
+        )
+        when Prism::SingletonClassNode then singleton(node)
+        else Nodes.one_of?(node, Nodes::BLOCKS) ? with(in_block: true) : self
+        end
+      end
+
+      def singleton(node)
+        node.expression.is_a?(Prism::SelfNode) ? nest(:sclass, nil) : with(foreign: true)
+      end
+
+      def nest(kind, name)
+        with(chain: chain + [[kind, name]])
+      end
+
+      def singleton_class?
+        chain.last == [:sclass, nil]
+      end
+
+      # Whether `name` is the enclosing class, as `More` in `def More.x` inside `class More`.
+      def enclosing?(name)
+        _, enclosing = chain.last
+        enclosing.to_s.split("::").last == name
+      end
+    end
+    private_constant :Scope
+  end
+end

data/lib/mutato/subjects.rb

@@ -0,0 +1,67 @@
+# frozen_string_literal: true
+
+require "prism"
+require_relative "subjects/scope"
+
+module Mutato
+  class Subjects
+    KINDS = { Prism::ModuleNode => :module, Prism::ClassNode => :class }.freeze
+    DEFINERS = {
+      %w[Data define] => :class,
+      %w[Struct new] => :class,
+      %w[Class new] => :class,
+      %w[Module new] => :module
+    }.freeze
+    private_constant :KINDS, :DEFINERS
+
+    # `Point = Struct.new(:x) do ... end`: the block is the body of the class the constant names.
+    def self.constant_body(node)
+      call = node.value if node.is_a?(Prism::ConstantWriteNode)
+      block = call.block if call.is_a?(Prism::CallNode)
+      return unless block.is_a?(Prism::BlockNode)
+
+      kind = DEFINERS[[call.receiver&.slice, call.name.to_s]]
+      [kind, node.name.to_s, block.body] if kind
+    end
+
+    def self.find(file, tree)
+      finder = new(file)
+      finder.walk(tree, Scope.top)
+      finder.found
+    end
+
+    attr_reader :found
+
+    def initialize(file)
+      @file = file
+      @found = []
+    end
+
+    def walk(node, scope)
+      return @found << define(node, scope) if node.is_a?(Prism::DefNode)
+
+      kind, name, body = Subjects.constant_body(node)
+      return walk(body, scope.nest(kind, name)) if body
+
+      node.compact_child_nodes.each { |child| walk(child, scope.into(node)) }
+    end
+
+    private
+
+    # `def More.x` inside `class More` is `def self.x`.
+    def define(node, scope)
+      receiver = Nodes.constant_receiver(node)
+      subject = Subject.new(
+        file: @file,
+        name: node.name,
+        chain: scope.chain,
+        node:,
+        receiver: (receiver unless scope.enclosing?(receiver)),
+        singleton: scope.singleton_class? || Nodes.singleton_receiver?(node),
+        skip: nil,
+        block_lines: []
+      )
+      subject.with(skip: Skips.reason(subject, scope), block_lines: Nodes.block_lines(node))
+    end
+  end
+end

data/lib/mutato/suite.rb

@@ -0,0 +1,6 @@
+# frozen_string_literal: true
+
+module Mutato
+  Suite = Data.define(:adapter, :baseline, :options, :output)
+  public_constant :Suite
+end

data/lib/mutato/trial.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+module Mutato
+  class Trial
+    KILLS = %i[caught crashed].freeze
+    private_constant :KILLS
+
+    # A child that died before any example ran names no tests: the whole sequence stands.
+    def self.sequence_of(result, remaining)
+      ran = result.fetch(:ran, [])
+      ran.empty? ? remaining : ran
+    end
+
+    def self.blame(result, sequence)
+      return if result.fetch(:outcome) == :missed
+
+      result[:failing] || result[:running] || sequence.last
+    end
+
+    def initialize(mutation, suite, experience)
+      @mutation = mutation
+      @suite = suite
+      @experience = experience
+    end
+
+    def judge
+      covering = Covering.new(@mutation, @suite.baseline)
+      ids = covering.ids
+      record = @mutation.record(ids)
+      return record.merge(outcome: covering.absence) if ids.empty?
+
+      record.merge(tried(ids))
+    end
+
+    private
+
+    def tried(ids)
+      return { outcome: :equivalent } if Equivalence.equivalent?(@mutation)
+
+      timed(ids)
+    end
+
+    def timed(ids)
+      budget = @suite.baseline.budget(ids, @suite.options.timeout_min)
+      started = Clock.now
+      result = verdict(ids - flaky, budget)
+      result.merge(seconds: Clock.since(started), budget:, flaky: ids & flaky)
+    end
+
+    def flaky
+      @experience.flaky.to_a
+    end
+
+    # A kill counts once the same tests pass unmutated; a test failing both ways is flaky.
+    def verdict(remaining, budget)
+      return { outcome: :unjudged } if remaining.empty?
+
+      result = Run.new(@suite, @mutation).go(remaining, budget, log:)
+      return result unless KILLS.include?(result.fetch(:outcome))
+
+      confirm(result, remaining, budget)
+    end
+
+    def confirm(result, remaining, budget)
+      sequence = Trial.sequence_of(result, remaining)
+      culprit = culprit_of(sequence, budget)
+      return result.merge(outcome: :caught) unless culprit
+
+      @experience.flaky << culprit
+      verdict(remaining - flaky, budget)
+    end
+
+    # The test the unmutated run failed or hung on; cached, as many kills share a sequence.
+    def culprit_of(sequence, budget)
+      culprits = @experience.culprits
+      return culprits[sequence] if culprits.key?(sequence)
+
+      result = unmutated(sequence, budget)
+      culprits[sequence] = Trial.blame(result, sequence)
+    end
+
+    def unmutated(sequence, budget)
+      result = Run.new(@suite, nil).go(sequence, budget, log: "#{log}.confirm")
+      return result unless result.fetch(:outcome) == :timeout
+
+      Run.new(@suite, nil).go(sequence, budget * 2, log: "#{log}.confirm")
+    end
+
+    def log
+      @suite.output.log_path(@mutation.id)
+    end
+  end
+end