kimera 0.1.0

data/lib/kimera/execution/composite_isolation.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+require_relative "isolation"
+
+class Kimera::Execution::CompositeIsolation < Kimera::Execution::Isolation
+  def initialize(strategies)
+    super()
+    @given = strategies
+  end
+
+  def around(&block)
+    strategies.reverse.reduce(block) do |inner, strategy|
+      -> { strategy.around(&inner) }
+    end.call
+  end
+
+  def reset!
+    strategies.each(&:reset!)
+  end
+
+  private
+
+  def strategies = @_strategies ||= Array(@given)
+end

data/lib/kimera/execution/harness.rb

@@ -0,0 +1,150 @@
+# frozen_string_literal: true
+
+require "english"
+require "json"
+require_relative "../error"
+require_relative "../results/result"
+require_relative "../results/run_report"
+require_relative "../runtime"
+require_relative "../self_protection"
+require_relative "baseline_pass"
+require_relative "boot"
+require_relative "child_process"
+require_relative "isolation"
+require_relative "null_progress"
+require_relative "parallel_test_databases"
+require_relative "pool_driver"
+require_relative "reload"
+require_relative "rig"
+require_relative "schedule"
+require_relative "schemata"
+require_relative "verdicts"
+require_relative "shift"
+require_relative "worker_pool"
+
+class Kimera::Execution::Harness
+  include Kimera::Execution::ChildProcess
+
+  Context = Data.define(:registry, :adapter, :options)
+  State = Struct.new(:skipped, :irrelevant, :isolation)
+
+  class << self
+    def new(registry:, adapter:, **options)
+      super(Context.new(registry, adapter, options), State.new({}, [], nil))
+    end
+  end
+
+  def initialize(context, state)
+    @context = context
+    @state = state
+  end
+
+  def warm!(test_files)
+    overlay!(test_files).tap do
+      measure!
+      notice
+    end
+  end
+
+  def without_coverage!(test_files)
+    overlay!(test_files).tap do
+      verify!
+    end
+  end
+
+  def overlay!(test_files)
+    boot.suite(test_files)
+    loader = Kimera::Execution::Schemata.new(registry, root: root, errors: errors)
+    loader.overlay!.tap { complete(loader) }
+  end
+
+  def load! = boot.load!
+
+  def isolate! = (state.isolation = boot.isolate(isolation))
+
+  def run(ids: mutants, label: "mutants")
+    schedule.run(ids, coverage: coverage, label: label)
+  end
+
+  def coverage = options[:coverage]
+
+  def skipped = state.skipped
+
+  private
+
+  attr_reader :context, :state
+
+  def registry = context.registry
+  def adapter = context.adapter
+  def options = context.options
+
+  def root = options.fetch(:source_root, ".")
+
+  def errors = options.fetch(:errio, $stderr)
+
+  def isolation = state.isolation ||= options.fetch(:isolation) { Kimera::Execution::Isolation.new }
+
+  def soft = options.fetch(:soft_timeout, 5.0)
+
+  def hard = @_hard ||= options.fetch(:hard_timeout) { soft ? (soft * 3) + 1 : 30.0 }
+
+  def jobs = @_jobs ||= [Integer(options.fetch(:jobs, 1)), 1].max
+
+  def progress = options.fetch(:progress, Kimera::Execution::NullProgress)
+
+  def driver = rig.driver
+
+  def spawner = rig.spawner
+
+  def schedule = rig.schedule
+
+  def rig
+    @_rig ||= Kimera::Execution::Rig.new(
+      registry: registry, adapter: adapter, isolation: isolation, root: root,
+      soft: soft, hard: hard, leak: options.fetch(:leak_every, 10), jobs: jobs,
+      coverage: coverage, spawner: options[:spawner], progress: progress
+    )
+  end
+
+  def boot
+    @_boot ||= Kimera::Execution::Boot.new(adapter: adapter, isolate: options[:isolate_db], errors: errors)
+  end
+
+  def complete(loader)
+    state.skipped = loader.skipped
+    isolate!
+  end
+
+  def mutants = registry.each.map { |mutant, _path| mutant.id }
+
+  def measure!
+    pass = baseline
+    measured = jobs > 1 ? parallel(pass) : pass.measure!
+    options[:coverage] = measured.coverage
+    driver.coverage = coverage
+    state.irrelevant = measured.irrelevant
+  end
+
+  def parallel(pass)
+    pass.parallel! do |resolve:, lost:|
+      driver.drive(adapter.test_ids, driver.method(:channel), resolve: resolve, lost: lost)
+    end
+  end
+
+  def verify! = baseline.check!
+
+  def baseline
+    Kimera::Execution::BaselinePass.new(adapter: adapter, registry: registry, progress: progress)
+  end
+
+  def notice
+    count = state.irrelevant.size
+    return if count.zero?
+    errors.puts(message(count))
+  end
+
+  def message(count)
+    "kimera: #{count} failing test(s) cover no in-scope mutant and were " \
+      "excluded from the baseline (they gate nothing this run)"
+  end
+end

data/lib/kimera/execution/isolated.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "tmpdir"
+require_relative "../results/result"
+require_relative "../results/run_report"
+require_relative "../synthesis/overlay"
+require_relative "isolated_plan"
+require_relative "isolated_scheduling"
+require_relative "isolated_verdict"
+require_relative "isolated_watchdog"
+require_relative "null_progress"
+
+module Kimera
+  module Execution
+  end
+end
+
+class Kimera::Execution::IsolatedExecution
+  include Kimera::Execution::IsolatedExecutionScheduling
+  include Kimera::Execution::IsolatedExecutionVerdict
+  include Kimera::Execution::IsolatedExecutionWatchdog
+
+  DEFAULT_HARD_TIMEOUT = 300.0
+
+  def initialize(registry:, root:, tests:, **options)
+    @registry = registry
+    @root = root
+    @tests = tests
+    @options = options
+  end
+
+  def run(ids: self.ids, label: "mutants")
+    progress.start(ids.size, label)
+    report(ids)
+  ensure
+    progress.finish
+  end
+
+  private
+
+  def progress = @options.fetch(:progress, Kimera::Execution::NullProgress)
+
+  def limit = @options.fetch(:hard_timeout, DEFAULT_HARD_TIMEOUT)
+
+  def jobs = @_jobs ||= [Integer(@options.fetch(:jobs, 1)), 1].max
+
+  def errors = @options.fetch(:errio, $stderr)
+
+  def plan
+    @_plan ||= Kimera::Execution::IsolatedPlan.new(
+      registry: @registry, root: @root, tests: @tests,
+      coverage: @options.fetch(:coverage, {}), framework: @options.fetch(:framework, "rspec"),
+      test_files: @options.fetch(:test_files, [])
+    )
+  end
+
+  def report(ids)
+    Kimera::RunReport.new(
+      results: (jobs > 1 ? concurrently(ids) : serially(ids)),
+      leaks: [], registry: plan.registry
+    )
+  end
+end

data/lib/kimera/execution/isolated_child.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "rspec/core"
+require "stringio"
+
+begin
+  RSpec::Core::ConfigurationOptions.new([]).configure(RSpec.configuration)
+rescue StandardError, ScriptError
+  nil
+end
+RSpec.configuration.output_stream = StringIO.new
+RSpec.configuration.error_stream = StringIO.new
+RSpec.configuration.deprecation_stream = StringIO.new
+
+exit RSpec::Core::Runner.run(ARGV)

data/lib/kimera/execution/isolated_child_command.rb

@@ -0,0 +1,44 @@
+# frozen_string_literal: true
+
+module Kimera
+  module Execution
+  end
+end
+
+class Kimera::Execution::IsolatedPlan; end
+
+class Kimera::Execution::IsolatedPlan::ChildCommand
+  CHILD = File.expand_path("isolated_child.rb", __dir__)
+  MINITEST_CHILD = File.expand_path("isolated_child_minitest.rb", __dir__)
+
+  def initialize(framework:, test_files:)
+    @framework = framework
+    @test_files = test_files
+  end
+
+  def command(mirror, locations, paths:)
+    bundled(mirror, {}, ["ruby", *paths.flat_map { |path| ["-I", path] }, "-I", helpers, *argv(locations)])
+  end
+
+  def argv(locations)
+    return [CHILD, *locations] unless minitest?
+    [MINITEST_CHILD, *@test_files, "--", *locations]
+  end
+
+  def helpers
+    minitest? ? "test" : "spec"
+  end
+
+  private
+
+  def bundled(mirror, env, argv)
+    gemfile = File.join(mirror, "Gemfile")
+    return [env, argv] unless File.file?(gemfile)
+    env["BUNDLE_GEMFILE"] = gemfile
+    [env, ["bundle", "exec", *argv]]
+  end
+
+  def minitest?
+    @framework.to_s == "minitest"
+  end
+end

data/lib/kimera/execution/isolated_child_minitest.rb

@@ -0,0 +1,34 @@
+# frozen_string_literal: true
+
+require "minitest"
+
+Minitest.seed ||= 1
+begin
+  Minitest.class_variable_set(:@@installed_at_exit, :installed)
+rescue NameError, ArgumentError
+  class << Minitest
+    def run(*) = true
+  end
+end
+
+split = ARGV.index("--") or abort("usage: <test files...> -- <ids...>")
+files = ARGV[0...split]
+ids = ARGV[(split + 1)..]
+
+files.each { |f| load File.expand_path(f) }
+
+methods = {}
+Minitest::Runnable.runnables.each do |runnable|
+  next unless runnable.respond_to?(:runnable_methods)
+  runnable.runnable_methods.each { |name| methods["#{runnable}##{name}"] = [runnable, name] }
+end
+
+killed =
+  ids.any? do |id|
+    klass, name = methods[id]
+    next false unless klass
+    result = klass.new(name).run
+    !result.passed? && !result.skipped?
+  end
+
+exit(killed ? 1 : 0)

data/lib/kimera/execution/isolated_plan.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative "../self_protection"
+require_relative "isolated_child_command"
+require_relative "isolated_test_selection"
+
+module Kimera
+  module Execution
+  end
+end
+
+class Kimera::Execution::IsolatedPlan
+  MIRROR_SKIP = %w[.git tmp vendor node_modules coverage log .bundle].freeze
+
+  attr_reader :registry
+
+  def initialize(registry:, root:, tests:, **options)
+    @registry = registry
+    @given = root
+    @tests = tests
+    @options = options
+  end
+
+  def root = @_root ||= File.expand_path(@given)
+
+  def tests(id, point)
+    all = selection.all
+    return all if Kimera::SelfProtection.protected?(File.join(root, point.file))
+    return all unless point.safe?
+    selection.recorded(id)
+  end
+
+  def command(mirror, locations)
+    child.command(mirror, locations, paths: mutables)
+  end
+
+  def mutables
+    @registry.files.map { |f| f.split(File::SEPARATOR).first }.uniq
+  end
+
+  class << self
+    def mirrors(entries)
+      entries.reject { |entry| MIRROR_SKIP.include?(entry) }
+    end
+  end
+
+  private
+
+  def selection = @_selection ||= TestSelection.new(all_tests: @tests, coverage: @options.fetch(:coverage, {}))
+
+  def child
+    @_child ||= ChildCommand.new(framework: @options.fetch(:framework), test_files: @options.fetch(:test_files))
+  end
+end

data/lib/kimera/execution/isolated_scheduling.rb

@@ -0,0 +1,54 @@
+# frozen_string_literal: true
+
+require_relative "sweep"
+require_relative "trial"
+
+module Kimera
+  module Execution
+    module IsolatedExecutionScheduling
+      private
+
+      def ids
+        plan.registry.each.map { |m, _p| m.id }
+      end
+
+      def serially(ids)
+        sweep = Kimera::Execution::Sweep.new(ids, Mutex.new)
+        with_mirror { |mirror| scan(sweep, mirror) }
+        sweep.results
+      end
+
+      def scan(sweep, mirror)
+        synth = Overlay.new(plan.registry)
+        sweep.each { |id| charge(sweep, evaluate(Kimera::Execution::Trial.new(id, mirror, synth))) }
+      end
+
+      def concurrently(ids)
+        sweep = Kimera::Execution::Sweep.new(ids, Mutex.new)
+        workers(sweep).each(&:join)
+        sweep.results
+      end
+
+      def workers(sweep)
+        Array.new(jobs) { Thread.new { work(sweep) } }
+      end
+
+      def work(sweep)
+        synth = Overlay.new(plan.registry)
+        with_mirror { |mirror| drain(sweep, mirror, synth) }
+      end
+
+      def drain(sweep, mirror, synth)
+        while (id = sweep.pop)
+          charge(sweep, evaluate(Kimera::Execution::Trial.new(id, mirror, synth)))
+        end
+      end
+
+      def charge(sweep, result)
+        done = sweep.record(result)
+        progress.tick(result.status)
+        trace(result, done, sweep.total)
+      end
+    end
+  end
+end

data/lib/kimera/execution/isolated_test_selection.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+module Kimera
+  module Execution
+  end
+end
+
+class Kimera::Execution::IsolatedPlan; end
+
+class Kimera::Execution::IsolatedPlan::TestSelection
+  attr_reader :all
+
+  def initialize(all_tests:, coverage:)
+    @all = all_tests
+    @coverage = coverage
+  end
+
+  def recorded(id)
+    tests = @coverage[id] || @coverage[id.to_s]
+    tests unless Array(tests).empty?
+  end
+end

data/lib/kimera/execution/isolated_verdict.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module Kimera
+  module Execution
+    module IsolatedExecutionVerdict
+      private
+
+      def evaluate(trial)
+        point = plan.registry.index[trial.id]
+        return error(trial, "unknown mutant") unless point
+        bake(trial, point)
+      rescue StandardError => error
+        self.error(trial, "#{error.class}: #{error.message}")
+      end
+
+      def bake(trial, point)
+        trial.file = point.file
+        tests = plan.tests(trial.id, point)
+        return result(trial, :no_coverage, nil) unless tests
+        status, duration = measure(trial, tests)
+        result(trial, status, duration, tests)
+      end
+
+      def measure(trial, tests)
+        started = now
+        [with_mutant(trial) { verdict(trial.mirror, tests) }, now - started]
+      end
+
+      def with_mutant(trial)
+        original, target = trial.stage(plan.root)
+        yield
+      ensure
+        File.write(target, original) if original
+      end
+
+      def verdict(mirror, locations)
+        env, cmd = plan.command(mirror, locations)
+        status = waitfor(Process.spawn(env, *cmd, chdir: mirror, pgroup: true, out: File::NULL, err: File::NULL))
+        return :timeout if status == :timeout
+        status.success? ? :survived : :killed
+      end
+
+      def with_mirror
+        Dir.mktmpdir("kimera-isolated") do |mirror|
+          populate(mirror)
+          yield(mirror)
+        end
+      end
+
+      def populate(mirror)
+        root = plan.root
+        IsolatedPlan.mirrors(Dir.children(root)).each do |entry|
+          FileUtils.cp_r(File.join(root, entry), File.join(mirror, entry))
+        end
+      end
+
+      def result(trial, status, duration, cover = nil)
+        MutantResult.new(
+          mutant_id: trial.id, status: status, file: trial.file, duration: duration, covering_tests: cover
+        )
+      end
+
+      def error(trial, detail)
+        MutantResult.new(mutant_id: trial.id, status: :error, file: trial.file, detail: detail)
+      end
+
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/kimera/execution/isolated_watchdog.rb

@@ -0,0 +1,59 @@
+# frozen_string_literal: true
+
+module Kimera
+  module Execution
+    module IsolatedExecutionWatchdog
+      private
+
+      def waitfor(pid)
+        deadline = now + limit
+        loop do
+          step = attempt(pid, deadline)
+          return step unless step == :continue
+        end
+      end
+
+      def attempt(pid, deadline)
+        done, status = Process.waitpid2(pid, Process::WNOHANG)
+        return status if done
+        return fail!(pid) unless (deadline - now).positive?
+        pause
+        :continue
+      end
+
+      def pause
+        sleep(@options.fetch(:poll_interval, 0.05))
+      end
+
+      def fail!(pid)
+        kill(pid)
+        reap(pid)
+        :timeout
+      end
+
+      def kill(pid)
+        Process.kill("KILL", -pid)
+      rescue Errno::ESRCH
+        nil
+      end
+
+      def reap(pid)
+        Process.waitpid(pid)
+      rescue Errno::ECHILD
+        nil
+      end
+
+      def trace(result, done, total)
+        return if progress.enabled?
+        errors.puts(line(result, done, total))
+      end
+
+      def line(result, done, total)
+        format(
+          "kimera: isolated %d/%d  #%s %s  %s  (%.1fs)", done, total,
+          *result.to_h.values_at("mutant_id", "status", "file", "duration")
+        )
+      end
+    end
+  end
+end

data/lib/kimera/execution/isolation.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Kimera
+  module Execution
+  end
+end
+
+class Kimera::Execution::Isolation
+  def around
+    yield
+  end
+
+  def reset!; end
+end
+
+require_relative "composite_isolation"
+require_relative "transaction_isolation"

data/lib/kimera/execution/null_progress.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+module Kimera
+  module Execution
+    module NullProgress
+      module_function
+
+      def start(_total, _label = nil); end
+
+      def tick(_status = nil); end
+
+      def finish; end
+
+      def enabled? = false
+    end
+  end
+end