kaisoku 0.1.0

data/lib/kaisoku/doctor.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+module Kaisoku
+  class Doctor
+    def initialize(test_map:, preload_server: nil)
+      @test_map = test_map
+      @preload_server = preload_server
+    end
+
+    def report
+      stats = @test_map.stats
+
+      [
+        'Kaisoku doctor',
+        "map: #{@test_map.path}",
+        "schema_version: #{stats[:schema_version]}",
+        "ruby_version: #{stats[:ruby_version]}",
+        "adapter: #{stats[:adapter]}",
+        "entities: #{stats[:entities]}",
+        "dependencies: #{stats[:dependencies]}",
+        preload_line,
+        warnings_line
+      ].join("\n")
+    end
+
+    private
+
+    def warnings_line
+      warnings = @test_map.health_warnings
+      return 'warnings: none' if warnings.empty?
+
+      "warnings: #{warnings.join(', ')}"
+    end
+
+    def preload_line
+      return 'preload: unavailable' unless @preload_server
+
+      status = @preload_server.status
+      line = "preload: generation=#{status[:generation]}, loaded_files=#{status[:loaded_files]}, " \
+             "last_restart=#{status[:last_restart_reason]}"
+      return line if status[:healthy]
+
+      "#{line}, disabled=#{status[:disabled_reason]}"
+    end
+  end
+end

data/lib/kaisoku/entity.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Kaisoku
+  Entity = Struct.new(
+    :adapter,
+    :identifier,
+    :file,
+    :line,
+    :avg_duration_ms,
+    :last_status,
+    :last_failed_at,
+    :failure_count,
+    keyword_init: true
+  ) do
+    def initialize(
+      adapter:,
+      identifier:,
+      file:,
+      line: nil,
+      avg_duration_ms: nil,
+      last_status: nil,
+      last_failed_at: nil,
+      failure_count: 0
+    )
+      super(
+        adapter: adapter.to_s,
+        identifier: identifier.to_s,
+        file: normalize_file(file),
+        line: line,
+        avg_duration_ms: avg_duration_ms,
+        last_status: last_status,
+        last_failed_at: last_failed_at,
+        failure_count: failure_count.to_i
+      )
+    end
+
+    def id
+      "#{adapter}:#{identifier}"
+    end
+
+    def failed?
+      last_status == 'failed'
+    end
+
+    def with_runtime(
+      avg_duration_ms: self.avg_duration_ms,
+      last_status: self.last_status,
+      last_failed_at: self.last_failed_at,
+      failure_count: self.failure_count
+    )
+      self.class.new(
+        adapter: adapter,
+        identifier: identifier,
+        file: file,
+        line: line,
+        avg_duration_ms: avg_duration_ms,
+        last_status: last_status,
+        last_failed_at: last_failed_at,
+        failure_count: failure_count
+      )
+    end
+
+    def eql?(other)
+      other.is_a?(Entity) && other.id == id
+    end
+
+    def hash
+      id.hash
+    end
+
+    private
+
+    def normalize_file(value)
+      value.to_s.sub(%r{\A\./}, '')
+    end
+  end
+end

data/lib/kaisoku/entity_result.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+module Kaisoku
+  EntityResult = Struct.new(:entity, :payload, :dependencies, :collection_skipped, :error, keyword_init: true) do
+    def passed?
+      payload[:status] == 'passed'
+    end
+
+    def failed?
+      payload[:status] == 'failed' || error
+    end
+
+    def status
+      return 'error' if error
+
+      payload[:status]
+    end
+  end
+end

data/lib/kaisoku/environment_tracker.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+require 'digest'
+
+module Kaisoku
+  class EnvironmentTracker
+    DEPENDENCY_KEY = '__ENV__'
+
+    def initialize(env: ENV)
+      @env = env
+      @before = fingerprint
+    end
+
+    def changed?
+      @before != fingerprint
+    end
+
+    def dependency
+      [DEPENDENCY_KEY, fingerprint]
+    end
+
+    private
+
+    def fingerprint
+      Digest::SHA256.hexdigest(@env.to_h.sort.map { |key, value| "#{key}=#{value}" }.join("\0"))
+    end
+  end
+end

data/lib/kaisoku/evaluator/cli.rb

@@ -0,0 +1,112 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+require 'optparse'
+
+module Kaisoku
+  module Evaluator
+    class CLI
+      def initialize(argv)
+        @argv = argv
+      end
+
+      def call
+        options = parse_options
+        return replay(options) if options[:commits]
+        return mutations(options) if options[:seed_mutations]
+        return usage_error unless options[:total] && options[:selected]
+
+        payload = ratio_payload(options)
+        write_json_report(json_report_path(options), payload) if json_report_path(options)
+        append_history(history_path(options), payload) if history_path(options)
+        puts "Selection ratio: #{format('%.2f', selection_ratio(options) * 100)}%"
+        puts "Reduction ratio: #{format('%.2f', reduction_ratio(options) * 100)}%"
+        0
+      end
+
+      private
+
+      def parse_options
+        options = {}
+        parser = OptionParser.new do |opts|
+          opts.on('--total N', Integer) { |value| options[:total] = value }
+          opts.on('--selected N', Integer) { |value| options[:selected] = value }
+          opts.on('--commits N', Integer) { |value| options[:commits] = value }
+          opts.on('--branch NAME') { |value| options[:branch] = value }
+          opts.on('--full-command COMMAND') { |value| options[:full_command] = value }
+          opts.on('--selected-command COMMAND') { |value| options[:selected_command] = value }
+          opts.on('--seed-mutations N', Integer) { |value| options[:seed_mutations] = value }
+          opts.on('--mutation-command COMMAND') { |value| options[:mutation_command] = value }
+          opts.on('--mutation-path PATH') { |value| (options[:mutation_paths] ||= []) << value }
+          opts.on('--json-report PATH') { |value| options[:json_report] = value }
+          opts.on('--history PATH') { |value| options[:history] = value }
+          opts.on('--ci') { options[:ci] = true }
+        end
+        parser.parse!(@argv)
+        options
+      end
+
+      def usage_error
+        warn 'usage: kaisoku eval --total N --selected N OR kaisoku eval --commits N'
+        1
+      end
+
+      def replay(options)
+        replay = Replay.new(
+          commits: options[:commits],
+          branch: options[:branch] || 'HEAD',
+          full_command: options[:full_command] || 'ruby -Ilib exe/kaisoku run --all --reporter json',
+          selected_command: options[:selected_command] || 'ruby -Ilib exe/kaisoku run --reporter json'
+        )
+        report = Report.new(replay.run)
+        payload = report.to_h
+        write_json_report(json_report_path(options), payload) if json_report_path(options)
+        append_history(history_path(options), payload) if history_path(options)
+        puts report.render
+        report.safe? ? 0 : 1
+      end
+
+      def mutations(options)
+        command = options[:mutation_command] || 'ruby -Ilib exe/kaisoku run --all'
+        paths = options[:mutation_paths] || ['lib']
+        puts MutationSeeder.new(paths: paths, command: command, limit: options[:seed_mutations]).report
+        0
+      end
+
+      def selection_ratio(options)
+        Metrics.selection_ratio(selected_count: options[:selected], total_count: options[:total])
+      end
+
+      def reduction_ratio(options)
+        Metrics.reduction_ratio(selected_count: options[:selected], total_count: options[:total])
+      end
+
+      def ratio_payload(options)
+        {
+          total_count: options[:total],
+          selected_count: options[:selected],
+          selection_ratio: selection_ratio(options),
+          reduction_ratio: reduction_ratio(options)
+        }
+      end
+
+      def write_json_report(path, payload)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, JSON.pretty_generate(payload))
+      end
+
+      def append_history(path, payload)
+        History.new.append(path, payload)
+      end
+
+      def json_report_path(options)
+        options[:json_report] || ('.kaisoku/eval-report.json' if options[:ci])
+      end
+
+      def history_path(options)
+        options[:history] || ('.kaisoku/eval-history.jsonl' if options[:ci])
+      end
+    end
+  end
+end

data/lib/kaisoku/evaluator/command_result.rb

@@ -0,0 +1,64 @@
+# frozen_string_literal: true
+
+require 'json'
+
+module Kaisoku
+  module Evaluator
+    class CommandResult
+      attr_reader :status, :time, :output
+
+      def initialize(status:, time:, output:)
+        @status = status
+        @time = time
+        @output = output
+      end
+
+      def failed_ids
+        results.select { |_id, payload| failed_payload?(payload) }.keys
+      end
+
+      def executed_ids
+        results.keys
+      end
+
+      def selected_count
+        selection['selected']
+      end
+
+      def total_count
+        selection['total']
+      end
+
+      def selection
+        parsed.fetch('selection', {})
+      end
+
+      def fallback?
+        selection['fallback'] || output.include?('fallback')
+      end
+
+      private
+
+      def results
+        parsed.fetch('results', {})
+      end
+
+      def parsed
+        @parsed ||= JSON.parse(json_payload)
+      rescue JSON::ParserError
+        { 'selection' => {}, 'results' => {} }
+      end
+
+      def json_payload
+        start = output.rindex("{\n  \"selection\"")
+        return output unless start
+
+        output[start..]
+      end
+
+      def failed_payload?(payload)
+        payload['status'] == 'failed' || payload[:status] == 'failed' || payload['error'] || payload[:error]
+      end
+    end
+  end
+end

data/lib/kaisoku/evaluator/history.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'json'
+
+module Kaisoku
+  module Evaluator
+    class History
+      def append(path, payload)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.open(path, 'a') do |file|
+          file.puts(JSON.generate(record(payload)))
+        end
+      end
+
+      private
+
+      def record(payload)
+        payload.merge(recorded_at: Time.now.to_i)
+      end
+    end
+  end
+end

data/lib/kaisoku/evaluator/metrics.rb

@@ -0,0 +1,45 @@
+# frozen_string_literal: true
+
+module Kaisoku
+  module Evaluator
+    module Metrics
+      module_function
+
+      def selection_ratio(selected_count:, total_count:)
+        return 0.0 if total_count.to_i.zero?
+
+        selected_count.to_f / total_count
+      end
+
+      def reduction_ratio(selected_count:, total_count:)
+        1.0 - selection_ratio(selected_count: selected_count, total_count: total_count)
+      end
+
+      def safety_violations(full_failures:, selected_ids:)
+        full_failures.map(&:to_s).uniq - selected_ids.map(&:to_s).uniq
+      end
+
+      def precision(selected_ids:, relevant_ids:)
+        selected = selected_ids.map(&:to_s).uniq
+        return 1.0 if selected.empty?
+
+        relevant = relevant_ids.map(&:to_s).uniq
+        (selected & relevant).length.to_f / selected.length
+      end
+
+      def apfd(order:, fault_detectors:)
+        ordered_ids = order.map { |entity| entity.respond_to?(:id) ? entity.id : entity.to_s }
+        fault_positions = fault_detectors.values.filter_map do |detectors|
+          Array(detectors).filter_map { |id| ordered_ids.index(id.to_s) }.min&.+(1)
+        end
+
+        return 1.0 if ordered_ids.empty? || fault_positions.empty?
+
+        n = ordered_ids.length.to_f
+        m = fault_positions.length.to_f
+
+        1.0 - (fault_positions.sum / (n * m)) + (1.0 / (2.0 * n))
+      end
+    end
+  end
+end

data/lib/kaisoku/evaluator/mutation_seeder.rb

@@ -0,0 +1,78 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module Kaisoku
+  module Evaluator
+    class MutationSeeder
+      Mutation = Struct.new(:file, :original, :replacement, :index, keyword_init: true)
+
+      PATTERNS = {
+        'true' => 'false',
+        'false' => 'true',
+        'nil' => 'false'
+      }.freeze
+
+      def initialize(paths:, command:, limit: 10)
+        @paths = Array(paths)
+        @command = command
+        @limit = limit.to_i
+      end
+
+      def run
+        mutations.first(@limit).map { |mutation| evaluate(mutation) }
+      end
+
+      def report
+        results = run
+        killed = results.count { |result| result[:killed] }
+        "Mutation score: #{killed}/#{results.length}"
+      end
+
+      def mutations
+        ruby_files.flat_map { |file| mutations_for_file(file) }
+      end
+
+      def mutations_for_source(file, source)
+        PATTERNS.flat_map do |original, replacement|
+          source.enum_for(:scan, /\b#{Regexp.escape(original)}\b/).map do
+            Mutation.new(file: file, original: original, replacement: replacement, index: Regexp.last_match.begin(0))
+          end
+        end
+      end
+
+      private
+
+      def ruby_files
+        @paths.flat_map do |path|
+          if File.directory?(path)
+            Dir.glob(File.join(path, '**', '*.rb'))
+          elsif File.file?(path) && path.end_with?('.rb')
+            [path]
+          else
+            []
+          end
+        end.uniq.sort
+      end
+
+      def mutations_for_file(file)
+        mutations_for_source(file, File.read(file))
+      end
+
+      def evaluate(mutation)
+        source = File.read(mutation.file)
+        File.write(mutation.file, mutated_source(source, mutation))
+        _output, status = Open3.capture2e(@command)
+        { mutation: mutation, killed: !status.success? }
+      ensure
+        File.write(mutation.file, source) if source
+      end
+
+      def mutated_source(source, mutation)
+        source.dup.tap do |copy|
+          copy[mutation.index, mutation.original.length] = mutation.replacement
+        end
+      end
+    end
+  end
+end

data/lib/kaisoku/evaluator/replay.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'open3'
+require 'shellwords'
+require 'tmpdir'
+
+module Kaisoku
+  module Evaluator
+    class Replay
+      Result = Struct.new(
+        :commit,
+        :full_status,
+        :selected_status,
+        :full_time,
+        :selected_time,
+        :fallback,
+        :safety_violations,
+        :precision,
+        :apfd,
+        :changed_files,
+        :selected_count,
+        :total_count,
+        keyword_init: true
+      )
+
+      def initialize(
+        commits:,
+        branch: 'HEAD',
+        full_command: 'ruby -Ilib exe/kaisoku run --all --reporter json',
+        selected_command: 'ruby -Ilib exe/kaisoku run --reporter json'
+      )
+        @commits = commits.to_i
+        @branch = branch
+        @full_command = full_command
+        @selected_command = selected_command
+      end
+
+      def run
+        commit_list.map { |commit| evaluate_commit(commit) }
+      end
+
+      def report
+        Report.new(run).render
+      end
+
+      private
+
+      def commit_list
+        output, status = Open3.capture2('git', 'rev-list', '--max-count', @commits.to_s, '--reverse', @branch)
+        raise Error, 'git rev-list failed' unless status.success?
+
+        output.lines.map(&:strip).reject(&:empty?)
+      end
+
+      def evaluate_commit(commit)
+        Dir.mktmpdir('kaisoku-eval') do |dir|
+          worktree = File.join(dir, 'worktree')
+          add_worktree(worktree, commit)
+          begin
+            changed = changed_files(commit)
+            full = timed_run(@full_command, worktree)
+            selected = timed_run(selected_command_for(changed), worktree)
+            violations = Metrics.safety_violations(
+              full_failures: full.failed_ids,
+              selected_ids: selected.executed_ids
+            )
+            Result.new(
+              commit: commit,
+              full_status: full.status,
+              selected_status: selected.status,
+              full_time: full.time,
+              selected_time: selected.time,
+              fallback: selected.fallback?,
+              safety_violations: violations,
+              precision: Metrics.precision(selected_ids: selected.executed_ids, relevant_ids: full.failed_ids),
+              apfd: Metrics.apfd(order: selected.executed_ids, fault_detectors: fault_detectors(full.failed_ids)),
+              changed_files: changed,
+              selected_count: selected.selected_count,
+              total_count: selected.total_count || full.executed_ids.length
+            )
+          ensure
+            remove_worktree(worktree)
+          end
+        end
+      end
+
+      def selected_command_for(changed_files)
+        files = changed_files.map { |file| Shellwords.escape(file) }.join(' ')
+        [@selected_command, files].reject(&:empty?).join(' ')
+      end
+
+      def changed_files(commit)
+        output, status = Open3.capture2('git', 'diff-tree', '--no-commit-id', '--name-only', '-r', commit)
+        return [] unless status.success?
+
+        output.lines.map(&:strip).reject(&:empty?)
+      end
+
+      def add_worktree(path, commit)
+        _output, status = Open3.capture2e('git', 'worktree', 'add', '--detach', path, commit)
+        raise Error, 'git worktree add failed' unless status.success?
+      end
+
+      def remove_worktree(path)
+        Open3.capture2e('git', 'worktree', 'remove', '--force', path)
+      end
+
+      def timed_run(command, chdir)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        output, status = Open3.capture2e(command, chdir: chdir)
+        elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+        CommandResult.new(output: output, status: status.exitstatus, time: elapsed)
+      end
+
+      def fault_detectors(failed_ids)
+        failed_ids.to_h { |id| [id, [id]] }
+      end
+    end
+  end
+end