minitest-impact 0.1.0

data/lib/minitest/impact/jev/client.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "json"
+require "net/http"
+require "uri"
+
+module Minitest
+  module Impact
+    module Jev
+      # A minimal client for TypeSafe's System One endpoint (docs.typesafe.ai/api): one POST with a
+      # state and named questions, answers under the same names. The `jev` gem (0.2.0) was
+      # considered first; it always sends "jev-latest" and one fixed URL, and a threshold tuned on
+      # one version needs that version pinned (docs.typesafe.ai/models).
+      class Client
+        DEFAULT_BASE_URL = "https://api.typesafe.ai"
+        # Pinned: the thresholds in Questions were set against this version.
+        DEFAULT_MODEL = "jev-1.13.0"
+        RETRY_STATUSES = [429, 529].freeze
+        ATTEMPTS = 3
+        TIMEOUT = 15
+
+        Response = Struct.new(:answers, :model, :input_tokens, keyword_init: true)
+
+        class Failure < Error; end
+
+        def self.from_env(env = ENV)
+          key = env["TYPESAFE_API_KEY"].to_s
+          return nil if key.empty?
+
+          new(api_key: key, base_url: env.fetch("TYPESAFE_BASE_URL", DEFAULT_BASE_URL),
+              model: env.fetch("MINITEST_IMPACT_JEV_MODEL", DEFAULT_MODEL))
+        end
+
+        attr_reader :model
+
+        def initialize(api_key:, base_url: DEFAULT_BASE_URL, model: DEFAULT_MODEL, sleeper: ->(seconds) { sleep(seconds) })
+          @api_key = api_key
+          @uri = URI.join(base_url.end_with?("/") ? base_url : "#{base_url}/", "v1/systemone")
+          @model = model
+          @sleeper = sleeper
+        end
+
+        def ask(state:, questions:)
+          attempt = 0
+          begin
+            attempt += 1
+            response = post(JSON.generate(model: model, state: state, questions: questions))
+            if RETRY_STATUSES.include?(response.code.to_i) && attempt < ATTEMPTS
+              @sleeper.call(retry_after(response, attempt))
+              raise RetryableStatus
+            end
+            parse(response)
+          rescue RetryableStatus
+            retry
+          rescue Net::OpenTimeout, Net::ReadTimeout, SocketError, SystemCallError => error
+            raise Failure, "Jev unreachable: #{error.class}"
+          end
+        end
+
+        private
+
+        class RetryableStatus < StandardError; end
+
+        def post(body)
+          request = Net::HTTP::Post.new(@uri, "Content-Type" => "application/json", "Authorization" => "Bearer #{@api_key}")
+          request.body = body
+          Net::HTTP.start(@uri.host, @uri.port, use_ssl: @uri.scheme == "https", open_timeout: TIMEOUT, read_timeout: TIMEOUT) do |http|
+            http.request(request)
+          end
+        end
+
+        def retry_after(response, attempt)
+          header = response["retry-after"].to_s
+          header.match?(/\A\d+(\.\d+)?\z/) ? header.to_f : 0.5 * (2**attempt)
+        end
+
+        def parse(response)
+          raise Failure, "Jev answered #{response.code}: #{response.body.to_s[0, 200]}" unless response.is_a?(Net::HTTPSuccess)
+
+          body = JSON.parse(response.body)
+          Response.new(answers: body.fetch("answers"), model: body.fetch("model"), input_tokens: body.dig("usage", "input_tokens").to_i)
+        rescue JSON::ParserError, KeyError => error
+          raise Failure, "Jev answered something unreadable: #{error.class}"
+        end
+      end
+    end
+  end
+end

data/lib/minitest/impact/jev/questions.rb

@@ -0,0 +1,58 @@
+# frozen_string_literal: true
+
+module Minitest
+  module Impact
+    module Jev
+      # Every question and threshold sent to Jev, in one file so they can be reviewed together
+      # (TypeSafe's agent skill). One narrow judgment per question, all asked over one state in one
+      # request. Rules (which files need the whole suite, what a test file is) stay in Rules.
+      module Questions
+        # Candidates per request: 48 Nouls and a 49-option Choice keep the state and the longest
+        # question well under jev-1.13's 32k-token bound.
+        MAX_CANDIDATES = 48
+        MAX_DIFF_CHARS = 12_000
+        MAX_TEST_NAMES = 8
+
+        # UNTUNED until `minitest-impact eval --jev` runs on labelled cases; see README, "Tuning".
+        KEEP = 0.5
+        MOST_DIRECT = 0.5
+        WHOLE_SUITE = 0.8
+
+        NONE = "none"
+
+        module_function
+
+        def exercises(index)
+          {
+            type: "noul",
+            instructions: "Do the tests in `candidates[#{index}]` call, render or assert on something that `change` adds, edits or removes?",
+            criteria: {
+              true: "They exercise code, pages, routes, copy or data that the change modifies.",
+              false: "They test other features. A shared word in the file names is not enough."
+            }
+          }
+        end
+
+        def most_direct(count)
+          {
+            type: "choice",
+            instructions: "Which candidate test file was written to check the behaviour that `change` modifies?",
+            criteria: Array.new(count) { |index| ["t#{index}", "The test file `candidates[#{index}]`."] }.to_h
+              .merge(NONE => "None of the candidates checks this behaviour.")
+          }
+        end
+
+        def whole_suite
+          {
+            type: "noul",
+            instructions: "Does `change` modify something that every test depends on, such as test setup, gem versions or how the application boots?",
+            criteria: {
+              true: "Any test could fail because of this change.",
+              false: "Only tests of particular features could fail."
+            }
+          }
+        end
+      end
+    end
+  end
+end

data/lib/minitest/impact/jev/ranker.rb

@@ -0,0 +1,121 @@
+# frozen_string_literal: true
+
+module Minitest
+  module Impact
+    module Jev
+      # Re-ranks and extends a map selection with Jev, for what the map cannot see: new files,
+      # views, config, copy, and selections too large to run in a loop. A fast word-overlap search
+      # over test paths and test names builds a shortlist; Jev reads the change once and answers one
+      # question per candidate, in one request (docs.typesafe.ai/cookbooks/rerank_typesafe).
+      #
+      # Exact map hits are never dropped: the map saw those tests run the changed code.
+      class Ranker
+        EXACT = 0.9
+
+        def initialize(client:)
+          @client = client
+        end
+
+        def call(picks:, changes:, intent:, repo:, map:, head:)
+          candidates = shortlist(picks, changes, intent, repo, map, head)
+          return [picks, nil] if candidates.empty?
+
+          response = @client.ask(state: state(candidates, changes, intent, repo, head), questions: questions(candidates.size))
+          [combine(picks, candidates, response.answers, map), report(response, candidates)]
+        rescue Client::Failure => error
+          [picks, { "error" => error.message }]
+        end
+
+        private
+
+        def shortlist(picks, changes, intent, repo, map, head)
+          chosen = picks.first(Questions::MAX_CANDIDATES).map(&:test)
+          room = Questions::MAX_CANDIDATES - chosen.size
+          return chosen if room <= 0
+
+          words = query_words(changes, intent)
+          pool = (map.test_files + repo.files(head).select { |file| Rules.test_file?(file) }).uniq - chosen
+          scored = pool.filter_map do |test|
+            overlap = (words & test_words(test, repo, head)).size
+            [test, overlap] if overlap.positive?
+          end
+          chosen + scored.sort_by { |test, overlap| [-overlap, test] }.first(room).map(&:first)
+        end
+
+        def query_words(changes, intent)
+          text = changes.map { |change| "#{change.path} #{change.added_text} #{change.removed_text}" }.join(" ") + " #{intent}"
+          words(text)
+        end
+
+        def test_words(test, repo, head)
+          words("#{test} #{test_names(test, repo, head).join(" ")}")
+        end
+
+        STOP = %w[test tests rb app the and for with that this should when from into def end do].freeze
+
+        def words(text)
+          text.downcase.scan(/[a-z][a-z0-9]{2,}/).map { |word| Rules.singularize(word) }.uniq - STOP
+        end
+
+        def test_names(test, repo, head)
+          source = repo.read(test, head) || ""
+          source.scan(/^\s*test\s+["'](.+?)["']|^\s*def\s+(test_\w+)/).map { |a, b| a || b.tr("_", " ") }.first(Questions::MAX_TEST_NAMES)
+        end
+
+        def state(candidates, changes, intent, repo, head)
+          budget = Questions::MAX_DIFF_CHARS
+          change = changes.map do |c|
+            excerpt = [c.removed_text.lines.map { |l| "-#{l}" }, c.added_text.lines.map { |l| "+#{l}" }].flatten.join
+            excerpt = excerpt[0, [budget, 0].max]
+            budget -= excerpt.size
+            { path: c.path, status: c.status.to_s, diff: excerpt }
+          end
+          {
+            intent: intent.to_s,
+            change: change,
+            candidates: candidates.map { |test| { path: test, tests: test_names(test, repo, head) } }
+          }
+        end
+
+        def questions(count)
+          asked = Array.new(count) { |index| ["t#{index}", Questions.exercises(index)] }.to_h
+          asked.merge("most_direct" => Questions.most_direct(count), "whole_suite" => Questions.whole_suite)
+        end
+
+        def combine(picks, candidates, answers, map)
+          by_test = picks.to_h { |pick| [pick.test, pick] }
+          best = answers["most_direct"]
+          direct = best && best["choice"] != Questions::NONE && best["confidence"].to_f >= Questions::MOST_DIRECT ? best["choice"] : nil
+
+          candidates.each_with_index do |test, index|
+            noul = answers.dig("t#{index}", "noul")
+            next if noul.nil?
+
+            pick = by_test[test]
+            if pick.nil?
+              next if noul < Questions::KEEP
+
+              by_test[test] = Pick.new(test: test, score: noul * 0.8, reasons: ["Jev: exercises the change (#{noul.round(2)})"], seconds: map.seconds(test))
+            elsif pick.score < EXACT && noul < Questions::KEEP
+              by_test.delete(test)
+            else
+              pick.score = pick.score >= EXACT ? pick.score : (pick.score + noul) / 2
+              pick.reasons += ["Jev: #{noul.round(2)}"]
+            end
+            by_test[test].score = 1.0 + (by_test[test].score / 10) if by_test[test] && direct == "t#{index}"
+          end
+          by_test.values.sort_by { |pick| [-pick.score, pick.seconds || Float::INFINITY, pick.test] }
+        end
+
+        def report(response, candidates)
+          {
+            "model" => response.model,
+            "input_tokens" => response.input_tokens,
+            "candidates" => candidates.size,
+            "whole_suite" => response.answers.dig("whole_suite", "noul").to_f >= Questions::WHOLE_SUITE
+          }
+        end
+      end
+    end
+  end
+end

data/lib/minitest/impact/locale_keys.rb

@@ -0,0 +1,39 @@
+# frozen_string_literal: true
+
+require "psych"
+
+module Minitest
+  module Impact
+    # The full i18n keys ("sessions.new.title", without the locale) whose lines a change touched.
+    module LocaleKeys
+      def self.at(source, lines)
+        return [] if source.nil? || lines.empty?
+
+        wanted = lines.to_set
+        keys = []
+        document = Psych.parse(source)
+        walk(document&.root, [], keys, wanted)
+        keys.uniq
+      rescue Psych::SyntaxError
+        []
+      end
+
+      def self.walk(node, path, keys, wanted)
+        return unless node.is_a?(Psych::Nodes::Mapping)
+
+        node.children.each_slice(2) do |key, value|
+          next unless key.is_a?(Psych::Nodes::Scalar)
+
+          full = path + [key.value]
+          if value.is_a?(Psych::Nodes::Mapping)
+            walk(value, full, keys, wanted)
+          else
+            span = (key.start_line + 1)..(value.end_line + 1)
+            keys << full.drop(1).join(".") if span.any? { |line| wanted.include?(line) } && full.size > 1
+          end
+        end
+      end
+      private_class_method :walk
+    end
+  end
+end

data/lib/minitest/impact/map.rb

@@ -0,0 +1,115 @@
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+require "set"
+require "time"
+
+module Minitest
+  module Impact
+    # The coverage map: for each test file, the project lines its tests executed, plus how long it
+    # took. Stored as JSON with line numbers folded into ranges ("3-7,12"), so a Rails app's map
+    # stays a few hundred kilobytes.
+    class Map
+      FORMAT = 1
+      DEFAULT_PATH = "tmp/minitest-impact/map.json"
+
+      TestEntry = Struct.new(:seconds, :count, :files, keyword_init: true)
+
+      attr_reader :commit, :recorded_at, :tests
+
+      def self.load(path)
+        data = JSON.parse(File.read(path))
+        raise Error, "#{path} is map format #{data["format"]}, this version reads #{FORMAT}" unless data["format"] == FORMAT
+
+        tests = data.fetch("tests").to_h do |test, entry|
+          files = entry.fetch("files").transform_values { |ranges| Ranges.parse(ranges) }
+          [test, TestEntry.new(seconds: entry["seconds"].to_f, count: entry["count"].to_i, files: files)]
+        end
+        new(commit: data["commit"], recorded_at: data["recorded_at"], tests: tests)
+      end
+
+      # Folds the per-process parts a recording wrote into one map.
+      def self.merge(parts_dir, commit:)
+        tests = Hash.new { |hash, key| hash[key] = TestEntry.new(seconds: 0.0, count: 0, files: Hash.new { |h, k| h[k] = Set.new }) }
+        Dir[File.join(parts_dir, "*.ndjson")].sort.each do |part|
+          File.foreach(part) do |line|
+            record = JSON.parse(line)
+            entry = tests[record.fetch("test")]
+            entry.seconds += record["seconds"].to_f
+            entry.count += 1
+            record.fetch("lines").each { |file, lines| entry.files[file].merge(lines) }
+          end
+        end
+        new(commit: commit, recorded_at: Time.now.utc.iso8601, tests: tests.to_h { |test, entry| [test, freeze_entry(entry)] })
+      end
+
+      def self.freeze_entry(entry)
+        TestEntry.new(seconds: entry.seconds.round(3), count: entry.count, files: entry.files.to_h { |f, l| [f, l.to_a.sort] })
+      end
+      private_class_method :freeze_entry
+
+      def initialize(commit:, recorded_at:, tests:)
+        @commit = commit
+        @recorded_at = recorded_at
+        @tests = tests
+      end
+
+      def save(path)
+        FileUtils.mkdir_p(File.dirname(path))
+        File.write(path, JSON.pretty_generate(to_h))
+      end
+
+      def to_h
+        {
+          "format" => FORMAT,
+          "commit" => commit,
+          "recorded_at" => recorded_at,
+          "tests" => tests.sort.to_h do |test, entry|
+            [test, { "seconds" => entry.seconds, "count" => entry.count,
+                     "files" => entry.files.sort.to_h { |file, lines| [file, Ranges.dump(lines)] } }]
+          end
+        }
+      end
+
+      def test_files = tests.keys
+
+      def known_file?(path) = by_file.key?(path)
+
+      # test file => executed lines of +path+
+      def tests_for(path) = by_file.fetch(path, {})
+
+      # Share of all test files that touch +path+: near 1 for a base class every test loads, near 0
+      # for a file only its own tests reach.
+      def spread(path)
+        return 0.0 if tests.empty?
+
+        tests_for(path).size.to_f / tests.size
+      end
+
+      def seconds(test) = tests[test]&.seconds
+
+      private
+
+      def by_file
+        @by_file ||= tests.each_with_object(Hash.new { |h, k| h[k] = {} }) do |(test, entry), index|
+          entry.files.each { |file, lines| index[file][test] = lines }
+        end.tap { |index| index.default_proc = nil }
+      end
+    end
+
+    # "1-3,7" <-> [1, 2, 3, 7]
+    module Ranges
+      def self.dump(lines)
+        lines.sort.slice_when { |a, b| b != a + 1 }.map { |run| run.size == 1 ? run.first.to_s : "#{run.first}-#{run.last}" }.join(",")
+      end
+
+      def self.parse(text)
+        text.split(",").flat_map do |part|
+          first, last = part.split("-").map(&:to_i)
+          last ? (first..last).to_a : [first]
+        end
+      end
+    end
+  end
+end

data/lib/minitest/impact/methods.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "prism"
+
+module Minitest
+  module Impact
+    # The methods of a Ruby file and the lines each spans, named with their namespace
+    # ("Billing::Invoice#total", "Billing::Invoice.build"). Naming a method lets a changed line in
+    # one version of a file find the same method in the version the map was recorded against,
+    # even after the lines around it moved.
+    class Methods
+      Definition = Struct.new(:name, :first_line, :last_line) do
+        def lines = (first_line..last_line)
+        # The body, without the `def` and `end` lines: those run when the file loads, not when a
+        # test calls the method. A one-line method runs on its only line.
+        def body = last_line - first_line >= 2 ? ((first_line + 1)..(last_line - 1)) : lines
+      end
+
+      def self.of(source)
+        return new([]) if source.nil?
+
+        result = Prism.parse(source)
+        definitions = []
+        walk(result.value, [], definitions)
+        new(definitions)
+      end
+
+      def self.walk(node, namespace, out)
+        case node
+        when Prism::ClassNode, Prism::ModuleNode
+          inner = namespace + [node.constant_path.slice]
+          node.body && walk(node.body, inner, out)
+          return
+        when Prism::SingletonClassNode
+          node.body && walk(node.body, namespace + ["<<self"], out)
+          return
+        when Prism::DefNode
+          separator = node.receiver ? "." : "#"
+          separator = "." if namespace.last == "<<self"
+          owner = namespace.reject { |part| part == "<<self" }.join("::")
+          out << Definition.new("#{owner}#{separator}#{node.name}", node.location.start_line, node.location.end_line)
+        end
+        node.compact_child_nodes.each { |child| walk(child, namespace, out) }
+      end
+      private_class_method :walk
+
+      attr_reader :definitions
+
+      def initialize(definitions)
+        @definitions = definitions
+      end
+
+      # The innermost method containing +line+, or nil for class-body code.
+      def enclosing(line)
+        definitions.select { |definition| definition.lines.cover?(line) }.min_by { |definition| definition.lines.size }
+      end
+
+      def named(name) = definitions.find { |definition| definition.name == name }
+    end
+  end
+end

data/lib/minitest/impact/railtie.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+module Minitest
+  module Impact
+    # Adds `bin/rails test:impact` to a Rails app that has the gem in its Gemfile.
+    class Railtie < Rails::Railtie
+      rake_tasks do
+        require_relative "rake_task"
+        RakeTask.new
+      end
+    end
+  end
+end

data/lib/minitest/impact/rake_task.rb

@@ -0,0 +1,30 @@
+# frozen_string_literal: true
+
+require "rake"
+require_relative "../impact"
+require_relative "cli"
+
+module Minitest
+  module Impact
+    # Defines `rake test:impact` (and so `bin/rails test:impact`): runs the tests the current
+    # change selects. SINCE=REV sets the base (default HEAD), INTENT="..." describes the change.
+    # When the change needs the whole suite, it runs `test` instead.
+    class RakeTask
+      include Rake::DSL
+
+      def initialize(name = "test:impact")
+        desc "Run the tests the current change is most likely to break (minitest-impact)"
+        task(name) do
+          args = ["run", "--since", ENV.fetch("SINCE", "HEAD")]
+          args += ["--intent", ENV["INTENT"]] if ENV["INTENT"]
+          status = CLI.start(args)
+          if status == CLI::WHOLE_SUITE_EXIT
+            Rake::Task["test"].invoke
+          elsif !status.zero?
+            abort
+          end
+        end
+      end
+    end
+  end
+end

data/lib/minitest/impact/record_bootstrap.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# Required through RUBYOPT by `minitest-impact record`, before the application or Bundler load, so
+# coverage sees every project file from its first line. Minitest 6 no longer loads plugins on its
+# own, so a TracePoint waits for Minitest::Test to be defined and wraps every test from there.
+require_relative "recorder"
+
+if (dir = ENV["MINITEST_IMPACT_RECORD"])
+  Minitest::Impact::Recorder.start(dir: dir, root: ENV.fetch("MINITEST_IMPACT_ROOT", Dir.pwd))
+
+  trace = TracePoint.new(:class) do |event|
+    next unless event.self.name == "Minitest::Test"
+
+    trace.disable
+    require_relative "recording_hooks"
+    event.self.prepend(Minitest::Impact::RecordingHooks)
+  end
+  trace.enable
+end

data/lib/minitest/impact/recorder.rb

@@ -0,0 +1,104 @@
+# frozen_string_literal: true
+
+require "coverage"
+require "json"
+
+module Minitest
+  module Impact
+    # Records which project lines each test executes. Loaded through RUBYOPT before the
+    # application boots (see `minitest-impact record`), so it depends on the standard library only.
+    #
+    # Each test appends one JSON line to a file named after its process, so forked parallel
+    # workers never share a file handle; `Map.merge` folds the parts into one map afterwards.
+    module Recorder
+      PARTS = "parts"
+
+      class << self
+        attr_reader :root, :dir, :mode
+
+        def start(dir:, root: Dir.pwd)
+          return if @started
+
+          @root = File.expand_path(root)
+          @dir = File.expand_path(dir)
+          Dir.mkdir(File.join(@dir, PARTS)) unless Dir.exist?(File.join(@dir, PARTS))
+          @mode = begin_coverage
+          @started = true
+        end
+
+        def started? = @started == true
+
+        def before_test
+          if mode == :clear
+            Coverage.result(stop: false, clear: true)
+          else
+            @baseline = Coverage.peek_result
+          end
+          @clock = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        end
+
+        def after_test(test_file:, name:)
+          seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @clock
+          lines = executed_lines
+          write(test: relative(test_file), name: name, seconds: seconds.round(4), lines: lines)
+        end
+
+        # Judged on the path inside the project: a project that itself lives under a `tmp`
+        # directory (Linux's temporary directories, some CI workspaces) must still be recorded.
+        def project_file?(path)
+          path.start_with?("#{root}/") &&
+            !"/#{relative(path)}".match?(%r{/(vendor|tmp|node_modules)/})
+        end
+
+        def relative(path) = path.delete_prefix("#{root}/")
+
+        private
+
+        # Coverage can be set up once per process. When another tool (SimpleCov) got there first,
+        # clearing its counters would corrupt its report, so each test is measured as the
+        # difference between two snapshots instead: slower, but harmless to the other tool.
+        def begin_coverage
+          if Coverage.running?
+            :peek
+          else
+            Coverage.setup(lines: true, eval: true)
+            Coverage.resume
+            :clear
+          end
+        end
+
+        def executed_lines
+          now = mode == :clear ? Coverage.result(stop: false, clear: true) : Coverage.peek_result
+          now.each_with_object({}) do |(path, data), acc|
+            next unless project_file?(path)
+
+            counts = data.is_a?(Hash) ? data[:lines] : data
+            next unless counts
+
+            before = mode == :peek ? line_counts(@baseline[path]) : nil
+            hit = []
+            counts.each_with_index do |count, index|
+              next unless count&.positive?
+              next if before && before[index].to_i >= count
+
+              hit << index + 1
+            end
+            acc[relative(path)] = hit unless hit.empty?
+          end
+        end
+
+        def line_counts(data)
+          return [] unless data
+
+          data.is_a?(Hash) ? (data[:lines] || []) : data
+        end
+
+        def write(record)
+          File.open(File.join(dir, PARTS, "#{Process.pid}.ndjson"), "a") do |file|
+            file.puts(JSON.generate(record))
+          end
+        end
+      end
+    end
+  end
+end

data/lib/minitest/impact/recording_hooks.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+module Minitest
+  module Impact
+    # Prepended to Minitest::Test while recording, so the measurement starts before every other
+    # before_setup (fixtures included) and ends after every other after_teardown.
+    module RecordingHooks
+      def before_setup
+        Recorder.before_test
+        super
+      end
+
+      def after_teardown
+        super
+      ensure
+        file = self.class.instance_method(name).source_location&.first
+        Recorder.after_test(test_file: file, name: "#{self.class.name}##{name}") if file
+      end
+    end
+  end
+end