branchout 0.1.0

data/lib/branchout/emitters.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require_relative "emitters/support"
+require_relative "emitters/minitest"
+require_relative "emitters/rspec"
+
+module Branchout
+  module Emitters
+    FORMATS = %w[minitest rspec].freeze
+
+    module_function
+
+    def resolve(explicit, path)
+      if explicit
+        unless FORMATS.include?(explicit)
+          raise Error, "format must be minitest or rspec"
+        end
+
+        return explicit
+      end
+      return "rspec" if path.to_s.end_with?("_spec.rb")
+
+      "minitest"
+    end
+
+    # Returns a warning for each file the test requires by a path out of its
+    # project, and for each path of this machine that the test passes
+    # as it is (Support.machine_paths), or pins in a returned value or an
+    # error message, for the caller to print.
+    def write(format, result, target, destination, require_by_name: false, pin_keys: [])
+      pins = project_pins(result.pins, destination)
+      Support.write_file(destination, render(format, result, target, destination, require_by_name: require_by_name, pin_keys: pin_keys))
+      varying = result.pins.select(&:varying).map { |pin| varying_warning(pin, pin_keys) }
+      passed = pins.flat_map { |pin| Support.machine_paths(pin.input) }.uniq
+      pinned = pins.flat_map do |pin|
+        message = pin.error.exact_message || pin.error.message if pin.error&.stable_message
+        Support.machine_paths_in([Support.pinned_value(pin.value, Support.keys_for(pin, pin_keys)), message])
+      end
+      existing, missing = passed.partition { |path| File.exist?(path) }
+      outside = Support.required_outside_project(result.requires, (target.path unless result.target_required), destination,
+                                                 by_name: require_by_name, load_paths: result.load_paths)
+      varying + outside.map do |path|
+        "branchout: the emitted test requires #{path} by its path, outside the test's project, so the test runs only where " \
+          "that file sits on this machine. Emit the test under that file's project, or #{require_by_name ? "pass -I with a directory that holds it" : "pass --require-by-name"} " \
+          "to require it through the load path."
+      end + existing.map do |path|
+        "branchout: the emitted test passes #{path}, a file of this machine outside the test's project, so the test passes " \
+          "only here. Put the file under the project, and the test names it from its own directory."
+      end + (missing - pinned).map do |path|
+        "branchout: the emitted test passes #{path}, a path of this machine outside the test's project that does not exist, " \
+          "so the test holds this machine's directories. Put the path under the project, and the test names it from its own directory."
+      end + (pinned.uniq - existing).map do |path|
+        "branchout: the emitted test pins #{path}, a path of this machine outside the test's project, in a returned value " \
+          "or an error message, so the test passes only here. Give #{target.entry} a path under the project."
+      end
+    end
+
+    # +pin_keys+ names the keys a returned Hash pins; empty pins it whole.
+    # +from+ is the directory the test's runner starts in.
+    # A file with no pin says whether no input ran or each one differed
+    # between two runs.
+    def render(format, result, target, destination, require_by_name: false, pin_keys: [], from: Dir.pwd)
+      pins = project_pins(result.pins, destination)
+      emitter_for(format).render(result.dup.tap { |copy| copy.pins = pins }, target, destination,
+                                 require_by_name: require_by_name, pin_keys: pin_keys, from: from,
+                                 all_varied: pins.empty? && result.pins.any?)
+    end
+
+    def varying_warning(pin, pin_keys)
+      return "branchout: #{pin.input.text} returned a value that differed between two runs, so it has no test." if pin.varying == :whole
+
+      "branchout: #{pin.input.text} returned a Hash whose #{pin.varying.join(", ")} differed between two runs, " \
+        "so its test pins only #{Support.keys_for(pin, pin_keys).join(", ")}."
+    end
+
+    # Each pin with the paths under the test's project in its input, its
+    # returned value, and its error message written from the test's file.
+    # A pin whose value differed as a whole between two runs is left out.
+    def project_pins(pins, destination)
+      pins.reject { |pin| pin.varying == :whole }.map do |pin|
+        pin.dup.tap do |copy|
+          copy.input = Support.project_paths(pin.input, destination)
+          copy.value = Support.project_value(pin.value, destination)
+          next unless pin.error
+
+          copy.error = pin.error.dup.tap do |error|
+            error.message = Support.project_value(error.message, destination)
+            error.exact_message = Support.project_value(error.exact_message, destination)
+          end
+        end
+      end
+    end
+
+    def emitter_for(format)
+      case resolve(format, nil)
+      when "minitest" then Minitest
+      when "rspec" then RSpec
+      end
+    end
+  end
+end

data/lib/branchout/engine.rb

@@ -0,0 +1,303 @@
+# frozen_string_literal: true
+
+module Branchout
+  class Engine
+    STRATEGY_ORDER = %w[inputs mutate random].freeze
+
+    Result = Struct.new(
+      :command, :target, :entry, :seed, :strategies, :random, :mutate_cap, :timeout,
+      :inputs_ran, :mutate_ran, :random_ran, :executions,
+      :covered, :frontier, :exceptions, :pins, :requires, :load_paths, :target_required, :private_calls, :const_expression, :repeated_inputs, :not_counted,
+      :operators, :memos, :boundaries,
+      :test_class, :spec_type,
+      keyword_init: true
+    )
+
+    # +pinned+ is false for a covered arm whose every input differed as a
+    # whole between two runs, so no emitted test checks it.
+    Arm = Struct.new(
+      :id, :loc, :line, :column, :cond, :sense, :input, :parent, :nearest, :path,
+      :covered, :pinned,
+      keyword_init: true
+    )
+
+    # +covers+ lists the arms the input took, in source order.
+    # +thawed+ marks an input that ran with unfrozen Strings (Runner::Execution).
+    # +varying+ is Runner::Execution's: what differed between two runs.
+    # +listed_at+ is the input's index in the inputs file, or nil for an
+    # input branchout made.
+    Pin = Struct.new(:input, :value, :error, :covers, :thawed, :varying, :listed_at, keyword_init: true)
+    Cover = Struct.new(:line, :column, :text, keyword_init: true)
+    Finding = Struct.new(:class_name, :message, :loc, :input, :stable_message, keyword_init: true)
+
+    def self.explore(target:, entry:, user_inputs: [], type_overrides: nil, seed: 0, strategies: STRATEGY_ORDER,
+                     random: 16, mutate_cap: 256, timeout: 1, command: "explore", requires: [], load_paths: [], host: nil)
+      new(
+        target: target, entry: entry, user_inputs: user_inputs, type_overrides: type_overrides,
+        seed: seed, strategies: strategies, random: random, mutate_cap: mutate_cap,
+        timeout: timeout, command: command, requires: requires, load_paths: load_paths, host: host
+      ).explore
+    end
+
+    def initialize(target:, entry:, user_inputs:, type_overrides:, seed:, strategies:, random:, mutate_cap:, timeout:, command:,
+                   requires:, load_paths:, host:)
+      @host = host
+      @path = target
+      @load_paths = load_paths.map { |dir| File.expand_path(dir) }
+      @requires = requires.map { |name| required_file(name) or raise Error, require_not_found(name) }
+      @entry = entry
+      @user_inputs = user_inputs
+      @type_overrides = type_overrides
+      @seed = seed
+      @strategies = STRATEGY_ORDER.select { |name| strategies.include?(name) }
+      @random = random
+      @mutate_cap = mutate_cap
+      @timeout = timeout
+      @command = command
+    end
+
+    def explore
+      target = Target.parse(@path, @entry, type_overrides: @type_overrides)
+      user = @strategies.include?("inputs") ? @user_inputs : []
+      mutated = @strategies.include?("mutate") ? Mutator.new(target, cap: @mutate_cap).inputs : []
+      randoms = @strategies.include?("random") ? Sampler.new(target, seed: @seed, count: @random).inputs : []
+
+      counts = {inputs: 0, mutate: 0, random: 0}
+      seen = {}
+      plan = []
+      enqueue = lambda do |list, origin|
+        list.each do |input|
+          key = input.canonical
+          next if seen[key]
+
+          seen[key] = true
+          plan << [input, origin]
+          counts[origin] += 1
+        end
+      end
+      enqueue.call(user, :inputs)
+      enqueue.call(mutated, :mutate)
+      enqueue.call(randoms, :random)
+
+      runner = Runner.new(target, timeout: @timeout, requires: @requires, load_paths: @load_paths, host: @host)
+      executions = plan.map do |input, origin|
+        execution = runner.execute(input)
+        execution.origin = origin
+        execution
+      end
+      build_result(target, runner, executions, counts, user.length - counts[:inputs])
+    end
+
+    private
+
+    # A require that names no file under the working directory is a
+    # feature, as Ruby's own -r reads it: found on an -I directory, then on
+    # the load path. Nil when it is found nowhere.
+    def required_file(name)
+      path = File.expand_path(name)
+      return path if File.file?(path) || File.file?("#{path}.rb")
+
+      feature = name.end_with?(".rb") ? name : "#{name}.rb"
+      found = @load_paths.map { |dir| File.join(dir, feature) }.find { |file| File.file?(file) }
+      found || $LOAD_PATH.resolve_feature_path(name)&.last
+    end
+
+    # In the words of the caller: Branchout.explore, the one caller that
+    # passes a host, takes requires: and no -I.
+    def require_not_found(name)
+      if @host
+        "requires: #{name.inspect} names no file under the working directory or on the load path"
+      else
+        "--require #{name} names no file under the working directory, an -I directory, or the load path"
+      end
+    end
+
+    def build_result(target, runner, executions, counts, repeated_inputs)
+      # Coverage lists every branch of the file, taken or not, so the table
+      # from loading holds every arm an execution can report.
+      arms = runner.template[:branches].flat_map do |branch_key, arm_counts|
+        arm_counts.keys.map { |arm_key| make_arm(target, branch_key, arm_key, executions) }
+      end
+      covered = arms.select(&:covered).sort_by(&:id)
+      frontier = arms.reject(&:covered).sort_by(&:id)
+      findings = collect_findings(executions)
+      pins = collect_pins(target, executions, covered, findings)
+
+      Result.new(
+        command: @command,
+        target: target.display_path,
+        entry: target.entry,
+        seed: @seed,
+        strategies: @strategies,
+        random: @random,
+        mutate_cap: @mutate_cap,
+        timeout: @timeout,
+        inputs_ran: counts[:inputs],
+        mutate_ran: counts[:mutate],
+        random_ran: counts[:random],
+        executions: executions.length,
+        covered: covered,
+        frontier: frontier,
+        exceptions: findings,
+        pins: pins,
+        requires: @requires,
+        load_paths: @load_paths,
+        target_required: runner.target_required,
+        private_calls: runner.private_calls,
+        const_expression: runner.const_expression,
+        repeated_inputs: repeated_inputs,
+        not_counted: runner.not_counted,
+        operators: target.operators,
+        memos: target.memos,
+        boundaries: target.boundaries,
+        test_class: @host&.test_class,
+        spec_type: @host&.spec_type
+      )
+    end
+
+    def make_arm(target, branch_key, arm_key, executions)
+      id = format_id(branch_key, arm_key)
+      line = arm_key[2]
+      column = arm_key[3]
+      takers = executions.select { |execution| count_of(execution, branch_key, arm_key).positive? }
+      if takers.empty?
+        nearest = best_execution(executions.select { |execution| parent_reached?(execution, branch_key) })
+        Arm.new(
+          id: id,
+          loc: "#{target.display_path}:#{line}:#{column}",
+          line: line,
+          column: column,
+          cond: target.condition_text(branch_key[0], branch_key[2], branch_key[3], arm_key[0], line, column),
+          sense: sense_for(branch_key[0], arm_key[0]),
+          parent: nearest ? "reached" : "not-reached",
+          nearest: nearest&.input,
+          path: nearest ? path_ids(nearest) : nil,
+          covered: false
+        )
+      else
+        pinnable = takers.reject { |execution| execution.varying == :whole }
+        witness = best_execution(pinnable.empty? ? takers : pinnable)
+        Arm.new(
+          id: id,
+          loc: "#{target.display_path}:#{line}:#{column}",
+          line: line,
+          column: column,
+          cond: target.condition_text(branch_key[0], branch_key[2], branch_key[3], arm_key[0], line, column),
+          sense: sense_for(branch_key[0], arm_key[0]),
+          input: witness.input,
+          parent: "reached",
+          path: path_ids(witness),
+          covered: true,
+          pinned: !pinnable.empty?
+        )
+      end
+    end
+
+    def collect_findings(executions)
+      chosen = {}
+      executions.each do |execution|
+        info = execution.error
+        next unless info
+
+        key = [info.class_name, info.message, info.loc.to_s, execution.input.canonical]
+        chosen[key] ||= execution
+      end
+      chosen.values.sort_by { |execution|
+        info = execution.error
+        [info.class_name, info.loc.to_s, info.message, execution.input.canonical]
+      }.map { |execution|
+        info = execution.error
+        Finding.new(
+          class_name: info.class_name,
+          message: info.message,
+          loc: info.loc,
+          input: execution.input,
+          stable_message: info.stable_message
+        )
+      }
+    end
+
+    def collect_pins(target, executions, covered, findings)
+      by_input = executions.to_h { |execution| [execution.input.canonical, execution] }
+      keys = executions.select { |execution| execution.origin == :inputs }.map { |execution| execution.input.canonical } |
+             covered.map { |arm| arm.input.canonical } | findings.map { |finding| finding.input.canonical }
+      listed = {}
+      @user_inputs.each_with_index { |input, at| listed[input.canonical] ||= at }
+      keys.sort.map do |key|
+        execution = by_input[key]
+        Pin.new(
+          listed_at: (listed[key] if execution.origin == :inputs),
+          input: execution.input,
+          value: execution.value,
+          error: execution.error,
+          covers: covers_of(target, execution),
+          thawed: execution.thawed,
+          varying: execution.varying
+        )
+      end
+    end
+
+    def covers_of(target, execution)
+      covers = execution.branches.flat_map do |branch_key, arms|
+        arms.filter_map do |arm_key, count|
+          next unless count.to_i.positive?
+
+          line, text = target.covered_arm(branch_key[0], branch_key[2], branch_key[3], arm_key[0], arm_key[2], arm_key[3])
+          Cover.new(line: line, column: branch_key[3], text: text)
+        end
+      end
+      covers.sort_by { |cover| [cover.line, cover.column, cover.text] }
+    end
+
+    def best_execution(executions)
+      executions.min_by { |execution| [path_ids(execution).length, execution.input.weight, execution.input.canonical] }
+    end
+
+    def parent_reached?(execution, branch_key)
+      return true if execution.branches[branch_key].values.any? { |count| count.to_i.positive? }
+      return false unless branch_key[0] == :while || branch_key[0] == :until
+
+      line = branch_key[2]
+      execution.lines[line - 1].to_i.positive?
+    end
+
+    def count_of(execution, branch_key, arm_key)
+      execution.branches[branch_key]&.[](arm_key).to_i
+    end
+
+    def path_ids(execution)
+      execution.path_ids ||= path_with_counts(execution)
+    end
+
+    def path_with_counts(execution)
+      ids = []
+      execution.branches.each do |branch_key, arms|
+        arms.each do |arm_key, count|
+          next unless count.to_i.positive?
+
+          id = format_id(branch_key, arm_key)
+          id = "#{id}*#{count}" if count.to_i > 1
+          ids << id
+        end
+      end
+      ids.sort
+    end
+
+    def format_id(branch_key, arm_key)
+      "#{branch_key[0]}@#{branch_key[2]}:#{branch_key[3]}/#{arm_key[0]}@#{arm_key[2]}:#{arm_key[3]}"
+    end
+
+    def sense_for(branch_type, arm_type)
+      case [branch_type, arm_type]
+      when [:if, :then], [:unless, :else] then "cond-true"
+      when [:if, :else], [:unless, :then] then "cond-false"
+      when [:while, :body], [:until, :body] then "body-entered"
+      when [:case, :when], [:case, :in] then "arm-matched"
+      when [:case, :else] then "arm-else"
+      when [:"&.", :then] then "receiver-present"
+      when [:"&.", :else] then "receiver-nil"
+      else "taken"
+      end
+    end
+  end
+end

data/lib/branchout/explore.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Branchout
+  # The test that runs explore from inside a suite. +test+ is the example or
+  # test instance whose helpers, factories, and fixtures the inputs' Ruby
+  # reaches. +around+ runs one input and undoes what it wrote. +test_class+
+  # is the class an emitted minitest file inherits from, and +spec_type+ the
+  # type metadata an emitted RSpec file gives its describe.
+  Host = Data.define(:test, :around, :test_class, :spec_type) do
+    # Runs the block with the test's lets forgotten, then puts them back.
+    # Returns the names of the lets the block read, the names among them
+    # that the test had not made before, and the block's result.
+    # RSpec keeps a let's value in @__memoized, minitest/spec in @_memoized.
+    def lets_read_by
+      holder, name = let_values
+      return [[], [], yield] unless holder
+
+      kept = holder.instance_variable_get(name)
+      holder.instance_variable_set(name, {})
+      result = yield
+      read = holder.instance_variable_get(name).keys
+      [read, read.reject { |let| kept&.key?(let) }, result]
+    ensure
+      holder&.instance_variable_set(name, kept)
+    end
+
+    private
+
+    def let_values
+      rspec = test.instance_variable_get(:@__memoized) if test.instance_variable_defined?(:@__memoized)
+      return [rspec, :@memoized] if rspec&.instance_variable_defined?(:@memoized)
+
+      [test, :@_memoized] if defined?(Minitest::Spec) && test.is_a?(Minitest::Spec)
+    end
+  end
+
+  # The helper files a suite loads before its tests, most specific first.
+  # An emitted test requires the first one loaded from the working directory.
+  HELPERS = %w[rails_helper.rb test_helper.rb spec_helper.rb].freeze
+
+  # Each input runs in a savepoint that is rolled back, so no input sees the
+  # rows another wrote, and the example's own rows are left as they were.
+  # It marks the input as rolled back, also when an around calls it, so the
+  # Runner knows that a second run of the input starts from the same rows.
+  ROLL_BACK = lambda do |&input|
+    ActiveRecord::Base.transaction(requires_new: true) do
+      Thread.current[:branchout_rolled_back] = true
+      input.call
+      raise ActiveRecord::Rollback
+    end
+  end
+
+  RUN_ONLY = ->(&input) { input.call }
+
+  module_function
+
+  # Explores +entry+ ("Loan#approve" or "Room.open!") from inside a test, and
+  # returns the report the CLI prints. +within+ is the running test: pass
+  # self. +inputs+ is the inputs file's list, or its path. The target is the
+  # file that defines the entry, unless +target+ names it.
+  def explore(entry, within:, inputs:, target: nil, emit_tests: nil, format: nil, json: false, verbose: false, pin_keys: [],
+              requires: nil, around: nil, strategies: %w[inputs], seed: 0, random: 16, mutate_cap: 256, timeout: 1)
+    if verbose && json
+      raise Error, "verbose: is only valid with the text report; json: true holds every arm and the run settings"
+    end
+    target ||= defining_file(entry)
+    user_inputs, type_overrides = inputs.is_a?(String) ? InputFile.load(inputs) : [parse_inputs(inputs), nil]
+    requires = requires.nil? ? [loaded_helper].compact : Array(requires)
+    format = Emitters.resolve(format&.to_s, emit_tests) if emit_tests
+    if emit_tests && requires.empty?
+      raise Error, "the suite loaded no #{HELPERS.join(", ")} from #{Dir.pwd}, so an emitted test would not load the app; " \
+                   "pass the helper file as requires:"
+    end
+    host = Host.new(test: within, around: around || (defined?(ActiveRecord::Base) ? ROLL_BACK : RUN_ONLY),
+                    test_class: within.class.superclass.name, spec_type: spec_type_of(within))
+    result = Engine.explore(
+      target: target, entry: entry, user_inputs: user_inputs, type_overrides: type_overrides, seed: seed,
+      strategies: strategies, random: random, mutate_cap: mutate_cap, timeout: timeout, requires: requires, host: host
+    )
+    if emit_tests
+      warnings = Emitters.write(format, result, Target.parse(target, entry, type_overrides: type_overrides),
+                                emit_tests, require_by_name: true, pin_keys: pin_keys)
+      warnings.each { |line| $stderr.puts line }
+    end
+    json ? Report.json(result, pin_keys: pin_keys) : Report.text(result, tests: emit_tests, verbose: verbose, verbose_flag: "verbose: true", pin_keys: pin_keys)
+  end
+
+  def defining_file(entry)
+    match = /\A(.+?)([.#])([^.#]+)\z/.match(entry)
+    raise Error, "entry #{entry.inspect} is not Const.method or Const#method" unless match
+
+    const = Object.const_get(match[1])
+    method = match[2] == "." ? const.method(match[3]) : const.instance_method(match[3])
+    file, = method.source_location
+    raise Error, "#{entry} has no Ruby source file; pass target:" unless file && File.file?(file)
+
+    file
+  rescue NameError => e
+    raise Error, "cannot resolve #{entry}: #{e.message.lines.first.strip}"
+  end
+
+  # Symbol keys and JSON values come back as an inputs file would give them.
+  def parse_inputs(inputs)
+    raise Error, "inputs must be a list or an inputs file path" unless inputs.is_a?(Array)
+
+    JSON.parse(JSON.generate(inputs)).each_with_index.map { |entry, index| InputFile.parse_entry(entry, "inputs[#{index}]") }
+  end
+
+  def loaded_helper
+    root = "#{Dir.pwd}/"
+    loaded = $LOADED_FEATURES.select { |path| path.start_with?(root) }
+    HELPERS.each do |name|
+      found = loaded.find { |path| File.basename(path) == name }
+      return found if found
+    end
+    nil
+  end
+
+  def spec_type_of(test)
+    test.class.metadata[:type] if test.class.respond_to?(:metadata)
+  end
+
+  private_class_method :spec_type_of, :defining_file, :parse_inputs, :loaded_helper
+end