branchout 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/AGENTS.md +38 -0
- data/LICENSE +21 -0
- data/README.ja.md +903 -0
- data/README.md +586 -0
- data/bin/branchout +15 -0
- data/examples/arithmetic.rb +24 -0
- data/examples/hidden_bug.rb +17 -0
- data/examples/inputs/arithmetic.json +10 -0
- data/examples/inputs/arithmetic.seed.json +5 -0
- data/examples/inputs/hidden_bug.json +9 -0
- data/examples/inputs/hidden_bug.seed.json +5 -0
- data/examples/inputs/magic_value.json +7 -0
- data/examples/inputs/magic_value.seed.json +5 -0
- data/examples/inputs/version_parser.json +11 -0
- data/examples/inputs/version_parser.seed.json +5 -0
- data/examples/magic_value.rb +11 -0
- data/examples/version_parser.rb +29 -0
- data/lib/branchout/canon.rb +59 -0
- data/lib/branchout/cli.rb +302 -0
- data/lib/branchout/coverage.rb +9 -0
- data/lib/branchout/coverage_session.rb +49 -0
- data/lib/branchout/emitters/minitest.rb +92 -0
- data/lib/branchout/emitters/rspec.rb +532 -0
- data/lib/branchout/emitters/support.rb +1248 -0
- data/lib/branchout/emitters.rb +101 -0
- data/lib/branchout/engine.rb +303 -0
- data/lib/branchout/explore.rb +124 -0
- data/lib/branchout/input.rb +215 -0
- data/lib/branchout/input_file.rb +125 -0
- data/lib/branchout/mutator.rb +124 -0
- data/lib/branchout/prism_choice.rb +25 -0
- data/lib/branchout/report.rb +449 -0
- data/lib/branchout/rng.rb +31 -0
- data/lib/branchout/runner.rb +591 -0
- data/lib/branchout/sampler.rb +70 -0
- data/lib/branchout/target.rb +696 -0
- data/lib/branchout/version.rb +8 -0
- data/lib/branchout.rb +31 -0
- data/skills/branchout/SKILL.md +230 -0
- data/skills/branchout/inputs.example.json +10 -0
- metadata +114 -0
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Arithmetic
|
|
4
|
+
# (3x + 7) % 1000 == 421 is true for x = 138.
|
|
5
|
+
# y == x + 5 then forces y = 143, and z == 7 is a third nested decision.
|
|
6
|
+
# x == 114514 && x == 0 is unsatisfiable; that arm stays uncovered.
|
|
7
|
+
def self.classify(x, y, z)
|
|
8
|
+
if (x * 3 + 7) % 1000 == 421
|
|
9
|
+
if y == x + 5
|
|
10
|
+
if z == 7
|
|
11
|
+
:hit
|
|
12
|
+
else
|
|
13
|
+
:need_z
|
|
14
|
+
end
|
|
15
|
+
else
|
|
16
|
+
:x_only
|
|
17
|
+
end
|
|
18
|
+
elsif x == 114514 && x == 0
|
|
19
|
+
:impossible
|
|
20
|
+
else
|
|
21
|
+
:miss
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module HiddenBug
|
|
4
|
+
# ZeroDivisionError only when a > 10, b > 10, and a - b * 2 == 0.
|
|
5
|
+
def self.scale(a, b)
|
|
6
|
+
if a > 10 && b > 10
|
|
7
|
+
denom = a - (b * 2)
|
|
8
|
+
if denom < 50
|
|
9
|
+
1000 / denom
|
|
10
|
+
else
|
|
11
|
+
:large_gap
|
|
12
|
+
end
|
|
13
|
+
else
|
|
14
|
+
:out_of_range
|
|
15
|
+
end
|
|
16
|
+
end
|
|
17
|
+
end
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module VersionParser
|
|
4
|
+
def self.parse(text)
|
|
5
|
+
if text.empty?
|
|
6
|
+
:blank
|
|
7
|
+
elsif text.include?("-")
|
|
8
|
+
_version, tag = text.split("-", 2)
|
|
9
|
+
if tag.start_with?("rc")
|
|
10
|
+
:rc
|
|
11
|
+
elsif tag.start_with?("beta")
|
|
12
|
+
:beta
|
|
13
|
+
else
|
|
14
|
+
:pre
|
|
15
|
+
end
|
|
16
|
+
elsif release?(text)
|
|
17
|
+
:release
|
|
18
|
+
else
|
|
19
|
+
:other
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def self.release?(text)
|
|
24
|
+
parts = text.split(".")
|
|
25
|
+
return false unless parts.length == 3
|
|
26
|
+
|
|
27
|
+
parts.all? { |part| part.match?(/\A[0-9]+\z/) }
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Branchout
|
|
4
|
+
# Byte-stable JSON. Key order is sorted, floats have a fixed spelling,
|
|
5
|
+
# and non-ASCII is escaped so the bytes do not depend on a JSON mode flag.
|
|
6
|
+
module Canon
|
|
7
|
+
module_function
|
|
8
|
+
|
|
9
|
+
def json(obj)
|
|
10
|
+
case obj
|
|
11
|
+
when nil then "null"
|
|
12
|
+
when true then "true"
|
|
13
|
+
when false then "false"
|
|
14
|
+
when Integer then obj.to_s
|
|
15
|
+
when Float then format_float(obj)
|
|
16
|
+
when String then JSON.generate(obj, ascii_only: true)
|
|
17
|
+
when Symbol then JSON.generate(obj.to_s, ascii_only: true)
|
|
18
|
+
when Code then json({"ruby" => obj.source})
|
|
19
|
+
when Array
|
|
20
|
+
"[#{obj.map { |item| json(item) }.join(",")}]"
|
|
21
|
+
when Hash
|
|
22
|
+
pairs = obj.map { |key, value| [key.to_s, value] }.sort_by { |key, _| key }
|
|
23
|
+
"{#{pairs.map { |key, value| "#{json(key)}:#{json(value)}" }.join(",")}}"
|
|
24
|
+
else
|
|
25
|
+
raise Error, "cannot encode #{obj.class} as canonical JSON"
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Float#to_s is the shortest text that reads back as the same float in
|
|
30
|
+
# Ruby and in JSON. A fixed precision is not: %.16g printed 0.1 + 0.2 as
|
|
31
|
+
# 0.3, and the largest float as a number that parses to Infinity.
|
|
32
|
+
def format_float(value)
|
|
33
|
+
raise Error, "non-finite float" unless value.finite?
|
|
34
|
+
return "0.0" if value.zero?
|
|
35
|
+
|
|
36
|
+
value.to_s
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def pretty(obj, indent = 0)
|
|
40
|
+
pad = " " * indent
|
|
41
|
+
case obj
|
|
42
|
+
when Hash
|
|
43
|
+
return "{}" if obj.empty?
|
|
44
|
+
|
|
45
|
+
inner = obj.map do |key, value|
|
|
46
|
+
"#{" " * (indent + 1)}#{json(key.to_s)}: #{pretty(value, indent + 1)}"
|
|
47
|
+
end
|
|
48
|
+
"{\n#{inner.join(",\n")}\n#{pad}}"
|
|
49
|
+
when Array
|
|
50
|
+
return "[]" if obj.empty?
|
|
51
|
+
|
|
52
|
+
inner = obj.map { |value| "#{" " * (indent + 1)}#{pretty(value, indent + 1)}" }
|
|
53
|
+
"[\n#{inner.join(",\n")}\n#{pad}]"
|
|
54
|
+
else
|
|
55
|
+
json(obj)
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
59
|
+
end
|
|
@@ -0,0 +1,302 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "fileutils"
|
|
4
|
+
|
|
5
|
+
module Branchout
|
|
6
|
+
module CLI
|
|
7
|
+
HELP = <<~TEXT
|
|
8
|
+
branchout #{VERSION}
|
|
9
|
+
|
|
10
|
+
Usage:
|
|
11
|
+
branchout explore TARGET ENTRY [options]
|
|
12
|
+
branchout baselines TARGET ENTRY [options]
|
|
13
|
+
|
|
14
|
+
ENTRY is Const.method or Const#method. --entry ENTRY is accepted too.
|
|
15
|
+
|
|
16
|
+
The tool does not call an LLM. It executes the inputs you pass, plus
|
|
17
|
+
optional literal mutation and seeded random samples, and prints the
|
|
18
|
+
branch arms that are still uncovered.
|
|
19
|
+
|
|
20
|
+
Options:
|
|
21
|
+
--inputs PATH JSON file of candidate inputs
|
|
22
|
+
--require PATH load PATH, or a feature on -I or the load path, before TARGET (repeatable)
|
|
23
|
+
-I DIR put DIR on the load path (repeatable)
|
|
24
|
+
--seed N RNG seed (default 0)
|
|
25
|
+
--random N random samples (default 16)
|
|
26
|
+
--mutate-cap N literal-mutation budget (default 256)
|
|
27
|
+
--strategy LIST comma list: inputs,mutate,random (default all)
|
|
28
|
+
--timeout SEC per-input timeout (default 1)
|
|
29
|
+
--verbose print the run settings and every arm in the text report
|
|
30
|
+
--json print JSON on stdout instead of text
|
|
31
|
+
--json-out PATH write JSON to a file
|
|
32
|
+
--md-out PATH write the text report as Markdown
|
|
33
|
+
--emit-tests PATH write characterization tests (explore only)
|
|
34
|
+
--format NAME minitest or rspec (default minitest; _spec.rb infers rspec)
|
|
35
|
+
--pin-keys LIST comma list of keys: a returned Hash pins only these keys
|
|
36
|
+
--require-by-name emitted tests require the --require files and the target
|
|
37
|
+
through the load path, named from the first -I directory or
|
|
38
|
+
--require file's directory that holds each (with neither, the
|
|
39
|
+
nearest lib directory above the target); a file none holds is
|
|
40
|
+
required by path, and the test and stderr say so; on by
|
|
41
|
+
default for a test under a .jitcatch directory
|
|
42
|
+
--help
|
|
43
|
+
--version
|
|
44
|
+
TEXT
|
|
45
|
+
|
|
46
|
+
module_function
|
|
47
|
+
|
|
48
|
+
def start(argv)
|
|
49
|
+
run(argv)
|
|
50
|
+
rescue Error => e
|
|
51
|
+
$stderr.puts "branchout: #{e.message}"
|
|
52
|
+
1
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
def run(argv)
|
|
56
|
+
args = argv.map(&:dup)
|
|
57
|
+
if args.empty? || args == ["--help"] || args == ["-h"]
|
|
58
|
+
puts HELP
|
|
59
|
+
return 0
|
|
60
|
+
end
|
|
61
|
+
if args == ["--version"] || args == ["-v"]
|
|
62
|
+
puts "branchout #{VERSION}"
|
|
63
|
+
return 0
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
command = args.shift
|
|
67
|
+
raise Error, "unknown command #{command.inspect}" unless %w[explore baselines].include?(command)
|
|
68
|
+
|
|
69
|
+
opts = {
|
|
70
|
+
seed: 0,
|
|
71
|
+
random: 16,
|
|
72
|
+
mutate_cap: 256,
|
|
73
|
+
timeout: 1,
|
|
74
|
+
strategies: Engine::STRATEGY_ORDER.dup,
|
|
75
|
+
json: false,
|
|
76
|
+
verbose: false,
|
|
77
|
+
inputs: nil,
|
|
78
|
+
entry: nil,
|
|
79
|
+
json_out: nil,
|
|
80
|
+
md_out: nil,
|
|
81
|
+
emit_tests: nil,
|
|
82
|
+
format: nil,
|
|
83
|
+
require_by_name: false,
|
|
84
|
+
pin_keys: [],
|
|
85
|
+
requires: [],
|
|
86
|
+
load_paths: []
|
|
87
|
+
}
|
|
88
|
+
positionals = []
|
|
89
|
+
until args.empty?
|
|
90
|
+
arg = args.shift
|
|
91
|
+
case arg
|
|
92
|
+
when "--inputs" then opts[:inputs] = take(args, arg)
|
|
93
|
+
when "--require" then opts[:requires] << take(args, arg)
|
|
94
|
+
when "-I" then opts[:load_paths] << take(args, arg)
|
|
95
|
+
when /\A-I(.+)/ then opts[:load_paths] << Regexp.last_match(1)
|
|
96
|
+
when "--seed" then opts[:seed] = parse_int(take(args, arg), "seed")
|
|
97
|
+
when "--random" then opts[:random] = parse_int(take(args, arg), "random")
|
|
98
|
+
when "--mutate-cap" then opts[:mutate_cap] = parse_int(take(args, arg), "mutate-cap")
|
|
99
|
+
when "--timeout" then opts[:timeout] = parse_timeout(take(args, arg))
|
|
100
|
+
when "--strategy" then opts[:strategies] = parse_strategies(take(args, arg))
|
|
101
|
+
when "--entry" then opts[:entry] = take(args, arg)
|
|
102
|
+
when "--json" then opts[:json] = true
|
|
103
|
+
when "--verbose" then opts[:verbose] = true
|
|
104
|
+
when "--json-out" then opts[:json_out] = take(args, arg)
|
|
105
|
+
when "--md-out" then opts[:md_out] = take(args, arg)
|
|
106
|
+
when "--emit-tests" then opts[:emit_tests] = take(args, arg)
|
|
107
|
+
when "--format" then opts[:format] = take(args, arg)
|
|
108
|
+
when "--require-by-name" then opts[:require_by_name] = true
|
|
109
|
+
when "--pin-keys" then opts[:pin_keys] = parse_keys(take(args, arg))
|
|
110
|
+
when "--help", "-h"
|
|
111
|
+
puts HELP
|
|
112
|
+
return 0
|
|
113
|
+
else
|
|
114
|
+
raise Error, "unknown option #{arg}" if arg.start_with?("-")
|
|
115
|
+
|
|
116
|
+
positionals << arg
|
|
117
|
+
end
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
target = positionals.shift
|
|
121
|
+
entry = opts[:entry] || positionals.shift
|
|
122
|
+
raise Error, "missing TARGET" unless target
|
|
123
|
+
raise Error, "missing ENTRY" unless entry
|
|
124
|
+
raise Error, "unexpected arguments: #{positionals.join(" ")}" unless positionals.empty?
|
|
125
|
+
raise Error, "seed must be >= 0" if opts[:seed].negative?
|
|
126
|
+
raise Error, "random count must be >= 0" if opts[:random].negative?
|
|
127
|
+
raise Error, "mutate-cap must be >= 0" if opts[:mutate_cap].negative?
|
|
128
|
+
if opts[:emit_tests] && command != "explore"
|
|
129
|
+
raise Error, "--emit-tests is only valid with explore"
|
|
130
|
+
end
|
|
131
|
+
raise Error, "--verbose is only valid with explore" if opts[:verbose] && command != "explore"
|
|
132
|
+
if opts[:verbose] && opts[:json]
|
|
133
|
+
raise Error, "--verbose is only valid with the text report; --json holds every arm and the run settings"
|
|
134
|
+
end
|
|
135
|
+
if opts[:require_by_name]
|
|
136
|
+
raise Error, "--require-by-name requires --emit-tests" unless opts[:emit_tests]
|
|
137
|
+
end
|
|
138
|
+
raise Error, "--pin-keys requires --emit-tests" if !opts[:pin_keys].empty? && !opts[:emit_tests]
|
|
139
|
+
if opts[:format] && !opts[:emit_tests]
|
|
140
|
+
raise Error, "--format requires --emit-tests"
|
|
141
|
+
end
|
|
142
|
+
if opts[:emit_tests]
|
|
143
|
+
opts[:format] = Emitters.resolve(opts[:format], opts[:emit_tests])
|
|
144
|
+
# jitcatch copies a test under .jitcatch/ to the root of each
|
|
145
|
+
# checkout, and its rerun line loads the test where it was written.
|
|
146
|
+
# A require_relative names a different file from each place.
|
|
147
|
+
opts[:require_by_name] ||= File.expand_path(opts[:emit_tests]).split("/").include?(".jitcatch")
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
user_inputs = []
|
|
151
|
+
type_overrides = nil
|
|
152
|
+
if opts[:inputs]
|
|
153
|
+
user_inputs, type_overrides = InputFile.load(opts[:inputs])
|
|
154
|
+
InputFile.quoted_strings(user_inputs).each { |line| $stderr.puts "branchout: #{line}" }
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
case command
|
|
158
|
+
when "explore"
|
|
159
|
+
result = Engine.explore(
|
|
160
|
+
target: target,
|
|
161
|
+
entry: entry,
|
|
162
|
+
user_inputs: user_inputs,
|
|
163
|
+
type_overrides: type_overrides,
|
|
164
|
+
seed: opts[:seed],
|
|
165
|
+
strategies: opts[:strategies],
|
|
166
|
+
random: opts[:random],
|
|
167
|
+
mutate_cap: opts[:mutate_cap],
|
|
168
|
+
timeout: opts[:timeout],
|
|
169
|
+
requires: opts[:requires],
|
|
170
|
+
load_paths: opts[:load_paths]
|
|
171
|
+
)
|
|
172
|
+
if result.repeated_inputs.positive?
|
|
173
|
+
$stderr.puts "branchout: #{result.repeated_inputs} of #{user_inputs.length} inputs repeat an earlier input, " \
|
|
174
|
+
"so they did not run again."
|
|
175
|
+
end
|
|
176
|
+
if result.covered.empty? && result.frontier.empty? && !result.operators.empty?
|
|
177
|
+
$stderr.puts "branchout: #{entry} and the methods it calls in #{result.target} have no arm that branch coverage counts, " \
|
|
178
|
+
"but it branches at the operators OPERATORS lists, and coverage does not see which side of " \
|
|
179
|
+
"||, &&, ||=, or &&= ran. Give inputs that make each left side truthy and falsy. " \
|
|
180
|
+
"--emit-tests still pins the return value or error of each input."
|
|
181
|
+
elsif result.covered.empty? && result.frontier.empty? && !Array(result.memos).empty?
|
|
182
|
+
memos = result.memos.map { |found| "`#{found.text}` at #{found.loc}" }
|
|
183
|
+
$stderr.puts "branchout: #{entry} and the methods it calls in #{result.target} have no arm that branch coverage counts, " \
|
|
184
|
+
"and their only #{memos.one? ? "branch is the memo" : "branches are the memos"} #{memos.join(", ")}. " \
|
|
185
|
+
"#{memos.one? ? "Its cached side runs" : "The cached side of each runs"} only on a second call on the same object, " \
|
|
186
|
+
"which no input makes. Explore the methods #{memos.one? ? "its right side calls" : "their right sides call"} " \
|
|
187
|
+
"(NOT COUNTED lists those in other files), and test #{memos.one? ? "the memo" : "each memo"} by hand: call the method twice on one object and check that the second call returns " \
|
|
188
|
+
"the first value without running the right side again. " \
|
|
189
|
+
"--emit-tests still pins the return value or error of each input."
|
|
190
|
+
elsif result.covered.empty? && result.frontier.empty?
|
|
191
|
+
$stderr.puts "branchout: #{entry} and the methods it calls in #{result.target} have no branches. " \
|
|
192
|
+
"Only a bare call, a self. call, and a call on new are followed; target the method that branches. " \
|
|
193
|
+
"--emit-tests still pins the return value or error of each input."
|
|
194
|
+
end
|
|
195
|
+
write_optional(opts[:json_out], Report.json(result, pin_keys: opts[:pin_keys]))
|
|
196
|
+
write_optional(opts[:md_out], Report.markdown(result))
|
|
197
|
+
if opts[:emit_tests]
|
|
198
|
+
parsed = Target.parse(target, entry, type_overrides: type_overrides)
|
|
199
|
+
format = Emitters.resolve(opts[:format], opts[:emit_tests])
|
|
200
|
+
warnings = Emitters.write(format, result, parsed, opts[:emit_tests], require_by_name: opts[:require_by_name],
|
|
201
|
+
pin_keys: opts[:pin_keys])
|
|
202
|
+
warnings.each { |line| $stderr.puts line }
|
|
203
|
+
warn_unnamed(result, target, opts[:emit_tests]) if opts[:require_by_name]
|
|
204
|
+
end
|
|
205
|
+
$stdout.write(opts[:json] ? Report.json(result, pin_keys: opts[:pin_keys]) : Report.text(result, tests: opts[:emit_tests], verbose: opts[:verbose], pin_keys: opts[:pin_keys]))
|
|
206
|
+
when "baselines"
|
|
207
|
+
info = {
|
|
208
|
+
target: Target.display_path(File.expand_path(target)),
|
|
209
|
+
entry: entry,
|
|
210
|
+
seed: opts[:seed],
|
|
211
|
+
random: opts[:random],
|
|
212
|
+
mutate_cap: opts[:mutate_cap],
|
|
213
|
+
timeout: opts[:timeout]
|
|
214
|
+
}
|
|
215
|
+
rows = [
|
|
216
|
+
["random", run_strategy(target, entry, user_inputs, type_overrides, opts, ["random"])],
|
|
217
|
+
["mutate", run_strategy(target, entry, user_inputs, type_overrides, opts, ["mutate"])],
|
|
218
|
+
["agent", run_strategy(target, entry, user_inputs, type_overrides, opts, ["inputs"])]
|
|
219
|
+
]
|
|
220
|
+
stdout = opts[:json] ? Report.baselines_json(info, rows) : Report.baselines_text(info, rows)
|
|
221
|
+
$stdout.write(stdout)
|
|
222
|
+
write_optional(opts[:json_out], Report.baselines_json(info, rows))
|
|
223
|
+
write_optional(opts[:md_out], Report.baselines_markdown(info, rows))
|
|
224
|
+
end
|
|
225
|
+
0
|
|
226
|
+
end
|
|
227
|
+
|
|
228
|
+
def run_strategy(target, entry, user_inputs, type_overrides, opts, strategies)
|
|
229
|
+
Engine.explore(
|
|
230
|
+
target: target,
|
|
231
|
+
entry: entry,
|
|
232
|
+
user_inputs: user_inputs,
|
|
233
|
+
type_overrides: type_overrides,
|
|
234
|
+
seed: opts[:seed],
|
|
235
|
+
strategies: strategies,
|
|
236
|
+
random: opts[:random],
|
|
237
|
+
mutate_cap: opts[:mutate_cap],
|
|
238
|
+
timeout: opts[:timeout],
|
|
239
|
+
command: "baselines",
|
|
240
|
+
requires: opts[:requires],
|
|
241
|
+
load_paths: opts[:load_paths]
|
|
242
|
+
)
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# A file outside the test's project has had its warning from
|
|
246
|
+
# Emitters.write already.
|
|
247
|
+
def warn_unnamed(result, target, destination)
|
|
248
|
+
required = (target unless result.target_required)
|
|
249
|
+
unnamed = Emitters::Support.unnamed_requires(result.requires, required, load_paths: result.load_paths).map { |path| File.expand_path(path) } -
|
|
250
|
+
Emitters::Support.required_outside_project(result.requires, required, destination)
|
|
251
|
+
unnamed.each do |path|
|
|
252
|
+
$stderr.puts "branchout: #{Target.display_path(path)} is under no -I or --require directory, so the emitted test requires it by path. " \
|
|
253
|
+
"Pass -I with a directory that holds it to require it by name."
|
|
254
|
+
end
|
|
255
|
+
end
|
|
256
|
+
|
|
257
|
+
def write_optional(path, content)
|
|
258
|
+
return unless path
|
|
259
|
+
|
|
260
|
+
FileUtils.mkdir_p(File.dirname(File.expand_path(path)))
|
|
261
|
+
File.write(path, content)
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
def take(args, flag)
|
|
265
|
+
value = args.shift
|
|
266
|
+
raise Error, "#{flag} needs a value" unless value
|
|
267
|
+
|
|
268
|
+
value
|
|
269
|
+
end
|
|
270
|
+
|
|
271
|
+
def parse_int(text, label)
|
|
272
|
+
Integer(text)
|
|
273
|
+
rescue ArgumentError
|
|
274
|
+
raise Error, "#{label} must be an integer"
|
|
275
|
+
end
|
|
276
|
+
|
|
277
|
+
def parse_timeout(text)
|
|
278
|
+
value = Float(text)
|
|
279
|
+
raise Error, "timeout must be > 0" unless value.positive?
|
|
280
|
+
|
|
281
|
+
value == value.to_i ? value.to_i : value
|
|
282
|
+
rescue ArgumentError
|
|
283
|
+
raise Error, "timeout must be a number"
|
|
284
|
+
end
|
|
285
|
+
|
|
286
|
+
def parse_keys(text)
|
|
287
|
+
keys = text.split(",").map(&:strip).reject(&:empty?)
|
|
288
|
+
raise Error, "--pin-keys list is empty" if keys.empty?
|
|
289
|
+
|
|
290
|
+
keys
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
def parse_strategies(text)
|
|
294
|
+
names = text.split(",").map(&:strip).reject(&:empty?)
|
|
295
|
+
unknown = names - Engine::STRATEGY_ORDER
|
|
296
|
+
raise Error, "unknown strategy #{unknown.join(", ")}" unless unknown.empty?
|
|
297
|
+
raise Error, "strategy list is empty" if names.empty?
|
|
298
|
+
|
|
299
|
+
Engine::STRATEGY_ORDER.select { |name| names.include?(name) }
|
|
300
|
+
end
|
|
301
|
+
end
|
|
302
|
+
end
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# The test file that explores requires this file first, before its helper
|
|
4
|
+
# loads the app, so that explore can count the arms of the app's files.
|
|
5
|
+
# Requiring branchout itself does not start coverage: an app whose Gemfile
|
|
6
|
+
# lists branchout without require: false loads it in every process.
|
|
7
|
+
require_relative "coverage_session"
|
|
8
|
+
|
|
9
|
+
Branchout::CoverageSession.install
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "coverage"
|
|
4
|
+
|
|
5
|
+
module Branchout
|
|
6
|
+
# Coverage counts a file's arms only when the file loads after Coverage
|
|
7
|
+
# was set up. Setup leaves measurement suspended: explore resumes it
|
|
8
|
+
# around each call.
|
|
9
|
+
module CoverageSession
|
|
10
|
+
LIB = File.join(File.expand_path("..", __dir__), "")
|
|
11
|
+
|
|
12
|
+
# Many suites start SimpleCov unconditionally at the top of a helper.
|
|
13
|
+
# From a helper that loads after the session, its Coverage.start would
|
|
14
|
+
# raise "coverage measurement is already setup", so the session keeps
|
|
15
|
+
# measuring, and the tool reads the session's result when it reports.
|
|
16
|
+
module LaterStart
|
|
17
|
+
def setup(...) = (super unless CoverageSession.installed?)
|
|
18
|
+
|
|
19
|
+
def start(...) = (super unless CoverageSession.installed?)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
module_function
|
|
23
|
+
|
|
24
|
+
# Coverage that another tool started earlier measures lines only, so
|
|
25
|
+
# the session stops it and sets Coverage up again with branches.
|
|
26
|
+
def install
|
|
27
|
+
return if installed?
|
|
28
|
+
|
|
29
|
+
Coverage.result unless Coverage.state == :idle
|
|
30
|
+
Coverage.setup(lines: true, branches: true)
|
|
31
|
+
Coverage.singleton_class.prepend(LaterStart)
|
|
32
|
+
@installed = true
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def installed? = @installed == true
|
|
36
|
+
|
|
37
|
+
# A file of branchout's own lib loads with the tool, so it has arms only
|
|
38
|
+
# when coverage was set up before the tool loaded. Only a run that names
|
|
39
|
+
# one sets it up that early: each call reads the coverage of every file
|
|
40
|
+
# loaded after setup, and the tool's own files made that 1.3 ms a call
|
|
41
|
+
# instead of 0.04 ms.
|
|
42
|
+
def names_own_file?(argv, from: Dir.pwd)
|
|
43
|
+
argv.any? do |arg|
|
|
44
|
+
path = File.expand_path(arg, from)
|
|
45
|
+
File.file?(path) && path.start_with?(LIB)
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Branchout
|
|
4
|
+
module Emitters
|
|
5
|
+
module Minitest
|
|
6
|
+
module_function
|
|
7
|
+
|
|
8
|
+
def render(result, target, destination, require_by_name: false, pin_keys: [], from: Dir.pwd, all_varied: false)
|
|
9
|
+
lines = Support.header(target, "tests", destination)
|
|
10
|
+
lines << ""
|
|
11
|
+
lines << 'require "minitest/autorun"'
|
|
12
|
+
lines.concat(Support.require_lines(result.requires, (target.path unless result.target_required), destination,
|
|
13
|
+
by_name: require_by_name, load_paths: result.load_paths, from: from))
|
|
14
|
+
lines << ""
|
|
15
|
+
lines << "class #{Support.class_name_for(target)} < #{result.test_class || "Minitest::Test"}"
|
|
16
|
+
if result.pins.empty?
|
|
17
|
+
lines << " def #{all_varied ? "test_no_input_pinned" : "test_no_inputs_discovered"}"
|
|
18
|
+
lines << " skip #{Support.no_pin_reason(all_varied).inspect}"
|
|
19
|
+
lines << " end"
|
|
20
|
+
else
|
|
21
|
+
names = Support.test_names(result.pins, pin_keys)
|
|
22
|
+
result.pins.each_with_index do |pin, index|
|
|
23
|
+
lines << "" unless index.zero?
|
|
24
|
+
lines.concat(example_lines(target, pin, names[index], result.private_calls, pin_keys, result.const_expression || target.const_name))
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
lines << "end"
|
|
28
|
+
"#{lines.join("\n")}\n"
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def example_lines(target, pin, name, private_calls, pin_keys, const = target.const_name)
|
|
32
|
+
lines = [" def #{name}"]
|
|
33
|
+
lines.concat(Support.cover_comment(pin, " "))
|
|
34
|
+
lines.concat(Support.thawed_comment(pin, " "))
|
|
35
|
+
arrange, call = Support.arrange(target, pin.input, private_calls: private_calls, const: const, thawed: pin.thawed)
|
|
36
|
+
if pin.error
|
|
37
|
+
lines << " error = assert_raises(#{pin.error.class_name}) do"
|
|
38
|
+
lines.concat(Support.indent(arrange + [call], " "))
|
|
39
|
+
lines << " end"
|
|
40
|
+
if pin.error.name
|
|
41
|
+
lines << " assert_equal #{pin.error.name.inspect}, error.name"
|
|
42
|
+
elsif pin.error.stable_message
|
|
43
|
+
lines << " assert_equal #{Support.ruby_literal(pin.error.exact_message || pin.error.message)}, error.message"
|
|
44
|
+
elsif pin.error.message_pattern
|
|
45
|
+
lines << " assert_match(#{Regexp.new(pin.error.message_pattern).inspect}, error.message)"
|
|
46
|
+
end
|
|
47
|
+
return lines << " end"
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
lines.concat(Support.indent(arrange, " "))
|
|
51
|
+
pinned = Support.pinned_value(pin.value, Support.keys_for(pin, pin_keys))
|
|
52
|
+
if !pinned.equal?(pin.value) && Support.literal?(pinned)
|
|
53
|
+
keys = pinned.keys.map { |key| Support.ruby_literal(key) }.join(", ")
|
|
54
|
+
lines << " assert_equal(#{Support.ruby_literal(pinned)}, #{call}.slice(#{keys}))"
|
|
55
|
+
elsif !pinned.equal?(pin.value)
|
|
56
|
+
lines.concat(part_lines(pinned, call))
|
|
57
|
+
elsif pin.value.nil?
|
|
58
|
+
# Minitest 6 fails assert_equal with an expected nil.
|
|
59
|
+
lines << " assert_nil #{call}"
|
|
60
|
+
elsif (pattern = Support.frame_pattern(pin.value))
|
|
61
|
+
lines << " assert_match(#{pattern}, #{call})"
|
|
62
|
+
elsif pin.value.is_a?(Hash) && Support.literal?(pin.value)
|
|
63
|
+
# Without parentheses Ruby reads a leading { as a block.
|
|
64
|
+
lines << " assert_equal(#{Support.ruby_literal(pin.value)}, #{call})"
|
|
65
|
+
elsif Support.literal?(pin.value)
|
|
66
|
+
lines << " assert_equal #{Support.ruby_literal(pin.value)}, #{call}"
|
|
67
|
+
elsif Support.class_pins(pin.value) == [["", Support.class_literal(pin.value)]]
|
|
68
|
+
lines << " assert_kind_of #{Support.class_literal(pin.value)}, #{call}"
|
|
69
|
+
else
|
|
70
|
+
lines << " value = #{call}"
|
|
71
|
+
lines.concat(Support.shape_assertions(pin.value).map { |line| " #{line}" })
|
|
72
|
+
end
|
|
73
|
+
lines << " end"
|
|
74
|
+
lines
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# The stable keys of a Hash with no literal: its literal entries by
|
|
78
|
+
# value, and each other one by its shape, as RSpec's include pins it.
|
|
79
|
+
# The call runs once, since a second call could make another record.
|
|
80
|
+
def part_lines(pinned, call)
|
|
81
|
+
literal, other = pinned.partition { |_, item| Support.literal?(item) }.map(&:to_h)
|
|
82
|
+
lines = [" value = #{call}"]
|
|
83
|
+
unless literal.empty?
|
|
84
|
+
keys = literal.keys.map { |key| Support.ruby_literal(key) }.join(", ")
|
|
85
|
+
lines << " assert_equal(#{Support.ruby_literal(literal)}, value.slice(#{keys}))"
|
|
86
|
+
end
|
|
87
|
+
other.each { |key, item| lines.concat(Support.shape_assertions(item, "[#{Support.ruby_literal(key)}]").map { |line| " #{line}" }) }
|
|
88
|
+
lines
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
end
|
|
92
|
+
end
|