branchout 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.ja.md +102 -6
- data/README.md +72 -7
- data/lib/branchout/cli.rb +11 -3
- data/lib/branchout/emitters/minitest.rb +22 -6
- data/lib/branchout/emitters/rspec.rb +25 -2
- data/lib/branchout/emitters/support.rb +68 -7
- data/lib/branchout/emitters.rb +19 -6
- data/lib/branchout/engine.rb +15 -6
- data/lib/branchout/explore.rb +2 -2
- data/lib/branchout/input.rb +38 -5
- data/lib/branchout/input_file.rb +38 -7
- data/lib/branchout/report.rb +142 -14
- data/lib/branchout/runner.rb +70 -13
- data/lib/branchout/target.rb +22 -0
- data/lib/branchout/version.rb +1 -1
- data/skills/branchout/SKILL.md +12 -11
- metadata +1 -1
data/lib/branchout/report.rb
CHANGED
|
@@ -33,6 +33,7 @@ module Branchout
|
|
|
33
33
|
frontier_brief(lines, result.frontier)
|
|
34
34
|
returned_brief(lines, returned(Array(result.pins), pin_keys))
|
|
35
35
|
exceptions_brief(lines, result.exceptions)
|
|
36
|
+
after_brief(lines, after(Array(result.pins)), after_unread(Array(result.pins)))
|
|
36
37
|
not_pinned_brief(lines, result.covered.reject(&:pinned))
|
|
37
38
|
pinned_by_class_brief(lines, result.pins, pin_keys) if tests
|
|
38
39
|
not_counted_brief(lines, Array(result.not_counted))
|
|
@@ -75,6 +76,8 @@ module Branchout
|
|
|
75
76
|
# input of the inputs file and each arm's input, grouped by what it
|
|
76
77
|
# returned, as [shown, value, pins]: +value+ is the Ruby literal of the
|
|
77
78
|
# whole value, or nil for one with no literal, and +shown+ fits a line.
|
|
79
|
+
# Both mark a path of this machine (Support.marked) and do not drop
|
|
80
|
+
# it, so neither reads as a value the method did not return.
|
|
78
81
|
def returned(pins, pin_keys)
|
|
79
82
|
groups = pins.reject(&:error).group_by do |pin|
|
|
80
83
|
next [:shown, "a value that differed between two runs"] if pin.varying == :whole
|
|
@@ -87,15 +90,23 @@ module Branchout
|
|
|
87
90
|
end
|
|
88
91
|
values = groups.keys.select { |kind, _| kind == :value }.map(&:last)
|
|
89
92
|
differences = values.zip(Emitters::Support.hash_differences(values)).to_h
|
|
90
|
-
groups.map do |(kind, value), group|
|
|
93
|
+
rows = groups.map do |(kind, value), group|
|
|
91
94
|
next [value, nil, group] if kind == :shown
|
|
92
95
|
|
|
93
|
-
whole = Emitters::Support.ruby_literal(Emitters::Support.
|
|
96
|
+
whole = Emitters::Support.ruby_literal(Emitters::Support.marked(value))
|
|
94
97
|
next [whole, whole, group] if whole.length <= SHOWN_WIDTH
|
|
95
98
|
|
|
96
|
-
shown =
|
|
99
|
+
shown = if Emitters::Support.keyed?(value)
|
|
100
|
+
shown_hash(value, differences.fetch(value))
|
|
101
|
+
elsif value.is_a?(Array) || value.is_a?(Struct) || value.is_a?(Data)
|
|
102
|
+
shown_composite(value, values)
|
|
103
|
+
end
|
|
97
104
|
[shown || "#{whole[0, SHOWN_WIDTH - 3]}...", whole, group]
|
|
98
105
|
end
|
|
106
|
+
# Two values that differ only in the directories a mark stands for
|
|
107
|
+
# would read as one value returned twice.
|
|
108
|
+
alike = rows.filter_map { |_, whole, _| whole }.tally
|
|
109
|
+
rows.map { |shown, whole, group| whole && alike[whole] > 1 ? ["#{shown} (another value differs from it in a path of this machine)", whole, group] : [shown, whole, group] }
|
|
99
110
|
end
|
|
100
111
|
|
|
101
112
|
SHOWN_WIDTH = 72
|
|
@@ -106,7 +117,7 @@ module Branchout
|
|
|
106
117
|
# way, then the rest of its top-level scalars that fit, or nil when
|
|
107
118
|
# none fits. A part with no literal shows its class.
|
|
108
119
|
def shown_hash(value, differing)
|
|
109
|
-
leaf_text = ->(item) { Emitters::Support.literal?(item) ? Emitters::Support.ruby_literal(Emitters::Support.
|
|
120
|
+
leaf_text = ->(item) { Emitters::Support.literal?(item) ? Emitters::Support.ruby_literal(Emitters::Support.marked(item)) : Emitters::Support.article(Emitters::Support.class_literal(item)) }
|
|
110
121
|
text = lambda do |entries|
|
|
111
122
|
omitted = entries.count { |path, _| path.one? } < value.size
|
|
112
123
|
"a Hash with #{Emitters::Support.entries_text(entries.map { |path, item| [path, leaf_text.(item)] })}#{", ..." if omitted}"
|
|
@@ -120,11 +131,85 @@ module Branchout
|
|
|
120
131
|
line.length <= SHOWN_WIDTH ? line : "#{line[0, SHOWN_WIDTH - 3]}..."
|
|
121
132
|
end
|
|
122
133
|
|
|
134
|
+
# An Array, a Struct, or a Data too long for a line says its size or
|
|
135
|
+
# its class, then the leaves that tell it from the other values of that
|
|
136
|
+
# size or class, since those start the same way and a cut start would
|
|
137
|
+
# read alike, then the rest of its leaves that fit, or nil when none
|
|
138
|
+
# fits. Values of another size or class are told apart by the start.
|
|
139
|
+
def shown_composite(value, values)
|
|
140
|
+
leaves = composite_leaves(value)
|
|
141
|
+
return if leaves.empty?
|
|
142
|
+
|
|
143
|
+
head = value.is_a?(Array) ? "an Array of #{value.length}" : Emitters::Support.article(Emitters::Support.class_literal(value))
|
|
144
|
+
differs = ->(peer, path) { !peer.key?(path) || !peer[path].eql?(leaves[path]) }
|
|
145
|
+
peers = values.reject { |other| other.eql?(value) }.select { |other| same_shape?(other, value) }.map { |other| composite_leaves(other) }
|
|
146
|
+
paths = leaves.keys
|
|
147
|
+
# The telling leaves come first, in their order in the value.
|
|
148
|
+
text = lambda do |told, rest = []|
|
|
149
|
+
entries = (paths.select { |path| told.include?(path) } + rest).map { |path| "#{Emitters::Support.path_text(path)}: #{leaf_text(leaves[path])}" }
|
|
150
|
+
"#{head} with #{entries.join(", ")}#{", ..." if told.length + rest.length < paths.length}"
|
|
151
|
+
end
|
|
152
|
+
told = []
|
|
153
|
+
until peers.empty?
|
|
154
|
+
best = paths.each_with_index.max_by { |path, index| [peers.count { |peer| differs.(peer, path) }, -index] }.first
|
|
155
|
+
break if peers.none? { |peer| differs.(peer, best) } || (told.any? && text.(told + [best]).length > SHOWN_WIDTH)
|
|
156
|
+
|
|
157
|
+
told << best
|
|
158
|
+
peers.reject! { |peer| differs.(peer, best) }
|
|
159
|
+
end
|
|
160
|
+
rest = []
|
|
161
|
+
paths.each { |path| rest << path if !told.include?(path) && text.(told, rest + [path]).length <= SHOWN_WIDTH }
|
|
162
|
+
return if told.empty? && rest.empty?
|
|
163
|
+
|
|
164
|
+
line = text.(told, rest)
|
|
165
|
+
line.length <= SHOWN_WIDTH ? line : "#{line[0, SHOWN_WIDTH - 3]}..."
|
|
166
|
+
end
|
|
167
|
+
|
|
168
|
+
def same_shape?(other, value)
|
|
169
|
+
return other.is_a?(Array) && other.length == value.length if value.is_a?(Array)
|
|
170
|
+
|
|
171
|
+
other.instance_of?(value.class)
|
|
172
|
+
end
|
|
173
|
+
|
|
174
|
+
# path => leaf under an Array, a Hash, a Struct, or a Data, with a
|
|
175
|
+
# member as a Symbol key, so a path reads as a reader writes it:
|
|
176
|
+
# [0].kind.
|
|
177
|
+
def composite_leaves(value, path = [], out = {})
|
|
178
|
+
children = case value
|
|
179
|
+
when Array then value.each_with_index.map { |item, index| [index, item] }
|
|
180
|
+
when Hash then value.map { |key, item| [key.is_a?(Symbol) ? Emitters::Support::SymbolKey.new(key.to_s) : key, item] }
|
|
181
|
+
when Struct, Data then value.to_h.map { |key, item| [Emitters::Support::SymbolKey.new(key.to_s), item] }
|
|
182
|
+
else []
|
|
183
|
+
end
|
|
184
|
+
if children.empty?
|
|
185
|
+
out[path] = value unless path.empty?
|
|
186
|
+
return out
|
|
187
|
+
end
|
|
188
|
+
|
|
189
|
+
children.each { |step, item| composite_leaves(item, path + [step], out) }
|
|
190
|
+
out
|
|
191
|
+
end
|
|
192
|
+
|
|
193
|
+
def leaf_text(item)
|
|
194
|
+
return Emitters::Support.ruby_literal(Emitters::Support.marked(item)) if Emitters::Support.literal?(item)
|
|
195
|
+
|
|
196
|
+
Emitters::Support.article(Emitters::Support.class_literal(item))
|
|
197
|
+
end
|
|
198
|
+
|
|
123
199
|
# An input from the inputs file too long for a line is named by its
|
|
124
|
-
# place there; --json and --verbose
|
|
200
|
+
# place there, and by its note when it has one; --json and --verbose
|
|
201
|
+
# keep the whole input.
|
|
125
202
|
def shown_input(pin)
|
|
126
203
|
text = pin.input.text
|
|
127
|
-
text.length > SHOWN_INPUT_WIDTH && pin.listed_at
|
|
204
|
+
return text unless text.length > SHOWN_INPUT_WIDTH && pin.listed_at
|
|
205
|
+
|
|
206
|
+
pin.input.note ? "inputs[#{pin.listed_at}] (#{pin.input.note})" : "inputs[#{pin.listed_at}]"
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# An input as the JSON report writes it: the call, and the note of the
|
|
210
|
+
# inputs file, so an entry pasted back keeps its reason.
|
|
211
|
+
def input_hash(input)
|
|
212
|
+
input.note ? input.to_h.merge("note" => input.note) : input.to_h
|
|
128
213
|
end
|
|
129
214
|
|
|
130
215
|
def returned_brief(lines, groups)
|
|
@@ -147,6 +232,34 @@ module Branchout
|
|
|
147
232
|
end
|
|
148
233
|
end
|
|
149
234
|
|
|
235
|
+
# What each after read, once its call returned or raised, as
|
|
236
|
+
# [source, shown, value, pins], grouped as RETURNED groups what the
|
|
237
|
+
# calls returned. A call's value and its after are told apart, so a
|
|
238
|
+
# raise that left the receiver changed reads as both.
|
|
239
|
+
def after(pins)
|
|
240
|
+
pins.select(&:observed).group_by { |pin| pin.input.after.source }.sort_by(&:first).flat_map do |source, group|
|
|
241
|
+
returned(group.map { |pin| Emitters::Support.after_pin(pin) }, []).map { |shown, whole, read| [source, shown, whole, read] }
|
|
242
|
+
end
|
|
243
|
+
end
|
|
244
|
+
|
|
245
|
+
# The inputs whose after read nothing, as [source, reason, pins].
|
|
246
|
+
def after_unread(pins)
|
|
247
|
+
unread = pins.select { |pin| pin.input.after && !pin.observed }
|
|
248
|
+
unread.group_by { |pin| [pin.input.after.source, unread_reason(pin)] }.sort_by(&:first).map { |(source, reason), group| [source, reason, group] }
|
|
249
|
+
end
|
|
250
|
+
|
|
251
|
+
def unread_reason(pin)
|
|
252
|
+
pin.error&.class_name == "Timeout::Error" ? "the call timed out" : "new or a setup call raised before the call"
|
|
253
|
+
end
|
|
254
|
+
|
|
255
|
+
def after_brief(lines, groups, unread)
|
|
256
|
+
return if groups.empty? && unread.empty?
|
|
257
|
+
|
|
258
|
+
lines << "" << "AFTER (what each after read once its call returned or raised; compare each with the state the call should leave)"
|
|
259
|
+
groups.each { |source, shown, _, group| lines << "#{source} is #{shown} for #{group.map { |pin| shown_input(pin) }.join(", ")}" }
|
|
260
|
+
unread.each { |source, reason, group| lines << "#{source} read nothing for #{group.map { |pin| shown_input(pin) }.join(", ")}: #{reason}" }
|
|
261
|
+
end
|
|
262
|
+
|
|
150
263
|
def not_pinned_brief(lines, arms)
|
|
151
264
|
return if arms.empty?
|
|
152
265
|
|
|
@@ -160,11 +273,16 @@ module Branchout
|
|
|
160
273
|
# Each part of a returned value that an emitted test can check only by
|
|
161
274
|
# its class, as "value[:at]" => [class name, inputs that returned it].
|
|
162
275
|
def pinned_by_class(pins, pin_keys)
|
|
163
|
-
|
|
164
|
-
Emitters::Support.class_pins(Emitters::Support.pinned_value(pin.value, Emitters::Support.keys_for(pin,
|
|
276
|
+
by_class = lambda do |pin, keys, root|
|
|
277
|
+
Emitters::Support.class_pins(Emitters::Support.pinned_value(pin.value, Emitters::Support.keys_for(pin, keys))).map { |path, name| [["#{root}#{path}", name], pin.input] }
|
|
165
278
|
end
|
|
279
|
+
parts = pins.reject { |pin| pin.error || pin.varying == :whole }.flat_map { |pin| by_class.(pin, pin_keys, "value") }
|
|
280
|
+
# An after's value is pinned as a returned one is, so its parts with
|
|
281
|
+
# no literal are checked by their class too.
|
|
282
|
+
parts += pins.filter_map { |pin| Emitters::Support.after_pin(pin) }.reject { |pin| pin.varying == :whole }
|
|
283
|
+
.flat_map { |pin| by_class.(pin, [], pin.input.after.source) }
|
|
166
284
|
parts.group_by(&:first).map do |(path, name), group|
|
|
167
|
-
[
|
|
285
|
+
[path, name, group.map(&:last).uniq(&:canonical)]
|
|
168
286
|
end
|
|
169
287
|
end
|
|
170
288
|
|
|
@@ -209,6 +327,11 @@ module Branchout
|
|
|
209
327
|
section(lines, "FRONTIER", result.frontier) { |arm| [*frontier_lines(arm), "path: #{format_path(arm.path)}"] }
|
|
210
328
|
section(lines, "RETURNED", returned(Array(result.pins), pin_keys)) { |shown, whole, group| ["returns: #{whole || shown}", *group.map { |pin| "input: #{pin.input.text}" }] }
|
|
211
329
|
section(lines, "EXCEPTIONS", result.exceptions) { |finding| finding_lines(finding) }
|
|
330
|
+
read = after(Array(result.pins)).map { |source, shown, whole, group| ["after: #{source}", "is: #{whole || shown}", group] } +
|
|
331
|
+
after_unread(Array(result.pins)).map { |source, reason, group| ["after: #{source}", "read nothing: #{reason}", group] }
|
|
332
|
+
unless read.empty?
|
|
333
|
+
section(lines, "AFTER", read) { |source, outcome, group| [source, outcome, *group.map { |pin| "input: #{pin.input.text}" }] }
|
|
334
|
+
end
|
|
212
335
|
section(lines, "NOT COUNTED", Array(result.not_counted)) { |callee| callee_lines(callee) }
|
|
213
336
|
section(lines, "OPERATORS", Array(result.operators)) { |found| operator_lines(found) }
|
|
214
337
|
section(lines, "BOUNDARIES", Array(result.boundaries)) { |found| operator_lines(found) }
|
|
@@ -236,14 +359,19 @@ module Branchout
|
|
|
236
359
|
"arms_not_pinned" => result.covered.reject(&:pinned).map { |arm| covered_hash(arm) },
|
|
237
360
|
"frontier" => result.frontier.map { |arm| frontier_hash(arm) },
|
|
238
361
|
"returned" => returned(Array(result.pins), pin_keys).map { |shown, whole, group|
|
|
239
|
-
{"returns" => shown, "value" => whole, "inputs" => group.map { |pin| pin.input
|
|
362
|
+
{"returns" => shown, "value" => whole, "inputs" => group.map { |pin| input_hash(pin.input) }}.compact
|
|
240
363
|
},
|
|
241
364
|
"exceptions" => result.exceptions.map { |finding| finding_hash(finding) },
|
|
365
|
+
"after" => after(Array(result.pins)).map { |source, shown, whole, group|
|
|
366
|
+
{"after" => source, "is" => shown, "value" => whole, "inputs" => group.map { |pin| input_hash(pin.input) }}.compact
|
|
367
|
+
} + after_unread(Array(result.pins)).map { |source, reason, group|
|
|
368
|
+
{"after" => source, "read_nothing" => reason, "inputs" => group.map { |pin| input_hash(pin.input) }}
|
|
369
|
+
},
|
|
242
370
|
"not_counted" => Array(result.not_counted).map { |callee| callee_hash(callee) },
|
|
243
371
|
"operators_not_counted" => Array(result.operators).map { |found| operator_hash(found) },
|
|
244
372
|
"boundaries_not_counted" => Array(result.boundaries).map { |found| operator_hash(found) },
|
|
245
373
|
"pinned_by_class" => pinned_by_class(Array(result.pins), pin_keys).map do |path, name, inputs|
|
|
246
|
-
{"path" => path, "class" => name, "inputs" => inputs.map(
|
|
374
|
+
{"path" => path, "class" => name, "inputs" => inputs.map { |input| input_hash(input) }}
|
|
247
375
|
end
|
|
248
376
|
}
|
|
249
377
|
"#{Canon.pretty(body)}\n"
|
|
@@ -411,7 +539,7 @@ module Branchout
|
|
|
411
539
|
"column" => arm.column,
|
|
412
540
|
"cond" => arm.cond,
|
|
413
541
|
"sense" => arm.sense,
|
|
414
|
-
"input" => arm.input
|
|
542
|
+
"input" => input_hash(arm.input)
|
|
415
543
|
}
|
|
416
544
|
end
|
|
417
545
|
|
|
@@ -424,7 +552,7 @@ module Branchout
|
|
|
424
552
|
"cond" => arm.cond,
|
|
425
553
|
"sense" => arm.sense,
|
|
426
554
|
"parent" => arm.parent,
|
|
427
|
-
"nearest" => arm.nearest
|
|
555
|
+
"nearest" => (input_hash(arm.nearest) if arm.nearest),
|
|
428
556
|
"path" => arm.path
|
|
429
557
|
}
|
|
430
558
|
end
|
|
@@ -433,7 +561,7 @@ module Branchout
|
|
|
433
561
|
{
|
|
434
562
|
"class" => finding.class_name,
|
|
435
563
|
"loc" => finding.loc,
|
|
436
|
-
"input" => finding.input
|
|
564
|
+
"input" => input_hash(finding.input),
|
|
437
565
|
"message" => finding.message
|
|
438
566
|
}
|
|
439
567
|
end
|
data/lib/branchout/runner.rb
CHANGED
|
@@ -13,14 +13,19 @@ module Branchout
|
|
|
13
13
|
# +varying+ is nil when a second run gave the same value, :whole when
|
|
14
14
|
# the value differed as a whole, or the keys of a returned Hash whose
|
|
15
15
|
# values differed.
|
|
16
|
-
|
|
16
|
+
# +observed+ is the Observation of what the input's after read, or nil.
|
|
17
|
+
Execution = Struct.new(:input, :value, :error, :lines, :branches, :path_ids, :origin, :thawed, :varying, :observed, keyword_init: true)
|
|
18
|
+
# What an input's after read once the call returned or raised. +varying+
|
|
19
|
+
# is as Execution's is for the returned value.
|
|
20
|
+
Observation = Struct.new(:value, :varying, keyword_init: true)
|
|
17
21
|
# +name+ is the missing name of a NameError, which is stable where its
|
|
18
22
|
# message is not: Ruby 3.4 reworded both NameError and NoMethodError.
|
|
19
23
|
# +message+ has its line breaks folded for reports; a test pins
|
|
20
24
|
# +exact_message+, the message as raised, when that is stable.
|
|
21
25
|
# +message_pattern+ is the source of a Regexp for a message that quotes
|
|
22
26
|
# a backtrace frame: Ruby 3.4 labels a frame 'Box.run' where 3.3 wrote
|
|
23
|
-
# `run', so the pattern keeps the rest and accepts either label.
|
|
27
|
+
# `run', so the pattern keeps the rest and accepts either label. It
|
|
28
|
+
# accepts any file descriptor (fd:\d+) where the message names one.
|
|
24
29
|
ExceptionInfo = Struct.new(:class_name, :message, :loc, :stable_message, :name, :exact_message, :message_pattern,
|
|
25
30
|
keyword_init: true)
|
|
26
31
|
# +arms+ is nil when the callee's file holds no coverage to count from.
|
|
@@ -89,6 +94,7 @@ module Branchout
|
|
|
89
94
|
return if run.error&.class_name == "Timeout::Error"
|
|
90
95
|
|
|
91
96
|
again = attempt(run.input, run.thawed)
|
|
97
|
+
run.observed.varying = again.observed ? Runner.differing(run.observed.value, again.observed.value) : :whole if run.observed
|
|
92
98
|
if run.error && again.error&.class_name == run.error.class_name
|
|
93
99
|
unless again.error.message == run.error.message && again.error.exact_message == run.error.exact_message
|
|
94
100
|
run.error = run.error.dup.tap do |error|
|
|
@@ -133,9 +139,11 @@ module Branchout
|
|
|
133
139
|
private_class_method :same_pin?
|
|
134
140
|
|
|
135
141
|
def attempt(input, thawed)
|
|
142
|
+
@observed = nil
|
|
136
143
|
value, error = @host ? refusing_lets(input) { outcome(input, thawed) } : outcome(input, thawed)
|
|
137
144
|
data = extract(Coverage.result(stop: false, clear: true))
|
|
138
|
-
Execution.new(input: input, value: value, error: error, lines: data[:lines], branches: data[:branches], thawed: thawed
|
|
145
|
+
Execution.new(input: input, value: value, error: error, lines: data[:lines], branches: data[:branches], thawed: thawed,
|
|
146
|
+
observed: @observed)
|
|
139
147
|
end
|
|
140
148
|
|
|
141
149
|
def outcome(input, thawed)
|
|
@@ -431,8 +439,13 @@ module Branchout
|
|
|
431
439
|
value
|
|
432
440
|
end
|
|
433
441
|
|
|
442
|
+
# An input with an after reads it once the entry returned or raised,
|
|
443
|
+
# in the input's scope, so it reads the receiver by the name the
|
|
444
|
+
# emitted test gives it, and in a suite before the input's savepoint
|
|
445
|
+
# is rolled back.
|
|
434
446
|
def run_entry(input, thawed)
|
|
435
447
|
scope = new_scope
|
|
448
|
+
entered = false
|
|
436
449
|
run = proc do
|
|
437
450
|
receiver, args, kwargs, block = built_by_input(input) do
|
|
438
451
|
scope.eval(input.before.source, "(input)") if input.before
|
|
@@ -440,16 +453,45 @@ module Branchout
|
|
|
440
453
|
args, kwargs = fresh_arguments(input, scope, thawed)
|
|
441
454
|
[receiver, args, kwargs, (scope.eval("proc #{input.block.source}", "(input)") if input.block)]
|
|
442
455
|
end
|
|
456
|
+
entered = true
|
|
443
457
|
if @target.singleton
|
|
444
458
|
measure { @method.call(*args, **kwargs, &block) }
|
|
445
459
|
else
|
|
446
460
|
measure { @method.bind_call(receiver, *args, **kwargs, &block) }
|
|
447
461
|
end
|
|
448
462
|
end
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
463
|
+
begin
|
|
464
|
+
value = timed(&run)
|
|
465
|
+
rescue Error, Timeout::Error
|
|
466
|
+
raise
|
|
467
|
+
rescue Exception # rubocop:disable Lint/RescueException
|
|
468
|
+
read_after(input, scope, entered)
|
|
469
|
+
raise
|
|
470
|
+
end
|
|
471
|
+
read_after(input, scope, entered)
|
|
472
|
+
value
|
|
473
|
+
end
|
|
474
|
+
|
|
475
|
+
def timed(&run)
|
|
476
|
+
@timeout ? Timeout.timeout(@timeout, &run) : run.call
|
|
477
|
+
end
|
|
478
|
+
|
|
479
|
+
# When new or a setup call raised, no call left a state to read, and
|
|
480
|
+
# the error is pinned as it is without an after; the report says the
|
|
481
|
+
# after read nothing. An after that raises reads nothing an emitted
|
|
482
|
+
# test could pin, so explore stops and says which input to fix.
|
|
483
|
+
def read_after(input, scope, entered)
|
|
484
|
+
return unless input.after && entered
|
|
485
|
+
|
|
486
|
+
begin
|
|
487
|
+
@observed = Observation.new(value: timed { scope.eval(input.after.source, "(input)") })
|
|
488
|
+
rescue Error
|
|
489
|
+
raise
|
|
490
|
+
rescue Timeout::Error
|
|
491
|
+
raise Error, "the after #{input.after.source.inspect} of the input #{Canon.json(input.to_h)} did not end within the timeout. Fix the after."
|
|
492
|
+
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
493
|
+
raise Error, "the after #{input.after.source.inspect} of the input #{Canon.json(input.to_h)} raised #{e.class}: " \
|
|
494
|
+
"#{ExceptionInfo.one_line(e.message)}. Fix the after."
|
|
453
495
|
end
|
|
454
496
|
end
|
|
455
497
|
|
|
@@ -492,10 +534,14 @@ module Branchout
|
|
|
492
534
|
else
|
|
493
535
|
new_args, new_kwargs = input.receiver ? fresh_arguments(input.receiver, scope, thawed) : [[], {}]
|
|
494
536
|
receiver = @const.send(:new, *new_args, **new_kwargs)
|
|
537
|
+
# With setup calls or an after the emitted test names the receiver
|
|
538
|
+
# too, so a setup block, an argument, or the after can read it.
|
|
539
|
+
scope.local_variable_set(Emitters::Support.built_receiver_name(@target), receiver) unless input.setup.empty? && input.after.nil?
|
|
495
540
|
end
|
|
496
541
|
input.setup.each do |call|
|
|
497
542
|
call_args, call_kwargs = fresh_arguments(call.arguments, scope, thawed)
|
|
498
|
-
|
|
543
|
+
block = scope.eval("proc #{call.arguments.block.source}", "(input)") if call.arguments.block
|
|
544
|
+
receiver.public_send(call.method_name, *call_args, **call_kwargs, &block)
|
|
499
545
|
end
|
|
500
546
|
receiver
|
|
501
547
|
end
|
|
@@ -567,13 +613,24 @@ module Branchout
|
|
|
567
613
|
end
|
|
568
614
|
|
|
569
615
|
FRAME_LABEL = /(:\d+:in )[`'][^`'\n]*'/
|
|
616
|
+
# An Errno message from an IO names its file descriptor (fd:5), whose
|
|
617
|
+
# number depends on what the process opened before, so a test that ran
|
|
618
|
+
# after other files were opened would see another one.
|
|
619
|
+
FILE_DESCRIPTOR = /\bfd:\d+/
|
|
570
620
|
|
|
571
621
|
def self.frame_pattern(raw)
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
source =
|
|
576
|
-
|
|
622
|
+
varying = Regexp.union(FRAME_LABEL, FILE_DESCRIPTOR)
|
|
623
|
+
return nil unless varying.match?(raw)
|
|
624
|
+
|
|
625
|
+
source = +""
|
|
626
|
+
at = 0
|
|
627
|
+
raw.to_enum(:scan, varying).map { Regexp.last_match }.each do |match|
|
|
628
|
+
source << Regexp.escape(raw[at...match.begin(0)])
|
|
629
|
+
source << (match[1] ? "#{Regexp.escape(match[1])}[`'][^`'\\n]*'" : "fd:\\d+")
|
|
630
|
+
at = match.end(0)
|
|
631
|
+
end
|
|
632
|
+
source << Regexp.escape(raw[at..])
|
|
633
|
+
"\\A#{source.gsub("\\ ", " ")}\\z"
|
|
577
634
|
end
|
|
578
635
|
|
|
579
636
|
def self.locate(exception, display_path, real_path)
|
data/lib/branchout/target.rb
CHANGED
|
@@ -40,6 +40,12 @@ module Branchout
|
|
|
40
40
|
# Each @ivar ||= or &&= that OPERATORS leaves out, for the hint on a
|
|
41
41
|
# method whose only branch is a memo.
|
|
42
42
|
attr_reader :memos
|
|
43
|
+
# Each if or unless on defined?(@ivar), a memo that caches nil and
|
|
44
|
+
# false too. Coverage counts its arms, but an input reaches the cached
|
|
45
|
+
# arm only after a call on the same object, through setup, and then
|
|
46
|
+
# returns what the first call stored, so a covered memo pins the same
|
|
47
|
+
# value as no memo at all.
|
|
48
|
+
attr_reader :defined_memos
|
|
43
49
|
# Each <, <=, >, and >= of the counted code. Coverage sees no side of
|
|
44
50
|
# them either, so every arm can be taken with no input where the sides
|
|
45
51
|
# are equal, where > and >= part.
|
|
@@ -146,6 +152,7 @@ module Branchout
|
|
|
146
152
|
@operators = bodies.flat_map { |body| operators_in(body, written) }.sort_by { |found| [found.line, found.column] }
|
|
147
153
|
@memos = bodies.flat_map { |body| operators_in(body, written, memos: true) }.sort_by { |found| [found.line, found.column] }
|
|
148
154
|
@boundaries = bodies.flat_map { |body| boundaries_in(body) }.sort_by { |found| [found.line, found.column] }
|
|
155
|
+
@defined_memos = bodies.flat_map { |body| defined_memos_in(body, written) }.sort_by { |found| [found.line, found.column] }
|
|
149
156
|
collect_literals(entry_def.node.parameters)
|
|
150
157
|
bodies.each { |body| collect_literals(body) }
|
|
151
158
|
@literals.ints = @literals.ints.uniq.sort
|
|
@@ -403,6 +410,21 @@ module Branchout
|
|
|
403
410
|
acc
|
|
404
411
|
end
|
|
405
412
|
|
|
413
|
+
# Each if or unless on defined?(@ivar) of an ivar the file sets, as in
|
|
414
|
+
# return @value if defined?(@value).
|
|
415
|
+
def defined_memos_in(node, written, acc = [])
|
|
416
|
+
return acc if node.type == :def_node
|
|
417
|
+
|
|
418
|
+
predicate = node.predicate if %i[if_node unless_node].include?(node.type)
|
|
419
|
+
if predicate&.type == :defined_node && predicate.value.type == :instance_variable_read_node && written.include?(predicate.value.name)
|
|
420
|
+
place = predicate.location
|
|
421
|
+
acc << Operator.new(loc: "#{display_path}:#{place.start_line}:#{place.start_column}", line: place.start_line,
|
|
422
|
+
column: place.start_column, op: "defined?", text: cut(squash(node.slice)))
|
|
423
|
+
end
|
|
424
|
+
node.compact_child_nodes.each { |child| defined_memos_in(child, written, acc) }
|
|
425
|
+
acc
|
|
426
|
+
end
|
|
427
|
+
|
|
406
428
|
def collect_literals(node)
|
|
407
429
|
return unless node.is_a?(Prism::Node)
|
|
408
430
|
return if node.type == :def_node
|
data/lib/branchout/version.rb
CHANGED
data/skills/branchout/SKILL.md
CHANGED
|
@@ -46,13 +46,14 @@ Start from one seed. [inputs.example.json](inputs.example.json) shows every acce
|
|
|
46
46
|
}
|
|
47
47
|
```
|
|
48
48
|
|
|
49
|
-
- An entry is `{"args": [...]}` or a bare array of arguments: `["0306406150"]` calls the method with that one String. A bare value such as `"0306406150"` is refused with the array to write. Keywords go in `kwargs`: `{"args": [0], "kwargs": {"flag": true}}`. An entry with `new`, `receiver`, `setup`, `kwargs`, `block`, or `
|
|
49
|
+
- An entry is `{"args": [...]}` or a bare array of arguments: `["0306406150"]` calls the method with that one String. A bare value such as `"0306406150"` is refused with the array to write. Keywords go in `kwargs`: `{"args": [0], "kwargs": {"flag": true}}`. An entry with `new`, `receiver`, `setup`, `kwargs`, `block`, `before`, or `after` may leave out `args` for a call with no positional argument.
|
|
50
50
|
- When the method reads rows that no argument holds, such as a record a class method finds, make them in `before`: `{"before": {"ruby": "FactoryBot.create(:part, sku: \"A\")"}, "args": ["A"]}`. It runs first, in the input's own savepoint. RSpec writes it as the `before` of the input's context, and minitest as the line before the call. Do not hide it in the argument with `.tap { ... }`: the call then runs inside the argument, and RSpec names the input by the block (`when fact.tap { FactoryBot.create(:part, sku: "A") }`), not by a `before` of its own.
|
|
51
51
|
- For `Const#method`, constructor arguments go in `new`, in the same shape as an entry: `{"args": [5], "new": [10]}` calls `Const.new(10).method(5)`, and `{"args": [5], "new": {"kwargs": {"main": 1}}}` calls `Const.new(main: 1).method(5)`. A required keyword of `initialize` needs `new` as a required argument does. Inputs differ by `new` too, so a method with no parameters is explored by varying `new`. When the target file defines `initialize`, `mutate` and `random` fill `new` from its parameters, in the same domains as the method's own.
|
|
52
|
-
- When the receiver is an object that `new` cannot build, such as a saved record from a factory, a fixture, or a test helper, give its Ruby in `receiver` in place of `new`: `{"receiver": {"ruby": "create(:loan, :approved)"}, "args": [{"ruby": "loan.borrower"}]}`. The receiver takes the name the emitted test gives it, the class in snake case (`loan`, `room`), so `args`, `kwargs`, and `
|
|
53
|
-
- When the method reads state that later calls build, such as `cart.add(...)` before `cart.total`, list those calls in `setup`, in order: `{"args": [], "setup": [{"method": "add", "args": ["apple", 3]}]}`. Each step has `method` and optional `args` and `
|
|
52
|
+
- When the receiver is an object that `new` cannot build, such as a saved record from a factory, a fixture, or a test helper, give its Ruby in `receiver` in place of `new`: `{"receiver": {"ruby": "create(:loan, :approved)"}, "args": [{"ruby": "loan.borrower"}]}`. The receiver takes the name the emitted test gives it, the class in snake case (`loan`, `room`), so `args`, `kwargs`, `setup`, and `block` can read from it (`room.acts.first`, `"block": "{ hold.pause(:read); :written }"`). Each input builds it again. The emitted minitest test assigns that name, and the RSpec spec declares it as a `let`. A bare `create` or `build` needs `FactoryBot::Syntax::Methods` included in the suite; without it write `FactoryBot.create`. An input whose own Ruby raises before the entry runs stops explore and names the input: fix that input, since its error is not the method's.
|
|
53
|
+
- When the method reads state that later calls build, such as `cart.add(...)` before `cart.total`, list those calls in `setup`, in order: `{"args": [], "setup": [{"method": "add", "args": ["apple", 3]}]}`. Each step has `method` and optional `args`, `kwargs`, and `block` (`{"method": "start", "args": ["r1"], "block": "{ hold.pause(:read); :end }"}`). With setup steps the receiver built by `new` takes the name the emitted test gives it too, so a step's arguments and block can read it. Coverage counts only the entry call. An exception raised in a setup call is still reported, with `loc` on that line. The emitted test names the receiver after the class (`cart = Shop::Cart.new`) and repeats the calls.
|
|
54
|
+
- When the call changes state that its value does not show, such as what a receiver holds after it returned or raised, read that state in `after`: one Ruby expression (`{"after": {"ruby": "hold.held"}}`), on an entry, or at the top of the file for every input, where an entry's own `after` replaces it and the inputs `mutate` and `random` make take it too. It runs once the call returned or raised, in the input's scope, so it reads the receiver by the name the emitted test gives it (`hold`), and inside a suite before the input's savepoint is rolled back. `AFTER` lists what each after read, grouped by value (`hold.held is [] for inputs[1] (ends without a pause), inputs[2] (raises before a pause)`), and the emitted test pins it after the call's value or error, as a value is pinned: `assert_equal [], hold.held` in minitest, after the `assert_raises` for a call that raised, and `expect(hold.held).to eq([])` in RSpec. A value that differed between two runs is not pinned, a Hash keeps only the keys that were the same, and stderr names each. Read several values as one Array or Hash (`[cart.items, cart.total]`); a modifier `if` or two statements are refused. An after that raises stops explore and names the input. When `new` or a setup call raised, there is no call to read after: `AFTER` says `read nothing`, and the test pins that error as before. For `QuintDriver::Hold#start`, `"after": {"ruby": "hold.held"}` with an input whose block raises before any pause gives `hold.held is [] ...`, and deleting the `rescue` that frees the name fails that input's test with `Expected: [] Actual: ["r1"]`, while the error it pins is the same.
|
|
54
55
|
- Seeds: `[0]` int, `[0.0]` float, `[""]` string, `[[]]` array, `[{}]` hash, `[false]` bool.
|
|
55
|
-
- `"types"` values are `int`, `float`, `string`, `array`, `hash`, or `bool`. A default of `[]`, `{}`, `true`, or `false` already gives the type. The array length must match the positional parameters. Extra top-level keys such as `meta` are ignored. An entry may carry a `note` (`{"args": [100], "note": "the refund limit"}`)
|
|
56
|
+
- `"types"` values are `int`, `float`, `string`, `array`, `hash`, or `bool`. A default of `[]`, `{}`, `true`, or `false` already gives the type. The array length must match the positional parameters. Extra top-level keys such as `meta` are ignored. An entry may carry a `note` (`{"args": [100], "note": "the refund limit"}`); put the reason for that input there. It takes no part in the call and names the input: `RETURNED` shows a long input as `inputs[3] (the refund limit)`, `--json` keeps it, a minitest name is made from its words (`test_the_refund_limit_returns_true`), and an RSpec example says it after the outcome (`returns true (the refund limit)`). Write it in plain English words, since a note with no Latin letter or digit gives a minitest name no words. An entry may carry `pin_keys` (`{"args": [3], "pin_keys": ["tax"]}`), which replaces `--pin-keys` for that input's test; `"pin_keys": []` pins its Hash whole.
|
|
56
57
|
- A JSON value is passed as the Ruby value it reads as: null is nil, a JSON array an Array, and a JSON object a Hash with String keys. `-0.0` is stored as `0.0`. A String is its text with no quotes of its own: `"\"0306406150\""` passes the quotes too, and stderr names such an argument and the String to write.
|
|
57
58
|
- Any other argument, such as a domain object (`RegisterCommands.apply(register, command)`), an ActiveRecord record, or an exception, is Ruby source in `{"ruby": "..."}`: `{"args": [{"ruby": "Factories.loan(3)"}]}`. Put a factory method in a `--require` file and call it here, so the test reads as a call (`Lending.approve(Factories.loan(3))`). Each call evaluates the source again at the top level, so every call gets a new object. A Hash argument whose one key is `"ruby"` is written as Ruby too (`{"ruby": "{\"ruby\" => \"3.4\"}"}`). Source that does not parse is refused with the entry's label.
|
|
58
59
|
- A file path is a plain String. Put the file under the project, such as `spec/fixtures/files/rates.csv`, and give its absolute path: the emitted test writes it from its own directory (`File.expand_path("../fixtures/files/rates.csv", __dir__)`), so it passes in every checkout, and its name says `spec/fixtures/files/rates.csv`. A path under the temporary or the home directory and outside the project is written as given, and stderr says `the emitted test passes <path>, a file of this machine outside the test's project`, or `a path of this machine outside the test's project that does not exist`: move that path into the project and run again. A returned value or an error message that holds a path under the project is written from the test's directory as well; one that holds a path of this machine outside it makes stderr say `the emitted test pins <path>, a path of this machine outside the test's project`. Names and descriptions say such a path by its file alone. Do not write `__dir__` in `{"ruby": ...}`: explore evaluates the source as `(input)`, where `__dir__` is `.`, and the test file has another directory.
|
|
@@ -112,23 +113,23 @@ The emitted test requires the helper the suite loaded (`require "rails_helper"`,
|
|
|
112
113
|
|
|
113
114
|
## Reading the frontier
|
|
114
115
|
|
|
115
|
-
Stdout is the contract. There is no timestamp. The report opens with `covered: 5 of 6 arms by 13 inputs`, `raised:`, and, with `--emit-tests`, `tests:`. `FRONTIER` follows: each arm that an input reached and did not take, in line order, one line each, its `loc` and what an input must do to take it (``app/models/loan.rb:7:4 makes `state == "approved"` true``), under a `nearest:` line that names the input to change, which consecutive arms share; then, under `not reached yet`, one line for each decision that no input ran (`file:line cond (2 arms)`). `RETURNED` follows, one line for each value: what the inputs of the inputs file and the input of each covered arm returned, grouped by value (`returns true for ["0306406150"], ["0306406152"]`). A value longer than a line shows the Hash leaves that tell it from the other values first, then the top-level scalars that fit (`returns a Hash with due_cents: 900, discount: 10, discount_cents: 100, ...`), and an input from the inputs file longer than a line shows its index there (`inputs[3]`); `--json` keeps each whole value as a Ruby literal (`value`) and each whole input. Coverage counts the arms an input took and never checks a value, so `FRONTIER` prints `(none: every arm ran, which does not mean each value is right; check RETURNED)` when every arm ran: read each `RETURNED` line against what the method should return, since a wrong value in a method with few branches shows nowhere else. `EXCEPTIONS` gives one line for each class and place that raised, with how many inputs raised it, one of them, and its message (`ArgumentError at shipping.rb:3 raised by 3 inputs, such as [-1]: weight must be positive`); a message that quotes each input does not make a line for each. `NOT PINNED` lists the covered arms that no emitted test checks, under the `input:` that took them: every input that took such an arm returned a value that differed as a whole between two runs. Give such an input the value it makes (an id, a token), or add one that takes the arm and returns the same value twice. With `--emit-tests`, `PINNED BY CLASS` lists each part of a returned value that the emitted tests check only by its class (`value[:start_at] is a Time: ...`, `value is an Enumerator: ...`), with the first three inputs that returned it; assert what matters in it by hand, or make the method return a value with a literal. `--json` holds the same parts as `pinned_by_class`, with or without `--emit-tests`. `NOT COUNTED` and `OPERATORS` appear when they have an entry. A section with no entry is left out, except `FRONTIER`.
|
|
116
|
+
Stdout is the contract. There is no timestamp. The report opens with `covered: 5 of 6 arms by 13 inputs`, `raised:`, and, with `--emit-tests`, `tests:`. `FRONTIER` follows: each arm that an input reached and did not take, in line order, one line each, its `loc` and what an input must do to take it (``app/models/loan.rb:7:4 makes `state == "approved"` true``), under a `nearest:` line that names the input to change, which consecutive arms share; then, under `not reached yet`, one line for each decision that no input ran (`file:line cond (2 arms)`). `RETURNED` follows, one line for each value: what the inputs of the inputs file and the input of each covered arm returned, grouped by value (`returns true for ["0306406150"], ["0306406152"]`). A value longer than a line shows the Hash leaves that tell it from the other values first, then the top-level scalars that fit (`returns a Hash with due_cents: 900, discount: 10, discount_cents: 100, ...`). An Array, a Struct, or a Data shows its size or class, then the leaves that tell it from the other values of that size or class, then the leaves that fit (`returns an Array of 2 with [1].kind: "skips-callbacks", [0].line: 2, ...`), and an input from the inputs file longer than a line shows its index there (`inputs[3]`); `--json` keeps each whole value as a Ruby literal (`value`) and each whole input. A path under the temporary or the home directory shows as `<tmp>/` or `<home>/` with `.../` for its directories (`"No hit under <tmp>/.../app."`), in `value` too, so the report is the same on every machine; two values that differ only there say `(another value differs from it in a path of this machine)`. Coverage counts the arms an input took and never checks a value, so `FRONTIER` prints `(none: every arm ran, which does not mean each value is right; check RETURNED)` when every arm ran: read each `RETURNED` line against what the method should return, since a wrong value in a method with few branches shows nowhere else. `EXCEPTIONS` gives one line for each class and place that raised, with how many inputs raised it, one of them, and its message (`ArgumentError at shipping.rb:3 raised by 3 inputs, such as [-1]: weight must be positive`); a message that quotes each input does not make a line for each. With an `after`, `AFTER` gives one line for each value it read, in the shape of `RETURNED` (`hold.held is ["r1"] for inputs[3] (name already held), inputs[0] (pauses)`), and `<after> read nothing for <inputs>: new or a setup call raised before the call` (or `the call timed out`); read each line against the state the call should leave. `NOT PINNED` lists the covered arms that no emitted test checks, under the `input:` that took them: every input that took such an arm returned a value that differed as a whole between two runs. Give such an input the value it makes (an id, a token), or add one that takes the arm and returns the same value twice. With `--emit-tests`, `PINNED BY CLASS` lists each part of a returned value that the emitted tests check only by its class (`value[:start_at] is a Time: ...`, `value is an Enumerator: ...`), with the first three inputs that returned it; assert what matters in it by hand, or make the method return a value with a literal. `--json` holds the same parts as `pinned_by_class`, with or without `--emit-tests`. `NOT COUNTED` and `OPERATORS` appear when they have an entry. A section with no entry is left out, except `FRONTIER`.
|
|
116
117
|
|
|
117
|
-
`--verbose` (`verbose: true` inside a suite) prints the whole report instead: the run settings of the strategies that ran (`seed` and `random` only with `random`, `mutate_cap` only with `mutate`) and the counts (`arms_covered`, `arms_total`, `exceptions`, `callees_not_counted`, `operators_not_counted`, `boundaries_not_counted`), then the sections `COVERED`, `FRONTIER`, `RETURNED` (`returns:` and an `input:` line for each input that returned it), `EXCEPTIONS`, `NOT COUNTED`, `OPERATORS`, and `BOUNDARIES`, each record on its own lines, every arm in id order, and each frontier arm with `path`. `--md-out` writes that report. `--json` always holds all of it, so `--verbose` with `--json` is refused.
|
|
118
|
+
`--verbose` (`verbose: true` inside a suite) prints the whole report instead: the run settings of the strategies that ran (`seed` and `random` only with `random`, `mutate_cap` only with `mutate`) and the counts (`arms_covered`, `arms_total`, `exceptions`, `callees_not_counted`, `operators_not_counted`, `boundaries_not_counted`), then the sections `COVERED`, `FRONTIER`, `RETURNED` (`returns:` and an `input:` line for each input that returned it), `EXCEPTIONS`, `AFTER` when an input has one (`after:`, `is:` or `read nothing:`, and an `input:` line for each input), `NOT COUNTED`, `OPERATORS`, and `BOUNDARIES`, each record on its own lines, every arm in id order, and each frontier arm with `path`. `--md-out` writes that report. `--json` always holds all of it, so `--verbose` with `--json` is refused.
|
|
118
119
|
|
|
119
120
|
|
|
120
121
|
- `id` is `kind@line:column/arm@line:column`. Columns are 0-based, matching Coverage and Prism. A loop count greater than 1 is appended as `*count`.
|
|
121
122
|
- `loc` is `file:line:column`. Open that line. `cond` is the predicate, or the `when` / `in` line. Longer text is cut at 160 characters.
|
|
122
123
|
- `sense` selects the arm: `cond-true` (`if` then, `unless` else), `cond-false` (`if` else, `unless` then), `arm-matched` (that `when` or `in`), `arm-else` (case else), `body-entered` (`while` or `until` body), `receiver-present` / `receiver-nil` (`&.`). Anything else is `taken`.
|
|
123
124
|
- `parent` (`--verbose` and `--json`): `reached` means some input executed the decision. Flip these first; the default report lists them one line each. `not-reached` means the parent has not run and `nearest` is `null`. Reach the parent first.
|
|
124
|
-
- `nearest` reached the decision with the fewest taken arms, then the smallest weight (absolute numeric size, plus string, Ruby source, and container length), then the smallest canonical JSON. With no keywords and no `new` it prints as the args array (`[0,1]`), which pastes into `args`. With keywords, `new`, `receiver`, `setup`,
|
|
125
|
+
- `nearest` reached the decision with the fewest taken arms, then the smallest weight (absolute numeric size, plus string, Ruby source, and container length), then the smallest canonical JSON. With no keywords and no `new` it prints as the args array (`[0,1]`), which pastes into `args`. With keywords, `new`, `receiver`, `setup`, a `block`, or an `after` it prints as `{"args":[...]}`, plus `"kwargs"`, `"new"`, `"receiver"`, `"setup"`, `"block"`, and `"after"` when set, which pastes back as an entry.
|
|
125
126
|
- `path` (`--verbose` and `--json`) is the sorted set of arms `nearest` took. Order is not recorded. An empty set prints `(none)`; a missing path prints `null`.
|
|
126
127
|
- `COVERED` (`--verbose`) uses `input` for the witness of that arm, an input that returned the same value on two runs when one did, and adds `pinned: no` when none did. In `--verbose`, `EXCEPTIONS` lists `class`, `loc`, `input`, and `message` for each input. An empty frontier can still have exceptions, and a covered method can still raise.
|
|
127
128
|
- `NOT COUNTED` lists the methods in other files that the counted code calls on self, such as a parent class's helper or a mixin's method, and the methods it calls on an object built with `Const.new` in a local or in an instance variable the class sets (`@ledger = Ledger.new`): `callee` (`Register#required_sku`), `loc` (its `def`), and `arms` (the arms in that `def`, or `unknown`). Their arms are not in `arms_total`, so an `arms_total` that dropped after a refactor can mean the logic moved there. Explore that callee as its own entry. Methods of installed gems and of Ruby's own library (an association reader, `update!`), C methods, attribute readers, methods with no arms, and calls on an object whose class the code does not show (a parameter, a method's return value) are not listed.
|
|
128
|
-
- `OPERATORS` lists each `||`, `&&`, `||=`, and `&&=` in the counted code: `loc` (the operator), `op`, and `text`. An `@ivar ||=` or `@ivar &&=` on an instance variable the file sets nowhere else is left out: it memoizes across calls on one object, and each input builds its own receiver, so no input can vary its left side. When such a memo is the only branch, stderr names it: explore the methods its right side calls, and test the memo by hand with two calls on one object. Branch coverage counts no arm for them, so `arms_total` leaves them out and the report cannot say which side ran. `arms_total: 0` with an `OPERATORS` entry is not full coverage: give inputs that make each left side truthy and falsy, and let the pinned results show the difference. `&.` has its own arms.
|
|
129
|
+
- `OPERATORS` lists each `||`, `&&`, `||=`, and `&&=` in the counted code: `loc` (the operator), `op`, and `text`. An `@ivar ||=` or `@ivar &&=` on an instance variable the file sets nowhere else is left out: it memoizes across calls on one object, and each input builds its own receiver, so no input can vary its left side. When such a memo is the only branch, stderr names it: explore the methods its right side calls, and test the memo by hand with two calls on one object. A memo on `defined?(@ivar)` (`return @value if defined?(@value)`) has arms that coverage counts, and an input with a `setup` call of the method covers the cached arm, but that arm returns what the first call stored, so the pinned values read the same without the memo; stderr names each such memo, and the same test by hand applies. Branch coverage counts no arm for them, so `arms_total` leaves them out and the report cannot say which side ran. `arms_total: 0` with an `OPERATORS` entry is not full coverage: give inputs that make each left side truthy and falsy, and let the pinned results show the difference. `&.` has its own arms.
|
|
129
130
|
- `BOUNDARIES` lists each `<`, `<=`, `>`, and `>=` in the counted code, in the same shape. Coverage sees no side of a comparison, so every arm can be covered while no input makes its two sides equal, and a mutant that turns `>` into `>=` survives. Add an input that makes the sides equal (the date on the boundary, the limit itself). The list comes from the source, so a comparison an input already made equal stays on it; check the inputs, not the list.
|
|
130
131
|
|
|
131
|
-
`--json` is the same report (`version`, counts, `covered`, `arms_not_pinned` (the `NOT PINNED` arms, in the shape of `covered`), `frontier`, `returned` (each value as `returns`, with the `inputs` that returned it), `exceptions`, `not_counted`, `operators_not_counted`, `boundaries_not_counted`, `pinned_by_class` (each part of a returned value that an emitted test checks only by its class, as `path`, `class`, and every input that returned it)). Each frontier object has `id`, `loc`, `cond`, `sense`, `parent`, `nearest` (`{"args","kwargs"}`, plus `"new"` and `"setup"` when set, or null), and `path`.
|
|
132
|
+
`--json` is the same report (`version`, counts, `covered`, `arms_not_pinned` (the `NOT PINNED` arms, in the shape of `covered`), `frontier`, `returned` (each value as `returns`, with the `inputs` that returned it), `exceptions`, `after` (each value an after read, as `after` (its source), `is`, `value`, and the `inputs` that read it, then each after that read nothing, as `after`, `read_nothing` (the reason), and `inputs`), `not_counted`, `operators_not_counted`, `boundaries_not_counted`, `pinned_by_class` (each part of a returned value, or of what an after read, that an emitted test checks only by its class, as `path` (`value[:at]`, `hold.held[0]`), `class`, and every input that returned it)). Each frontier object has `id`, `loc`, `cond`, `sense`, `parent`, `nearest` (`{"args","kwargs"}`, plus `"new"` and `"setup"` when set, or null), and `path`.
|
|
132
133
|
|
|
133
134
|
## Rounds
|
|
134
135
|
|
|
@@ -137,13 +138,13 @@ Stdout is the contract. There is no timestamp. The report opens with `covered: 5
|
|
|
137
138
|
3. If you have not yet added an input meant to flip that arm, add one. Copy the `nearest` above its line and change it only enough to do what the line says. `&&` and `||` are not separate arms.
|
|
138
139
|
4. If a covered arm divides by a non-constant denominator, indexes out of range, or `raise`s, and you have not yet added an input that fires that fault while staying in the arm, add one.
|
|
139
140
|
5. Save and rerun the same command.
|
|
140
|
-
6. Stop when `FRONTIER` is `(none: ...)`, each `RETURNED` value is what the method should return, and no fault remains to aim at, or every remaining reached arm has already been tried and did not flip (unsatisfiable, such as `x == 1 && x == 0`), or you have completed 8 rounds.
|
|
141
|
+
6. Stop when `FRONTIER` is `(none: ...)`, each `RETURNED` value is what the method should return, each `AFTER` value is the state the call should leave, and no fault remains to aim at, or every remaining reached arm has already been tried and did not flip (unsatisfiable, such as `x == 1 && x == 0`), or you have completed 8 rounds.
|
|
141
142
|
|
|
142
143
|
In one round, add inputs only for arms that are reached now and for faults you can see now. Do not batch the `not reached yet` decisions before their parent has run.
|
|
143
144
|
|
|
144
145
|
## Emit tests
|
|
145
146
|
|
|
146
|
-
When you stop, emit characterization tests. Each one pins a discovered input's return value, or the exception class and message, a message over several lines as it was raised (the report puts each line break as one space). A `NameError` or `NoMethodError` is pinned by its class and `error.name`, so the test passes on Ruby 3.3 and 3.4+ alike. A message that quotes a backtrace frame (`lib/x.rb:3:in 'Box.run'`) is matched by a Regexp that leaves the frame's label open, since 3.3 writes `` `run' `` there; so is a returned String that quotes a frame. A `Set` is pinned by its members (`eq(Set[1, 2])`), and a `Date` by its value (`Date.new(2026, 4, 1)`). A `Struct` or `Data` whose members all have a literal is pinned by its class and members (`eq(Rule.new(action: "borrow", on: Date.new(2026, 4, 1)))`). A return value with no Ruby literal, such as an `Object`, a `Time`, or a record, is pinned by its class only. In RSpec a `Struct` or `Data` that holds such a value is pinned by its class and members, and an Array or Hash that holds one item by item (`match([a_kind_of(Rule).and(have_attributes(at: a_kind_of(Time)))])`); minitest checks the same parts: each part with no literal by its class (`assert_kind_of Time, value[0].at`), a `Struct` or `Data` by its class and members, and an Array or Hash by comparing it with a literal that reads those parts back from the value (`assert_equal([value[0]], value)`), so a missing key or item fails too. A Hash with a key that has no literal is pinned by its class in both formats. An expected Hash keeps the value's key order, so a failure diff reads key by key. A private or protected entry, and a private `new`, are called through `send`, as `explore` calls them. A String argument is written as a plain literal and reaches the method frozen, as in the emitted file. Only an input that raises `FrozenError` with frozen Strings, and not with unfrozen ones, gets `+"..."`, under a comment that says why. That value is from a second run in the same process, which saw any global, class variable, or file the first run changed before it raised; run the emitted test on its own before you trust it. A class or module that `private_constant` hides is reached through `const_get` (`Shop.const_get(:Ledger)`), and the RSpec describe names it as a string, since a bare name fails when the spec file loads.
|
|
147
|
+
When you stop, emit characterization tests. Each one pins a discovered input's return value, or the exception class and message, a message over several lines as it was raised (the report puts each line break as one space). A `NameError` or `NoMethodError` is pinned by its class and `error.name`, so the test passes on Ruby 3.3 and 3.4+ alike. A message that quotes a backtrace frame (`lib/x.rb:3:in 'Box.run'`) is matched by a Regexp that leaves the frame's label open, since 3.3 writes `` `run' `` there; so is a returned String that quotes a frame. An Errno message that names a file descriptor (`fd:5`) is matched by a Regexp that accepts any descriptor, since the number depends on what the process opened before. A `Set` is pinned by its members (`eq(Set[1, 2])`), and a `Date` by its value (`Date.new(2026, 4, 1)`). A `Struct` or `Data` whose members all have a literal is pinned by its class and members (`eq(Rule.new(action: "borrow", on: Date.new(2026, 4, 1)))`). A return value with no Ruby literal, such as an `Object`, a `Time`, or a record, is pinned by its class only. In RSpec a `Struct` or `Data` that holds such a value is pinned by its class and members, and an Array or Hash that holds one item by item (`match([a_kind_of(Rule).and(have_attributes(at: a_kind_of(Time)))])`); minitest checks the same parts: each part with no literal by its class (`assert_kind_of Time, value[0].at`), a `Struct` or `Data` by its class and members, and an Array or Hash by comparing it with a literal that reads those parts back from the value (`assert_equal([value[0]], value)`), so a missing key or item fails too. A Hash with a key that has no literal is pinned by its class in both formats. An expected Hash keeps the value's key order, so a failure diff reads key by key. A private or protected entry, and a private `new`, are called through `send`, as `explore` calls them. A String argument is written as a plain literal and reaches the method frozen, as in the emitted file. Only an input that raises `FrozenError` with frozen Strings, and not with unfrozen ones, gets `+"..."`, under a comment that says why. That value is from a second run in the same process, which saw any global, class variable, or file the first run changed before it raised; run the emitted test on its own before you trust it. A class or module that `private_constant` hides is reached through `const_get` (`Shop.const_get(:Ledger)`), and the RSpec describe names it as a string, since a bare name fails when the spec file loads.
|
|
147
148
|
|
|
148
149
|
A minitest test is named after its input and what it pins: the input's values and the outcome (`test_0_0_returns_out_of_range`). Inputs whose minitest names would be the same are named by the leaves that differ between them instead: the deepest Hash key above each leaf, with the keys above it when another leaf ends in the same key (`items_id` beside `id`), and its value, or `none` where an input lacks the key and every key above or below it (an input whose Array is empty says `empty`, and one whose Array holds items is named by them) (`test_status_fail_and_error_class_none_returns_ok`). Strings at one leaf drop the words they all start with, so two failure diffs that part at `fail` and `pass` are named from there; Ruby source, such as two factory calls in `receiver`, keeps the word before they part (`limit_100`). Only a name that still collides gets six hex digits of its input's digest. The same inputs file gives the same names, and adding an input can rename the inputs it now shares a name with. Names are cut between words: a blank string argument shows as `blank`, an empty Array or Hash argument as `empty_array` or `empty_hash`, a Hash outcome is named by the fewest leaves, nested ones too, that tell every two different sibling Hashes apart, a leaf every Hash has first (`returns_due_cents_0_and_discount_100`, `returns_discarded_1` for `counts.discarded`), the same leaves the RSpec description says, or, with no sibling, by its keys, never cut, each after the first after `and` (`returns_hash_lines_and_discount_cents`), an Array of collections by its size (`returns_array_of_2`), a Hash or an Array that holds a part with no literal, as `--pin-keys` or a varying id may leave one, the same way by its parts that have one (`returns_due_cents_100`, `returns_hash_name_and_start_at`, `returns_array_of_2`, as RSpec says `returns due_cents: 100`, `returns a Hash of name, start_at, end_at`, `returns an Array of 2`), and an Array of scalars after `array` (`[nil]` is `returns_array_nil`, never `returns_nil`); an Array that holds scalars and collections says `array`, then each scalar and each Hash's keys (`returns_array_ok_hash_id_and_name`, RSpec `returns [:ok, a Hash of id, name]`; `["Done", []]` is `returns_array_done_empty_array`). A String or a Symbol that reads as `nil`, `true`, or `false` says `text` (`returns_text_nil`), and a Hash key that reads so says `key` (`returns_key_nil_0` for `{"NIL" => 0}`). A number of more than 15 digits is named by its first three digits and its power of ten (`123e46`). An input whose key holds another shape than its sibling's (Hashes where the sibling has Arrays) is named by its own leaves, not `none`, and a missing key says `none` once. A Hash argument is named by its values, a collection in it by its key and size (`groups_3`, `tags_empty`), and an Array of collections by its size (`array_of_2`). A test emitted outside the target's directory tree requires the target by absolute path. When a test requires a file by its path from out of its own project (the nearest directory with a `Gemfile`, a gemspec, or `.git`), by absolute path or by a climb into another project, stderr warns that the test runs only where that file sits on this machine, and names the fix. Emit the test under the app (`spec/`), or pass `--require-by-name`, so it runs on another machine. With `--require-by-name`, the emitted test requires each `--require` file and the target through the load path instead, named from the first `-I` directory or `--require` file's directory that holds it: `--require lib/shop.rb` on `lib/shop/cart.rb` emits `require "shop"` and `require "shop/cart"`. A target outside them, such as a Rails model, needs `-I`: `--require lib/shop.rb -I app/models` on `app/models/cart.rb` emits `require "cart"`, and the explore run has `app/models` on its load path too. Without that `-I`, the target is required by path, under a `# Required by path:` comment, and stderr names the file. With no `--require` file and no `-I`, the name comes from the nearest `lib` directory above the target, or from the target's own directory, so a library's entry file `lib/shop.rb` needs no `--require` of itself. Run such a test with those directories on the load path (`ruby -Ilib -Iapp/models`); a `# Load path: -I lib -I app/models` comment above its requires names them, from the working directory, or, for a directory outside it, from its project root with the project named (`-I lib (in shop)`). When a `--require` file already loads the target, the test requires the `--require` files only. That holds for a target it loaded with `Kernel#load`, as Rails loads an initializer, and explore does not load it a second time. The `# target:` header is relative to the working directory, or, for a target outside it, to the nearest directory with a `Gemfile`, a gemspec, or `.git`.
|
|
149
150
|
|