rgot 1.1.0 → 1.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
data/lib/rgot/f.rb ADDED
@@ -0,0 +1,230 @@
1
+ module Rgot
2
+ # def fuzz_foo(f)
3
+ # f.add(5, "hello")
4
+ # f.fuzz do |t, i, s|
5
+ # ...
6
+ # end
7
+ class F < Common
8
+ class Options
9
+ # @dynamic fuzz, fuzz=, fuzztime, fuzztime=
10
+ attr_accessor :fuzz
11
+ attr_accessor :fuzztime
12
+ def initialize(fuzz:, fuzztime:)
13
+ @fuzz = fuzz
14
+ @fuzztime = fuzztime
15
+ end
16
+ end
17
+
18
+ class CorpusEntry
19
+ # @dynamic values, values=, is_seed, is_seed=, path, path=
20
+ attr_accessor :values
21
+ attr_accessor :is_seed
22
+ attr_accessor :path
23
+ def initialize(values:, is_seed:, path:)
24
+ @values = values
25
+ @is_seed = is_seed
26
+ @path = path
27
+ end
28
+
29
+ def mutate_values
30
+ @values.map do |value|
31
+ if generator = SUPPORTED_TYPES[value.class]
32
+ generator.call(value)
33
+ else
34
+ raise "unsupported type #{value.class}"
35
+ end
36
+ end
37
+ end
38
+ end
39
+
40
+ class Coordinator
41
+ # @dynamic count, count=, interesting_count, interesting_count=
42
+ attr_accessor :count
43
+ attr_accessor :interesting_count
44
+
45
+ def initialize(warmup_input_count:)
46
+ @warmup_input_count = warmup_input_count
47
+ @before_cov = 0
48
+ @start_time = Rgot.now
49
+ @count = 0
50
+ @interesting_count = 0
51
+ @count_last_log = 0
52
+ @time_last_log = 0.0
53
+ end
54
+
55
+ def start_logger
56
+ Thread.new do
57
+ loop do
58
+ log_stats
59
+ sleep 3
60
+ end
61
+ end
62
+ end
63
+
64
+ def diff_coverage
65
+ current_cov = Coverage.peek_result.sum do |path, hash|
66
+ hash.map do |_, covs|
67
+ covs.length
68
+ end.sum
69
+ end
70
+ (current_cov - @before_cov).tap { @before_cov = current_cov }
71
+ end
72
+
73
+ def log_stats
74
+ rate = Float(count - @count_last_log) / (Rgot.now - @time_last_log)
75
+ total = @warmup_input_count + interesting_count
76
+ printf "fuzz: elapsed: %ds, execs: %d (%d/sec), new interesting: %d (total: %d)\n",
77
+ elapsed, count, rate, interesting_count, total
78
+
79
+ duration = Rgot.now - @time_last_log
80
+ @count_last_log = count
81
+ @time_last_log = Rgot.now
82
+ end
83
+
84
+ private
85
+
86
+ def elapsed
87
+ (Rgot.now - @start_time).round
88
+ end
89
+ end
90
+
91
+ SUPPORTED_TYPES = {
92
+ TrueClass => ->(v) { [true, false].sample },
93
+ FalseClass => ->(v) { [true, false].sample },
94
+ Integer => ->(v) { Random.rand(v) },
95
+ Float => ->(v) { Random.rand(v) },
96
+ String => ->(v) { Random.bytes(v.length) },
97
+ }
98
+
99
+ # @dynamic name
100
+ attr_reader :name
101
+
102
+ def initialize(fuzz_target:, opts:)
103
+ super()
104
+ @opts = opts
105
+ @fuzz_target = fuzz_target
106
+ @fuzz_block = nil
107
+ @module = fuzz_target.module
108
+ @name = fuzz_target.name
109
+ @corpus = []
110
+ end
111
+
112
+ # TODO: DRY with T
113
+ def run
114
+ catch(:skip) { call }
115
+ finish!
116
+ rescue => e
117
+ fail!
118
+ raise e
119
+ end
120
+
121
+ def run_testing
122
+ run
123
+ report if !fuzz? || failed?
124
+ end
125
+
126
+ def run_fuzzing
127
+ return unless fuzz?
128
+ raise("must call after #fuzz") unless @fuzz_block
129
+
130
+ coordinator = Coordinator.new(
131
+ warmup_input_count: @corpus.length
132
+ )
133
+ coordinator.start_logger
134
+
135
+ t = T.new(@fuzz_target.module, @fuzz_target.name)
136
+
137
+ begin
138
+ Timeout.timeout(@opts.fuzztime.to_f) do
139
+ loop do
140
+ @corpus.each do |entry|
141
+ values = entry.mutate_values
142
+
143
+ @fuzz_block.call(t, *values)
144
+
145
+ if 0 < coordinator.diff_coverage
146
+ coordinator.interesting_count += 1
147
+ end
148
+ coordinator.count += 1
149
+ fail! if t.failed?
150
+ end
151
+ end
152
+ end
153
+ rescue Timeout::Error, Interrupt
154
+ coordinator.log_stats
155
+ end
156
+
157
+ report
158
+ end
159
+
160
+ def add(*args)
161
+ args.each do |arg|
162
+ unless SUPPORTED_TYPES.key?(arg.class)
163
+ raise "unsupported type to Add #{arg.class}"
164
+ end
165
+ end
166
+ entry = CorpusEntry.new(
167
+ values: args.dup,
168
+ is_seed: true,
169
+ path: "seed##{@corpus.length}"
170
+ )
171
+ @corpus.push(entry)
172
+ end
173
+
174
+ def fuzz(&block)
175
+ unless block
176
+ raise LocalJumpError, "must set block"
177
+ end
178
+ unless 2 <= block.arity
179
+ raise "fuzz target must receive at least two arguments"
180
+ end
181
+
182
+ t = T.new(@fuzz_target.module, @fuzz_target.name)
183
+
184
+ @corpus.each do |entry|
185
+ unless entry.values.length == (block.arity - 1)
186
+ raise "wrong number of values in corpus entry: #{entry.values.length}, want #{block.arity - 1}"
187
+ end
188
+ block.call(t, *entry.values.dup)
189
+ fail! if t.failed?
190
+ end
191
+
192
+ @fuzz_block = block
193
+
194
+ nil
195
+ end
196
+
197
+ def fuzz?
198
+ return false unless @opts.fuzz
199
+ return false unless Regexp.new(@opts.fuzz.to_s).match?(@fuzz_target.name)
200
+ true
201
+ end
202
+
203
+ def report
204
+ puts @output if Rgot.verbose? && !@output.empty?
205
+ duration = Rgot.now - @start
206
+ template = "--- \e[%sm%s\e[m: %s (%.2fs)\n"
207
+ if failed?
208
+ printf template, [41, 1].join(';'), "FAIL", @name, duration
209
+ elsif Rgot.verbose?
210
+ if skipped?
211
+ printf template, [44, 1].join(';'), "SKIP", @name, duration
212
+ else
213
+ printf template, [42, 1].join(';'), "PASS", @name, duration
214
+ end
215
+ end
216
+ end
217
+
218
+ private
219
+
220
+ def call
221
+ test_method = @module.instance_method(@name).bind(@module)
222
+ if test_method.arity == 0
223
+ path, line = test_method.source_location
224
+ warn "#{path}:#{line} `#{test_method.name}' is not running. It's a testing method name, But not have argument"
225
+ else
226
+ test_method.call(self)
227
+ end
228
+ end
229
+ end
230
+ end
data/lib/rgot/m.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  require 'stringio'
2
4
  require 'etc'
3
5
  require 'timeout'
@@ -10,45 +12,101 @@ module Rgot
10
12
  :timeout,
11
13
  :cpu,
12
14
  :thread,
15
+ :fuzz,
16
+ :fuzztime,
13
17
  ); end
14
18
 
15
- def initialize(tests:, benchmarks:, examples:, opts: Options.new)
16
- cpu = opts.cpu || (Etc.respond_to?(:nprocessors) ? Etc.nprocessors : '1').to_s
17
- @cpu_list = cpu.split(',').map { |i|
18
- j = i.to_i
19
- raise Rgot::OptionError, "invalid value #{i.inspect} for --cpu" unless 0 < j
20
- j
21
- }
22
- @thread_list = (opts.thread || "1").split(',').map { |i|
23
- j = i.to_i
24
- raise Rgot::OptionError, "invalid value #{i.inspect} for --thread" unless 0 < j
25
- j
26
- }
19
+ def initialize(tests:, benchmarks:, examples:, fuzz_targets: nil, test_module: nil, opts: Options.new)
20
+ unless fuzz_targets
21
+ raise "Require `fuzz_targets` keyword" if Gem::Version.new("2.0") <= Gem::Version.new(Rgot::VERSION)
22
+ warn "`Rgot::M#initialize` will require the `fuzz_targets` keyword in the next major version."
23
+ end
24
+ unless test_module
25
+ raise "Require `test_module` keyword" if Gem::Version.new("2.0") <= Gem::Version.new(Rgot::VERSION)
26
+ warn "`Rgot::M#initialize` will require the `test_module` keyword in the next major version."
27
+ end
28
+
27
29
  @tests = tests
28
30
  @benchmarks = benchmarks
29
31
  @examples = examples
32
+ @fuzz_targets = fuzz_targets || []
33
+ @test_module = test_module
30
34
  @opts = opts
35
+
36
+ @cpu_list = []
37
+ @thread_list = []
38
+ @fs = @fuzz_targets.map do |fuzz_target|
39
+ F.new(
40
+ fuzz_target: fuzz_target,
41
+ opts: F::Options.new(
42
+ fuzz: opts.fuzz,
43
+ fuzztime: opts.fuzztime,
44
+ )
45
+ )
46
+ end
31
47
  end
32
48
 
33
49
  def run
50
+ duration = Rgot.now
34
51
  test_ok = false
52
+ fuzz_targets_ok = false
35
53
  example_ok = false
36
54
 
55
+ if @tests.empty? && @benchmarks.empty? && @examples.empty? && @fuzz_targets.empty?
56
+ warn "rgot: warning: no tests to run"
57
+ end
58
+
59
+ begin
60
+ parse_option
61
+ rescue Rgot::OptionError
62
+ puts sprintf("%s\t%s\t%.3fs", "FAIL", @test_module, Rgot.now - duration)
63
+ raise
64
+ end
65
+
37
66
  Timeout.timeout(@opts.timeout.to_f) do
38
67
  test_ok = run_tests
68
+ fuzz_targets_ok = run_fuzz_tests
39
69
  example_ok = run_examples
40
70
  end
41
- if !test_ok || !example_ok
71
+
72
+ if !test_ok || !example_ok || !fuzz_targets_ok
73
+ puts "FAIL"
74
+ puts "exit status 1"
75
+ puts sprintf("%s\t%s\t%.3fs", "FAIL", @test_module, Rgot.now - duration)
76
+ return 1
77
+ end
78
+
79
+ if !run_fuzzing()
42
80
  puts "FAIL"
81
+ puts "exit status 1"
82
+ puts sprintf("%s\t%s\t%.3fs", "FAIL", @test_module, Rgot.now - duration)
43
83
  return 1
44
84
  end
85
+
45
86
  puts "PASS"
46
87
  run_benchmarks
88
+ puts sprintf("%s\t%s\t%.3fs", "ok ", @test_module, Rgot.now - duration)
89
+
47
90
  0
48
91
  end
49
92
 
50
93
  private
51
94
 
95
+ def parse_option
96
+ cpu = @opts.cpu || (Etc.respond_to?(:nprocessors) ? Etc.nprocessors : '1').to_s
97
+ @cpu_list = cpu.split(',').map { |i|
98
+ j = i.to_i
99
+ raise Rgot::OptionError, "invalid value #{i.inspect} for --cpu" unless 0 < j
100
+ j
101
+ }
102
+
103
+ @thread_list = (@opts.thread || "1").split(',').map { |i|
104
+ j = i.to_i
105
+ raise Rgot::OptionError, "invalid value #{i.inspect} for --thread" unless 0 < j
106
+ j
107
+ }
108
+ end
109
+
52
110
  def run_tests
53
111
  ok = true
54
112
  @tests.each do |test|
@@ -57,10 +115,7 @@ module Rgot
57
115
  puts "=== RUN #{test.name}"
58
116
  end
59
117
  t.run
60
- t.report
61
- if t.failed?
62
- ok = false
63
- end
118
+ ok = ok && !t.failed?
64
119
  end
65
120
  ok
66
121
  end
@@ -98,6 +153,50 @@ module Rgot
98
153
  ok
99
154
  end
100
155
 
156
+ def run_fuzz_tests
157
+ ok = true
158
+ @fs.each do |f|
159
+ if Rgot.verbose?
160
+ if f.fuzz?
161
+ puts "=== FUZZ #{f.name}"
162
+ else
163
+ puts "=== RUN #{f.name}"
164
+ end
165
+ end
166
+ f.run_testing
167
+ ok = ok && !f.failed?
168
+ end
169
+ ok
170
+ end
171
+
172
+ def run_fuzzing
173
+ if @fuzz_targets.empty? || @opts.fuzz.nil?
174
+ return true
175
+ end
176
+
177
+ fuzzing_fs = @fs.select(&:fuzz?)
178
+
179
+ if fuzzing_fs.empty?
180
+ puts "rgot: warning: no fuzz tests to fuzz"
181
+ return true
182
+ end
183
+
184
+ if fuzzing_fs.length > 1
185
+ names = fuzzing_fs.map(&:name)
186
+ puts "rgot: will not fuzz, --fuzz matches more than one fuzz test: #{names.inspect}"
187
+ return false
188
+ end
189
+
190
+ ok = true
191
+
192
+ fuzzing_fs.each do |f|
193
+ f.run_fuzzing
194
+ ok = ok && !f.failed?
195
+ end
196
+
197
+ ok
198
+ end
199
+
101
200
  def run_examples
102
201
  ok = true
103
202
  @examples.each do |example|
@@ -111,13 +210,13 @@ module Rgot
111
210
  out, _ = capture do
112
211
  method.call
113
212
  end
114
- file = method.source_location[0]
115
- r = ExampleParser.new(File.read(file))
116
- r.parse
117
- e = r.examples.find { |re| re.name == example.name }
213
+ file = method.source_location&.[](0) or raise("bug")
214
+ example_parser = ExampleParser.new(File.read(file))
215
+ example_parser.parse
216
+ e = example_parser.examples.find { |er| er.name == example.name } or raise("bug")
118
217
 
119
218
  duration = Rgot.now - start
120
- if e && e.output.strip != out.strip
219
+ if e.output.strip != out.strip
121
220
  printf("--- FAIL: %s (%.2fs)\n", e.name, duration)
122
221
  ok = false
123
222
  puts "got:"
data/lib/rgot/pb.rb CHANGED
@@ -1,7 +1,7 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Rgot
2
4
  class PB
3
- attr_accessor :bn
4
-
5
5
  def initialize(bn:)
6
6
  @bn = bn
7
7
  end
data/lib/rgot/t.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Rgot
2
4
  class T < Common
3
5
  def initialize(test_module, name)
@@ -10,6 +12,7 @@ module Rgot
10
12
  def run
11
13
  catch(:skip) { call }
12
14
  finish!
15
+ report
13
16
  rescue => e
14
17
  fail!
15
18
  report
@@ -17,15 +20,16 @@ module Rgot
17
20
  end
18
21
 
19
22
  def report
23
+ puts @output if Rgot.verbose? && !@output.empty?
20
24
  duration = Rgot.now - @start
21
- template = "--- %s: %s (%.2fs)\n%s"
25
+ template = "--- \e[%sm%s\e[m: %s (%.2fs)\n"
22
26
  if failed?
23
- printf template, "FAIL", @name, duration, @output
27
+ printf template, [41, 1].join(';'), "FAIL", @name, duration
24
28
  elsif Rgot.verbose?
25
29
  if skipped?
26
- printf template, "SKIP", @name, duration, @output
30
+ printf template, [44, 1].join(';'), "SKIP", @name, duration
27
31
  else
28
- printf template, "PASS", @name, duration, @output
32
+ printf template, [42, 1].join(';'), "PASS", @name, duration
29
33
  end
30
34
  end
31
35
  end
data/lib/rgot/version.rb CHANGED
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Rgot
2
- VERSION = "1.1.0"
4
+ VERSION = "1.3.0"
3
5
  end
data/lib/rgot.rb CHANGED
@@ -1,17 +1,21 @@
1
+ # frozen_string_literal: true
2
+
1
3
  module Rgot
2
- require 'rgot/version'
3
- require 'rgot/common'
4
- require 'rgot/m'
5
- require 'rgot/t'
6
- require 'rgot/b'
7
- require 'rgot/pb'
8
- require 'rgot/benchmark_result'
9
- require 'rgot/example_parser'
4
+ autoload :VERSION, 'rgot/version'
5
+ autoload :Common, 'rgot/common'
6
+ autoload :M, 'rgot/m'
7
+ autoload :T, 'rgot/t'
8
+ autoload :B, 'rgot/b'
9
+ autoload :PB, 'rgot/pb'
10
+ autoload :BenchmarkResult, 'rgot/benchmark_result'
11
+ autoload :F, 'rgot/f'
12
+ autoload :ExampleParser, 'rgot/example_parser'
10
13
 
11
14
  OptionError = Class.new(StandardError)
12
15
  InternalTest = Struct.new(:module, :name)
13
16
  InternalBenchmark = Struct.new(:module, :name)
14
17
  InternalExample = Struct.new(:module, :name)
18
+ InternalFuzzTarget = Struct.new(:module, :name)
15
19
  ExampleOutput = Struct.new(:name, :output)
16
20
 
17
21
  class << self
@@ -0,0 +1,49 @@
1
+ ---
2
+ sources:
3
+ - name: ruby/gem_rbs_collection
4
+ remote: https://github.com/ruby/gem_rbs_collection.git
5
+ revision: main
6
+ repo_dir: gems
7
+ path: ".gem_rbs_collection"
8
+ gems:
9
+ - name: pathname
10
+ version: '0'
11
+ source:
12
+ type: stdlib
13
+ - name: optparse
14
+ version: '0'
15
+ source:
16
+ type: stdlib
17
+ - name: timeout
18
+ version: '0'
19
+ source:
20
+ type: stdlib
21
+ - name: etc
22
+ version: '0'
23
+ source:
24
+ type: stdlib
25
+ - name: coverage
26
+ version: '0'
27
+ source:
28
+ type: stdlib
29
+ - name: concurrent-ruby
30
+ version: '1.1'
31
+ source:
32
+ type: git
33
+ name: ruby/gem_rbs_collection
34
+ revision: 9576ce5b109170f1ba8a42671bfafb64ab95bd23
35
+ remote: https://github.com/ruby/gem_rbs_collection.git
36
+ repo_dir: gems
37
+ - name: io-console
38
+ version: '0'
39
+ source:
40
+ type: stdlib
41
+ - name: logger
42
+ version: '0'
43
+ source:
44
+ type: stdlib
45
+ - name: monitor
46
+ version: '0'
47
+ source:
48
+ type: stdlib
49
+ gemfile_lock_path: Gemfile.lock
@@ -0,0 +1,53 @@
1
+ # Download sources
2
+ sources:
3
+ - name: ruby/gem_rbs_collection
4
+ remote: https://github.com/ruby/gem_rbs_collection.git
5
+ revision: main
6
+ repo_dir: gems
7
+
8
+ # A directory to install the downloaded RBSs
9
+ path: .gem_rbs_collection
10
+
11
+ gems:
12
+ - name: pathname
13
+ - name: optparse
14
+ - name: timeout
15
+ - name: etc
16
+ - name: coverage
17
+
18
+ # Skip loading rbs gem's RBS.
19
+ # It's unnecessary if you don't use rbs as a library.
20
+ - name: rbs
21
+ ignore: true
22
+ - name: steep
23
+ ignore: true
24
+ - name: activesupport
25
+ ignore: true
26
+ - name: ast
27
+ ignore: true
28
+ - name: csv
29
+ ignore: true
30
+ - name: i18n
31
+ ignore: true
32
+ - name: json
33
+ ignore: true
34
+ - name: listen
35
+ ignore: true
36
+ - name: fileutils
37
+ ignore: true
38
+ - name: minitest
39
+ ignore: true
40
+ - name: parallel
41
+ ignore: true
42
+ - name: rainbow
43
+ ignore: true
44
+ - name: rgot
45
+ ignore: true
46
+ - name: securerandom
47
+ ignore: true
48
+ - name: strscan
49
+ ignore: true
50
+ - name: forwardable
51
+ ignore: true
52
+ - name: mutex_m
53
+ ignore: true
data/rgot.gemspec CHANGED
@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
14
14
  spec.homepage = "https://github.com/ksss/rgot"
15
15
  spec.license = "MIT"
16
16
 
17
- spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
17
+ spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|go)/}) }
18
18
  spec.bindir = "bin"
19
19
  spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
20
20
  spec.require_paths = ["lib"]
data/sig/patch.rbs ADDED
@@ -0,0 +1,3 @@
1
+ # patch
2
+ class Ripper
3
+ end