rgot 1.1.0 → 1.3.0

data/lib/rgot/f.rb

@@ -0,0 +1,230 @@
+module Rgot
+  # def fuzz_foo(f)
+  #   f.add(5, "hello")
+  #   f.fuzz do |t, i, s|
+  #     ...
+  #   end
+  class F < Common
+    class Options
+      # @dynamic fuzz, fuzz=, fuzztime, fuzztime=
+      attr_accessor :fuzz
+      attr_accessor :fuzztime
+      def initialize(fuzz:, fuzztime:)
+        @fuzz = fuzz
+        @fuzztime = fuzztime
+      end
+    end
+
+    class CorpusEntry
+      # @dynamic values, values=, is_seed, is_seed=, path, path=
+      attr_accessor :values
+      attr_accessor :is_seed
+      attr_accessor :path
+      def initialize(values:, is_seed:, path:)
+        @values = values
+        @is_seed = is_seed
+        @path = path
+      end
+
+      def mutate_values
+        @values.map do |value|
+          if generator = SUPPORTED_TYPES[value.class]
+            generator.call(value)
+          else
+            raise "unsupported type #{value.class}"
+          end
+        end
+      end
+    end
+
+    class Coordinator
+      # @dynamic count, count=, interesting_count, interesting_count=
+      attr_accessor :count
+      attr_accessor :interesting_count
+
+      def initialize(warmup_input_count:)
+        @warmup_input_count = warmup_input_count
+        @before_cov = 0
+        @start_time = Rgot.now
+        @count = 0
+        @interesting_count = 0
+        @count_last_log = 0
+        @time_last_log = 0.0
+      end
+
+      def start_logger
+        Thread.new do
+          loop do
+            log_stats
+            sleep 3
+          end
+        end
+      end
+
+      def diff_coverage
+        current_cov = Coverage.peek_result.sum do |path, hash|
+          hash.map do |_, covs|
+            covs.length
+          end.sum
+        end
+        (current_cov - @before_cov).tap { @before_cov = current_cov }
+      end
+
+      def log_stats
+        rate = Float(count - @count_last_log) / (Rgot.now - @time_last_log)
+        total = @warmup_input_count + interesting_count
+        printf "fuzz: elapsed: %ds, execs: %d (%d/sec), new interesting: %d (total: %d)\n",
+          elapsed, count, rate, interesting_count, total
+
+        duration = Rgot.now - @time_last_log
+        @count_last_log = count
+        @time_last_log = Rgot.now
+      end
+
+      private
+
+      def elapsed
+        (Rgot.now - @start_time).round
+      end
+    end
+
+    SUPPORTED_TYPES = {
+      TrueClass => ->(v) { [true, false].sample },
+      FalseClass => ->(v) { [true, false].sample },
+      Integer => ->(v) { Random.rand(v) },
+      Float => ->(v) { Random.rand(v) },
+      String => ->(v) { Random.bytes(v.length) },
+    }
+
+    # @dynamic name
+    attr_reader :name
+
+    def initialize(fuzz_target:, opts:)
+      super()
+      @opts = opts
+      @fuzz_target = fuzz_target
+      @fuzz_block = nil
+      @module = fuzz_target.module
+      @name = fuzz_target.name
+      @corpus = []
+    end
+
+    # TODO: DRY with T
+    def run
+      catch(:skip) { call }
+      finish!
+    rescue => e
+      fail!
+      raise e
+    end
+
+    def run_testing
+      run
+      report if !fuzz? || failed?
+    end
+
+    def run_fuzzing
+      return unless fuzz?
+      raise("must call after #fuzz") unless @fuzz_block
+
+      coordinator = Coordinator.new(
+        warmup_input_count: @corpus.length
+      )
+      coordinator.start_logger
+
+      t = T.new(@fuzz_target.module, @fuzz_target.name)
+
+      begin
+        Timeout.timeout(@opts.fuzztime.to_f) do
+          loop do
+            @corpus.each do |entry|
+              values = entry.mutate_values
+
+              @fuzz_block.call(t, *values)
+
+              if 0 < coordinator.diff_coverage
+                coordinator.interesting_count += 1
+              end
+              coordinator.count += 1
+              fail! if t.failed?
+            end
+          end
+        end
+      rescue Timeout::Error, Interrupt
+        coordinator.log_stats
+      end
+
+      report
+    end
+
+    def add(*args)
+      args.each do |arg|
+        unless SUPPORTED_TYPES.key?(arg.class)
+          raise "unsupported type to Add #{arg.class}"
+        end
+      end
+      entry = CorpusEntry.new(
+        values: args.dup,
+        is_seed: true,
+        path: "seed##{@corpus.length}"
+      )
+      @corpus.push(entry)
+    end
+
+    def fuzz(&block)
+      unless block
+        raise LocalJumpError, "must set block"
+      end
+      unless 2 <= block.arity
+        raise "fuzz target must receive at least two arguments"
+      end
+
+      t = T.new(@fuzz_target.module, @fuzz_target.name)
+
+      @corpus.each do |entry|
+        unless entry.values.length == (block.arity - 1)
+          raise "wrong number of values in corpus entry: #{entry.values.length}, want #{block.arity - 1}"
+        end
+        block.call(t, *entry.values.dup)
+        fail! if t.failed?
+      end
+
+      @fuzz_block = block
+
+      nil
+    end
+
+    def fuzz?
+      return false unless @opts.fuzz
+      return false unless Regexp.new(@opts.fuzz.to_s).match?(@fuzz_target.name)
+      true
+    end
+
+    def report
+      puts @output if Rgot.verbose? && !@output.empty?
+      duration = Rgot.now - @start
+      template = "--- \e[%sm%s\e[m: %s (%.2fs)\n"
+      if failed?
+        printf template, [41, 1].join(';'), "FAIL", @name, duration
+      elsif Rgot.verbose?
+        if skipped?
+          printf template, [44, 1].join(';'), "SKIP", @name, duration
+        else
+          printf template, [42, 1].join(';'), "PASS", @name, duration
+        end
+      end
+    end
+
+    private
+
+    def call
+      test_method = @module.instance_method(@name).bind(@module)
+      if test_method.arity == 0
+        path, line = test_method.source_location
+        warn "#{path}:#{line} `#{test_method.name}' is not running. It's a testing method name, But not have argument"
+      else
+        test_method.call(self)
+      end
+    end
+  end
+end

data/lib/rgot/m.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'stringio'
 require 'etc'
 require 'timeout'
@@ -10,45 +12,101 @@ module Rgot
       :timeout,
       :cpu,
       :thread,
+      :fuzz,
+      :fuzztime,
     ); end
 
-    def initialize(tests:, benchmarks:, examples:, opts: Options.new)
-      cpu = opts.cpu || (Etc.respond_to?(:nprocessors) ? Etc.nprocessors : '1').to_s
-      @cpu_list = cpu.split(',').map { |i|
-        j = i.to_i
-        raise Rgot::OptionError, "invalid value #{i.inspect} for --cpu" unless 0 < j
-        j
-      }
-      @thread_list = (opts.thread || "1").split(',').map { |i|
-        j = i.to_i
-        raise Rgot::OptionError, "invalid value #{i.inspect} for --thread" unless 0 < j
-        j
-      }
+    def initialize(tests:, benchmarks:, examples:, fuzz_targets: nil, test_module: nil, opts: Options.new)
+      unless fuzz_targets
+        raise "Require `fuzz_targets` keyword" if Gem::Version.new("2.0") <= Gem::Version.new(Rgot::VERSION)
+        warn "`Rgot::M#initialize` will require the `fuzz_targets` keyword in the next major version."
+      end
+      unless test_module
+        raise "Require `test_module` keyword" if Gem::Version.new("2.0") <= Gem::Version.new(Rgot::VERSION)
+        warn "`Rgot::M#initialize` will require the `test_module` keyword in the next major version."
+      end
+
       @tests = tests
       @benchmarks = benchmarks
       @examples = examples
+      @fuzz_targets = fuzz_targets || []
+      @test_module = test_module
       @opts = opts
+
+      @cpu_list = []
+      @thread_list = []
+      @fs = @fuzz_targets.map do |fuzz_target|
+        F.new(
+          fuzz_target: fuzz_target,
+          opts: F::Options.new(
+            fuzz: opts.fuzz,
+            fuzztime: opts.fuzztime,
+          )
+        )
+      end
     end
 
     def run
+      duration = Rgot.now
       test_ok = false
+      fuzz_targets_ok = false
       example_ok = false
 
+      if @tests.empty? && @benchmarks.empty? && @examples.empty? && @fuzz_targets.empty?
+        warn "rgot: warning: no tests to run"
+      end
+
+      begin
+        parse_option
+      rescue Rgot::OptionError
+        puts sprintf("%s\t%s\t%.3fs", "FAIL", @test_module, Rgot.now - duration)
+        raise
+      end
+
       Timeout.timeout(@opts.timeout.to_f) do
         test_ok = run_tests
+        fuzz_targets_ok = run_fuzz_tests
         example_ok = run_examples
       end
-      if !test_ok || !example_ok
+
+      if !test_ok || !example_ok || !fuzz_targets_ok
+        puts "FAIL"
+        puts "exit status 1"
+        puts sprintf("%s\t%s\t%.3fs", "FAIL", @test_module, Rgot.now - duration)
+        return 1
+      end
+
+      if !run_fuzzing()
         puts "FAIL"
+        puts "exit status 1"
+        puts sprintf("%s\t%s\t%.3fs", "FAIL", @test_module, Rgot.now - duration)
         return 1
       end
+
       puts "PASS"
       run_benchmarks
+      puts sprintf("%s\t%s\t%.3fs", "ok  ", @test_module, Rgot.now - duration)
+
       0
     end
 
     private
 
+    def parse_option
+      cpu = @opts.cpu || (Etc.respond_to?(:nprocessors) ? Etc.nprocessors : '1').to_s
+      @cpu_list = cpu.split(',').map { |i|
+        j = i.to_i
+        raise Rgot::OptionError, "invalid value #{i.inspect} for --cpu" unless 0 < j
+        j
+      }
+
+      @thread_list = (@opts.thread || "1").split(',').map { |i|
+        j = i.to_i
+        raise Rgot::OptionError, "invalid value #{i.inspect} for --thread" unless 0 < j
+        j
+      }
+    end
+
     def run_tests
       ok = true
       @tests.each do |test|
@@ -57,10 +115,7 @@ module Rgot
           puts "=== RUN   #{test.name}"
         end
         t.run
-        t.report
-        if t.failed?
-          ok = false
-        end
+        ok = ok && !t.failed?
       end
       ok
     end
@@ -98,6 +153,50 @@ module Rgot
       ok
     end
 
+    def run_fuzz_tests
+      ok = true
+      @fs.each do |f|
+        if Rgot.verbose?
+          if f.fuzz?
+            puts "=== FUZZ  #{f.name}"
+          else
+            puts "=== RUN   #{f.name}"
+          end
+        end
+        f.run_testing
+        ok = ok && !f.failed?
+      end
+      ok
+    end
+
+    def run_fuzzing
+      if @fuzz_targets.empty? || @opts.fuzz.nil?
+        return true
+      end
+
+      fuzzing_fs = @fs.select(&:fuzz?)
+
+      if fuzzing_fs.empty?
+        puts "rgot: warning: no fuzz tests to fuzz"
+        return true
+      end
+
+      if fuzzing_fs.length > 1
+        names = fuzzing_fs.map(&:name)
+        puts "rgot: will not fuzz, --fuzz matches more than one fuzz test: #{names.inspect}"
+        return false
+      end
+
+      ok = true
+
+      fuzzing_fs.each do |f|
+        f.run_fuzzing
+        ok = ok && !f.failed?
+      end
+
+      ok
+    end
+
     def run_examples
       ok = true
       @examples.each do |example|
@@ -111,13 +210,13 @@ module Rgot
         out, _ = capture do
           method.call
         end
-        file = method.source_location[0]
-        r = ExampleParser.new(File.read(file))
-        r.parse
-        e = r.examples.find { |re| re.name == example.name }
+        file = method.source_location&.[](0) or raise("bug")
+        example_parser = ExampleParser.new(File.read(file))
+        example_parser.parse
+        e = example_parser.examples.find { |er| er.name == example.name } or raise("bug")
 
         duration = Rgot.now - start
-        if e && e.output.strip != out.strip
+        if e.output.strip != out.strip
           printf("--- FAIL: %s (%.2fs)\n", e.name, duration)
           ok = false
           puts "got:"

data/lib/rgot/pb.rb

@@ -1,7 +1,7 @@
+# frozen_string_literal: true
+
 module Rgot
   class PB
-    attr_accessor :bn
-
     def initialize(bn:)
       @bn = bn
     end

data/lib/rgot/t.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rgot
   class T < Common
     def initialize(test_module, name)
@@ -10,6 +12,7 @@ module Rgot
     def run
       catch(:skip) { call }
       finish!
+      report
     rescue => e
       fail!
       report
@@ -17,15 +20,16 @@ module Rgot
     end
 
     def report
+      puts @output if Rgot.verbose? && !@output.empty?
       duration = Rgot.now - @start
-      template = "--- %s: %s (%.2fs)\n%s"
+      template = "--- \e[%sm%s\e[m: %s (%.2fs)\n"
       if failed?
-        printf template, "FAIL", @name, duration, @output
+        printf template, [41, 1].join(';'), "FAIL", @name, duration
       elsif Rgot.verbose?
         if skipped?
-          printf template, "SKIP", @name, duration, @output
+          printf template, [44, 1].join(';'), "SKIP", @name, duration
         else
-          printf template, "PASS", @name, duration, @output
+          printf template, [42, 1].join(';'), "PASS", @name, duration
         end
       end
     end

data/lib/rgot/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Rgot
-  VERSION = "1.1.0"
+  VERSION = "1.3.0"
 end

data/lib/rgot.rb

@@ -1,17 +1,21 @@
+# frozen_string_literal: true
+
 module Rgot
-  require 'rgot/version'
-  require 'rgot/common'
-  require 'rgot/m'
-  require 'rgot/t'
-  require 'rgot/b'
-  require 'rgot/pb'
-  require 'rgot/benchmark_result'
-  require 'rgot/example_parser'
+  autoload :VERSION, 'rgot/version'
+  autoload :Common, 'rgot/common'
+  autoload :M, 'rgot/m'
+  autoload :T, 'rgot/t'
+  autoload :B, 'rgot/b'
+  autoload :PB, 'rgot/pb'
+  autoload :BenchmarkResult, 'rgot/benchmark_result'
+  autoload :F, 'rgot/f'
+  autoload :ExampleParser, 'rgot/example_parser'
 
   OptionError = Class.new(StandardError)
   InternalTest = Struct.new(:module, :name)
   InternalBenchmark = Struct.new(:module, :name)
   InternalExample = Struct.new(:module, :name)
+  InternalFuzzTarget = Struct.new(:module, :name)
   ExampleOutput = Struct.new(:name, :output)
 
   class << self

data/rbs_collection.lock.yaml

@@ -0,0 +1,49 @@
+---
+sources:
+- name: ruby/gem_rbs_collection
+  remote: https://github.com/ruby/gem_rbs_collection.git
+  revision: main
+  repo_dir: gems
+path: ".gem_rbs_collection"
+gems:
+- name: pathname
+  version: '0'
+  source:
+    type: stdlib
+- name: optparse
+  version: '0'
+  source:
+    type: stdlib
+- name: timeout
+  version: '0'
+  source:
+    type: stdlib
+- name: etc
+  version: '0'
+  source:
+    type: stdlib
+- name: coverage
+  version: '0'
+  source:
+    type: stdlib
+- name: concurrent-ruby
+  version: '1.1'
+  source:
+    type: git
+    name: ruby/gem_rbs_collection
+    revision: 9576ce5b109170f1ba8a42671bfafb64ab95bd23
+    remote: https://github.com/ruby/gem_rbs_collection.git
+    repo_dir: gems
+- name: io-console
+  version: '0'
+  source:
+    type: stdlib
+- name: logger
+  version: '0'
+  source:
+    type: stdlib
+- name: monitor
+  version: '0'
+  source:
+    type: stdlib
+gemfile_lock_path: Gemfile.lock

data/rbs_collection.yaml

@@ -0,0 +1,53 @@
+# Download sources
+sources:
+  - name: ruby/gem_rbs_collection
+    remote: https://github.com/ruby/gem_rbs_collection.git
+    revision: main
+    repo_dir: gems
+
+# A directory to install the downloaded RBSs
+path: .gem_rbs_collection
+
+gems:
+  - name: pathname
+  - name: optparse
+  - name: timeout
+  - name: etc
+  - name: coverage
+
+  # Skip loading rbs gem's RBS.
+  # It's unnecessary if you don't use rbs as a library.
+  - name: rbs
+    ignore: true
+  - name: steep
+    ignore: true
+  - name: activesupport
+    ignore: true
+  - name: ast
+    ignore: true
+  - name: csv
+    ignore: true
+  - name: i18n
+    ignore: true
+  - name: json
+    ignore: true
+  - name: listen
+    ignore: true
+  - name: fileutils
+    ignore: true
+  - name: minitest
+    ignore: true
+  - name: parallel
+    ignore: true
+  - name: rainbow
+    ignore: true
+  - name: rgot
+    ignore: true
+  - name: securerandom
+    ignore: true
+  - name: strscan
+    ignore: true
+  - name: forwardable
+    ignore: true
+  - name: mutex_m
+    ignore: true

data/rgot.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |spec|
   spec.homepage      = "https://github.com/ksss/rgot"
   spec.license       = "MIT"
 
-  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  spec.files         = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|go)/}) }
   spec.bindir        = "bin"
   spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
   spec.require_paths = ["lib"]

data/sig/patch.rbs

@@ -0,0 +1,3 @@
+# patch
+class Ripper
+end