startup-time 1.1.1 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 377344af2f7a7802b54695b2027e04f43d7c87bd6d9af5bfc4292300c3b226e2
-  data.tar.gz: e2dc97fca65d09d04df5c132a8b5553f675be80b564b9dc2bc7f3a4dde1b5d01
+  metadata.gz: 5edbc737357924e55026244d924546b02dd43c3f875c710a4174027abb0bfc82
+  data.tar.gz: 8c79da2fbbbf34be5dc0bae8115b33a7ccbe681493c4efea6f86028c20ad1f2b
 SHA512:
-  metadata.gz: ff0b44c9fac210362dab891ee279f9b9cfeaffb878e96007bc3d44cb180db835c43b86c82bd5214eb04e48ea8a8ec92616c04fd09b57b8daaf049f22c33dbcd3
-  data.tar.gz: 5fe70d4707bee6f8c8e33957170c2d1e99b239ac921b6d4c94cb6c2a927c3840a62418a6e23335beae722ae50a687fd759949a5a34ae038164d10ff14dec2327
+  metadata.gz: d37390eae5748fde5435c8a261a7b6888f12c870424500124d6aa0d8ba6baa357c4de0846685013bb6f4cc3fac9b5470a4f167e265c775e1f2dcdcc520e7cb9a
+  data.tar.gz: 4eb78e5f8a43691d8f898b9fab7a23219e831c583836cf43fece6b3c9d1dc1d8af24170a0df7096a14a244cb4bdcb398035d7d8337a944606c0f964fa79abe97

data/CHANGELOG.md

@@ -1,7 +1,13 @@
+## 1.2.0 - 2019-07-15
+
+- add -t/--time option specifying the minimum length of time to run tests for
+ (default: 5s)
+- format the ID -> group table as JSON if the --json option is provided
+
 ## 1.1.1 - 2019-07-13
 
 - add QuickJS
-- fix deno version command
+- update deno version command
 
 ## 1.1.0 - 2019-02-27
 

data/README.md

@@ -49,8 +49,8 @@ $ startup-time --only jvm
 # only run tests which finish quickly
 $ startup-time --only fast --omit slow-compile
 
-# increase the number of times each test is run (default: 10)
-$ startup-time --count 100
+# minimum number of seconds to run the test suite for (default: 5)
+$ startup-time --time 10
 ```
 
 ### Sample Output
@@ -106,7 +106,7 @@ USAGE:
 
 OPTIONS:
 
-    -c, --count, --rounds INTEGER    The number of times to run each test (default: 10)
+    -c, --count, --rounds INTEGER    The number of times to run each program
         --clean                      Remove the build directory and exit
                                      (targets will be recompiled on the next run)
     -d, --dir PATH                   Specify the build directory
@@ -117,6 +117,8 @@ OPTIONS:
     -o, --only LIST                  Only run the specified tests (comma-separated list of IDs/groups)
     -O, --omit LIST                  Don't run the specified tests (comma-separated list of IDs/groups)
     -q, --quiet                      Suppress all inessential output
+    -t, --time INTEGER               The minimum number of seconds to run the test suite for
+                                     (minimum: 2, default: 5)
     -v, --verbose                    Enable verbose logging
     -V, --version                    Display the version and exit
 ```
@@ -142,7 +144,7 @@ OPTIONS:
 
 ## VERSION
 
-1.1.1
+1.2.0
 
 ## COPYRIGHT AND LICENSE
 

data/lib/startup_time/app.rb

@@ -1,13 +1,14 @@
 # frozen_string_literal: true
 
 require 'benchmark'
+require 'json'
 require 'komenda'
 require 'shellwords' # for Array#shelljoin
 require 'tty/table'
 
-# FIXME we only need bundler/setup here, but it appears to create an incomplete
-# Bundler object which (sometimes) confuses Komenda as well as causing a
-# Gem::LoadError (for unicode-display_width)
+# FIXME we only need bundler/setup here (for Bundler.with_clean_env), but it appears
+# to create an incomplete Bundler object which (sometimes) confuses Komenda as well
+# as causing a Gem::LoadError (for unicode-display_width)
 #
 # require 'bundler/setup'
 require 'bundler'
@@ -20,10 +21,11 @@ module StartupTime
 
     include FileUtils # for `sh`
     include Util # for `which`
-    include Services.mixin %i[builder ids_to_groups selected_tests]
+    include Services.mixin %i[builder selected_tests]
 
     def initialize(args = ARGV)
       @options = Options.new(args)
+      @json = @options.format == :json
       @verbosity = @options.verbosity
       @times = []
 
@@ -37,20 +39,15 @@ module StartupTime
     # or print a help message) or the default command, which runs
     # the selected benchmark-tests
     def run
-      if @verbosity == :verbose
-        # used by StartupTime::App#time to dump the command line
-        require 'shellwords'
-      end
-
       case @options.action
       when :clean
         builder.clean!
       when :help
         puts @options.usage
-      when :show_ids
-        puts render_ids_to_groups
       when :version
         puts VERSION
+      when :show_ids
+        render_ids_to_groups
       else
         benchmark
       end
@@ -67,16 +64,55 @@ module StartupTime
     def benchmark
       builder.build!
 
-      selected_tests.entries.shuffle.each do |id, test|
-        time(id, test)
+      # run a test if:
+      #
+      #   - its interpreter exists
+      #   - it's a compiled executable (i.e. its compiler exists)
+      #
+      # otherwise, skip it
+      runnable_tests = selected_tests.each_with_object([]) do |(id, test), tests|
+        args = Array(test[:command])
+
+        if args.length == 1 # native executable
+          compiler = test[:compiler] || id
+          path = File.absolute_path(args.first)
+          next unless File.exist?(path)
+        else # interpreter + source
+          compiler = args.first
+          path = which(compiler)
+          next unless path
+        end
+
+        tests << {
+          id: id,
+          test: test,
+          args: args,
+          compiler: compiler,
+          path: path,
+        }
+      end
+
+      if runnable_tests.empty?
+        puts '[]' if @json
+        return
+      end
+
+      spec = @options.spec
+
+      if spec.type == :duration
+        spec = spec.with(value: spec.value.to_f / runnable_tests.length)
+      end
+
+      runnable_tests.shuffle.each do |config|
+        config[:spec] = spec
+        time(config)
       end
 
       sorted = @times.sort_by { |result| result[:time] }
 
-      if @options.format == :json
-        require 'json'
+      if @json
         puts sorted.to_json
-      elsif !sorted.empty?
+      else
         pairs = sorted.map { |result| [result[:name], '%.02f' % result[:time]] }
         table = TTY::Table.new(['Test', 'Time (ms)'], pairs)
         puts unless @verbosity == :quiet
@@ -84,33 +120,20 @@ module StartupTime
       end
     end
 
-    # an ASCII table representation of the mapping from test IDs (e.g. "scala")
-    # to group IDs (e.g. "compiled, jvm, slow")
+    # print a JSON or ASCII-table representation of the mapping from test IDs
+    # (e.g. "scala") to group IDs (e.g. ["compiled", "jvm", "slow"])
     def render_ids_to_groups
-      table = TTY::Table.new(%w[Test Groups], ids_to_groups)
-      table.render
-    end
-
-    # takes a test ID and a test spec and measures how long it takes to execute
-    # the test if either:
-    #
-    #   - its interpreter exists
-    #   - it's a compiled executable (i.e. its compiler exists)
-    #
-    # otherwise, skip the test
-    def time(id, test)
-      args = Array(test[:command])
-
-      if args.size == 1 # native executable
-        compiler = test[:compiler] || id
-        cmd = File.absolute_path(args.first)
-        return unless File.exist?(cmd)
-      else # interpreter + source
-        compiler = args.first
-        cmd = which(compiler)
-        return unless cmd
+      if @json
+        puts Registry.ids_to_groups(format: :json).to_json
+      else
+        table = TTY::Table.new(%w[Test Groups], Registry.ids_to_groups)
+        puts table.render
       end
+    end
 
+    # takes a test configuration and measures how long it takes to execute the
+    # test
+    def time(id:, test:, args:, compiler:, path:, spec:)
       # dump the compiler/interpreter's version if running in verbose mode
       if @verbosity == :verbose
         puts
@@ -133,7 +156,7 @@ module StartupTime
       end
 
       argv0 = args.shift
-      command = [cmd, *args]
+      command = [path, *args]
 
       unless @verbosity == :quiet
         if @verbosity == :verbose
@@ -160,9 +183,26 @@ module StartupTime
       # the bundler environment slows down ruby and breaks truffle-ruby,
       # so make sure it's disabled for the benchmark
       Bundler.with_clean_env do
-        @options.rounds.times do
-          times << Benchmark.realtime do
-            system([cmd, argv0], *args, out: File::NULL)
+        if spec.type == :duration # how long to run the tests for
+          duration = spec.value
+          elapsed = 0
+          start = Time.now
+
+          loop do
+            time = Benchmark.realtime do
+              system([path, argv0], *args, out: File::NULL)
+            end
+
+            elapsed = Time.now - start
+            times << time
+
+            break if elapsed >= duration
+          end
+        else # how many times to run the tests
+          spec.value.times do
+            times << Benchmark.realtime do
+              system([path, argv0], *args, out: File::NULL)
+            end
           end
         end
       end

data/lib/startup_time/builder.rb

@@ -108,12 +108,12 @@ module StartupTime
       # absolute path (which may be mocked)
       test = test.merge(compiler: compiler_path)
 
-      # pass the test object as the block's second argument. Rake passes an
-      # instance of +Rake::TaskArguments+, a Hash-like object which provides
-      # access to the command-line arguments for a Rake task e.g. { name:
-      # "world" } for `rake greet[world]`. since we're not relying on Rake's
-      # limited option-handling support, we have no use for that here, so we
-      # simply replace it with the test data.
+      # pass the test object as the `file(...) { ... }` block's second
+      # argument. Rake passes an instance of +Rake::TaskArguments+, a Hash-like
+      # object which provides access to the command-line arguments for a Rake
+      # task e.g. { name: "world" } for `rake greet[world]`. since we're not
+      # relying on Rake's limited option-handling support, we have no use for
+      # that here, so we simply replace it with the test data.
       wrapper = ->(task, _) { yield(task, test) }
 
       # declare the prerequisites for the target file.
@@ -138,8 +138,8 @@ module StartupTime
     end
 
     # ensure each file in the source directory is mirrored to the build
-    # directory, and add each task which ensures this as a prerequisite
-    # of the master task (:build)
+    # directory, and add each task which ensures this as a prerequisite of the
+    # master task (:build)
     def copy_source_files
       Dir["#{SRC_DIR}/*.*"].each do |path|
         filename = File.basename(path)
@@ -153,7 +153,7 @@ module StartupTime
     end
 
     # run a shell command (string) by substituting the compiler path, source
-    # file, and target file into the supplied template and executing the
+    # file, and target file into the supplied template string and executing the
     # resulting command with the test's (optional) environment hash
     def run(template, task, test)
       replacements = {
@@ -192,11 +192,12 @@ module StartupTime
       end
 
       if java_native
-        javac = compile_if(:javac, force: true) do |task, test|
+        javac = compile_if(:javac, connect: false, force: true) do |task, test|
           run('%{compiler} -d . %{source}', task, test)
         end
 
         if javac
+          task java_native => javac
           task :build => java_native
         end
       else

data/lib/startup_time/options.rb

@@ -2,13 +2,17 @@
 
 require 'env_paths'
 require 'optparse'
+require 'values'
 
 module StartupTime
   # StartupTime::Options - a struct-like interface to the app options set or
   # overridden on the command line
   class Options
     BUILD_DIR = EnvPaths.get('startup-time', suffix: false).cache
-    ROUNDS = 10
+    DEFAULT_DURATION = 5
+    MINIMUM_DURATION = 2
+
+    Spec = Value.new(:type, :value)
 
     attr_reader :action, :build_dir, :format, :rounds, :verbosity
 
@@ -17,14 +21,24 @@ module StartupTime
     def initialize(args)
       @action = :benchmark
       @build_dir = BUILD_DIR
+      @duration = DEFAULT_DURATION
       @format = :default
       @parser = nil
-      @rounds = ROUNDS
+      @rounds = nil
+      @spec = nil
       @verbosity = :default
 
       parse! args
     end
 
+    def spec
+      @spec ||= if @rounds
+        Spec.with(type: :count, value: @rounds)
+      else
+        Spec.with(type: :duration, value: @duration)
+      end
+    end
+
     # the usage message (string) generated by the option parser for this tool
     def usage
       @parser.to_s
@@ -41,7 +55,7 @@ module StartupTime
           '--count',
           '--rounds INTEGER',
           Integer,
-          "The number of times to execute each test (default: #{ROUNDS})"
+          'The number of times to run each program'
         ) do |value|
           @rounds = value
         end
@@ -94,7 +108,7 @@ module StartupTime
           '-o',
           '--only LIST',
           Array, # comma-separated strings
-          'Only execute the specified tests (comma-separated list of IDs/groups)'
+          'Only run the specified tests (comma-separated list of IDs/groups)'
         ) do |values|
           values.each { |value| registry.only(value.strip) }
         end
@@ -103,7 +117,7 @@ module StartupTime
           '-O',
           '--omit LIST',
           Array, # comma-separated strings
-          "Don't execute the specified tests (comma-separated list of IDs/groups)"
+          "Don't run the specified tests (comma-separated list of IDs/groups)"
         ) do |values|
           values.each { |value| registry.omit(value.strip) }
         end
@@ -116,6 +130,16 @@ module StartupTime
           @verbosity = :quiet
         end
 
+        opts.on(
+          '-t',
+          '--time INTEGER',
+          Integer,
+          'Specify the minimum number of seconds to run tests for',
+          "(minimum: #{MINIMUM_DURATION}, default: #{DEFAULT_DURATION})"
+        ) do |value|
+          @duration = [value, MINIMUM_DURATION].max
+        end
+
         opts.on(
           '-v',
           '--verbose',

data/lib/startup_time/registry.rb

@@ -4,7 +4,6 @@ require 'active_support'
 require 'active_support/core_ext/hash/indifferent_access'
 require 'active_support/core_ext/hash/slice' # XXX in core since 2.5
 require 'set'
-require 'shellwords'
 require 'yaml'
 
 module StartupTime
@@ -35,10 +34,16 @@ module StartupTime
       end
     end
 
-    # returns a hash which maps test-ID keys (e.g. "scala") to their
-    # corresponding group names (e.g. "compiled, jvm, slow")
-    def self.ids_to_groups
-      TESTS.entries.map { |id, test| [id, test[:groups].sort.join(', ')] }
+    # a hash which maps test IDs (e.g. "scala") to their corresponding group
+    # names (e.g. ["compiled", "jvm", "slow"])
+    def self.ids_to_groups(format: :ascii)
+      if format == :json
+        TESTS.entries.each_with_object({}) do |(id, test), target|
+          target[id] = test[:groups].sort
+        end
+      else # ASCII
+        TESTS.entries.map { |id, test| [id, test[:groups].sort.join(', ')] }
+      end
     end
 
     def initialize

data/lib/startup_time/services.rb

@@ -8,10 +8,6 @@ module StartupTime
     # the component responsible for managing the build directory
     once(:builder) { Builder.new }
 
-    # a hash which maps test IDs (e.g. "scala") to group names
-    # (e.g. "compiled, jvm, slow")
-    once(:ids_to_groups) { Registry.ids_to_groups }
-
     # an interface to the tests configured in resources/tests.yaml
     once(:registry) { Registry.new }
 

data/lib/startup_time/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module StartupTime
-  VERSION = '1.1.1'
+  VERSION = '1.2.0'
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: startup-time
 version: !ruby/object:Gem::Version
-  version: 1.1.1
+  version: 1.2.0
 platform: ruby
 authors:
 - chocolateboy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-12 00:00:00.000000000 Z
+date: 2019-07-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -122,6 +122,20 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: 0.4.0
+- !ruby/object:Gem::Dependency
+  name: values
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - "~>"
+      - !ruby/object:Gem::Version
+        version: '1.8'
 - !ruby/object:Gem::Dependency
   name: wireless
   requirement: !ruby/object:Gem::Requirement