startup-time 1.2.0 → 1.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 5edbc737357924e55026244d924546b02dd43c3f875c710a4174027abb0bfc82
-  data.tar.gz: 8c79da2fbbbf34be5dc0bae8115b33a7ccbe681493c4efea6f86028c20ad1f2b
+  metadata.gz: fb5446ede5073351faeb7fee768f11d0bbbf8aa223cf5ad8f69b7782b63f91e3
+  data.tar.gz: 699c670e0386d4701a589d25e4467e3f36ad8c6d9baf2c49fc1b4819b35318c5
 SHA512:
-  metadata.gz: d37390eae5748fde5435c8a261a7b6888f12c870424500124d6aa0d8ba6baa357c4de0846685013bb6f4cc3fac9b5470a4f167e265c775e1f2dcdcc520e7cb9a
-  data.tar.gz: 4eb78e5f8a43691d8f898b9fab7a23219e831c583836cf43fece6b3c9d1dc1d8af24170a0df7096a14a244cb4bdcb398035d7d8337a944606c0f964fa79abe97
+  metadata.gz: c7f577bdf5e4d5782825efe6562e97b2cc3210fd0471faf61fc052f1fc6ed6f798b93d8349a13cd652953f68a69096c2fe9cd45109ba3d3f15bb81207d3360a6
+  data.tar.gz: a8a74489a53efc48456fd20af5533e716fbef5731a1b853fefd1b9565734c07ea8e8b1ae28ac065862aec9a4ad5b05e06364941575f2dfc2f17a8963bf6067aa

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 1.3.0 - 2019-09-11
+
+- bump the default minimum duration from 5s to 10s
+- disable the Java tests (javac, java-native) if `java` is installed but
+  `javac` isn't
+- add [Wren](http://wren.io/)
+
 ## 1.2.0 - 2019-07-15
 
 - add -t/--time option specifying the minimum length of time to run tests for
@@ -6,7 +13,7 @@
 
 ## 1.1.1 - 2019-07-13
 
-- add QuickJS
+- add [QuickJS](https://bellard.org/quickjs/)
 - update deno version command
 
 ## 1.1.0 - 2019-02-27

data/README.md

@@ -49,8 +49,11 @@ $ startup-time --only jvm
 # only run tests which finish quickly
 $ startup-time --only fast --omit slow-compile
 
-# minimum number of seconds to run the test suite for (default: 5)
+# run the test suite for at least 10 seconds (default: 5)
 $ startup-time --time 10
+
+# run each program 10 times
+$ startup-time --count 10
 ```
 
 ### Sample Output
@@ -62,24 +65,25 @@ $ startup-time --time 10
     LuaJIT                     0.64
     Go                         0.66
     Rust                       0.67
+    JavaScript (QuickJS)       0.83
     D (DMD)                    0.88
-    JavaScript (QuickJS)       0.92
-    Lua                        0.98
+    Lua                        0.94
+    Java Native (GraalVM)      0.99
     D (GDC)                    1.10
     Haskell (GHC)              1.14
-    C++ (g++)                  1.22
-    Crystal                    1.63
+    Wren                       1.14
+    C++ (g++)                  1.19
+    Crystal                    1.55
     Perl                       1.66
-    Java Native (GraalVM)      1.95
-    Bash                       2.67
+    Bash                       2.63
+    JavaScript (GraalVM)       3.88
     JavaScript (Deno)         11.15
     Python 3                  22.09
-    JavaScript (GraalVM)      32.38
+    Ruby (TruffleRuby)        33.59
     JavaScript (Node.js)      35.20
     Python 2                  43.43
     Java                      54.59
-    Ruby                      67.40
-    Ruby (TruffleRuby)        81.77
+    Ruby                      68.68
     Kotlin                   103.02
     Scala                    801.21
 
@@ -118,7 +122,7 @@ OPTIONS:
     -O, --omit LIST                  Don't run the specified tests (comma-separated list of IDs/groups)
     -q, --quiet                      Suppress all inessential output
     -t, --time INTEGER               The minimum number of seconds to run the test suite for
-                                     (minimum: 2, default: 5)
+                                     (minimum: 2, default: 10)
     -v, --verbose                    Enable verbose logging
     -V, --version                    Display the version and exit
 ```
@@ -134,6 +138,7 @@ OPTIONS:
 
 ## SEE ALSO
 
+- [Gabriel439/bench](https://github.com/Gabriel439/bench) - a command-line benchmark tool written in Haskell
 - [gnustavo/startup-times](https://github.com/gnustavo/startup-times) - a script to investigate the startup times of several programming languages
 - [jwiegley/helloworld](https://github.com/jwiegley/helloworld) - a comparison of "Hello, world" startup times in various languages
 - [sharkdp/hyperfine](https://github.com/sharkdp/hyperfine) - a command-line benchmarking tool
@@ -144,7 +149,7 @@ OPTIONS:
 
 ## VERSION
 
-1.2.0
+1.3.0
 
 ## COPYRIGHT AND LICENSE
 

data/lib/startup_time/app.rb

@@ -64,35 +64,9 @@ module StartupTime
     def benchmark
       builder.build!
 
-      # run a test if:
-      #
-      #   - its interpreter exists
-      #   - it's a compiled executable (i.e. its compiler exists)
-      #
-      # otherwise, skip it
-      runnable_tests = selected_tests.each_with_object([]) do |(id, test), tests|
-        args = Array(test[:command])
-
-        if args.length == 1 # native executable
-          compiler = test[:compiler] || id
-          path = File.absolute_path(args.first)
-          next unless File.exist?(path)
-        else # interpreter + source
-          compiler = args.first
-          path = which(compiler)
-          next unless path
-        end
-
-        tests << {
-          id: id,
-          test: test,
-          args: args,
-          compiler: compiler,
-          path: path,
-        }
-      end
+      tests = runnable_tests
 
-      if runnable_tests.empty?
+      if tests.empty?
         puts '[]' if @json
         return
       end
@@ -100,12 +74,11 @@ module StartupTime
       spec = @options.spec
 
       if spec.type == :duration
-        spec = spec.with(value: spec.value.to_f / runnable_tests.length)
+        spec = spec.with(value: spec.value.to_f / tests.length)
       end
 
-      runnable_tests.shuffle.each do |config|
-        config[:spec] = spec
-        time(config)
+      tests.shuffle.each do |test|
+        time(spec, test)
       end
 
       sorted = @times.sort_by { |result| result[:time] }
@@ -131,9 +104,54 @@ module StartupTime
       end
     end
 
+    # return the subset of selected tests that are runnable.
+    #
+    # a test is runnable if these requirements are met:
+    #
+    #   - (if it's interpreted): the interpreter exists
+    #   - (if it's compiled): the compiler exists
+    #
+    # otherwise, skip it
+    #
+    # the compiler path is resolved in the builder, which can also disable a test
+    # (by setting test[:disabled] = true) if the compilation prerequisites aren't
+    # installed
+    #
+    # note that some tests are both interpreted and compiled, so both the
+    # compiler and the interpreter must exist, e.g.:
+    #
+    #   - interpreter: java
+    #   - compiler: javac
+
+    def runnable_tests
+      selected_tests.each_with_object([]) do |(id, test), tests|
+        next if test[:disabled]
+
+        args = Array(test[:command])
+        compiler = test[:compiler]
+
+        if args.length == 1 # native executable
+          executable = File.absolute_path(args.first)
+          next unless File.exist?(executable)
+        else # interpreter + source/bytecode
+          executable = which(args.first)
+          next unless (interpreter = executable)
+        end
+
+        tests << {
+          args:        args,
+          compiler:    compiler,
+          executable:  executable,
+          id:          id,
+          interpreter: interpreter,
+          test:        test,
+        }
+      end
+    end
+
     # takes a test configuration and measures how long it takes to execute the
     # test
-    def time(id:, test:, args:, compiler:, path:, spec:)
+    def time(spec, args:, compiler:, executable:, id:, interpreter:, test:)
       # dump the compiler/interpreter's version if running in verbose mode
       if @verbosity == :verbose
         puts
@@ -144,19 +162,22 @@ module StartupTime
         version = test[:version]
 
         unless version == false
-          version ||= '%{compiler} --version'
+          # if the test is both interpreted and compiled, default to
+          # dumping the compiler version
+          version ||= '%{compiler} --version' if compiler
+          version ||= '%{interpreter} --version' if interpreter
 
-          # the compiler may have been uninstalled since the target was
-          # built, so make sure it still exists
-          if (compiler_path = which(compiler))
-            version_command = version % { compiler: compiler_path }
-            sh version_command
-          end
+          version_command = version % {
+            compiler: compiler || interpreter,
+            interpreter: interpreter,
+          }
+
+          sh version_command
         end
       end
 
       argv0 = args.shift
-      command = [path, *args]
+      command = [executable, *args]
 
       unless @verbosity == :quiet
         if @verbosity == :verbose
@@ -190,7 +211,7 @@ module StartupTime
 
           loop do
             time = Benchmark.realtime do
-              system([path, argv0], *args, out: File::NULL)
+              system([executable, argv0], *args, out: File::NULL)
             end
 
             elapsed = Time.now - start
@@ -201,7 +222,7 @@ module StartupTime
         else # how many times to run the tests
           spec.value.times do
             times << Benchmark.realtime do
-              system([path, argv0], *args, out: File::NULL)
+              system([executable, argv0], *args, out: File::NULL)
             end
           end
         end

data/lib/startup_time/builder.rb

@@ -1,9 +1,14 @@
 # frozen_string_literal: true
 
 require 'rake'
+require 'set'
 require 'shellwords'
+
+require_relative 'refinements'
 require_relative 'util'
 
+using StartupTime::Refinements
+
 module StartupTime
   # StartupTime::Builder - clean and prepare the build directory
   #
@@ -16,6 +21,11 @@ module StartupTime
   # once these tasks are complete, everything required to run the benchmark tests
   # will be available in the build directory
   class Builder
+    CUSTOM_COMPILER = {
+      'kotlinc-native': :compile_kotlinc_native,
+      'java-native': :compile_java_native,
+    }
+
     SRC_DIR = File.absolute_path('../../resources/src', __dir__)
 
     include Rake::DSL
@@ -23,17 +33,12 @@ module StartupTime
     include Services.mixin %i[options selected_tests]
 
     def initialize
-      @verbosity = options.verbosity
       @build_dir = options.build_dir
+      @verbosity = options.verbosity
 
       Rake.verbose(@verbosity != :quiet)
     end
 
-    # remove the build directory and its contents
-    def clean!
-      rm_rf @build_dir
-    end
-
     # ensure the build directory is in a fit state to run the tests i.e. copy
     # source files and compile target files
     def build!
@@ -47,27 +52,13 @@ module StartupTime
       Rake::Task[:build].invoke
     end
 
-    private
-
-    # a wrapper for Rake's +FileUtils#sh+ method (which wraps +Kernel#spawn+)
-    # which allows the command's environment to be included in the final options
-    # hash rather than cramming it in as the first argument i.e.:
-    #
-    # before:
-    #
-    #   sh FOO_VERBOSE: "0", "foo -c hello.foo -o hello", out: File::NULL
-    #
-    # after:
-    #
-    #   shell "foo -c hello.foo -o hello", env: { FOO_VERBOSE: "0" }, out: File::NULL
-    #
-    def shell(args, **options)
-      args = Array(args) # args is a string or array
-      env = options.delete(:env)
-      args.unshift(env) if env
-      sh(*args, options)
+    # remove the build directory and its contents
+    def clean!
+      rm_rf @build_dir
     end
 
+    private
+
     # a conditional version of Rake's `file` task which compiles a source file to
     # a target file via the block provided. if the compiler isn't installed, the
     # task is skipped.
@@ -88,6 +79,12 @@ module StartupTime
 
       return unless (compiler_path = which(compiler))
 
+      # update the test spec's compiler field to point to the compiler's
+      # absolute path
+      #
+      # XXX mutation/side-effect
+      test[:compiler] = compiler_path
+
       # the source filename must be supplied
       source = test.fetch(:source)
 
@@ -97,17 +94,13 @@ module StartupTime
 
         if command.length == 1
           target = command.first
-        elsif source.match?(/^[A-Z]/) # JVM language
+        elsif source.match?(/\A[A-Z]/) # JVM language
           target = source.pathmap('%n.class')
         else # native executable
           target = '%s.out' % source
         end
       end
 
-      # update the test spec's compiler field to point to the compiler's
-      # absolute path (which may be mocked)
-      test = test.merge(compiler: compiler_path)
-
       # pass the test object as the `file(...) { ... }` block's second
       # argument. Rake passes an instance of +Rake::TaskArguments+, a Hash-like
       # object which provides access to the command-line arguments for a Rake
@@ -119,22 +112,96 @@ module StartupTime
       # declare the prerequisites for the target file.
       # compiler_path: recompile if the compiler has been
       # updated since the target was last built
-      file(target => [source, compiler_path], &wrapper)
+      file_task = file(target => [source, compiler_path], &wrapper)
 
-      # add the target file to the build task
-      task :build => target
+      # register the task under the supplied ID so it can be referenced by name
+      # rather than by filename
+      compile_task = task(id => file_task)
 
-      target
+      # add the task which builds the target file to the build task as a
+      # prerequisite
+      # task(:build => target) unless options[:connect] == false
+      task(:build => compile_task) unless options[:connect] == false
+
+      compile_task
     end
 
-    # register the prerequisites of the :build task. creates file tasks which:
-    #
-    # a) keep the build directory sources in sync with the source directory
-    # b) rebuild target files if their source files are modified
-    # c) rebuild target files if their compilers are updated
-    def register_tasks
-      copy_source_files
-      compile_target_files
+    # native-image compiles .class files to native binaries. it differs from
+    # the other tasks because it depends on a target file rather than a
+    # source file i.e. it depends on the target of the javac task
+    def compile_java_native
+      java_native = compile_if('java-native', connect: false) do |t, test|
+        # XXX native-image doesn't provide a way to silence its output, so
+        # send it to /dev/null
+        shell [test[:compiler], "-H:Name=#{t.target}", '--no-server', '-O1', t.source.ext], {
+          out: File::NULL
+        }
+      end
+
+      return unless java_native # return a falsey value i.e. disable the test
+
+      javac = Rake.application.lookup(:javac) || begin
+        compile_if(:javac, connect: false, force: true) do |task, test|
+          run(test[:compile], task, test)
+        end
+      end
+
+      return unless javac # disable this test if javac is not available
+
+      # prepend the javac task to this task as a prerequisite
+      java_native.prepend(javac)
+
+      # register this task as a dependency of the root (:build) task
+      task(:build => java_native)
+
+      # uncomment this to see the dependency graph
+      # pp Rake::Task.tasks
+
+      java_native
+    end
+
+    # implement the compilation step for the kotlinc-native test manually. we
+    # need to do this to work around the compiler's non-standard behavior
+    def compile_kotlinc_native
+      compile_if 'kotlinc-native' do |t, test|
+        # XXX kotlinc-native doesn't provide a way to silence
+        # its debug messages, so file them under /dev/null
+        shell %W[#{test[:compiler]} -opt -o #{t.target} #{t.source}], out: File::NULL
+
+        # XXX work around a kotlinc-native "feature"
+        # https://github.com/JetBrains/kotlin-native/issues/967
+        exe = "#{t.target}.kexe" # XXX or .exe, or...
+        verbose(@verbosity == :verbose) { mv exe, t.target } if File.exist?(exe)
+      end
+    end
+
+    # make sure the target files (e.g. native executables and JVM .class files)
+    # are built if their compilers are installed
+    def compile_target_files
+      selected_tests.each do |id, test|
+        enabled = true
+
+        # handle the tests which have compile templates by a) turning them into
+        # blocks which substitute the compiler, source file and target file into
+        # the corresponding placeholders in the template, then b) executing the
+        # resulting command via +shell+
+
+        if (command = test[:compile])
+          block = ->(task, test_) { run(command, task, test_) }
+          enabled = compile_if(id, &block)
+        end
+
+        test[:disabled] = !enabled
+      end
+
+      # do these after the main pass so they can reuse tasks (if available)
+      # e.g. the javac task
+
+      CUSTOM_COMPILER.each do |id, meth|
+        selected_tests[id].tap do |test|
+          test[:disabled] = !send(meth) if test
+        end
+      end
     end
 
     # ensure each file in the source directory is mirrored to the build
@@ -152,6 +219,16 @@ module StartupTime
       end
     end
 
+    # register the prerequisites of the :build task. creates file tasks which:
+    #
+    # a) keep the build directory sources in sync with the source directory
+    # b) rebuild target files if their source files are modified
+    # c) rebuild target files if their compilers are updated
+    def register_tasks
+      copy_source_files
+      compile_target_files
+    end
+
     # run a shell command (string) by substituting the compiler path, source
     # file, and target file into the supplied template string and executing the
     # resulting command with the test's (optional) environment hash
@@ -159,63 +236,30 @@ module StartupTime
       replacements = {
         compiler: Shellwords.escape(test[:compiler]),
         source: Shellwords.escape(task.source),
-        target: Shellwords.escape(task.name),
+        target: Shellwords.escape(task.target),
       }
 
       command = template % replacements
       shell(command, env: test[:env])
     end
 
-    # make sure the target files (e.g. native executables and JVM .class files)
-    # are built if their compilers are installed
-    def compile_target_files
-      # handle the tests which have compile templates by turning them into
-      # blocks which substitute the compiler, source file and target file into
-      # the corresponding placeholders in the template and then execute the
-      # command via `shell`
-      selected_tests.each do |id, selected|
-        if (command = selected[:compile])
-          block = ->(task, test) { run(command, task, test) }
-          compile_if(id, &block)
-        end
-      end
-
-      # native-image compiles .class files to native binaries. it differs from
-      # the other tasks because it depends on a target file rather than a
-      # source file i.e. it depends on the target of the javac task
-      java_native = compile_if('java-native', connect: false) do |t, test|
-        # XXX native-image doesn't provide a way to silence its output, so
-        # send it to /dev/null
-        shell [test[:compiler], "-H:Name=#{t.name}", '--no-server', '-O1', t.source.ext], {
-          out: File::NULL
-        }
-      end
-
-      if java_native
-        javac = compile_if(:javac, connect: false, force: true) do |task, test|
-          run('%{compiler} -d . %{source}', task, test)
-        end
-
-        if javac
-          task java_native => javac
-          task :build => java_native
-        end
-      else
-        compile_if :javac do |task, test|
-          run('%{compiler} -d . %{source}', task, test)
-        end
-      end
-
-      compile_if 'kotlinc-native' do |t, test|
-        # XXX kotlinc-native doesn't provide a way to silence
-        # its debug messages, so file them under /dev/null
-        shell %W[#{test[:compiler]} -opt -o #{t.name} #{t.source}], out: File::NULL
-
-        # XXX work around a kotlinc-native "feature"
-        # https://github.com/JetBrains/kotlin-native/issues/967
-        exe = "#{t.name}.kexe" # XXX or .exe, or...
-        verbose(@verbosity == :verbose) { mv exe, t.name } if File.exist?(exe)
-      end
+    # a wrapper for Rake's +FileUtils#sh+ method (which wraps +Kernel#spawn+)
+    # which allows the command's environment to be included in the final options
+    # hash rather than cramming it in as the first argument i.e.:
+    #
+    # before:
+    #
+    #   sh FOO_VERBOSE: "0", "foo -c hello.foo -o hello", out: File::NULL
+    #
+    # after:
+    #
+    #   shell "foo -c hello.foo -o hello", env: { FOO_VERBOSE: "0" }, out: File::NULL
+    #
+    def shell(args, **options)
+      args = Array(args) # args is a string or array
+      env = options.delete(:env)
+      args.unshift(env) if env
+      sh(*args, options)
     end
   end
 end

data/lib/startup_time/options.rb

@@ -9,7 +9,7 @@ module StartupTime
   # overridden on the command line
   class Options
     BUILD_DIR = EnvPaths.get('startup-time', suffix: false).cache
-    DEFAULT_DURATION = 5
+    DEFAULT_DURATION = 10
     MINIMUM_DURATION = 2
 
     Spec = Value.new(:type, :value)

data/lib/startup_time/refinements.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+require 'rake'
+
+module StartupTime
+  # add some missing methods/aliases to Rake tasks
+  # TODO move to a gem (or PR)
+  module Refinements
+    refine Rake::FileTask do
+      alias_method :target, :name
+    end
+
+    refine Rake::Task do
+      alias_method :append, :enhance
+
+      def prepend(deps = nil, &block)
+        prerequisites.replace(Array(deps) | prerequisites) if deps
+        actions.unshift(block) if block_given?
+        self
+      end
+    end
+  end
+end

data/lib/startup_time/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module StartupTime
-  VERSION = '1.2.0'
+  VERSION = '1.3.0'
 end

data/resources/src/hello.wren

@@ -0,0 +1 @@
+System.print("Hello, world!")

data/resources/tests.yaml

@@ -21,6 +21,7 @@ crystal:
     - slow-compile
   source: hello.cr
   compile: '%{compiler} build --release -o %{target} %{source}'
+  version: '%{compiler} --version | head -n1'
   command: hello.cr.out
   env:
       CRYSTAL_CACHE_DIR: .crystal
@@ -123,7 +124,8 @@ javac:
     - java
     - jvm
   source: HelloJava.java
-  version: '%{compiler} -version'
+  compile: '%{compiler} -d . %{source}'
+  version: '%{compiler} -version' # we could dump the interpeter version as well/instead
   command:
     - java
     - HelloJava
@@ -313,3 +315,12 @@ truffle-ruby:
   command:
     - truffleruby
     - hello.rb
+
+wren:
+  name: Wren
+  groups:
+    - fast
+    - script
+  command:
+    - wren
+    - hello.wren

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: startup-time
 version: !ruby/object:Gem::Version
-  version: 1.2.0
+  version: 1.3.0
 platform: ruby
 authors:
 - chocolateboy
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-07-15 00:00:00.000000000 Z
+date: 2019-09-11 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: activesupport
@@ -180,6 +180,7 @@ files:
 - lib/startup_time/app.rb
 - lib/startup_time/builder.rb
 - lib/startup_time/options.rb
+- lib/startup_time/refinements.rb
 - lib/startup_time/registry.rb
 - lib/startup_time/services.rb
 - lib/startup_time/util.rb
@@ -202,6 +203,7 @@ files:
 - resources/src/hello.py
 - resources/src/hello.rb
 - resources/src/hello.rs
+- resources/src/hello.wren
 - resources/tests.yaml
 homepage: https://github.com/chocolateboy/startup-time
 licenses:
@@ -226,7 +228,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.4
+rubygems_version: 3.0.6
 signing_key: 
 specification_version: 4
 summary: A command-line tool to measure the startup times of programs in various languages