startup-time 1.0.0

data/lib/startup_time/builder.rb

@@ -0,0 +1,220 @@
+# frozen_string_literal: true
+
+require 'rake'
+require 'shellwords'
+require_relative 'util'
+
+module StartupTime
+  # StartupTime::Builder - clean and prepare the build directory
+  #
+  # this class provides two methods which clean (i.e. remove) and prepare the build
+  # directory. the latter is done by executing the following tasks:
+  #
+  # 1) copy source files from the source directory to the build directory
+  # 2) compile all of the target files that need to be compiled from source files
+  #
+  # once these tasks are complete, everything required to run the benchmark tests
+  # will be available in the build directory
+  class Builder
+    SRC_DIR = File.absolute_path('../../resources/src', __dir__)
+
+    include Rake::DSL
+    include Util # for `which`
+    include Services.mixin %i[options selected_tests]
+
+    def initialize
+      @verbosity = options.verbosity
+      @build_dir = options.build_dir
+
+      Rake.verbose(@verbosity != :quiet)
+    end
+
+    # remove the build directory and its contents
+    def clean!
+      rm_rf @build_dir
+    end
+
+    # ensure the build directory is in a fit state to run the tests i.e. copy
+    # source files and compile target files
+    def build!
+      verbose(@verbosity == :verbose) do
+        mkdir_p(@build_dir) unless Dir.exist?(@build_dir)
+        cd @build_dir
+      end
+
+      register_tasks
+
+      Rake::Task[:build].invoke
+    end
+
+    private
+
+    # a wrapper for Rake's +FileUtils#sh+ method (which wraps +Kernel#spawn+)
+    # which allows the command's environment to be included in the final options
+    # hash rather than cramming it in as the first argument i.e.:
+    #
+    # before:
+    #
+    #   sh FOO_VERBOSE: "0", "foo -c hello.foo -o hello", out: File::NULL
+    #
+    # after:
+    #
+    #   shell "foo -c hello.foo -o hello", env: { FOO_VERBOSE: "0" }, out: File::NULL
+    #
+    def shell(args, **options)
+      args = Array(args) # args is a string or array
+      env = options.delete(:env)
+      args.unshift(env) if env
+      sh(*args, options)
+    end
+
+    # a conditional version of Rake's `file` task which compiles a source file to
+    # a target file via the block provided. if the compiler isn't installed, the
+    # task is skipped.
+    #
+    # returns a truthy value (the target filename) if the task is created, or nil
+    # otherwise
+    def compile_if(id, **options)
+      tests = options[:force] ? Registry::TESTS : selected_tests
+
+      # look up the test's spec among the remaining tests which haven't been
+      # excluded by --omit or --only
+      return unless (test = tests[id])
+
+      # the compiler name (e.g. "crystal") is usually the same as the ID for
+      # the test (e.g. "crystal"), but can be supplied explicitly in the test
+      # spec e.g. { id: "java-native", compiler: "native-image" }
+      compiler = test[:compiler] || id
+
+      return unless (compiler_path = which(compiler))
+
+      # the source filename must be supplied
+      source = test.fetch(:source)
+
+      # infer the target if not specified
+      unless (target = test[:target])
+        command = Array(test[:command])
+
+        if command.length == 1
+          target = command.first
+        elsif source.match?(/^[A-Z]/) # JVM language
+          target = source.pathmap('%n.class')
+        else # native executable
+          target = '%s.out' % source
+        end
+      end
+
+      # update the test hash's compiler field to point to the compiler's
+      # absolute path (which may be mocked)
+      test = test.merge(compiler: compiler_path)
+
+      # pass the test object as the block's second argument.  Rake passes an
+      # instance of +Rake::TaskArguments+, a Hash-like object which provides
+      # access to the command-line arguments for a Rake task e.g. { name:
+      # "world" } for `rake greet[world]`.  since we're not relying on Rake's
+      # limited option-handling support, we have no use for that here, so we
+      # simply replace it with the test data.
+      wrapper = ->(task, _) { yield(task, test) }
+
+      # declare the prerequisites for the target file.
+      # compiler_path: recompile if the compiler has been
+      # updated since the target was last built
+      file(target => [source, compiler_path], &wrapper)
+
+      # add the target file to the build task
+      task :build => target
+
+      target
+    end
+
+    # register the prerequisites of the :build task. creates file tasks which:
+    #
+    # a) keep the build directory sources in sync with the source directory
+    # b) rebuild target files if their source files are modified
+    # c) rebuild target files if their compilers are updated
+    def register_tasks
+      copy_source_files
+      compile_target_files
+    end
+
+    # ensure each file in the source directory is mirrored to the build
+    # directory, and add each task which ensures this as a prerequisite
+    # of the master task (:build)
+    def copy_source_files
+      Dir["#{SRC_DIR}/*.*"].each do |path|
+        filename = File.basename(path)
+
+        source = file(filename => path) do
+          verbose(@verbosity == :verbose) { cp path, filename }
+        end
+
+        task build: source
+      end
+    end
+
+    # run a shell command (string) by substituting the compiler path, source
+    # file, and target file into the supplied template and executing the
+    # resulting command with the test's (optional) environment hash
+    def run(template, task, test)
+      replacements = {
+        compiler: Shellwords.escape(test[:compiler]),
+        source: Shellwords.escape(task.source),
+        target: Shellwords.escape(task.name),
+      }
+
+      command = template % replacements
+      shell(command, env: test[:env])
+    end
+
+    # make sure the target files (e.g. native executables and JVM .class files)
+    # are built if their compilers are installed
+    def compile_target_files
+      # handle the tests which have compile templates by turning them into
+      # blocks which substitute the compiler, source file and target file into
+      # the corresponding placeholders in the template and then execute the
+      # command via `shell`
+      selected_tests.each do |id, selected|
+        if (command = selected[:compile])
+          block = ->(task, test) { run(command, task, test) }
+          compile_if(id, &block)
+        end
+      end
+
+      # native-image compiles .class files to native binaries. it differs from
+      # the other tasks because it depends on a target file rather than a
+      # source file i.e. it depends on the target of the javac task
+      java_native = compile_if('java-native', connect: false) do |t, test|
+        # XXX native-image doesn't provide a way to silence its output, so
+        # send it to /dev/null
+        shell [test[:compiler], "-H:Name=#{t.name}", '--no-server', '-O1', t.source.ext], {
+          out: File::NULL
+        }
+      end
+
+      if java_native
+        javac = compile_if(:javac, force: true) do |task, test|
+          run('%{compiler} -d . %{source}', task, test)
+        end
+
+        if javac
+          task :build => java_native
+        end
+      else
+        compile_if :javac do |task, test|
+          run('%{compiler} -d . %{source}', task, test)
+        end
+      end
+
+      compile_if 'kotlinc-native' do |t, test|
+        # XXX kotlinc-native doesn't provide a way to silence
+        # its debug messages, so file them under /dev/null
+        shell %W[#{test[:compiler]} -opt -o #{t.name} #{t.source}], out: File::NULL
+
+        # XXX work around a kotlinc-native "feature"
+        # https://github.com/JetBrains/kotlin-native/issues/967
+        exe = "#{t.name}.kexe" # XXX or .exe, or...
+        verbose(@verbosity == :verbose) { mv exe, t.name } if File.exist?(exe)
+      end
+    end
+  end
+end

data/lib/startup_time/options.rb

@@ -0,0 +1,140 @@
+# frozen_string_literal: true
+
+require 'env_paths'
+require 'optparse'
+
+module StartupTime
+  # StartupTime::Options - a struct-like interface to the app options set or
+  # overridden on the command line
+  class Options
+    BUILD_DIR = EnvPaths.get('startup-time', suffix: false).cache
+    ROUNDS = 10
+
+    attr_reader :action, :build_dir, :format, :rounds, :verbosity
+
+    include Services.mixin %i[registry]
+
+    def initialize(args)
+      @action = :benchmark
+      @build_dir = BUILD_DIR
+      @format = :default
+      @parser = nil
+      @rounds = ROUNDS
+      @verbosity = :default
+
+      parse! args
+    end
+
+    # the usage message (string) generated by the option parser for this tool
+    def usage
+      @parser.to_s
+    end
+
+    private
+
+    # process the command-line options and assign values to the corresponding
+    # instance variables
+    def parse!(args)
+      @parser = OptionParser.new do |opts|
+        opts.on(
+          '-c',
+          '--count',
+          '--rounds INTEGER',
+          Integer,
+          "The number of times to execute each test (default: #{ROUNDS})"
+        ) do |value|
+          @rounds = value
+        end
+
+        opts.on(
+          '--clean',
+          'Remove the build directory and exit (targets will be ',
+          'recompiled on the next run)'
+        ) do
+          @action = :clean
+        end
+
+        opts.on(
+          '-d',
+          '--dir PATH',
+          String,
+          "Specify the build directory (default: #{BUILD_DIR})"
+        ) do |value|
+          @build_dir = value
+        end
+
+        opts.on(
+          '-h',
+          '--help',
+          'Show this help message and exit'
+        ) do
+          @action = :help
+        end
+
+        opts.on(
+          '-H',
+          '--help-only',
+          '--help-omit',
+          'Show the IDs and groups that can be passed to --only and --omit'
+        ) do
+          @action = :show_ids
+        end
+
+        opts.on(
+          '-j',
+          '--json',
+          'Output the results in JSON format (implies --quiet)'
+        ) do
+          @format = :json
+          @verbosity = :quiet
+        end
+
+        opts.on(
+          '-o',
+          '--only LIST',
+          Array, # comma-separated strings
+          'Only execute the specified tests (comma-separated list of ',
+          'IDs/groups)'
+        ) do |values|
+          values.each { |value| registry.only(value.strip) }
+        end
+
+        opts.on(
+          '-O',
+          '--omit LIST',
+          Array, # comma-separated strings
+          "Don't execute the specified tests (comma-separated list ",
+          'of IDs/groups)'
+        ) do |values|
+          values.each { |value| registry.omit(value.strip) }
+        end
+
+        opts.on(
+          '-q',
+          '--quiet',
+          'Suppress all inessential output'
+        ) do
+          @verbosity = :quiet
+        end
+
+        opts.on(
+          '-v',
+          '--verbose',
+          'Enable verbose logging'
+        ) do
+          @verbosity = :verbose
+        end
+
+        opts.on(
+          '-V',
+          '--version',
+          'Display the version and exit'
+        ) do
+          @action = :version
+        end
+      end
+
+      @parser.parse!(args)
+    end
+  end
+end

data/lib/startup_time/registry.rb

@@ -0,0 +1,83 @@
+# frozen_string_literal: true
+
+require 'active_support'
+require 'active_support/core_ext/hash/indifferent_access'
+require 'active_support/core_ext/hash/slice' # XXX in core since 2.5
+require 'set'
+require 'shellwords'
+require 'yaml'
+
+module StartupTime
+  # StartupTime::Registry - an interface to the tests configured in
+  # resources/tests.yaml
+  class Registry
+    # the path to the test configuration file
+    TEST_DATA = File.join(__dir__, '../../resources/tests.yaml')
+
+    # map each group (e.g. "jvm") to an array of its member IDs
+    # (e.g. ["java", "kotlin", "scala"])
+    GROUPS = Hash.new { |h, k| h[k] = [] }.with_indifferent_access
+
+    # the top-level hash in tests.yaml. keys are test IDs (e.g. "ruby"); values
+    # are test specs
+    TESTS = YAML.load_file(TEST_DATA).with_indifferent_access
+
+    # perform some basic sanity checks on the test specs and populate the group
+    # -> tests map
+    TESTS.each do |id, test|
+      if test[:compile] && !test[:source]
+        abort "invalid test spec (#{id}): compiled tests must define a source file"
+      end
+
+      test[:groups].each do |group|
+        abort "invalid test spec (#{id}): group ID (#{group}) conflicts with test ID" if TESTS[group]
+        GROUPS[group] << id
+      end
+    end
+
+    # returns a hash which maps test-ID keys (e.g. "scala") to their
+    # corresponding group names (e.g. "compiled, jvm, slow")
+    def self.ids_to_groups
+      TESTS.entries.map { |id, test| [id, test[:groups].sort.join(', ')] }
+    end
+
+    def initialize
+      @omit = Set.new
+      @only = Set.new
+    end
+
+    # remove the specified test(s) from the set of enabled tests
+    def omit(id)
+      @omit.merge(ids_for(id))
+    end
+
+    # add the specified test(s) to the set of enabled tests
+    def only(id)
+      @only.merge(ids_for(id))
+    end
+
+    # the subset of candidate tests which satisfy the `only` and `omit` criteria
+    def selected_tests
+      only = !@only.empty? ? @only : TESTS.keys.to_set
+      test_ids = (only - @omit).to_a
+      TESTS.slice(*test_ids)
+    end
+
+    private
+
+    # takes a test ID or group ID and resolves it into its corresponding test IDs.
+    # if it's already a test ID, it's wrapped in an array; otherwise, an array
+    # of test IDs belonging to the group is returned.
+    def ids_for(id)
+      if TESTS[id]
+        ids = [id]
+      elsif GROUPS.key?(id)
+        ids = GROUPS[id]
+      else
+        abort "Can't resolve IDs for: #{id.inspect}"
+      end
+
+      ids
+    end
+  end
+end

data/lib/startup_time/services.rb

@@ -0,0 +1,22 @@
+# frozen_string_literal: true
+
+require 'wireless'
+
+module StartupTime
+  # shared dependencies required by multiple components
+  Services = Wireless.new do
+    # the component responsible for managing the build directory
+    once(:builder) { Builder.new }
+
+    # a hash which maps test IDs (e.g. "scala") to group names
+    # (e.g. "compiled, jvm, slow")
+    once(:ids_to_groups) { Registry.ids_to_groups }
+
+    # an interface to the tests configured in resources/tests.yaml
+    once(:registry) { Registry.new }
+
+    # the tests which remain after the --only and --omit filters have been
+    # applied
+    once(:selected_tests) { |wl| wl[:registry].selected_tests }
+  end
+end