bootsnap 1.4.5 → 1.6.0

data/lib/bootsnap/cli/worker_pool.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Bootsnap
+  class CLI
+    class WorkerPool
+      class << self
+        def create(size:, jobs:)
+          if size > 0 && Process.respond_to?(:fork)
+            new(size: size, jobs: jobs)
+          else
+            Inline.new(jobs: jobs)
+          end
+        end
+      end
+
+      class Inline
+        def initialize(jobs: {})
+          @jobs = jobs
+        end
+
+        def push(job, *args)
+          @jobs.fetch(job).call(*args)
+          nil
+        end
+
+        def spawn
+          # noop
+        end
+
+        def shutdown
+          # noop
+        end
+      end
+
+      class Worker
+        attr_reader :to_io, :pid
+
+        def initialize(jobs)
+          @jobs = jobs
+          @pipe_out, @to_io = IO.pipe
+          @pid = nil
+        end
+
+        def write(message, block: true)
+          payload = Marshal.dump(message)
+          if block
+            to_io.write(payload)
+            true
+          else
+            to_io.write_nonblock(payload, exception: false) != :wait_writable
+          end
+        end
+
+        def close
+          to_io.close
+        end
+
+        def work_loop
+          loop do
+            job, *args = Marshal.load(@pipe_out)
+            return if job == :exit
+            @jobs.fetch(job).call(*args)
+          end
+        rescue IOError
+          nil
+        end
+
+        def spawn
+          @pid = Process.fork do
+            to_io.close
+            work_loop
+            exit!(0)
+          end
+          @pipe_out.close
+          true
+        end
+      end
+
+      def initialize(size:, jobs: {})
+        @size = size
+        @jobs = jobs
+        @queue = Queue.new
+        @pids = []
+      end
+
+      def spawn
+        @workers = @size.times.map { Worker.new(@jobs) }
+        @workers.each(&:spawn)
+        @dispatcher_thread = Thread.new { dispatch_loop }
+        @dispatcher_thread.abort_on_exception = true
+        true
+      end
+
+      def dispatch_loop
+        loop do
+          case job = @queue.pop
+          when nil
+            @workers.each do |worker|
+              worker.write([:exit])
+              worker.close
+            end
+            return true
+          else
+            unless @workers.sample.write(job, block: false)
+              free_worker.write(job)
+            end
+          end
+        end
+      end
+
+      def free_worker
+        IO.select(nil, @workers)[1].sample
+      end
+
+      def push(*args)
+        @queue.push(args)
+        nil
+      end
+
+      def shutdown
+        @queue.close
+        @dispatcher_thread.join
+        @workers.each do |worker|
+          _pid, status = Process.wait2(worker.pid)
+          return status.exitstatus unless status.success?
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/bootsnap/cli.rb

@@ -0,0 +1,246 @@
+# frozen_string_literal: true
+
+require 'bootsnap'
+require 'bootsnap/cli/worker_pool'
+require 'optparse'
+require 'fileutils'
+require 'etc'
+
+module Bootsnap
+  class CLI
+    unless Regexp.method_defined?(:match?)
+      module RegexpMatchBackport
+        refine Regexp do
+          def match?(string)
+            !!match(string)
+          end
+        end
+      end
+      using RegexpMatchBackport
+    end
+
+    attr_reader :cache_dir, :argv
+
+    attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :jobs
+
+    def initialize(argv)
+      @argv = argv
+      self.cache_dir = ENV.fetch('BOOTSNAP_CACHE_DIR', 'tmp/cache')
+      self.compile_gemfile = false
+      self.exclude = nil
+      self.verbose = false
+      self.jobs = Etc.nprocessors
+      self.iseq = true
+      self.yaml = true
+    end
+
+    def precompile_command(*sources)
+      require 'bootsnap/compile_cache/iseq'
+      require 'bootsnap/compile_cache/yaml'
+
+      fix_default_encoding do
+        Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
+        Bootsnap::CompileCache::YAML.init!
+        Bootsnap::CompileCache::YAML.cache_dir = self.cache_dir
+
+        @work_pool = WorkerPool.create(size: jobs, jobs: {
+          ruby: method(:precompile_ruby),
+          yaml: method(:precompile_yaml),
+        })
+        @work_pool.spawn
+
+        main_sources = sources.map { |d| File.expand_path(d) }
+        precompile_ruby_files(main_sources)
+        precompile_yaml_files(main_sources)
+
+        if compile_gemfile
+          # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
+          gem_exclude = Regexp.union([exclude, '/spec/', '/test/'].compact)
+          precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
+
+          # Gems that include YAML files usually don't put them in `lib/`.
+          # So we look at the gem root.
+          gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gem\/[^/]+}
+          gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
+          precompile_yaml_files(gem_paths, exclude: gem_exclude)
+        end
+
+        if exitstatus = @work_pool.shutdown
+          exit(exitstatus)
+        end
+      end
+      0
+    end
+
+    dir_sort = begin
+      Dir[__FILE__, sort: false]
+      true
+    rescue ArgumentError, TypeError
+      false
+    end
+
+    if dir_sort
+      def list_files(path, pattern)
+        if File.directory?(path)
+          Dir[File.join(path,  pattern), sort: false]
+        elsif File.exist?(path)
+          [path]
+        else
+          []
+        end
+      end
+    else
+      def list_files(path, pattern)
+        if File.directory?(path)
+          Dir[File.join(path,  pattern)]
+        elsif File.exist?(path)
+          [path]
+        else
+          []
+        end
+      end
+    end
+
+    def run
+      parser.parse!(argv)
+      command = argv.shift
+      method = "#{command}_command"
+      if respond_to?(method)
+        public_send(method, *argv)
+      else
+        invalid_usage!("Unknown command: #{command}")
+      end
+    end
+
+    private
+
+    def precompile_yaml_files(load_paths, exclude: self.exclude)
+      return unless yaml
+
+      load_paths.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_files(path, '**/*.{yml,yaml}').each do |yaml_file|
+            # We ignore hidden files to not match the various .ci.yml files
+            if !yaml_file.include?('/.') && (!exclude || !exclude.match?(yaml_file))
+              @work_pool.push(:yaml, yaml_file)
+            end
+          end
+        end
+      end
+    end
+
+    def precompile_yaml(*yaml_files)
+      Array(yaml_files).each do |yaml_file|
+        if CompileCache::YAML.precompile(yaml_file, cache_dir: cache_dir)
+          STDERR.puts(yaml_file) if verbose
+        end
+      end
+    end
+
+    def precompile_ruby_files(load_paths, exclude: self.exclude)
+      return unless iseq
+
+      load_paths.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_files(path, '**/*.rb').each do |ruby_file|
+            if !exclude || !exclude.match?(ruby_file)
+              @work_pool.push(:ruby, ruby_file)
+            end
+          end
+        end
+      end
+    end
+
+    def precompile_ruby(*ruby_files)
+      Array(ruby_files).each do |ruby_file|
+        if CompileCache::ISeq.precompile(ruby_file, cache_dir: cache_dir)
+          STDERR.puts(ruby_file) if verbose
+        end
+      end
+    end
+
+    def fix_default_encoding
+      if Encoding.default_external == Encoding::US_ASCII
+        Encoding.default_external = Encoding::UTF_8
+        begin
+          yield
+        ensure
+          Encoding.default_external = Encoding::US_ASCII
+        end
+      else
+        yield
+      end
+    end
+
+    def invalid_usage!(message)
+      STDERR.puts message
+      STDERR.puts
+      STDERR.puts parser
+      1
+    end
+
+    def cache_dir=(dir)
+      @cache_dir = File.expand_path(File.join(dir, 'bootsnap/compile-cache'))
+    end
+
+    def exclude_pattern(pattern)
+      (@exclude_patterns ||= []) << Regexp.new(pattern)
+      self.exclude = Regexp.union(@exclude_patterns)
+    end
+
+    def parser
+      @parser ||= OptionParser.new do |opts|
+        opts.banner = "Usage: bootsnap COMMAND [ARGS]"
+        opts.separator ""
+        opts.separator "GLOBAL OPTIONS"
+        opts.separator ""
+
+        help = <<~EOS
+          Path to the bootsnap cache directory. Defaults to tmp/cache
+        EOS
+        opts.on('--cache-dir DIR', help.strip) do |dir|
+          self.cache_dir = dir
+        end
+
+        help = <<~EOS
+          Print precompiled paths.
+        EOS
+        opts.on('--verbose', '-v', help.strip) do
+          self.verbose = true
+        end
+
+        help = <<~EOS
+          Number of workers to use. Default to number of processors, set to 0 to disable multi-processing.
+        EOS
+        opts.on('--jobs JOBS', '-j', help.strip) do |jobs|
+          self.jobs = Integer(jobs)
+        end
+
+        opts.separator ""
+        opts.separator "COMMANDS"
+        opts.separator ""
+        opts.separator "    precompile [DIRECTORIES...]: Precompile all .rb files in the passed directories"
+
+        help = <<~EOS
+          Precompile the gems in Gemfile
+        EOS
+        opts.on('--gemfile', help) { self.compile_gemfile = true }
+
+        help = <<~EOS
+          Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api'
+        EOS
+        opts.on('--exclude PATTERN', help) { |pattern| exclude_pattern(pattern) }
+
+        help = <<~EOS
+          Disable ISeq (.rb) precompilation.
+        EOS
+        opts.on('--no-iseq', help) { self.iseq = false }
+
+        help = <<~EOS
+          Disable YAML precompilation.
+        EOS
+        opts.on('--no-yaml', help) { self.yaml = false }
+      end
+    end
+  end
+end

data/lib/bootsnap/compile_cache/iseq.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require('bootsnap/bootsnap')
 require('zlib')
 
@@ -14,7 +15,7 @@ module Bootsnap
         raise(Uncompilable, 'syntax error')
       end
 
-      def self.storage_to_output(binary)
+      def self.storage_to_output(binary, _args)
         RubyVM::InstructionSequence.load_from_binary(binary)
       rescue RuntimeError => e
         if e.message == 'broken binary format'
@@ -25,7 +26,24 @@ module Bootsnap
         end
       end
 
-      def self.input_to_output(_)
+      def self.fetch(path, cache_dir: ISeq.cache_dir)
+        Bootsnap::CompileCache::Native.fetch(
+          cache_dir,
+          path.to_s,
+          Bootsnap::CompileCache::ISeq,
+          nil,
+        )
+      end
+
+      def self.precompile(path, cache_dir: ISeq.cache_dir)
+        Bootsnap::CompileCache::Native.precompile(
+          cache_dir,
+          path.to_s,
+          Bootsnap::CompileCache::ISeq,
+        )
+      end
+
+      def self.input_to_output(_data, _kwargs)
         nil # ruby handles this
       end
 
@@ -34,11 +52,7 @@ module Bootsnap
           # Having coverage enabled prevents iseq dumping/loading.
           return nil if defined?(Coverage) && Bootsnap::CompileCache::Native.coverage_running?
 
-          Bootsnap::CompileCache::Native.fetch(
-            Bootsnap::CompileCache::ISeq.cache_dir,
-            path.to_s,
-            Bootsnap::CompileCache::ISeq
-          )
+          Bootsnap::CompileCache::ISeq.fetch(path.to_s)
         rescue Errno::EACCES
           Bootsnap::CompileCache.permission_error(path)
         rescue RuntimeError => e
@@ -59,6 +73,7 @@ module Bootsnap
         crc = Zlib.crc32(option.inspect)
         Bootsnap::CompileCache::Native.compile_option_crc32 = crc
       end
+      compile_option_updated
 
       def self.install!(cache_dir)
         Bootsnap::CompileCache::ISeq.cache_dir = cache_dir

data/lib/bootsnap/compile_cache/yaml.rb

@@ -1,61 +1,111 @@
+# frozen_string_literal: true
 require('bootsnap/bootsnap')
 
 module Bootsnap
   module CompileCache
     module YAML
       class << self
-        attr_accessor(:msgpack_factory)
-      end
+        attr_accessor(:msgpack_factory, :cache_dir, :supported_options)
 
-      def self.input_to_storage(contents, _)
-        raise(Uncompilable) if contents.index("!ruby/object")
-        obj = ::YAML.load(contents)
-        msgpack_factory.packer.write(obj).to_s
-      rescue NoMethodError, RangeError
-        # if the object included things that we can't serialize, fall back to
-        # Marshal. It's a bit slower, but can encode anything yaml can.
-        # NoMethodError is unexpected types; RangeError is Bignums
-        Marshal.dump(obj)
-      end
+        def input_to_storage(contents, _)
+          raise(Uncompilable) if contents.index("!ruby/object")
+          obj = ::YAML.load(contents)
+          msgpack_factory.dump(obj)
+        rescue NoMethodError, RangeError
+          # The object included things that we can't serialize
+          raise(Uncompilable)
+        end
 
-      def self.storage_to_output(data)
-        # This could have a meaning in messagepack, and we're being a little lazy
-        # about it. -- but a leading 0x04 would indicate the contents of the YAML
-        # is a positive integer, which is rare, to say the least.
-        if data[0] == 0x04.chr && data[1] == 0x08.chr
-          Marshal.load(data)
-        else
-          msgpack_factory.unpacker.feed(data).read
+        def storage_to_output(data, kwargs)
+          if kwargs && kwargs.key?(:symbolize_names)
+            kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
+          end
+          msgpack_factory.load(data, kwargs)
         end
-      end
 
-      def self.input_to_output(data)
-        ::YAML.load(data)
-      end
+        def input_to_output(data, kwargs)
+          ::YAML.load(data, **(kwargs || {}))
+        end
+
+        def precompile(path, cache_dir: YAML.cache_dir)
+          Bootsnap::CompileCache::Native.precompile(
+            cache_dir,
+            path.to_s,
+            Bootsnap::CompileCache::YAML,
+          )
+        end
+
+        def install!(cache_dir)
+          self.cache_dir = cache_dir
+          init!
+          ::YAML.singleton_class.prepend(Patch)
+        end
 
-      def self.install!(cache_dir)
-        require('yaml')
-        require('msgpack')
+        def init!
+          require('yaml')
+          require('msgpack')
+          require('date')
 
-        # MessagePack serializes symbols as strings by default.
-        # We want them to roundtrip cleanly, so we use a custom factory.
-        # see: https://github.com/msgpack/msgpack-ruby/pull/122
-        factory = MessagePack::Factory.new
-        factory.register_type(0x00, Symbol)
-        Bootsnap::CompileCache::YAML.msgpack_factory = factory
+          # MessagePack serializes symbols as strings by default.
+          # We want them to roundtrip cleanly, so we use a custom factory.
+          # see: https://github.com/msgpack/msgpack-ruby/pull/122
+          factory = MessagePack::Factory.new
+          factory.register_type(0x00, Symbol)
+          factory.register_type(
+            MessagePack::Timestamp::TYPE, # or just -1
+            Time,
+            packer: MessagePack::Time::Packer,
+            unpacker: MessagePack::Time::Unpacker
+          )
+
+          marshal_fallback = {
+            packer: ->(value) { Marshal.dump(value) },
+            unpacker: ->(payload) { Marshal.load(payload) },
+          }
+          {
+            Date => 0x01,
+            Regexp => 0x02,
+          }.each do |type, code|
+            factory.register_type(code, type, marshal_fallback)
+          end
+
+          self.msgpack_factory = factory
+
+          self.supported_options = []
+          params = ::YAML.method(:load).parameters
+          if params.include?([:key, :symbolize_names])
+            self.supported_options << :symbolize_names
+          end
+          if params.include?([:key, :freeze])
+            if factory.load(factory.dump('yaml'), freeze: true).frozen?
+              self.supported_options << :freeze
+            end
+          end
+          self.supported_options.freeze
+        end
+      end
+
+      module Patch
+        def load_file(path, *args)
+          return super if args.size > 1
+          if kwargs = args.first
+            return super unless kwargs.is_a?(Hash)
+            return super unless (kwargs.keys - ::Bootsnap::CompileCache::YAML.supported_options).empty?
+          end
 
-        klass = class << ::YAML; self; end
-        klass.send(:define_method, :load_file) do |path|
           begin
-            Bootsnap::CompileCache::Native.fetch(
-              cache_dir,
+            ::Bootsnap::CompileCache::Native.fetch(
+              Bootsnap::CompileCache::YAML.cache_dir,
               path,
-              Bootsnap::CompileCache::YAML
+              ::Bootsnap::CompileCache::YAML,
+              kwargs,
             )
           rescue Errno::EACCES
-            Bootsnap::CompileCache.permission_error(path)
+            ::Bootsnap::CompileCache.permission_error(path)
           end
         end
+
+        ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
       end
     end
   end

data/lib/bootsnap/compile_cache.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Bootsnap
   module CompileCache
     Error           = Class.new(StandardError)
@@ -33,9 +34,9 @@ module Bootsnap
     end
 
     def self.supported?
-      # only enable on 'ruby' (MRI), POSIX (darwin, linux, *bsd), and >= 2.3.0
+      # only enable on 'ruby' (MRI), POSIX (darwin, linux, *bsd), Windows (RubyInstaller2) and >= 2.3.0
       RUBY_ENGINE == 'ruby' &&
-      RUBY_PLATFORM =~ /darwin|linux|bsd/ &&
+      RUBY_PLATFORM =~ /darwin|linux|bsd|mswin|mingw|cygwin/ &&
       Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.3.0")
     end
   end

data/lib/bootsnap/explicit_require.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Bootsnap
   module ExplicitRequire
     ARCHDIR    = RbConfig::CONFIG['archdir']

data/lib/bootsnap/load_path_cache/cache.rb

@@ -46,7 +46,7 @@ module Bootsnap
       # loadpath.
       def find(feature)
         reinitialize if (@has_relative_paths && dir_changed?) || stale?
-        feature = feature.to_s
+        feature = feature.to_s.freeze
         return feature if absolute_path?(feature)
         return expand_path(feature) if feature.start_with?('./')
         @mutex.synchronize do
@@ -67,7 +67,7 @@ module Bootsnap
           # native dynamic extension, e.g. .bundle or .so), we know it was a
           # failure and there's nothing more we can do to find the file.
           # no extension, .rb, (.bundle or .so)
-          when '', *CACHED_EXTENSIONS # rubocop:disable Performance/CaseWhenSplat
+          when '', *CACHED_EXTENSIONS
             nil
           # Ruby allows specifying native extensions as '.so' even when DLEXT
           # is '.bundle'. This is where we handle that case.
@@ -144,7 +144,7 @@ module Bootsnap
             expanded_path = p.expanded_path
             entries, dirs = p.entries_and_dirs(@store)
             # push -> low precedence -> set only if unset
-            dirs.each    { |dir| @dirs[dir]  ||= path }
+            dirs.each    { |dir| @dirs[dir] ||= path }
             entries.each { |rel| @index[rel] ||= expanded_path }
           end
         end
@@ -178,25 +178,25 @@ module Bootsnap
 
       if DLEXT2
         def search_index(f)
-          try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f + DLEXT2) || try_index(f)
+          try_index("#{f}#{DOT_RB}") || try_index("#{f}#{DLEXT}") || try_index("#{f}#{DLEXT2}") || try_index(f)
         end
 
         def maybe_append_extension(f)
-          try_ext(f + DOT_RB) || try_ext(f + DLEXT) || try_ext(f + DLEXT2) || f
+          try_ext("#{f}#{DOT_RB}") || try_ext("#{f}#{DLEXT}") || try_ext("#{f}#{DLEXT2}") || f
         end
       else
         def search_index(f)
-          try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f)
+          try_index("#{f}#{DOT_RB}") || try_index("#{f}#{DLEXT}") || try_index(f)
         end
 
         def maybe_append_extension(f)
-          try_ext(f + DOT_RB) || try_ext(f + DLEXT) || f
+          try_ext("#{f}#{DOT_RB}") || try_ext("#{f}#{DLEXT}") || f
         end
       end
 
       def try_index(f)
         if (p = @index[f])
-          p + '/' + f
+          "#{p}/#{f}"
         end
       end
 

data/lib/bootsnap/load_path_cache/change_observer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Bootsnap
   module LoadPathCache
     module ChangeObserver
@@ -25,7 +26,7 @@ module Bootsnap
           super
         end
 
-        # uniq! keeps the first occurance of each path, otherwise preserving
+        # uniq! keeps the first occurrence of each path, otherwise preserving
         # order, preserving the effective load path
         def uniq!(*args)
           ret = super

data/lib/bootsnap/load_path_cache/core_ext/active_support.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Bootsnap
   module LoadPathCache
     module CoreExt