bootsnap 1.4.9 → 1.7.0

data/lib/bootsnap/cli.rb

@@ -0,0 +1,246 @@
+# frozen_string_literal: true
+
+require 'bootsnap'
+require 'bootsnap/cli/worker_pool'
+require 'optparse'
+require 'fileutils'
+require 'etc'
+
+module Bootsnap
+  class CLI
+    unless Regexp.method_defined?(:match?)
+      module RegexpMatchBackport
+        refine Regexp do
+          def match?(string)
+            !!match(string)
+          end
+        end
+      end
+      using RegexpMatchBackport
+    end
+
+    attr_reader :cache_dir, :argv
+
+    attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :jobs
+
+    def initialize(argv)
+      @argv = argv
+      self.cache_dir = ENV.fetch('BOOTSNAP_CACHE_DIR', 'tmp/cache')
+      self.compile_gemfile = false
+      self.exclude = nil
+      self.verbose = false
+      self.jobs = Etc.nprocessors
+      self.iseq = true
+      self.yaml = true
+    end
+
+    def precompile_command(*sources)
+      require 'bootsnap/compile_cache/iseq'
+      require 'bootsnap/compile_cache/yaml'
+
+      fix_default_encoding do
+        Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
+        Bootsnap::CompileCache::YAML.init!
+        Bootsnap::CompileCache::YAML.cache_dir = self.cache_dir
+
+        @work_pool = WorkerPool.create(size: jobs, jobs: {
+          ruby: method(:precompile_ruby),
+          yaml: method(:precompile_yaml),
+        })
+        @work_pool.spawn
+
+        main_sources = sources.map { |d| File.expand_path(d) }
+        precompile_ruby_files(main_sources)
+        precompile_yaml_files(main_sources)
+
+        if compile_gemfile
+          # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
+          gem_exclude = Regexp.union([exclude, '/spec/', '/test/'].compact)
+          precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
+
+          # Gems that include YAML files usually don't put them in `lib/`.
+          # So we look at the gem root.
+          gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems\/[^/]+}
+          gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
+          precompile_yaml_files(gem_paths, exclude: gem_exclude)
+        end
+
+        if exitstatus = @work_pool.shutdown
+          exit(exitstatus)
+        end
+      end
+      0
+    end
+
+    dir_sort = begin
+      Dir[__FILE__, sort: false]
+      true
+    rescue ArgumentError, TypeError
+      false
+    end
+
+    if dir_sort
+      def list_files(path, pattern)
+        if File.directory?(path)
+          Dir[File.join(path,  pattern), sort: false]
+        elsif File.exist?(path)
+          [path]
+        else
+          []
+        end
+      end
+    else
+      def list_files(path, pattern)
+        if File.directory?(path)
+          Dir[File.join(path,  pattern)]
+        elsif File.exist?(path)
+          [path]
+        else
+          []
+        end
+      end
+    end
+
+    def run
+      parser.parse!(argv)
+      command = argv.shift
+      method = "#{command}_command"
+      if respond_to?(method)
+        public_send(method, *argv)
+      else
+        invalid_usage!("Unknown command: #{command}")
+      end
+    end
+
+    private
+
+    def precompile_yaml_files(load_paths, exclude: self.exclude)
+      return unless yaml
+
+      load_paths.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_files(path, '**/*.{yml,yaml}').each do |yaml_file|
+            # We ignore hidden files to not match the various .ci.yml files
+            if !File.basename(yaml_file).start_with?('.') && (!exclude || !exclude.match?(yaml_file))
+              @work_pool.push(:yaml, yaml_file)
+            end
+          end
+        end
+      end
+    end
+
+    def precompile_yaml(*yaml_files)
+      Array(yaml_files).each do |yaml_file|
+        if CompileCache::YAML.precompile(yaml_file, cache_dir: cache_dir)
+          STDERR.puts(yaml_file) if verbose
+        end
+      end
+    end
+
+    def precompile_ruby_files(load_paths, exclude: self.exclude)
+      return unless iseq
+
+      load_paths.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_files(path, '**/*.rb').each do |ruby_file|
+            if !exclude || !exclude.match?(ruby_file)
+              @work_pool.push(:ruby, ruby_file)
+            end
+          end
+        end
+      end
+    end
+
+    def precompile_ruby(*ruby_files)
+      Array(ruby_files).each do |ruby_file|
+        if CompileCache::ISeq.precompile(ruby_file, cache_dir: cache_dir)
+          STDERR.puts(ruby_file) if verbose
+        end
+      end
+    end
+
+    def fix_default_encoding
+      if Encoding.default_external == Encoding::US_ASCII
+        Encoding.default_external = Encoding::UTF_8
+        begin
+          yield
+        ensure
+          Encoding.default_external = Encoding::US_ASCII
+        end
+      else
+        yield
+      end
+    end
+
+    def invalid_usage!(message)
+      STDERR.puts message
+      STDERR.puts
+      STDERR.puts parser
+      1
+    end
+
+    def cache_dir=(dir)
+      @cache_dir = File.expand_path(File.join(dir, 'bootsnap/compile-cache'))
+    end
+
+    def exclude_pattern(pattern)
+      (@exclude_patterns ||= []) << Regexp.new(pattern)
+      self.exclude = Regexp.union(@exclude_patterns)
+    end
+
+    def parser
+      @parser ||= OptionParser.new do |opts|
+        opts.banner = "Usage: bootsnap COMMAND [ARGS]"
+        opts.separator ""
+        opts.separator "GLOBAL OPTIONS"
+        opts.separator ""
+
+        help = <<~EOS
+          Path to the bootsnap cache directory. Defaults to tmp/cache
+        EOS
+        opts.on('--cache-dir DIR', help.strip) do |dir|
+          self.cache_dir = dir
+        end
+
+        help = <<~EOS
+          Print precompiled paths.
+        EOS
+        opts.on('--verbose', '-v', help.strip) do
+          self.verbose = true
+        end
+
+        help = <<~EOS
+          Number of workers to use. Default to number of processors, set to 0 to disable multi-processing.
+        EOS
+        opts.on('--jobs JOBS', '-j', help.strip) do |jobs|
+          self.jobs = Integer(jobs)
+        end
+
+        opts.separator ""
+        opts.separator "COMMANDS"
+        opts.separator ""
+        opts.separator "    precompile [DIRECTORIES...]: Precompile all .rb files in the passed directories"
+
+        help = <<~EOS
+          Precompile the gems in Gemfile
+        EOS
+        opts.on('--gemfile', help) { self.compile_gemfile = true }
+
+        help = <<~EOS
+          Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api'
+        EOS
+        opts.on('--exclude PATTERN', help) { |pattern| exclude_pattern(pattern) }
+
+        help = <<~EOS
+          Disable ISeq (.rb) precompilation.
+        EOS
+        opts.on('--no-iseq', help) { self.iseq = false }
+
+        help = <<~EOS
+          Disable YAML precompilation.
+        EOS
+        opts.on('--no-yaml', help) { self.yaml = false }
+      end
+    end
+  end
+end

data/lib/bootsnap/compile_cache/iseq.rb

@@ -9,7 +9,7 @@ module Bootsnap
         attr_accessor(:cache_dir)
       end
 
-      def self.input_to_storage(_, path, _args)
+      def self.input_to_storage(_, path)
         RubyVM::InstructionSequence.compile_file(path).to_binary
       rescue SyntaxError
         raise(Uncompilable, 'syntax error')
@@ -26,7 +26,24 @@ module Bootsnap
         end
       end
 
-      def self.input_to_output(_, _)
+      def self.fetch(path, cache_dir: ISeq.cache_dir)
+        Bootsnap::CompileCache::Native.fetch(
+          cache_dir,
+          path.to_s,
+          Bootsnap::CompileCache::ISeq,
+          nil,
+        )
+      end
+
+      def self.precompile(path, cache_dir: ISeq.cache_dir)
+        Bootsnap::CompileCache::Native.precompile(
+          cache_dir,
+          path.to_s,
+          Bootsnap::CompileCache::ISeq,
+        )
+      end
+
+      def self.input_to_output(_data, _kwargs)
         nil # ruby handles this
       end
 
@@ -35,12 +52,7 @@ module Bootsnap
           # Having coverage enabled prevents iseq dumping/loading.
           return nil if defined?(Coverage) && Bootsnap::CompileCache::Native.coverage_running?
 
-          Bootsnap::CompileCache::Native.fetch(
-            Bootsnap::CompileCache::ISeq.cache_dir,
-            path.to_s,
-            Bootsnap::CompileCache::ISeq,
-            nil,
-          )
+          Bootsnap::CompileCache::ISeq.fetch(path.to_s)
         rescue Errno::EACCES
           Bootsnap::CompileCache.permission_error(path)
         rescue RuntimeError => e
@@ -61,6 +73,7 @@ module Bootsnap
         crc = Zlib.crc32(option.inspect)
         Bootsnap::CompileCache::Native.compile_option_crc32 = crc
       end
+      compile_option_updated
 
       def self.install!(cache_dir)
         Bootsnap::CompileCache::ISeq.cache_dir = cache_dir

data/lib/bootsnap/compile_cache/yaml.rb

@@ -7,32 +7,34 @@ module Bootsnap
       class << self
         attr_accessor(:msgpack_factory, :cache_dir, :supported_options)
 
-        def input_to_storage(contents, _, kwargs)
+        def input_to_storage(contents, _)
           raise(Uncompilable) if contents.index("!ruby/object")
-          obj = ::YAML.load(contents, **(kwargs || {}))
+          obj = ::YAML.load(contents)
           msgpack_factory.dump(obj)
         rescue NoMethodError, RangeError
-          # if the object included things that we can't serialize, fall back to
-          # Marshal. It's a bit slower, but can encode anything yaml can.
-          # NoMethodError is unexpected types; RangeError is Bignums
-          Marshal.dump(obj)
+          # The object included things that we can't serialize
+          raise(Uncompilable)
         end
 
         def storage_to_output(data, kwargs)
-          # This could have a meaning in messagepack, and we're being a little lazy
-          # about it. -- but a leading 0x04 would indicate the contents of the YAML
-          # is a positive integer, which is rare, to say the least.
-          if data[0] == 0x04.chr && data[1] == 0x08.chr
-            Marshal.load(data)
-          else
-            msgpack_factory.load(data, **(kwargs || {}))
+          if kwargs && kwargs.key?(:symbolize_names)
+            kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
           end
+          msgpack_factory.load(data, kwargs)
         end
 
         def input_to_output(data, kwargs)
           ::YAML.load(data, **(kwargs || {}))
         end
 
+        def precompile(path, cache_dir: YAML.cache_dir)
+          Bootsnap::CompileCache::Native.precompile(
+            cache_dir,
+            path.to_s,
+            Bootsnap::CompileCache::YAML,
+          )
+        end
+
         def install!(cache_dir)
           self.cache_dir = cache_dir
           init!
@@ -42,12 +44,31 @@ module Bootsnap
         def init!
           require('yaml')
           require('msgpack')
+          require('date')
 
           # MessagePack serializes symbols as strings by default.
           # We want them to roundtrip cleanly, so we use a custom factory.
           # see: https://github.com/msgpack/msgpack-ruby/pull/122
           factory = MessagePack::Factory.new
           factory.register_type(0x00, Symbol)
+          factory.register_type(
+            MessagePack::Timestamp::TYPE, # or just -1
+            Time,
+            packer: MessagePack::Time::Packer,
+            unpacker: MessagePack::Time::Unpacker
+          )
+
+          marshal_fallback = {
+            packer: ->(value) { Marshal.dump(value) },
+            unpacker: ->(payload) { Marshal.load(payload) },
+          }
+          {
+            Date => 0x01,
+            Regexp => 0x02,
+          }.each do |type, code|
+            factory.register_type(code, type, marshal_fallback)
+          end
+
           self.msgpack_factory = factory
 
           self.supported_options = []
@@ -65,8 +86,6 @@ module Bootsnap
       end
 
       module Patch
-        extend self
-
         def load_file(path, *args)
           return super if args.size > 1
           if kwargs = args.first
@@ -77,7 +96,7 @@ module Bootsnap
           begin
             ::Bootsnap::CompileCache::Native.fetch(
               Bootsnap::CompileCache::YAML.cache_dir,
-              path,
+              File.realpath(path),
               ::Bootsnap::CompileCache::YAML,
               kwargs,
             )
@@ -85,6 +104,8 @@ module Bootsnap
             ::Bootsnap::CompileCache.permission_error(path)
           end
         end
+
+        ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
       end
     end
   end

data/lib/bootsnap/load_path_cache.rb

@@ -28,10 +28,9 @@ module Bootsnap
     CACHED_EXTENSIONS = DLEXT2 ? [DOT_RB, DLEXT, DLEXT2] : [DOT_RB, DLEXT]
 
     class << self
-      attr_reader(:load_path_cache, :autoload_paths_cache,
-        :loaded_features_index, :realpath_cache)
+      attr_reader(:load_path_cache, :loaded_features_index, :realpath_cache)
 
-      def setup(cache_path:, development_mode:, active_support: true)
+      def setup(cache_path:, development_mode:)
         unless supported?
           warn("[bootsnap/setup] Load path caching is not supported on this implementation of Ruby") if $VERBOSE
           return
@@ -45,18 +44,6 @@ module Bootsnap
         @load_path_cache = Cache.new(store, $LOAD_PATH, development_mode: development_mode)
         require_relative('load_path_cache/core_ext/kernel_require')
         require_relative('load_path_cache/core_ext/loaded_features')
-
-        if active_support
-          # this should happen after setting up the initial cache because it
-          # loads a lot of code. It's better to do after +require+ is optimized.
-          require('active_support/dependencies')
-          @autoload_paths_cache = Cache.new(
-            store,
-            ::ActiveSupport::Dependencies.autoload_paths,
-            development_mode: development_mode
-          )
-          require_relative('load_path_cache/core_ext/active_support')
-        end
       end
 
       def supported?

data/lib/bootsnap/setup.rb

@@ -1,39 +1,4 @@
 # frozen_string_literal: true
 require_relative('../bootsnap')
 
-env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || ENV['ENV']
-development_mode = ['', nil, 'development'].include?(env)
-
-cache_dir = ENV['BOOTSNAP_CACHE_DIR']
-unless cache_dir
-  config_dir_frame = caller.detect do |line|
-    line.include?('/config/')
-  end
-
-  unless config_dir_frame
-    $stderr.puts("[bootsnap/setup] couldn't infer cache directory! Either:")
-    $stderr.puts("[bootsnap/setup]   1. require bootsnap/setup from your application's config directory; or")
-    $stderr.puts("[bootsnap/setup]   2. Define the environment variable BOOTSNAP_CACHE_DIR")
-
-    raise("couldn't infer bootsnap cache directory")
-  end
-
-  path = config_dir_frame.split(/:\d+:/).first
-  path = File.dirname(path) until File.basename(path) == 'config'
-  app_root = File.dirname(path)
-
-  cache_dir = File.join(app_root, 'tmp', 'cache')
-end
-
-ruby_version = Gem::Version.new(RUBY_VERSION)
-iseq_cache_enabled = ruby_version < Gem::Version.new('2.5.0') || ruby_version >= Gem::Version.new('2.6.0')
-
-Bootsnap.setup(
-  cache_dir:            cache_dir,
-  development_mode:     development_mode,
-  load_path_cache:      true,
-  autoload_paths_cache: true, # assume rails. open to PRs to impl. detection
-  disable_trace:        false,
-  compile_cache_iseq:   iseq_cache_enabled,
-  compile_cache_yaml:   true,
-)
+Bootsnap.default_setup

data/lib/bootsnap/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Bootsnap
-  VERSION = "1.4.9"
+  VERSION = "1.7.0"
 end

data/lib/bootsnap.rb

@@ -8,42 +8,104 @@ require_relative('bootsnap/compile_cache')
 module Bootsnap
   InvalidConfiguration = Class.new(StandardError)
 
+  class << self
+    attr_reader :logger
+  end
+
+  def self.log!
+    self.logger = $stderr.method(:puts)
+  end
+
+  def self.logger=(logger)
+    @logger = logger
+    if logger.respond_to?(:debug)
+      self.instrumentation = ->(event, path) { @logger.debug("[Bootsnap] #{event} #{path}") }
+    else
+      self.instrumentation = ->(event, path) { @logger.call("[Bootsnap] #{event} #{path}") }
+    end
+  end
+
+  def self.instrumentation=(callback)
+    @instrumentation = callback
+    self.instrumentation_enabled = !!callback
+  end
+
+  def self._instrument(event, path)
+    @instrumentation.call(event, path)
+  end
+
   def self.setup(
     cache_dir:,
     development_mode: true,
     load_path_cache: true,
-    autoload_paths_cache: true,
-    disable_trace: false,
+    autoload_paths_cache: nil,
+    disable_trace: nil,
     compile_cache_iseq: true,
     compile_cache_yaml: true
   )
-    if autoload_paths_cache && !load_path_cache
-      raise(InvalidConfiguration, "feature 'autoload_paths_cache' depends on feature 'load_path_cache'")
+    unless autoload_paths_cache.nil?
+      warn "[DEPRECATED] Bootsnap's `autoload_paths_cache:` option is deprecated and will be removed. " \
+        "If you use Zeitwerk this option is useless, and if you are still using the classic autoloader " \
+        "upgrading is recommended."
     end
 
-    setup_disable_trace if disable_trace
+    unless disable_trace.nil?
+      warn "[DEPRECATED] Bootsnap's `disable_trace:` option is deprecated and will be removed. " \
+        "If you use Ruby 2.5 or newer this option is useless, if not upgrading is recommended."
+    end
 
     Bootsnap::LoadPathCache.setup(
-      cache_path:       cache_dir + '/bootsnap-load-path-cache',
+      cache_path:       cache_dir + '/bootsnap/load-path-cache',
       development_mode: development_mode,
-      active_support:   autoload_paths_cache
     ) if load_path_cache
 
     Bootsnap::CompileCache.setup(
-      cache_dir: cache_dir + '/bootsnap-compile-cache',
+      cache_dir: cache_dir + '/bootsnap/compile-cache',
       iseq: compile_cache_iseq,
       yaml: compile_cache_yaml
     )
   end
 
-  def self.setup_disable_trace
-    if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.5.0')
-      warn(
-        "from #{caller_locations(1, 1)[0]}: The 'disable_trace' method is not allowed with this Ruby version. " \
-        "current: #{RUBY_VERSION}, allowed version: < 2.5.0",
+  def self.default_setup
+    env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || ENV['ENV']
+    development_mode = ['', nil, 'development'].include?(env)
+
+    unless ENV['DISABLE_BOOTSNAP']
+      cache_dir = ENV['BOOTSNAP_CACHE_DIR']
+      unless cache_dir
+        config_dir_frame = caller.detect do |line|
+          line.include?('/config/')
+        end
+
+        unless config_dir_frame
+          $stderr.puts("[bootsnap/setup] couldn't infer cache directory! Either:")
+          $stderr.puts("[bootsnap/setup]   1. require bootsnap/setup from your application's config directory; or")
+          $stderr.puts("[bootsnap/setup]   2. Define the environment variable BOOTSNAP_CACHE_DIR")
+
+          raise("couldn't infer bootsnap cache directory")
+        end
+
+        path = config_dir_frame.split(/:\d+:/).first
+        path = File.dirname(path) until File.basename(path) == 'config'
+        app_root = File.dirname(path)
+
+        cache_dir = File.join(app_root, 'tmp', 'cache')
+      end
+
+      ruby_version = Gem::Version.new(RUBY_VERSION)
+      iseq_cache_enabled = ruby_version < Gem::Version.new('2.5.0') || ruby_version >= Gem::Version.new('2.5.4')
+
+      setup(
+        cache_dir:            cache_dir,
+        development_mode:     development_mode,
+        load_path_cache:      !ENV['DISABLE_BOOTSNAP_LOAD_PATH_CACHE'],
+        compile_cache_iseq:   !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'] && iseq_cache_enabled,
+        compile_cache_yaml:   !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'],
       )
-    else
-      RubyVM::InstructionSequence.compile_option = { trace_instruction: false }
+
+      if ENV['BOOTSNAP_LOG']
+        log!
+      end
     end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bootsnap
 version: !ruby/object:Gem::Version
-  version: 1.4.9
+  version: 1.7.0
 platform: ruby
 authors:
 - Burke Libbey
 autorequire: 
-bindir: bin
+bindir: exe
 cert_chain: []
-date: 2020-10-26 00:00:00.000000000 Z
+date: 2021-02-01 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -97,7 +97,8 @@ dependencies:
 description: Boot large ruby/rails apps faster
 email:
 - burke.libbey@shopify.com
-executables: []
+executables:
+- bootsnap
 extensions:
 - ext/bootsnap/extconf.rb
 extra_rdoc_files: []
@@ -105,11 +106,14 @@ files:
 - CHANGELOG.md
 - LICENSE.txt
 - README.md
+- exe/bootsnap
 - ext/bootsnap/bootsnap.c
 - ext/bootsnap/bootsnap.h
 - ext/bootsnap/extconf.rb
 - lib/bootsnap.rb
 - lib/bootsnap/bundler.rb
+- lib/bootsnap/cli.rb
+- lib/bootsnap/cli/worker_pool.rb
 - lib/bootsnap/compile_cache.rb
 - lib/bootsnap/compile_cache/iseq.rb
 - lib/bootsnap/compile_cache/yaml.rb
@@ -117,7 +121,6 @@ files:
 - lib/bootsnap/load_path_cache.rb
 - lib/bootsnap/load_path_cache/cache.rb
 - lib/bootsnap/load_path_cache/change_observer.rb
-- lib/bootsnap/load_path_cache/core_ext/active_support.rb
 - lib/bootsnap/load_path_cache/core_ext/kernel_require.rb
 - lib/bootsnap/load_path_cache/core_ext/loaded_features.rb
 - lib/bootsnap/load_path_cache/loaded_features_index.rb