bootsnap-pr-184 1.3.1.pr.pre.184.1

data/ext/bootsnap/bootsnap.h

@@ -0,0 +1,6 @@
+#ifndef BOOTSNAP_H
+#define BOOTSNAP_H 1
+
+/* doesn't expose anything */
+
+#endif /* BOOTSNAP_H */

data/ext/bootsnap/extconf.rb

@@ -0,0 +1,17 @@
+require "mkmf"
+$CFLAGS << ' -O3 '
+$CFLAGS << ' -std=c99'
+
+# ruby.h has some -Wpedantic fails in some cases
+# (e.g. https://github.com/Shopify/bootsnap/issues/15)
+unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC'])
+  $CFLAGS << ' -Wall'
+  $CFLAGS << ' -Werror'
+  $CFLAGS << ' -Wextra'
+  $CFLAGS << ' -Wpedantic'
+
+  $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
+  $CFLAGS << ' -Wno-keyword-macro' # hiding return
+end
+
+create_makefile("bootsnap/bootsnap")

data/lib/bootsnap.rb

@@ -0,0 +1,44 @@
+require_relative 'bootsnap/version'
+require_relative 'bootsnap/bundler'
+require_relative 'bootsnap/load_path_cache'
+require_relative 'bootsnap/compile_cache'
+
+module Bootsnap
+  InvalidConfiguration = Class.new(StandardError)
+
+  def self.setup(
+    cache_dir:,
+    development_mode: true,
+    load_path_cache: true,
+    autoload_paths_cache: true,
+    disable_trace: false,
+    compile_cache_iseq: true,
+    compile_cache_yaml: true
+  )
+    if autoload_paths_cache && !load_path_cache
+      raise InvalidConfiguration, "feature 'autoload_paths_cache' depends on feature 'load_path_cache'"
+    end
+
+    setup_disable_trace if disable_trace
+
+    Bootsnap::LoadPathCache.setup(
+      cache_path:       cache_dir + '/bootsnap-load-path-cache',
+      development_mode: development_mode,
+      active_support:   autoload_paths_cache
+    ) if load_path_cache
+
+    Bootsnap::CompileCache.setup(
+      cache_dir: cache_dir + '/bootsnap-compile-cache',
+      iseq: compile_cache_iseq,
+      yaml: compile_cache_yaml
+    )
+  end
+
+  def self.setup_disable_trace
+    if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.5.0')
+      warn("from #{caller_locations(1, 1)[0]}: The 'disable_trace' method is not allowed with this Ruby version. current: #{RUBY_VERSION}, allowed version: < 2.5.0")
+    else
+      RubyVM::InstructionSequence.compile_option = { trace_instruction: false }
+    end
+  end
+end

data/lib/bootsnap/bundler.rb

@@ -0,0 +1,12 @@
+module Bootsnap
+  module_function
+
+  def bundler?
+    # Bundler environment variable
+    ['BUNDLE_BIN_PATH', 'BUNDLE_GEMFILE'].each do |current|
+      return true if ENV.key?(current)
+    end
+    
+    false
+  end
+end

data/lib/bootsnap/compile_cache.rb

@@ -0,0 +1,15 @@
+module Bootsnap
+  module CompileCache
+    def self.setup(cache_dir:, iseq:, yaml:)
+      if iseq
+        require_relative 'compile_cache/iseq'
+        Bootsnap::CompileCache::ISeq.install!(cache_dir)
+      end
+
+      if yaml
+        require_relative 'compile_cache/yaml'
+        Bootsnap::CompileCache::YAML.install!(cache_dir)
+      end
+    end
+  end
+end

data/lib/bootsnap/compile_cache/iseq.rb

@@ -0,0 +1,70 @@
+require 'bootsnap/bootsnap'
+require 'zlib'
+
+module Bootsnap
+  module CompileCache
+    module ISeq
+      class << self
+        attr_accessor :cache_dir
+      end
+
+      def self.input_to_storage(_, path)
+        RubyVM::InstructionSequence.compile_file(path).to_binary
+      rescue SyntaxError
+        raise Uncompilable, 'syntax error'
+      end
+
+      def self.storage_to_output(binary)
+        RubyVM::InstructionSequence.load_from_binary(binary)
+      rescue RuntimeError => e
+        if e.message == 'broken binary format'
+          STDERR.puts "[Bootsnap::CompileCache] warning: rejecting broken binary"
+          return nil
+        else
+          raise
+        end
+      end
+
+      def self.input_to_output(_)
+        nil # ruby handles this
+      end
+
+      module InstructionSequenceMixin
+        def load_iseq(path)
+          # Having coverage enabled prevents iseq dumping/loading.
+          return nil if defined?(Coverage) && Bootsnap::CompileCache::Native.coverage_running?
+
+          Bootsnap::CompileCache::Native.fetch(
+            Bootsnap::CompileCache::ISeq.cache_dir,
+            path.to_s,
+            Bootsnap::CompileCache::ISeq
+          )
+        rescue RuntimeError => e
+          if e.message =~ /unmatched platform/
+            puts "unmatched platform for file #{path}"
+          end
+          raise
+        end
+
+        def compile_option=(hash)
+          super(hash)
+          Bootsnap::CompileCache::ISeq.compile_option_updated
+        end
+      end
+
+      def self.compile_option_updated
+        option = RubyVM::InstructionSequence.compile_option
+        crc = Zlib.crc32(option.inspect)
+        Bootsnap::CompileCache::Native.compile_option_crc32 = crc
+      end
+
+      def self.install!(cache_dir)
+        Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
+        Bootsnap::CompileCache::ISeq.compile_option_updated
+        class << RubyVM::InstructionSequence
+          prepend InstructionSequenceMixin
+        end
+      end
+    end
+  end
+end

data/lib/bootsnap/compile_cache/yaml.rb

@@ -0,0 +1,58 @@
+require 'bootsnap/bootsnap'
+
+module Bootsnap
+  module CompileCache
+    module YAML
+      class << self
+        attr_accessor :msgpack_factory
+      end
+
+      def self.input_to_storage(contents, _)
+        raise Uncompilable if contents.index("!ruby/object")
+        obj = ::YAML.load(contents)
+        msgpack_factory.packer.write(obj).to_s
+      rescue NoMethodError, RangeError
+        # if the object included things that we can't serialize, fall back to
+        # Marshal. It's a bit slower, but can encode anything yaml can.
+        # NoMethodError is unexpected types; RangeError is Bignums
+        return Marshal.dump(obj)
+      end
+
+      def self.storage_to_output(data)
+        # This could have a meaning in messagepack, and we're being a little lazy
+        # about it. -- but a leading 0x04 would indicate the contents of the YAML
+        # is a positive integer, which is rare, to say the least.
+        if data[0] == 0x04.chr && data[1] == 0x08.chr
+          Marshal.load(data)
+        else
+          msgpack_factory.unpacker.feed(data).read
+        end
+      end
+
+      def self.input_to_output(data)
+        ::YAML.load(data)
+      end
+
+      def self.install!(cache_dir)
+        require 'yaml'
+        require 'msgpack'
+
+        # MessagePack serializes symbols as strings by default.
+        # We want them to roundtrip cleanly, so we use a custom factory.
+        # see: https://github.com/msgpack/msgpack-ruby/pull/122
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        Bootsnap::CompileCache::YAML.msgpack_factory = factory
+
+        klass = class << ::YAML; self; end
+        klass.send(:define_method, :load_file) do |path|
+          Bootsnap::CompileCache::Native.fetch(
+            cache_dir,
+            path.to_s,
+            Bootsnap::CompileCache::YAML
+          )
+        end
+      end
+    end
+  end
+end

data/lib/bootsnap/explicit_require.rb

@@ -0,0 +1,49 @@
+module Bootsnap
+  module ExplicitRequire
+    ARCHDIR    = RbConfig::CONFIG['archdir']
+    RUBYLIBDIR = RbConfig::CONFIG['rubylibdir']
+    DLEXT      = RbConfig::CONFIG['DLEXT']
+
+    def self.from_self(feature)
+      require_relative "../#{feature}"
+    end
+
+    def self.from_rubylibdir(feature)
+      require(File.join(RUBYLIBDIR, "#{feature}.rb"))
+    end
+
+    def self.from_archdir(feature)
+      require(File.join(ARCHDIR, "#{feature}.#{DLEXT}"))
+    end
+
+    # Given a set of gems, run a block with the LOAD_PATH narrowed to include
+    # only core ruby source paths and these gems -- that is, roughly,
+    # temporarily remove all gems not listed in this call from the LOAD_PATH.
+    #
+    # This is useful before bootsnap is fully-initialized to load gems that it
+    # depends on, without forcing full LOAD_PATH traversals.
+    def self.with_gems(*gems)
+      orig = $LOAD_PATH.dup
+      $LOAD_PATH.clear
+      gems.each do |gem|
+        pat = %r{
+          /
+          (gems|extensions/[^/]+/[^/]+)          # "gems" or "extensions/x64_64-darwin16/2.3.0"
+          /
+          #{Regexp.escape(gem)}-(\h{12}|(\d+\.)) # msgpack-1.2.3 or msgpack-1234567890ab
+        }x
+        $LOAD_PATH.concat(orig.grep(pat))
+      end
+      $LOAD_PATH << ARCHDIR
+      $LOAD_PATH << RUBYLIBDIR
+      begin
+        yield
+      rescue LoadError
+        $LOAD_PATH.replace(orig)
+        yield
+      end
+    ensure
+      $LOAD_PATH.replace(orig)
+    end
+  end
+end

data/lib/bootsnap/load_path_cache.rb

@@ -0,0 +1,58 @@
+module Bootsnap
+  module LoadPathCache
+    ReturnFalse = Class.new(StandardError)
+    FallbackScan = Class.new(StandardError)
+
+    DOT_RB = '.rb'
+    DOT_SO = '.so'
+    SLASH  = '/'
+
+    DL_EXTENSIONS = ::RbConfig::CONFIG
+      .values_at('DLEXT', 'DLEXT2')
+      .reject { |ext| !ext || ext.empty? }
+      .map    { |ext| ".#{ext}" }
+      .freeze
+    DLEXT = DL_EXTENSIONS[0]
+    # This is nil on linux and darwin, but I think it's '.o' on some other
+    # platform.  I'm not really sure which, but it seems better to replicate
+    # ruby's semantics as faithfully as possible.
+    DLEXT2 = DL_EXTENSIONS[1]
+
+    CACHED_EXTENSIONS = DLEXT2 ? [DOT_RB, DLEXT, DLEXT2] : [DOT_RB, DLEXT]
+
+    class << self
+      attr_reader :load_path_cache, :autoload_paths_cache,
+                  :loaded_features_index, :realpath_cache
+
+      def setup(cache_path:, development_mode:, active_support: true)
+        store = Store.new(cache_path)
+
+        @loaded_features_index = LoadedFeaturesIndex.new
+        @realpath_cache = RealpathCache.new
+
+        @load_path_cache = Cache.new(store, $LOAD_PATH, development_mode: development_mode)
+        require_relative 'load_path_cache/core_ext/kernel_require'
+
+        if active_support
+          # this should happen after setting up the initial cache because it
+          # loads a lot of code. It's better to do after +require+ is optimized.
+          require 'active_support/dependencies'
+          @autoload_paths_cache = Cache.new(
+            store,
+            ::ActiveSupport::Dependencies.autoload_paths,
+            development_mode: development_mode
+          )
+          require_relative 'load_path_cache/core_ext/active_support'
+        end
+      end
+    end
+  end
+end
+
+require_relative 'load_path_cache/path_scanner'
+require_relative 'load_path_cache/path'
+require_relative 'load_path_cache/cache'
+require_relative 'load_path_cache/store'
+require_relative 'load_path_cache/change_observer'
+require_relative 'load_path_cache/loaded_features_index'
+require_relative 'load_path_cache/realpath_cache'

data/lib/bootsnap/load_path_cache/cache.rb

@@ -0,0 +1,181 @@
+require_relative '../explicit_require'
+
+module Bootsnap
+  module LoadPathCache
+    class Cache
+      AGE_THRESHOLD = 30 # seconds
+
+      def initialize(store, path_obj, development_mode: false)
+        @development_mode = development_mode
+        @store = store
+        @mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new # TODO: Remove once Ruby 2.2 support is dropped.
+        @path_obj = path_obj.map! { |f| File.exist?(f) ? File.realpath(f) : f }
+        @has_relative_paths = nil
+        reinitialize
+      end
+
+      # Does this directory exist as a child of one of the path items?
+      # e.g. given "/a/b/c/d" exists, and the path is ["/a/b"], has_dir?("c/d")
+      # is true.
+      def has_dir?(dir)
+        reinitialize if stale?
+        @mutex.synchronize { @dirs[dir] }
+      end
+
+      # { 'enumerator' => nil, 'enumerator.so' => nil, ... }
+      BUILTIN_FEATURES = $LOADED_FEATURES.each_with_object({}) do |feat, features|
+        # Builtin features are of the form 'enumerator.so'.
+        # All others include paths.
+        next unless feat.size < 20 && !feat.include?('/')
+
+        base = File.basename(feat, '.*') # enumerator.so -> enumerator
+        ext  = File.extname(feat) # .so
+
+        features[feat] = nil # enumerator.so
+        features[base] = nil # enumerator
+
+        next unless [DOT_SO, *DL_EXTENSIONS].include?(ext)
+        DL_EXTENSIONS.each do |dl_ext|
+          features["#{base}#{dl_ext}"] = nil # enumerator.bundle
+        end
+      end.freeze
+
+      # Try to resolve this feature to an absolute path without traversing the
+      # loadpath.
+      def find(feature)
+        reinitialize if (@has_relative_paths && dir_changed?) || stale?
+        feature = feature.to_s
+        return feature if absolute_path?(feature)
+        return File.expand_path(feature) if feature.start_with?('./')
+        @mutex.synchronize do
+          x = search_index(feature)
+          return x if x
+
+          # Ruby has some built-in features that require lies about.
+          # For example, 'enumerator' is built in. If you require it, ruby
+          # returns false as if it were already loaded; however, there is no
+          # file to find on disk. We've pre-built a list of these, and we
+          # return false if any of them is loaded.
+          raise LoadPathCache::ReturnFalse if BUILTIN_FEATURES.key?(feature)
+
+          # The feature wasn't found on our preliminary search through the index.
+          # We resolve this differently depending on what the extension was.
+          case File.extname(feature)
+          # If the extension was one of the ones we explicitly cache (.rb and the
+          # native dynamic extension, e.g. .bundle or .so), we know it was a
+          # failure and there's nothing more we can do to find the file.
+          # no extension, .rb, (.bundle or .so)
+          when '', *CACHED_EXTENSIONS # rubocop:disable Performance/CaseWhenSplat
+            nil
+          # Ruby allows specifying native extensions as '.so' even when DLEXT
+          # is '.bundle'. This is where we handle that case.
+          when DOT_SO
+            x = search_index(feature[0..-4] + DLEXT)
+            return x if x
+            if DLEXT2
+              search_index(feature[0..-4] + DLEXT2)
+            end
+          else
+            # other, unknown extension. For example, `.rake`. Since we haven't
+            # cached these, we legitimately need to run the load path search.
+            raise LoadPathCache::FallbackScan
+          end
+        end
+      end
+
+      if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
+        def absolute_path?(path)
+          path[1] == ':'
+        end
+      else
+        def absolute_path?(path)
+          path.start_with?(SLASH)
+        end
+      end
+
+      def unshift_paths(sender, *paths)
+        return unless sender == @path_obj
+        @mutex.synchronize { unshift_paths_locked(*paths) }
+      end
+
+      def push_paths(sender, *paths)
+        return unless sender == @path_obj
+        @mutex.synchronize { push_paths_locked(*paths) }
+      end
+
+      def reinitialize(path_obj = @path_obj)
+        @mutex.synchronize do
+          @path_obj = path_obj
+          ChangeObserver.register(self, @path_obj)
+          @index = {}
+          @dirs = Hash.new(false)
+          @generated_at = now
+          push_paths_locked(*@path_obj)
+        end
+      end
+
+      private
+
+      def dir_changed?
+        @prev_dir ||= Dir.pwd
+        if @prev_dir == Dir.pwd
+          false
+        else
+          @prev_dir = Dir.pwd
+          true
+        end
+      end
+
+      def push_paths_locked(*paths)
+        @store.transaction do
+          paths.map(&:to_s).each do |path|
+            p = Path.new(path)
+            @has_relative_paths = true if p.relative?
+            next if p.non_directory?
+            entries, dirs = p.entries_and_dirs(@store)
+            # push -> low precedence -> set only if unset
+            dirs.each    { |dir| @dirs[dir]  ||= true }
+            entries.each { |rel| @index[rel] ||= p.expanded_path }
+          end
+        end
+      end
+
+      def unshift_paths_locked(*paths)
+        @store.transaction do
+          paths.map(&:to_s).reverse_each do |path|
+            p = Path.new(path)
+            next if p.non_directory?
+            entries, dirs = p.entries_and_dirs(@store)
+            # unshift -> high precedence -> unconditional set
+            dirs.each    { |dir| @dirs[dir]  = true }
+            entries.each { |rel| @index[rel] = p.expanded_path }
+          end
+        end
+      end
+
+      def stale?
+        @development_mode && @generated_at + AGE_THRESHOLD < now
+      end
+
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC).to_i
+      end
+
+      if DLEXT2
+        def search_index(f)
+          try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f + DLEXT2) || try_index(f)
+        end
+      else
+        def search_index(f)
+          try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f)
+        end
+      end
+
+      def try_index(f)
+        if p = @index[f]
+          p + '/' + f
+        end
+      end
+    end
+  end
+end