bootsnap 1.4.4 → 1.9.4

data/lib/bootsnap/compile_cache/yaml.rb

@@ -1,61 +1,163 @@
+# frozen_string_literal: true
 require('bootsnap/bootsnap')
 
 module Bootsnap
   module CompileCache
     module YAML
       class << self
-        attr_accessor(:msgpack_factory)
-      end
+        attr_accessor(:msgpack_factory, :cache_dir, :supported_options)
 
-      def self.input_to_storage(contents, _)
-        raise(Uncompilable) if contents.index("!ruby/object")
-        obj = ::YAML.load(contents)
-        msgpack_factory.packer.write(obj).to_s
-      rescue NoMethodError, RangeError
-        # if the object included things that we can't serialize, fall back to
-        # Marshal. It's a bit slower, but can encode anything yaml can.
-        # NoMethodError is unexpected types; RangeError is Bignums
-        Marshal.dump(obj)
-      end
+        def input_to_storage(contents, _)
+          obj = strict_load(contents)
+          msgpack_factory.dump(obj)
+        rescue NoMethodError, RangeError
+          # The object included things that we can't serialize
+          raise(Uncompilable)
+        end
 
-      def self.storage_to_output(data)
-        # This could have a meaning in messagepack, and we're being a little lazy
-        # about it. -- but a leading 0x04 would indicate the contents of the YAML
-        # is a positive integer, which is rare, to say the least.
-        if data[0] == 0x04.chr && data[1] == 0x08.chr
-          Marshal.load(data)
-        else
-          msgpack_factory.unpacker.feed(data).read
+        def storage_to_output(data, kwargs)
+          if kwargs && kwargs.key?(:symbolize_names)
+            kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
+          end
+          msgpack_factory.load(data, kwargs)
+        end
+
+        def input_to_output(data, kwargs)
+          if ::YAML.respond_to?(:unsafe_load)
+            ::YAML.unsafe_load(data, **(kwargs || {}))
+          else
+            ::YAML.load(data, **(kwargs || {}))
+          end
+        end
+
+        def strict_load(payload, *args)
+          ast = ::YAML.parse(payload)
+          return ast unless ast
+          strict_visitor.create(*args).visit(ast)
+        end
+        ruby2_keywords :strict_load if respond_to?(:ruby2_keywords, true)
+
+        def precompile(path, cache_dir: YAML.cache_dir)
+          Bootsnap::CompileCache::Native.precompile(
+            cache_dir,
+            path.to_s,
+            Bootsnap::CompileCache::YAML,
+          )
+        end
+
+        def install!(cache_dir)
+          self.cache_dir = cache_dir
+          init!
+          ::YAML.singleton_class.prepend(Patch)
+        end
+
+        def init!
+          require('yaml')
+          require('msgpack')
+          require('date')
+
+          if Patch.method_defined?(:unsafe_load_file) && !::YAML.respond_to?(:unsafe_load_file)
+            Patch.send(:remove_method, :unsafe_load_file)
+          end
+          if Patch.method_defined?(:load_file) && ::YAML::VERSION >= '4'
+            Patch.send(:remove_method, :load_file)
+          end
+
+          # MessagePack serializes symbols as strings by default.
+          # We want them to roundtrip cleanly, so we use a custom factory.
+          # see: https://github.com/msgpack/msgpack-ruby/pull/122
+          factory = MessagePack::Factory.new
+          factory.register_type(0x00, Symbol)
+
+          if defined? MessagePack::Timestamp
+            factory.register_type(
+              MessagePack::Timestamp::TYPE, # or just -1
+              Time,
+              packer: MessagePack::Time::Packer,
+              unpacker: MessagePack::Time::Unpacker
+            )
+
+            marshal_fallback = {
+              packer: ->(value) { Marshal.dump(value) },
+              unpacker: ->(payload) { Marshal.load(payload) },
+            }
+            {
+              Date => 0x01,
+              Regexp => 0x02,
+            }.each do |type, code|
+              factory.register_type(code, type, marshal_fallback)
+            end
+          end
+
+          self.msgpack_factory = factory
+
+          self.supported_options = []
+          params = ::YAML.method(:load).parameters
+          if params.include?([:key, :symbolize_names])
+            self.supported_options << :symbolize_names
+          end
+          if params.include?([:key, :freeze])
+            if factory.load(factory.dump('yaml'), freeze: true).frozen?
+              self.supported_options << :freeze
+            end
+          end
+          self.supported_options.freeze
         end
-      end
 
-      def self.input_to_output(data)
-        ::YAML.load(data)
+        def strict_visitor
+          self::NoTagsVisitor ||= Class.new(Psych::Visitors::ToRuby) do
+            def visit(target)
+              if target.tag
+                raise Uncompilable, "YAML tags are not supported: #{target.tag}"
+              end
+              super
+            end
+          end
+        end
       end
 
-      def self.install!(cache_dir)
-        require('yaml')
-        require('msgpack')
+      module Patch
+        def load_file(path, *args)
+          return super if args.size > 1
+          if kwargs = args.first
+            return super unless kwargs.is_a?(Hash)
+            return super unless (kwargs.keys - ::Bootsnap::CompileCache::YAML.supported_options).empty?
+          end
+
+          begin
+            ::Bootsnap::CompileCache::Native.fetch(
+              Bootsnap::CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              ::Bootsnap::CompileCache::YAML,
+              kwargs,
+            )
+          rescue Errno::EACCES
+            ::Bootsnap::CompileCache.permission_error(path)
+          end
+        end
 
-        # MessagePack serializes symbols as strings by default.
-        # We want them to roundtrip cleanly, so we use a custom factory.
-        # see: https://github.com/msgpack/msgpack-ruby/pull/122
-        factory = MessagePack::Factory.new
-        factory.register_type(0x00, Symbol)
-        Bootsnap::CompileCache::YAML.msgpack_factory = factory
+        ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
+
+        def unsafe_load_file(path, *args)
+          return super if args.size > 1
+          if kwargs = args.first
+            return super unless kwargs.is_a?(Hash)
+            return super unless (kwargs.keys - ::Bootsnap::CompileCache::YAML.supported_options).empty?
+          end
 
-        klass = class << ::YAML; self; end
-        klass.send(:define_method, :load_file) do |path|
           begin
-            Bootsnap::CompileCache::Native.fetch(
-              cache_dir,
-              path,
-              Bootsnap::CompileCache::YAML
+            ::Bootsnap::CompileCache::Native.fetch(
+              Bootsnap::CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              ::Bootsnap::CompileCache::YAML,
+              kwargs,
             )
           rescue Errno::EACCES
-            Bootsnap::CompileCache.permission_error(path)
+            ::Bootsnap::CompileCache.permission_error(path)
           end
         end
+
+        ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true)
       end
     end
   end

data/lib/bootsnap/compile_cache.rb

@@ -1,9 +1,10 @@
+# frozen_string_literal: true
 module Bootsnap
   module CompileCache
     Error           = Class.new(StandardError)
     PermissionError = Class.new(Error)
 
-    def self.setup(cache_dir:, iseq:, yaml:)
+    def self.setup(cache_dir:, iseq:, yaml:, json:)
       if iseq
         if supported?
           require_relative('compile_cache/iseq')
@@ -21,6 +22,15 @@ module Bootsnap
           warn("[bootsnap/setup] YAML parsing caching is not supported on this implementation of Ruby")
         end
       end
+
+      if json
+        if supported?
+          require_relative('compile_cache/json')
+          Bootsnap::CompileCache::JSON.install!(cache_dir)
+        elsif $VERBOSE
+          warn("[bootsnap/setup] JSON parsing caching is not supported on this implementation of Ruby")
+        end
+      end
     end
 
     def self.permission_error(path)
@@ -28,14 +38,14 @@ module Bootsnap
       raise(
         PermissionError,
         "bootsnap doesn't have permission to write cache entries in '#{cpath}' " \
-        "(or, less likely, doesn't have permisison to read '#{path}')",
+        "(or, less likely, doesn't have permission to read '#{path}')",
       )
     end
 
     def self.supported?
-      # only enable on 'ruby' (MRI), POSIX (darwin, linux, *bsd), and >= 2.3.0
+      # only enable on 'ruby' (MRI), POSIX (darwin, linux, *bsd), Windows (RubyInstaller2) and >= 2.3.0
       RUBY_ENGINE == 'ruby' &&
-      RUBY_PLATFORM =~ /darwin|linux|bsd/ &&
+      RUBY_PLATFORM =~ /darwin|linux|bsd|mswin|mingw|cygwin/ &&
       Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.3.0")
     end
   end

data/lib/bootsnap/explicit_require.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Bootsnap
   module ExplicitRequire
     ARCHDIR    = RbConfig::CONFIG['archdir']

data/lib/bootsnap/load_path_cache/cache.rb

@@ -10,8 +10,8 @@ module Bootsnap
       def initialize(store, path_obj, development_mode: false)
         @development_mode = development_mode
         @store = store
-        @mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new # TODO: Remove once Ruby 2.2 support is dropped.
-        @path_obj = path_obj.map! { |f| File.exist?(f) ? File.realpath(f) : f }
+        @mutex = Mutex.new
+        @path_obj = path_obj.map! { |f| PathScanner.os_path(File.exist?(f) ? File.realpath(f) : f.dup) }
         @has_relative_paths = nil
         reinitialize
       end
@@ -44,14 +44,20 @@ module Bootsnap
 
       # Try to resolve this feature to an absolute path without traversing the
       # loadpath.
-      def find(feature)
+      def find(feature, try_extensions: true)
         reinitialize if (@has_relative_paths && dir_changed?) || stale?
-        feature = feature.to_s
-        return feature if absolute_path?(feature)
-        return expand_path(feature) if feature.start_with?('./')
+        feature = feature.to_s.freeze
+
+        return feature if Bootsnap.absolute_path?(feature)
+
+        if feature.start_with?('./', '../')
+          return try_extensions ? expand_path(feature) : File.expand_path(feature).freeze
+        end
+
         @mutex.synchronize do
-          x = search_index(feature)
+          x = search_index(feature, try_extensions: try_extensions)
           return x if x
+          return unless try_extensions
 
           # Ruby has some built-in features that require lies about.
           # For example, 'enumerator' is built in. If you require it, ruby
@@ -67,7 +73,7 @@ module Bootsnap
           # native dynamic extension, e.g. .bundle or .so), we know it was a
           # failure and there's nothing more we can do to find the file.
           # no extension, .rb, (.bundle or .so)
-          when '', *CACHED_EXTENSIONS # rubocop:disable Performance/CaseWhenSplat
+          when '', *CACHED_EXTENSIONS
             nil
           # Ruby allows specifying native extensions as '.so' even when DLEXT
           # is '.bundle'. This is where we handle that case.
@@ -92,16 +98,6 @@ module Bootsnap
         raise(LoadPathCache::FallbackScan, '', []) if @development_mode
       end
 
-      if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
-        def absolute_path?(path)
-          path[1] == ':'
-        end
-      else
-        def absolute_path?(path)
-          path.start_with?(SLASH)
-        end
-      end
-
       def unshift_paths(sender, *paths)
         return unless sender == @path_obj
         @mutex.synchronize { unshift_paths_locked(*paths) }
@@ -144,7 +140,7 @@ module Bootsnap
             expanded_path = p.expanded_path
             entries, dirs = p.entries_and_dirs(@store)
             # push -> low precedence -> set only if unset
-            dirs.each    { |dir| @dirs[dir]  ||= path }
+            dirs.each    { |dir| @dirs[dir] ||= path }
             entries.each { |rel| @index[rel] ||= expanded_path }
           end
         end
@@ -177,16 +173,24 @@ module Bootsnap
       end
 
       if DLEXT2
-        def search_index(f)
-          try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f + DLEXT2) || try_index(f)
+        def search_index(f, try_extensions: true)
+          if try_extensions
+            try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f + DLEXT2) || try_index(f)
+          else
+            try_index(f)
+          end
         end
 
         def maybe_append_extension(f)
           try_ext(f + DOT_RB) || try_ext(f + DLEXT) || try_ext(f + DLEXT2) || f
         end
       else
-        def search_index(f)
-          try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f)
+        def search_index(f, try_extensions: true)
+          if try_extensions
+            try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f)
+          else
+            try_index(f)
+          end
         end
 
         def maybe_append_extension(f)
@@ -194,9 +198,26 @@ module Bootsnap
         end
       end
 
-      def try_index(f)
-        if (p = @index[f])
-          p + '/' + f
+      s = rand.to_s.force_encoding(Encoding::US_ASCII).freeze
+      if s.respond_to?(:-@)
+        if (-s).equal?(s) && (-s.dup).equal?(s) || RUBY_VERSION >= '2.7'
+          def try_index(f)
+            if (p = @index[f])
+              -(File.join(p, f).freeze)
+            end
+          end
+        else
+          def try_index(f)
+            if (p = @index[f])
+              -File.join(p, f).untaint
+            end
+          end
+        end
+      else
+        def try_index(f)
+          if (p = @index[f])
+            File.join(p, f)
+          end
         end
       end
 

data/lib/bootsnap/load_path_cache/change_observer.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Bootsnap
   module LoadPathCache
     module ChangeObserver
@@ -14,18 +15,20 @@ module Bootsnap
           @lpc_observer.push_paths(self, *entries.map(&:to_s))
           super
         end
+        alias_method :append, :push
 
         def unshift(*entries)
           @lpc_observer.unshift_paths(self, *entries.map(&:to_s))
           super
         end
+        alias_method :prepend, :unshift
 
         def concat(entries)
           @lpc_observer.push_paths(self, *entries.map(&:to_s))
           super
         end
 
-        # uniq! keeps the first occurance of each path, otherwise preserving
+        # uniq! keeps the first occurrence of each path, otherwise preserving
         # order, preserving the effective load path
         def uniq!(*args)
           ret = super

data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb

@@ -1,8 +1,9 @@
+# frozen_string_literal: true
 module Bootsnap
   module LoadPathCache
     module CoreExt
       def self.make_load_error(path)
-        err = LoadError.new("cannot load such file -- #{path}")
+        err = LoadError.new(+"cannot load such file -- #{path}")
         err.instance_variable_set(Bootsnap::LoadPathCache::ERROR_TAG_IVAR, true)
         err.define_singleton_method(:path) { path }
         err
@@ -37,36 +38,29 @@ module Kernel
   rescue Bootsnap::LoadPathCache::ReturnFalse
     false
   rescue Bootsnap::LoadPathCache::FallbackScan
-    require_with_bootsnap_lfi(path)
+    fallback = true
+  ensure
+    if fallback
+      require_with_bootsnap_lfi(path)
+    end
   end
 
   alias_method(:require_relative_without_bootsnap, :require_relative)
   def require_relative(path)
+    location = caller_locations(1..1).first
     realpath = Bootsnap::LoadPathCache.realpath_cache.call(
-      caller_locations(1..1).first.absolute_path, path
+      location.absolute_path || location.path, path
     )
     require(realpath)
   end
 
   alias_method(:load_without_bootsnap, :load)
   def load(path, wrap = false)
-    if (resolved = Bootsnap::LoadPathCache.load_path_cache.find(path))
-      return load_without_bootsnap(resolved, wrap)
-    end
-
-    # load also allows relative paths from pwd even when not in $:
-    if File.exist?(relative = File.expand_path(path))
-      return load_without_bootsnap(relative, wrap)
+    if (resolved = Bootsnap::LoadPathCache.load_path_cache.find(path, try_extensions: false))
+      load_without_bootsnap(resolved, wrap)
+    else
+      load_without_bootsnap(path, wrap)
     end
-
-    raise(Bootsnap::LoadPathCache::CoreExt.make_load_error(path))
-  rescue LoadError => e
-    e.instance_variable_set(Bootsnap::LoadPathCache::ERROR_TAG_IVAR, true)
-    raise(e)
-  rescue Bootsnap::LoadPathCache::ReturnFalse
-    false
-  rescue Bootsnap::LoadPathCache::FallbackScan
-    load_without_bootsnap(path, wrap)
   end
 end
 
@@ -87,6 +81,10 @@ class Module
   rescue Bootsnap::LoadPathCache::ReturnFalse
     false
   rescue Bootsnap::LoadPathCache::FallbackScan
-    autoload_without_bootsnap(const, path)
+    fallback = true
+  ensure
+    if fallback
+      autoload_without_bootsnap(const, path)
+    end
   end
 end

data/lib/bootsnap/load_path_cache/core_ext/loaded_features.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class << $LOADED_FEATURES
   alias_method(:delete_without_bootsnap, :delete)
   def delete(key)

data/lib/bootsnap/load_path_cache/loaded_features_index.rb

@@ -26,7 +26,7 @@ module Bootsnap
     class LoadedFeaturesIndex
       def initialize
         @lfi = {}
-        @mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new # TODO: Remove once Ruby 2.2 support is dropped.
+        @mutex = Mutex.new
 
         # In theory the user could mutate $LOADED_FEATURES and invalidate our
         # cache. If this ever comes up in practice — or if you, the
@@ -40,7 +40,7 @@ module Bootsnap
             # /a/b/lib/my/foo.rb
             #          ^^^^^^^^^
             short = feat[(lpe.length + 1)..-1]
-            stripped = strip_extension(short)
+            stripped = strip_extension_if_elidable(short)
             @lfi[short] = hash
             @lfi[stripped] = hash
           end
@@ -58,9 +58,9 @@ module Bootsnap
       end
 
       def purge_multi(features)
-        rejected_hashes = features.map(&:hash).to_set
+        rejected_hashes = features.each_with_object({}) { |f, h| h[f.hash] = true }
         @mutex.synchronize do
-          @lfi.reject! { |_, hash| rejected_hashes.include?(hash) }
+          @lfi.reject! { |_, hash| rejected_hashes.key?(hash) }
         end
       end
 
@@ -83,24 +83,38 @@ module Bootsnap
       # 2. Inspect $LOADED_FEATURES upon return from yield to find the matching
       #    entry.
       def register(short, long = nil)
+        # Absolute paths are not a concern.
+        if Bootsnap.absolute_path?(short.to_s)
+          return yield
+        end
+
         if long.nil?
-          pat = %r{/#{Regexp.escape(short)}(\.[^/]+)?$}
           len = $LOADED_FEATURES.size
           ret = yield
-          long = $LOADED_FEATURES[len..-1].detect { |feat| feat =~ pat }
+          long = $LOADED_FEATURES[len..-1].detect do |feat|
+            offset = 0
+            while offset = feat.index(short, offset)
+              if feat.index(".", offset + 1) && !feat.index("/", offset + 2)
+                break true
+              else
+                offset += 1
+              end
+            end
+          end
         else
           ret = yield
         end
 
         hash = long.hash
 
-        # do we have 'bundler' or 'bundler.rb'?
-        altname = if File.extname(short) != ''
-          # strip the path from 'bundler.rb' -> 'bundler'
-          strip_extension(short)
-        elsif long && (ext = File.extname(long))
-          # get the extension from the expanded path if given
-          # 'bundler' + '.rb'
+        # Do we have a filename with an elidable extension, e.g.,
+        # 'bundler.rb', or 'libgit2.so'?
+        altname = if extension_elidable?(short)
+          # Strip the extension off, e.g. 'bundler.rb' -> 'bundler'.
+          strip_extension_if_elidable(short)
+        elsif long && (ext = File.extname(long.freeze))
+          # We already know the extension of the actual file this
+          # resolves to, so put that back on.
           short + ext
         end
 
@@ -117,8 +131,30 @@ module Bootsnap
       STRIP_EXTENSION = /\.[^.]*?$/
       private_constant(:STRIP_EXTENSION)
 
-      def strip_extension(f)
-        f.sub(STRIP_EXTENSION, '')
+      # Might Ruby automatically search for this extension if
+      # someone tries to 'require' the file without it? E.g. Ruby
+      # will implicitly try 'x.rb' if you ask for 'x'.
+      #
+      # This is complex and platform-dependent, and the Ruby docs are a little
+      # handwavy about what will be tried when and in what order.
+      # So optimistically pretend that all known elidable extensions
+      # will be tried on all platforms, and that people are unlikely
+      # to name files in a way that assumes otherwise.
+      # (E.g. It's unlikely that someone will know that their code
+      # will _never_ run on MacOS, and therefore think they can get away
+      # with calling a Ruby file 'x.dylib.rb' and then requiring it as 'x.dylib'.)
+      #
+      # See <https://ruby-doc.org/core-2.6.4/Kernel.html#method-i-require>.
+      def extension_elidable?(f)
+        f.to_s.end_with?('.rb', '.so', '.o', '.dll', '.dylib')
+      end
+
+      def strip_extension_if_elidable(f)
+        if extension_elidable?(f)
+          f.sub(STRIP_EXTENSION, '')
+        else
+          f
+        end
       end
     end
   end

data/lib/bootsnap/load_path_cache/path.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require_relative('path_scanner')
 
 module Bootsnap
@@ -20,7 +21,7 @@ module Bootsnap
       attr_reader(:path)
 
       def initialize(path)
-        @path = path.to_s
+        @path = path.to_s.freeze
       end
 
       # True if the path exists, but represents a non-directory object
@@ -59,7 +60,7 @@ module Bootsnap
       end
 
       def expanded_path
-        File.expand_path(path)
+        File.expand_path(path).freeze
       end
 
       private