bootsnap 1.4.8 → 1.16.0

data/lib/bootsnap/compile_cache/yaml.rb

@@ -1,61 +1,351 @@
 # frozen_string_literal: true
-require('bootsnap/bootsnap')
+
+require("bootsnap/bootsnap")
 
 module Bootsnap
   module CompileCache
     module YAML
+      Uncompilable = Class.new(StandardError)
+      UnsupportedTags = Class.new(Uncompilable)
+
+      SUPPORTED_INTERNAL_ENCODINGS = [
+        nil, # UTF-8
+        Encoding::UTF_8,
+        Encoding::ASCII,
+        Encoding::BINARY,
+      ].freeze
+
       class << self
-        attr_accessor(:msgpack_factory)
-      end
+        attr_accessor(:msgpack_factory, :supported_options)
+        attr_reader(:implementation, :cache_dir)
 
-      def self.input_to_storage(contents, _)
-        raise(Uncompilable) if contents.index("!ruby/object")
-        obj = ::YAML.load(contents)
-        msgpack_factory.packer.write(obj).to_s
-      rescue NoMethodError, RangeError
-        # if the object included things that we can't serialize, fall back to
-        # Marshal. It's a bit slower, but can encode anything yaml can.
-        # NoMethodError is unexpected types; RangeError is Bignums
-        Marshal.dump(obj)
-      end
+        def cache_dir=(cache_dir)
+          @cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}yaml" : "#{cache_dir}-yaml"
+        end
 
-      def self.storage_to_output(data)
-        # This could have a meaning in messagepack, and we're being a little lazy
-        # about it. -- but a leading 0x04 would indicate the contents of the YAML
-        # is a positive integer, which is rare, to say the least.
-        if data[0] == 0x04.chr && data[1] == 0x08.chr
-          Marshal.load(data)
-        else
-          msgpack_factory.unpacker.feed(data).read
+        def precompile(path)
+          return false unless CompileCache::YAML.supported_internal_encoding?
+
+          CompileCache::Native.precompile(
+            cache_dir,
+            path.to_s,
+            @implementation,
+          )
         end
-      end
 
-      def self.input_to_output(data)
-        ::YAML.load(data)
+        def install!(cache_dir)
+          self.cache_dir = cache_dir
+          init!
+          ::YAML.singleton_class.prepend(@implementation::Patch)
+        end
+
+        # Psych coerce strings to `Encoding.default_internal` but Message Pack only support
+        # UTF-8, US-ASCII and BINARY. So if Encoding.default_internal is set to anything else
+        # we can't safely use the cache
+        def supported_internal_encoding?
+          SUPPORTED_INTERNAL_ENCODINGS.include?(Encoding.default_internal)
+        end
+
+        module EncodingAwareSymbols
+          extend self
+
+          def unpack(payload)
+            (+payload).force_encoding(Encoding::UTF_8).to_sym
+          end
+        end
+
+        def init!
+          require("yaml")
+          require("msgpack")
+          require("date")
+
+          @implementation = ::YAML::VERSION >= "4" ? Psych4 : Psych3
+          if @implementation::Patch.method_defined?(:unsafe_load_file) && !::YAML.respond_to?(:unsafe_load_file)
+            @implementation::Patch.send(:remove_method, :unsafe_load_file)
+          end
+
+          unless const_defined?(:NoTagsVisitor)
+            visitor = Class.new(Psych::Visitors::NoAliasRuby) do
+              def visit(target)
+                if target.tag
+                  raise UnsupportedTags, "YAML tags are not supported: #{target.tag}"
+                end
+
+                super
+              end
+            end
+            const_set(:NoTagsVisitor, visitor)
+          end
+
+          # MessagePack serializes symbols as strings by default.
+          # We want them to roundtrip cleanly, so we use a custom factory.
+          # see: https://github.com/msgpack/msgpack-ruby/pull/122
+          factory = MessagePack::Factory.new
+          factory.register_type(
+            0x00,
+            Symbol,
+            packer: :to_msgpack_ext,
+            unpacker: EncodingAwareSymbols.method(:unpack).to_proc,
+          )
+
+          if defined? MessagePack::Timestamp
+            factory.register_type(
+              MessagePack::Timestamp::TYPE, # or just -1
+              Time,
+              packer: MessagePack::Time::Packer,
+              unpacker: MessagePack::Time::Unpacker,
+            )
+
+            marshal_fallback = {
+              packer: ->(value) { Marshal.dump(value) },
+              unpacker: ->(payload) { Marshal.load(payload) },
+            }
+            {
+              Date => 0x01,
+              Regexp => 0x02,
+            }.each do |type, code|
+              factory.register_type(code, type, marshal_fallback)
+            end
+          end
+
+          self.msgpack_factory = factory
+
+          self.supported_options = []
+          params = ::YAML.method(:load).parameters
+          if params.include?([:key, :symbolize_names])
+            supported_options << :symbolize_names
+          end
+          if params.include?([:key, :freeze]) && factory.load(factory.dump("yaml"), freeze: true).frozen?
+            supported_options << :freeze
+          end
+          supported_options.freeze
+        end
+
+        def patch
+          @implementation::Patch
+        end
+
+        def strict_load(payload)
+          ast = ::YAML.parse(payload)
+          return ast unless ast
+
+          loader = ::Psych::ClassLoader::Restricted.new(["Symbol"], [])
+          scanner = ::Psych::ScalarScanner.new(loader)
+
+          NoTagsVisitor.new(scanner, loader).visit(ast)
+        end
       end
 
-      def self.install!(cache_dir)
-        require('yaml')
-        require('msgpack')
+      module Psych4
+        extend self
+
+        def input_to_storage(contents, _)
+          obj = SafeLoad.input_to_storage(contents, nil)
+          if UNCOMPILABLE.equal?(obj)
+            obj = UnsafeLoad.input_to_storage(contents, nil)
+          end
+          obj
+        end
+
+        module UnsafeLoad
+          extend self
+
+          def input_to_storage(contents, _)
+            obj = ::YAML.unsafe_load(contents)
+            packer = CompileCache::YAML.msgpack_factory.packer
+            packer.pack(false) # not safe loaded
+            begin
+              packer.pack(obj)
+            rescue NoMethodError, RangeError
+              return UNCOMPILABLE # The object included things that we can't serialize
+            end
+            packer.to_s
+          end
+
+          def storage_to_output(data, kwargs)
+            if kwargs&.key?(:symbolize_names)
+              kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
+            end
+
+            unpacker = CompileCache::YAML.msgpack_factory.unpacker(kwargs)
+            unpacker.feed(data)
+            _safe_loaded = unpacker.unpack
+            unpacker.unpack
+          end
+
+          def input_to_output(data, kwargs)
+            ::YAML.unsafe_load(data, **(kwargs || {}))
+          end
+        end
+
+        module SafeLoad
+          extend self
+
+          def input_to_storage(contents, _)
+            obj = begin
+              CompileCache::YAML.strict_load(contents)
+            rescue Psych::DisallowedClass, Psych::BadAlias, Uncompilable
+              return UNCOMPILABLE
+            end
+
+            packer = CompileCache::YAML.msgpack_factory.packer
+            packer.pack(true) # safe loaded
+            begin
+              packer.pack(obj)
+            rescue NoMethodError, RangeError
+              return UNCOMPILABLE
+            end
+            packer.to_s
+          end
+
+          def storage_to_output(data, kwargs)
+            if kwargs&.key?(:symbolize_names)
+              kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
+            end
+
+            unpacker = CompileCache::YAML.msgpack_factory.unpacker(kwargs)
+            unpacker.feed(data)
+            safe_loaded = unpacker.unpack
+            if safe_loaded
+              unpacker.unpack
+            else
+              UNCOMPILABLE
+            end
+          end
+
+          def input_to_output(data, kwargs)
+            ::YAML.load(data, **(kwargs || {}))
+          end
+        end
+
+        module Patch
+          def load_file(path, *args)
+            return super unless CompileCache::YAML.supported_internal_encoding?
+
+            return super if args.size > 1
+
+            if (kwargs = args.first)
+              return super unless kwargs.is_a?(Hash)
+              return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
+            end
+
+            begin
+              CompileCache::Native.fetch(
+                CompileCache::YAML.cache_dir,
+                File.realpath(path),
+                CompileCache::YAML::Psych4::SafeLoad,
+                kwargs,
+              )
+            rescue Errno::EACCES
+              CompileCache.permission_error(path)
+            end
+          end
+
+          ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
+
+          def unsafe_load_file(path, *args)
+            return super unless CompileCache::YAML.supported_internal_encoding?
+
+            return super if args.size > 1
+
+            if (kwargs = args.first)
+              return super unless kwargs.is_a?(Hash)
+              return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
+            end
+
+            begin
+              CompileCache::Native.fetch(
+                CompileCache::YAML.cache_dir,
+                File.realpath(path),
+                CompileCache::YAML::Psych4::UnsafeLoad,
+                kwargs,
+              )
+            rescue Errno::EACCES
+              CompileCache.permission_error(path)
+            end
+          end
+
+          ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true)
+        end
+      end
 
-        # MessagePack serializes symbols as strings by default.
-        # We want them to roundtrip cleanly, so we use a custom factory.
-        # see: https://github.com/msgpack/msgpack-ruby/pull/122
-        factory = MessagePack::Factory.new
-        factory.register_type(0x00, Symbol)
-        Bootsnap::CompileCache::YAML.msgpack_factory = factory
+      module Psych3
+        extend self
 
-        klass = class << ::YAML; self; end
-        klass.send(:define_method, :load_file) do |path|
+        def input_to_storage(contents, _)
+          obj = ::YAML.load(contents)
+          packer = CompileCache::YAML.msgpack_factory.packer
+          packer.pack(false) # not safe loaded
           begin
-            Bootsnap::CompileCache::Native.fetch(
-              cache_dir,
-              path,
-              Bootsnap::CompileCache::YAML
-            )
-          rescue Errno::EACCES
-            Bootsnap::CompileCache.permission_error(path)
+            packer.pack(obj)
+          rescue NoMethodError, RangeError
+            return UNCOMPILABLE # The object included things that we can't serialize
+          end
+          packer.to_s
+        end
+
+        def storage_to_output(data, kwargs)
+          if kwargs&.key?(:symbolize_names)
+            kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
           end
+          unpacker = CompileCache::YAML.msgpack_factory.unpacker(kwargs)
+          unpacker.feed(data)
+          _safe_loaded = unpacker.unpack
+          unpacker.unpack
+        end
+
+        def input_to_output(data, kwargs)
+          ::YAML.load(data, **(kwargs || {}))
+        end
+
+        module Patch
+          def load_file(path, *args)
+            return super unless CompileCache::YAML.supported_internal_encoding?
+
+            return super if args.size > 1
+
+            if (kwargs = args.first)
+              return super unless kwargs.is_a?(Hash)
+              return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
+            end
+
+            begin
+              CompileCache::Native.fetch(
+                CompileCache::YAML.cache_dir,
+                File.realpath(path),
+                CompileCache::YAML::Psych3,
+                kwargs,
+              )
+            rescue Errno::EACCES
+              CompileCache.permission_error(path)
+            end
+          end
+
+          ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
+
+          def unsafe_load_file(path, *args)
+            return super unless CompileCache::YAML.supported_internal_encoding?
+
+            return super if args.size > 1
+
+            if (kwargs = args.first)
+              return super unless kwargs.is_a?(Hash)
+              return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
+            end
+
+            begin
+              CompileCache::Native.fetch(
+                CompileCache::YAML.cache_dir,
+                File.realpath(path),
+                CompileCache::YAML::Psych3,
+                kwargs,
+              )
+            rescue Errno::EACCES
+              CompileCache.permission_error(path)
+            end
+          end
+
+          ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true)
         end
       end
     end

data/lib/bootsnap/compile_cache.rb

@@ -1,13 +1,19 @@
 # frozen_string_literal: true
+
 module Bootsnap
   module CompileCache
-    Error           = Class.new(StandardError)
+    UNCOMPILABLE = BasicObject.new
+    def UNCOMPILABLE.inspect
+      "<Bootsnap::UNCOMPILABLE>"
+    end
+
+    Error = Class.new(StandardError)
     PermissionError = Class.new(Error)
 
-    def self.setup(cache_dir:, iseq:, yaml:)
+    def self.setup(cache_dir:, iseq:, yaml:, json:, readonly: false)
       if iseq
         if supported?
-          require_relative('compile_cache/iseq')
+          require_relative("compile_cache/iseq")
           Bootsnap::CompileCache::ISeq.install!(cache_dir)
         elsif $VERBOSE
           warn("[bootsnap/setup] bytecode caching is not supported on this implementation of Ruby")
@@ -16,12 +22,25 @@ module Bootsnap
 
       if yaml
         if supported?
-          require_relative('compile_cache/yaml')
+          require_relative("compile_cache/yaml")
           Bootsnap::CompileCache::YAML.install!(cache_dir)
         elsif $VERBOSE
           warn("[bootsnap/setup] YAML parsing caching is not supported on this implementation of Ruby")
         end
       end
+
+      if json
+        if supported?
+          require_relative("compile_cache/json")
+          Bootsnap::CompileCache::JSON.install!(cache_dir)
+        elsif $VERBOSE
+          warn("[bootsnap/setup] JSON parsing caching is not supported on this implementation of Ruby")
+        end
+      end
+
+      if supported? && defined?(Bootsnap::CompileCache::Native)
+        Bootsnap::CompileCache::Native.readonly = readonly
+      end
     end
 
     def self.permission_error(path)
@@ -34,10 +53,8 @@ module Bootsnap
     end
 
     def self.supported?
-      # only enable on 'ruby' (MRI), POSIX (darwin, linux, *bsd), and >= 2.3.0
-      RUBY_ENGINE == 'ruby' &&
-      RUBY_PLATFORM =~ /darwin|linux|bsd/ &&
-      Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.3.0")
+      # only enable on 'ruby' (MRI), POSIX (darwin, linux, *bsd), Windows (RubyInstaller2) and >= 2.3.0
+      RUBY_ENGINE == "ruby" && RUBY_PLATFORM.match?(/darwin|linux|bsd|mswin|mingw|cygwin/)
     end
   end
 end

data/lib/bootsnap/explicit_require.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
+
 module Bootsnap
   module ExplicitRequire
-    ARCHDIR    = RbConfig::CONFIG['archdir']
-    RUBYLIBDIR = RbConfig::CONFIG['rubylibdir']
-    DLEXT      = RbConfig::CONFIG['DLEXT']
+    ARCHDIR    = RbConfig::CONFIG["archdir"]
+    RUBYLIBDIR = RbConfig::CONFIG["rubylibdir"]
+    DLEXT      = RbConfig::CONFIG["DLEXT"]
 
     def self.from_self(feature)
       require_relative("../#{feature}")

data/lib/bootsnap/load_path_cache/cache.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative('../explicit_require')
+require_relative("../explicit_require")
 
 module Bootsnap
   module LoadPathCache
@@ -10,8 +10,8 @@ module Bootsnap
       def initialize(store, path_obj, development_mode: false)
         @development_mode = development_mode
         @store = store
-        @mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new # TODO: Remove once Ruby 2.2 support is dropped.
-        @path_obj = path_obj.map! { |f| File.exist?(f) ? File.realpath(f) : f }
+        @mutex = Mutex.new
+        @path_obj = path_obj.map! { |f| PathScanner.os_path(File.exist?(f) ? File.realpath(f) : f.dup) }
         @has_relative_paths = nil
         reinitialize
       end
@@ -28,15 +28,16 @@ module Bootsnap
       BUILTIN_FEATURES = $LOADED_FEATURES.each_with_object({}) do |feat, features|
         # Builtin features are of the form 'enumerator.so'.
         # All others include paths.
-        next unless feat.size < 20 && !feat.include?('/')
+        next unless feat.size < 20 && !feat.include?("/")
 
-        base = File.basename(feat, '.*') # enumerator.so -> enumerator
+        base = File.basename(feat, ".*") # enumerator.so -> enumerator
         ext  = File.extname(feat) # .so
 
         features[feat] = nil # enumerator.so
         features[base] = nil # enumerator
 
         next unless [DOT_SO, *DL_EXTENSIONS].include?(ext)
+
         DL_EXTENSIONS.each do |dl_ext|
           features["#{base}#{dl_ext}"] = nil # enumerator.bundle
         end
@@ -47,8 +48,13 @@ module Bootsnap
       def find(feature)
         reinitialize if (@has_relative_paths && dir_changed?) || stale?
         feature = feature.to_s.freeze
-        return feature if absolute_path?(feature)
-        return expand_path(feature) if feature.start_with?('./')
+
+        return feature if Bootsnap.absolute_path?(feature)
+
+        if feature.start_with?("./", "../")
+          return expand_path(feature)
+        end
+
         @mutex.synchronize do
           x = search_index(feature)
           return x if x
@@ -58,7 +64,7 @@ module Bootsnap
           # returns false as if it were already loaded; however, there is no
           # file to find on disk. We've pre-built a list of these, and we
           # return false if any of them is loaded.
-          raise(LoadPathCache::ReturnFalse, '', []) if BUILTIN_FEATURES.key?(feature)
+          return false if BUILTIN_FEATURES.key?(feature)
 
           # The feature wasn't found on our preliminary search through the index.
           # We resolve this differently depending on what the extension was.
@@ -67,13 +73,14 @@ module Bootsnap
           # native dynamic extension, e.g. .bundle or .so), we know it was a
           # failure and there's nothing more we can do to find the file.
           # no extension, .rb, (.bundle or .so)
-          when '', *CACHED_EXTENSIONS
+          when "", *CACHED_EXTENSIONS
             nil
           # Ruby allows specifying native extensions as '.so' even when DLEXT
           # is '.bundle'. This is where we handle that case.
           when DOT_SO
             x = search_index(feature[0..-4] + DLEXT)
             return x if x
+
             if DLEXT2
               x = search_index(feature[0..-4] + DLEXT2)
               return x if x
@@ -81,7 +88,7 @@ module Bootsnap
           else
             # other, unknown extension. For example, `.rake`. Since we haven't
             # cached these, we legitimately need to run the load path search.
-            raise(LoadPathCache::FallbackScan, '', [])
+            return FALLBACK_SCAN
           end
         end
 
@@ -89,33 +96,25 @@ module Bootsnap
         # cases where the file doesn't appear to be on the load path. We should
         # be able to detect newly-created files without rebooting the
         # application.
-        raise(LoadPathCache::FallbackScan, '', []) if @development_mode
-      end
-
-      if RbConfig::CONFIG['host_os'] =~ /mswin|mingw|cygwin/
-        def absolute_path?(path)
-          path[1] == ':'
-        end
-      else
-        def absolute_path?(path)
-          path.start_with?(SLASH)
-        end
+        return FALLBACK_SCAN if @development_mode
       end
 
       def unshift_paths(sender, *paths)
         return unless sender == @path_obj
+
         @mutex.synchronize { unshift_paths_locked(*paths) }
       end
 
       def push_paths(sender, *paths)
         return unless sender == @path_obj
+
         @mutex.synchronize { push_paths_locked(*paths) }
       end
 
       def reinitialize(path_obj = @path_obj)
         @mutex.synchronize do
           @path_obj = path_obj
-          ChangeObserver.register(self, @path_obj)
+          ChangeObserver.register(@path_obj, self)
           @index = {}
           @dirs = {}
           @generated_at = now
@@ -141,6 +140,9 @@ module Bootsnap
             p = Path.new(path)
             @has_relative_paths = true if p.relative?
             next if p.non_directory?
+
+            p = p.to_realpath
+
             expanded_path = p.expanded_path
             entries, dirs = p.entries_and_dirs(@store)
             # push -> low precedence -> set only if unset
@@ -155,6 +157,9 @@ module Bootsnap
           paths.map(&:to_s).reverse_each do |path|
             p = Path.new(path)
             next if p.non_directory?
+
+            p = p.to_realpath
+
             expanded_path = p.expanded_path
             entries, dirs = p.entries_and_dirs(@store)
             # unshift -> high precedence -> unconditional set
@@ -177,31 +182,54 @@ module Bootsnap
       end
 
       if DLEXT2
-        def search_index(f)
-          try_index("#{f}#{DOT_RB}") || try_index("#{f}#{DLEXT}") || try_index("#{f}#{DLEXT2}") || try_index(f)
+        def search_index(feature)
+          try_index(feature + DOT_RB) ||
+            try_index(feature + DLEXT) ||
+            try_index(feature + DLEXT2) ||
+            try_index(feature)
         end
 
-        def maybe_append_extension(f)
-          try_ext("#{f}#{DOT_RB}") || try_ext("#{f}#{DLEXT}") || try_ext("#{f}#{DLEXT2}") || f
+        def maybe_append_extension(feature)
+          try_ext(feature + DOT_RB) ||
+            try_ext(feature + DLEXT) ||
+            try_ext(feature + DLEXT2) ||
+            feature
         end
       else
-        def search_index(f)
-          try_index("#{f}#{DOT_RB}") || try_index("#{f}#{DLEXT}") || try_index(f)
+        def search_index(feature)
+          try_index(feature + DOT_RB) || try_index(feature + DLEXT) || try_index(feature)
         end
 
-        def maybe_append_extension(f)
-          try_ext("#{f}#{DOT_RB}") || try_ext("#{f}#{DLEXT}") || f
+        def maybe_append_extension(feature)
+          try_ext(feature + DOT_RB) || try_ext(feature + DLEXT) || feature
         end
       end
 
-      def try_index(f)
-        if (p = @index[f])
-          "#{p}/#{f}"
+      s = rand.to_s.force_encoding(Encoding::US_ASCII).freeze
+      if s.respond_to?(:-@)
+        if ((-s).equal?(s) && (-s.dup).equal?(s)) || RUBY_VERSION >= "2.7"
+          def try_index(feature)
+            if (path = @index[feature])
+              -File.join(path, feature).freeze
+            end
+          end
+        else
+          def try_index(feature)
+            if (path = @index[feature])
+              -File.join(path, feature).untaint
+            end
+          end
+        end
+      else
+        def try_index(feature)
+          if (path = @index[feature])
+            File.join(path, feature)
+          end
         end
       end
 
-      def try_ext(f)
-        f if File.exist?(f)
+      def try_ext(feature)
+        feature if File.exist?(feature)
       end
     end
   end