bootsnap 1.8.1 → 1.10.3

data/lib/bootsnap/compile_cache/iseq.rb

@@ -1,24 +1,54 @@
 # frozen_string_literal: true
-require('bootsnap/bootsnap')
-require('zlib')
+
+require("bootsnap/bootsnap")
+require("zlib")
 
 module Bootsnap
   module CompileCache
     module ISeq
       class << self
-        attr_accessor(:cache_dir)
+        attr_reader(:cache_dir)
+
+        def cache_dir=(cache_dir)
+          @cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}iseq" : "#{cache_dir}-iseq"
+        end
+      end
+
+      has_ruby_bug_18250 = begin # https://bugs.ruby-lang.org/issues/18250
+        if defined? RubyVM::InstructionSequence
+          RubyVM::InstructionSequence.compile("def foo(*); ->{ super }; end; def foo(**); ->{ super }; end").to_binary
+        end
+        false
+      rescue TypeError
+        true
       end
 
-      def self.input_to_storage(_, path)
-        RubyVM::InstructionSequence.compile_file(path).to_binary
-      rescue SyntaxError
-        raise(Uncompilable, 'syntax error')
+      if has_ruby_bug_18250
+        def self.input_to_storage(_, path)
+          iseq = begin
+            RubyVM::InstructionSequence.compile_file(path)
+          rescue SyntaxError
+            return UNCOMPILABLE # syntax error
+          end
+
+          begin
+            iseq.to_binary
+          rescue TypeError
+            return UNCOMPILABLE # ruby bug #18250
+          end
+        end
+      else
+        def self.input_to_storage(_, path)
+          RubyVM::InstructionSequence.compile_file(path).to_binary
+        rescue SyntaxError
+          return UNCOMPILABLE # syntax error
+        end
       end
 
       def self.storage_to_output(binary, _args)
         RubyVM::InstructionSequence.load_from_binary(binary)
-      rescue RuntimeError => e
-        if e.message == 'broken binary format'
+      rescue RuntimeError => error
+        if error.message == "broken binary format"
           STDERR.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
           nil
         else
@@ -35,7 +65,7 @@ module Bootsnap
         )
       end
 
-      def self.precompile(path, cache_dir: ISeq.cache_dir)
+      def self.precompile(path)
         Bootsnap::CompileCache::Native.precompile(
           cache_dir,
           path.to_s,
@@ -55,8 +85,8 @@ module Bootsnap
           Bootsnap::CompileCache::ISeq.fetch(path.to_s)
         rescue Errno::EACCES
           Bootsnap::CompileCache.permission_error(path)
-        rescue RuntimeError => e
-          if e.message =~ /unmatched platform/
+        rescue RuntimeError => error
+          if error.message =~ /unmatched platform/
             puts("unmatched platform for file #{path}")
           end
           raise

data/lib/bootsnap/compile_cache/json.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require("bootsnap/bootsnap")
+
+module Bootsnap
+  module CompileCache
+    module JSON
+      class << self
+        attr_accessor(:msgpack_factory, :supported_options)
+        attr_reader(:cache_dir)
+
+        def cache_dir=(cache_dir)
+          @cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}json" : "#{cache_dir}-json"
+        end
+
+        def input_to_storage(payload, _)
+          obj = ::JSON.parse(payload)
+          msgpack_factory.dump(obj)
+        end
+
+        def storage_to_output(data, kwargs)
+          if kwargs&.key?(:symbolize_names)
+            kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
+          end
+          msgpack_factory.load(data, kwargs)
+        end
+
+        def input_to_output(data, kwargs)
+          ::JSON.parse(data, **(kwargs || {}))
+        end
+
+        def precompile(path)
+          Bootsnap::CompileCache::Native.precompile(
+            cache_dir,
+            path.to_s,
+            self,
+          )
+        end
+
+        def install!(cache_dir)
+          self.cache_dir = cache_dir
+          init!
+          if ::JSON.respond_to?(:load_file)
+            ::JSON.singleton_class.prepend(Patch)
+          end
+        end
+
+        def init!
+          require("json")
+          require("msgpack")
+
+          self.msgpack_factory = MessagePack::Factory.new
+          self.supported_options = [:symbolize_names]
+          if ::JSON.parse('["foo"]', freeze: true).first.frozen?
+            if MessagePack.load(MessagePack.dump("foo"), freeze: true).frozen?
+              self.supported_options = [:freeze]
+            end
+          end
+          supported_options.freeze
+        end
+      end
+
+      module Patch
+        def load_file(path, *args)
+          return super if args.size > 1
+
+          if (kwargs = args.first)
+            return super unless kwargs.is_a?(Hash)
+            return super unless (kwargs.keys - ::Bootsnap::CompileCache::JSON.supported_options).empty?
+          end
+
+          begin
+            ::Bootsnap::CompileCache::Native.fetch(
+              Bootsnap::CompileCache::JSON.cache_dir,
+              File.realpath(path),
+              ::Bootsnap::CompileCache::JSON,
+              kwargs,
+            )
+          rescue Errno::EACCES
+            ::Bootsnap::CompileCache.permission_error(path)
+          end
+        end
+
+        ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
+      end
+    end
+  end
+end

data/lib/bootsnap/compile_cache/yaml.rb

@@ -1,80 +1,86 @@
 # frozen_string_literal: true
-require('bootsnap/bootsnap')
+
+require("bootsnap/bootsnap")
 
 module Bootsnap
   module CompileCache
     module YAML
-      class << self
-        attr_accessor(:msgpack_factory, :cache_dir, :supported_options)
+      Uncompilable = Class.new(StandardError)
+      UnsupportedTags = Class.new(Uncompilable)
 
-        def input_to_storage(contents, _)
-          obj = strict_load(contents)
-          msgpack_factory.dump(obj)
-        rescue NoMethodError, RangeError
-          # The object included things that we can't serialize
-          raise(Uncompilable)
-        end
+      SUPPORTED_INTERNAL_ENCODINGS = [
+        nil, # UTF-8
+        Encoding::UTF_8,
+        Encoding::ASCII,
+        Encoding::BINARY,
+      ].freeze
 
-        def storage_to_output(data, kwargs)
-          if kwargs && kwargs.key?(:symbolize_names)
-            kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
-          end
-          msgpack_factory.load(data, kwargs)
-        end
+      class << self
+        attr_accessor(:msgpack_factory, :supported_options)
+        attr_reader(:implementation, :cache_dir)
 
-        def input_to_output(data, kwargs)
-          if ::YAML.respond_to?(:unsafe_load)
-            ::YAML.unsafe_load(data, **(kwargs || {}))
-          else
-            ::YAML.load(data, **(kwargs || {}))
-          end
+        def cache_dir=(cache_dir)
+          @cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}yaml" : "#{cache_dir}-yaml"
         end
 
-        def strict_load(payload, *args)
-          ast = ::YAML.parse(payload)
-          return ast unless ast
-          strict_visitor.create(*args).visit(ast)
-        end
-        ruby2_keywords :strict_load if respond_to?(:ruby2_keywords, true)
+        def precompile(path)
+          return false unless CompileCache::YAML.supported_internal_encoding?
 
-        def precompile(path, cache_dir: YAML.cache_dir)
-          Bootsnap::CompileCache::Native.precompile(
+          CompileCache::Native.precompile(
             cache_dir,
             path.to_s,
-            Bootsnap::CompileCache::YAML,
+            @implementation,
           )
         end
 
         def install!(cache_dir)
           self.cache_dir = cache_dir
           init!
-          ::YAML.singleton_class.prepend(Patch)
+          ::YAML.singleton_class.prepend(@implementation::Patch)
         end
 
-        def init!
-          require('yaml')
-          require('msgpack')
-          require('date')
+        # Psych coerce strings to `Encoding.default_internal` but Message Pack only support
+        # UTF-8, US-ASCII and BINARY. So if Encoding.default_internal is set to anything else
+        # we can't safely use the cache
+        def supported_internal_encoding?
+          SUPPORTED_INTERNAL_ENCODINGS.include?(Encoding.default_internal)
+        end
+
+        module EncodingAwareSymbols
+          extend self
 
-          if Patch.method_defined?(:unsafe_load_file) && !::YAML.respond_to?(:unsafe_load_file)
-            Patch.send(:remove_method, :unsafe_load_file)
+          def unpack(payload)
+            (+payload).force_encoding(Encoding::UTF_8).to_sym
           end
-          if Patch.method_defined?(:load_file) && ::YAML::VERSION >= '4'
-            Patch.send(:remove_method, :load_file)
+        end
+
+        def init!
+          require("yaml")
+          require("msgpack")
+          require("date")
+
+          @implementation = ::YAML::VERSION >= "4" ? Psych4 : Psych3
+          if @implementation::Patch.method_defined?(:unsafe_load_file) && !::YAML.respond_to?(:unsafe_load_file)
+            @implementation::Patch.send(:remove_method, :unsafe_load_file)
           end
 
           # MessagePack serializes symbols as strings by default.
           # We want them to roundtrip cleanly, so we use a custom factory.
           # see: https://github.com/msgpack/msgpack-ruby/pull/122
           factory = MessagePack::Factory.new
-          factory.register_type(0x00, Symbol)
+          factory.register_type(
+            0x00,
+            Symbol,
+            packer: :to_msgpack_ext,
+            unpacker: EncodingAwareSymbols.method(:unpack).to_proc,
+          )
 
           if defined? MessagePack::Timestamp
             factory.register_type(
               MessagePack::Timestamp::TYPE, # or just -1
               Time,
               packer: MessagePack::Time::Packer,
-              unpacker: MessagePack::Time::Unpacker
+              unpacker: MessagePack::Time::Unpacker,
             )
 
             marshal_fallback = {
@@ -94,70 +100,251 @@ module Bootsnap
           self.supported_options = []
           params = ::YAML.method(:load).parameters
           if params.include?([:key, :symbolize_names])
-            self.supported_options << :symbolize_names
+            supported_options << :symbolize_names
           end
           if params.include?([:key, :freeze])
-            if factory.load(factory.dump('yaml'), freeze: true).frozen?
-              self.supported_options << :freeze
+            if factory.load(factory.dump("yaml"), freeze: true).frozen?
+              supported_options << :freeze
             end
           end
-          self.supported_options.freeze
+          supported_options.freeze
+        end
+
+        def patch
+          @implementation::Patch
+        end
+
+        def strict_load(payload)
+          ast = ::YAML.parse(payload)
+          return ast unless ast
+
+          strict_visitor.create.visit(ast)
         end
 
         def strict_visitor
           self::NoTagsVisitor ||= Class.new(Psych::Visitors::ToRuby) do
             def visit(target)
               if target.tag
-                raise Uncompilable, "YAML tags are not supported: #{target.tag}"
+                raise UnsupportedTags, "YAML tags are not supported: #{target.tag}"
               end
+
               super
             end
           end
         end
       end
 
-      module Patch
-        def load_file(path, *args)
-          return super if args.size > 1
-          if kwargs = args.first
-            return super unless kwargs.is_a?(Hash)
-            return super unless (kwargs.keys - ::Bootsnap::CompileCache::YAML.supported_options).empty?
+      module Psych4
+        extend self
+
+        def input_to_storage(contents, _)
+          obj = SafeLoad.input_to_storage(contents, nil)
+          if UNCOMPILABLE.equal?(obj)
+            obj = UnsafeLoad.input_to_storage(contents, nil)
           end
+          obj
+        end
 
-          begin
-            ::Bootsnap::CompileCache::Native.fetch(
-              Bootsnap::CompileCache::YAML.cache_dir,
-              File.realpath(path),
-              ::Bootsnap::CompileCache::YAML,
-              kwargs,
-            )
-          rescue Errno::EACCES
-            ::Bootsnap::CompileCache.permission_error(path)
+        module UnsafeLoad
+          extend self
+
+          def input_to_storage(contents, _)
+            obj = ::YAML.unsafe_load(contents)
+            packer = CompileCache::YAML.msgpack_factory.packer
+            packer.pack(false) # not safe loaded
+            begin
+              packer.pack(obj)
+            rescue NoMethodError, RangeError
+              return UNCOMPILABLE # The object included things that we can't serialize
+            end
+            packer.to_s
+          end
+
+          def storage_to_output(data, kwargs)
+            if kwargs&.key?(:symbolize_names)
+              kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
+            end
+
+            unpacker = CompileCache::YAML.msgpack_factory.unpacker(kwargs)
+            unpacker.feed(data)
+            _safe_loaded = unpacker.unpack
+            unpacker.unpack
+          end
+
+          def input_to_output(data, kwargs)
+            ::YAML.unsafe_load(data, **(kwargs || {}))
           end
         end
 
-        ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
+        module SafeLoad
+          extend self
 
-        def unsafe_load_file(path, *args)
-          return super if args.size > 1
-          if kwargs = args.first
-            return super unless kwargs.is_a?(Hash)
-            return super unless (kwargs.keys - ::Bootsnap::CompileCache::YAML.supported_options).empty?
+          def input_to_storage(contents, _)
+            obj = begin
+              CompileCache::YAML.strict_load(contents)
+            rescue Psych::DisallowedClass, Psych::BadAlias, Uncompilable
+              return UNCOMPILABLE
+            end
+
+            packer = CompileCache::YAML.msgpack_factory.packer
+            packer.pack(true) # safe loaded
+            begin
+              packer.pack(obj)
+            rescue NoMethodError, RangeError
+              return UNCOMPILABLE
+            end
+            packer.to_s
           end
 
+          def storage_to_output(data, kwargs)
+            if kwargs&.key?(:symbolize_names)
+              kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
+            end
+
+            unpacker = CompileCache::YAML.msgpack_factory.unpacker(kwargs)
+            unpacker.feed(data)
+            safe_loaded = unpacker.unpack
+            if safe_loaded
+              unpacker.unpack
+            else
+              UNCOMPILABLE
+            end
+          end
+
+          def input_to_output(data, kwargs)
+            ::YAML.load(data, **(kwargs || {}))
+          end
+        end
+
+        module Patch
+          def load_file(path, *args)
+            return super unless CompileCache::YAML.supported_internal_encoding?
+
+            return super if args.size > 1
+
+            if (kwargs = args.first)
+              return super unless kwargs.is_a?(Hash)
+              return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
+            end
+
+            begin
+              CompileCache::Native.fetch(
+                CompileCache::YAML.cache_dir,
+                File.realpath(path),
+                CompileCache::YAML::Psych4::SafeLoad,
+                kwargs,
+              )
+            rescue Errno::EACCES
+              CompileCache.permission_error(path)
+            end
+          end
+
+          ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
+
+          def unsafe_load_file(path, *args)
+            return super unless CompileCache::YAML.supported_internal_encoding?
+
+            return super if args.size > 1
+
+            if (kwargs = args.first)
+              return super unless kwargs.is_a?(Hash)
+              return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
+            end
+
+            begin
+              CompileCache::Native.fetch(
+                CompileCache::YAML.cache_dir,
+                File.realpath(path),
+                CompileCache::YAML::Psych4::UnsafeLoad,
+                kwargs,
+              )
+            rescue Errno::EACCES
+              CompileCache.permission_error(path)
+            end
+          end
+
+          ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true)
+        end
+      end
+
+      module Psych3
+        extend self
+
+        def input_to_storage(contents, _)
+          obj = ::YAML.load(contents)
+          packer = CompileCache::YAML.msgpack_factory.packer
+          packer.pack(false) # not safe loaded
           begin
-            ::Bootsnap::CompileCache::Native.fetch(
-              Bootsnap::CompileCache::YAML.cache_dir,
-              File.realpath(path),
-              ::Bootsnap::CompileCache::YAML,
-              kwargs,
-            )
-          rescue Errno::EACCES
-            ::Bootsnap::CompileCache.permission_error(path)
+            packer.pack(obj)
+          rescue NoMethodError, RangeError
+            return UNCOMPILABLE # The object included things that we can't serialize
           end
+          packer.to_s
         end
 
-        ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true)
+        def storage_to_output(data, kwargs)
+          if kwargs&.key?(:symbolize_names)
+            kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
+          end
+          unpacker = CompileCache::YAML.msgpack_factory.unpacker(kwargs)
+          unpacker.feed(data)
+          _safe_loaded = unpacker.unpack
+          unpacker.unpack
+        end
+
+        def input_to_output(data, kwargs)
+          ::YAML.load(data, **(kwargs || {}))
+        end
+
+        module Patch
+          def load_file(path, *args)
+            return super unless CompileCache::YAML.supported_internal_encoding?
+
+            return super if args.size > 1
+
+            if (kwargs = args.first)
+              return super unless kwargs.is_a?(Hash)
+              return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
+            end
+
+            begin
+              CompileCache::Native.fetch(
+                CompileCache::YAML.cache_dir,
+                File.realpath(path),
+                CompileCache::YAML::Psych3,
+                kwargs,
+              )
+            rescue Errno::EACCES
+              CompileCache.permission_error(path)
+            end
+          end
+
+          ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
+
+          def unsafe_load_file(path, *args)
+            return super unless CompileCache::YAML.supported_internal_encoding?
+
+            return super if args.size > 1
+
+            if (kwargs = args.first)
+              return super unless kwargs.is_a?(Hash)
+              return super unless (kwargs.keys - CompileCache::YAML.supported_options).empty?
+            end
+
+            begin
+              CompileCache::Native.fetch(
+                CompileCache::YAML.cache_dir,
+                File.realpath(path),
+                CompileCache::YAML::Psych3,
+                kwargs,
+              )
+            rescue Errno::EACCES
+              CompileCache.permission_error(path)
+            end
+          end
+
+          ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true)
+        end
       end
     end
   end

data/lib/bootsnap/compile_cache.rb

@@ -1,13 +1,19 @@
 # frozen_string_literal: true
+
 module Bootsnap
   module CompileCache
-    Error           = Class.new(StandardError)
+    UNCOMPILABLE = BasicObject.new
+    def UNCOMPILABLE.inspect
+      "<Bootsnap::UNCOMPILABLE>"
+    end
+
+    Error = Class.new(StandardError)
     PermissionError = Class.new(Error)
 
-    def self.setup(cache_dir:, iseq:, yaml:)
+    def self.setup(cache_dir:, iseq:, yaml:, json:)
       if iseq
         if supported?
-          require_relative('compile_cache/iseq')
+          require_relative("compile_cache/iseq")
           Bootsnap::CompileCache::ISeq.install!(cache_dir)
         elsif $VERBOSE
           warn("[bootsnap/setup] bytecode caching is not supported on this implementation of Ruby")
@@ -16,12 +22,21 @@ module Bootsnap
 
       if yaml
         if supported?
-          require_relative('compile_cache/yaml')
+          require_relative("compile_cache/yaml")
           Bootsnap::CompileCache::YAML.install!(cache_dir)
         elsif $VERBOSE
           warn("[bootsnap/setup] YAML parsing caching is not supported on this implementation of Ruby")
         end
       end
+
+      if json
+        if supported?
+          require_relative("compile_cache/json")
+          Bootsnap::CompileCache::JSON.install!(cache_dir)
+        elsif $VERBOSE
+          warn("[bootsnap/setup] JSON parsing caching is not supported on this implementation of Ruby")
+        end
+      end
     end
 
     def self.permission_error(path)
@@ -35,9 +50,9 @@ module Bootsnap
 
     def self.supported?
       # only enable on 'ruby' (MRI), POSIX (darwin, linux, *bsd), Windows (RubyInstaller2) and >= 2.3.0
-      RUBY_ENGINE == 'ruby' &&
-      RUBY_PLATFORM =~ /darwin|linux|bsd|mswin|mingw|cygwin/ &&
-      Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.3.0")
+      RUBY_ENGINE == "ruby" &&
+        RUBY_PLATFORM =~ /darwin|linux|bsd|mswin|mingw|cygwin/ &&
+        Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.3.0")
     end
   end
 end

data/lib/bootsnap/explicit_require.rb

@@ -1,9 +1,10 @@
 # frozen_string_literal: true
+
 module Bootsnap
   module ExplicitRequire
-    ARCHDIR    = RbConfig::CONFIG['archdir']
-    RUBYLIBDIR = RbConfig::CONFIG['rubylibdir']
-    DLEXT      = RbConfig::CONFIG['DLEXT']
+    ARCHDIR    = RbConfig::CONFIG["archdir"]
+    RUBYLIBDIR = RbConfig::CONFIG["rubylibdir"]
+    DLEXT      = RbConfig::CONFIG["DLEXT"]
 
     def self.from_self(feature)
       require_relative("../#{feature}")