bootsnap 1.8.0 → 1.9.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a12930656b479ba9d0675f13bf1edd2a52e0d84314f1e528866110a1a36776df
-  data.tar.gz: 33f4c8ea77733e461c9fe95ba75110e9da0b707e818727ebf5c13605d58eec62
+  metadata.gz: de52d100052f98bd60ccbc71d3172b7f7951df2fb45fb8dab4a5c4f8e95e4528
+  data.tar.gz: a09dacba13131d47509d41d729748a7d80703779bcbbb0a4c0474f0960cf5025
 SHA512:
-  metadata.gz: c366c8c65f4ce3fd73979b68c1201c2d7872d059a71ac79a561bc464b7fdaf1d0ba00acbe59e491d25d66efe967b3c0a25a9542727a43c101c4d2b719b82407c
-  data.tar.gz: 39d2ec6734c6234114c27f8665efe8a6170e6bcc58f98cec25f1c4386a644d0eec135ababc66b771dc0744c7df961f63d95669ebdb08db14c3b4982b320b5224
+  metadata.gz: 31fe33d4e05c9c85f4950434481131396dcc06485651144783767485da5ff887a6293aeb21d6baee22d3beeeb4c8e0a1ddb869f6f50b148bb241c674ee2b9f6a
+  data.tar.gz: 3e9a66ee5d26a54fa9c36515806c89f0757232b3191768f9769333155f08b5eda43d426c23a7ac144bd84ebd6d352042562bce8c828cbb2e8d624484d5adeb72

data/CHANGELOG.md

@@ -1,5 +1,22 @@
 # Unreleased
 
+# 1.9.2
+
+* Disable compile cache if Ruby 3.0.3's ISeq cache bug is detected.
+* Fix `Kernel.load` behavior: before `load 'a'` would load `a.rb` (and other tried extensions) and wouldn't load `a` unless `development_mode: true`, now only `a` would be loaded and files with extensions wouldn't be.
+
+# 1.9.1
+
+* Removed a forgotten debug statement in JSON precompilation.
+
+# 1.9.0
+
+* Added a compilation cache for `JSON.load_file`. (#370)
+
+# 1.8.1
+
+* Fixed support for older Psych. (#369)
+
 # 1.8.0
 
 * Improve support for Pysch 4. (#368)

data/lib/bootsnap/cli.rb

@@ -21,7 +21,7 @@ module Bootsnap
 
     attr_reader :cache_dir, :argv
 
-    attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :jobs
+    attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :json, :jobs
 
     def initialize(argv)
       @argv = argv
@@ -32,37 +32,44 @@ module Bootsnap
       self.jobs = Etc.nprocessors
       self.iseq = true
       self.yaml = true
+      self.json = true
     end
 
     def precompile_command(*sources)
       require 'bootsnap/compile_cache/iseq'
       require 'bootsnap/compile_cache/yaml'
+      require 'bootsnap/compile_cache/json'
 
       fix_default_encoding do
         Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
         Bootsnap::CompileCache::YAML.init!
         Bootsnap::CompileCache::YAML.cache_dir = self.cache_dir
+        Bootsnap::CompileCache::JSON.init!
+        Bootsnap::CompileCache::JSON.cache_dir = self.cache_dir
 
         @work_pool = WorkerPool.create(size: jobs, jobs: {
           ruby: method(:precompile_ruby),
           yaml: method(:precompile_yaml),
+          json: method(:precompile_json),
         })
         @work_pool.spawn
 
         main_sources = sources.map { |d| File.expand_path(d) }
         precompile_ruby_files(main_sources)
         precompile_yaml_files(main_sources)
+        precompile_json_files(main_sources)
 
         if compile_gemfile
           # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
           gem_exclude = Regexp.union([exclude, '/spec/', '/test/'].compact)
           precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
 
-          # Gems that include YAML files usually don't put them in `lib/`.
+          # Gems that include JSON or YAML files usually don't put them in `lib/`.
           # So we look at the gem root.
           gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems\/[^/]+}
           gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
           precompile_yaml_files(gem_paths, exclude: gem_exclude)
+          precompile_json_files(gem_paths, exclude: gem_exclude)
         end
 
         if exitstatus = @work_pool.shutdown
@@ -137,6 +144,29 @@ module Bootsnap
       end
     end
 
+    def precompile_json_files(load_paths, exclude: self.exclude)
+      return unless json
+
+      load_paths.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_files(path, '**/*.json').each do |json_file|
+            # We ignore hidden files to not match the various .config.json files
+            if !File.basename(json_file).start_with?('.') && (!exclude || !exclude.match?(json_file))
+              @work_pool.push(:json, json_file)
+            end
+          end
+        end
+      end
+    end
+
+    def precompile_json(*json_files)
+      Array(json_files).each do |json_file|
+        if CompileCache::JSON.precompile(json_file, cache_dir: cache_dir)
+          STDERR.puts(json_file) if verbose
+        end
+      end
+    end
+
     def precompile_ruby_files(load_paths, exclude: self.exclude)
       return unless iseq
 
@@ -240,6 +270,11 @@ module Bootsnap
           Disable YAML precompilation.
         EOS
         opts.on('--no-yaml', help) { self.yaml = false }
+
+        help = <<~EOS
+          Disable JSON precompilation.
+        EOS
+        opts.on('--no-json', help) { self.json = false }
       end
     end
   end

data/lib/bootsnap/compile_cache/json.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+require('bootsnap/bootsnap')
+
+module Bootsnap
+  module CompileCache
+    module JSON
+      class << self
+        attr_accessor(:msgpack_factory, :cache_dir, :supported_options)
+
+        def input_to_storage(payload, _)
+          obj = ::JSON.parse(payload)
+          msgpack_factory.dump(obj)
+        end
+
+        def storage_to_output(data, kwargs)
+          if kwargs && kwargs.key?(:symbolize_names)
+            kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
+          end
+          msgpack_factory.load(data, kwargs)
+        end
+
+        def input_to_output(data, kwargs)
+          ::JSON.parse(data, **(kwargs || {}))
+        end
+
+        def precompile(path, cache_dir: self.cache_dir)
+          Bootsnap::CompileCache::Native.precompile(
+            cache_dir,
+            path.to_s,
+            self,
+          )
+        end
+
+        def install!(cache_dir)
+          self.cache_dir = cache_dir
+          init!
+          if ::JSON.respond_to?(:load_file)
+            ::JSON.singleton_class.prepend(Patch)
+          end
+        end
+
+        def init!
+          require('json')
+          require('msgpack')
+
+          self.msgpack_factory = MessagePack::Factory.new
+          self.supported_options = [:symbolize_names]
+          if ::JSON.parse('["foo"]', freeze: true).first.frozen?
+            self.supported_options = [:freeze]
+          end
+          self.supported_options.freeze
+        end
+      end
+
+      module Patch
+        def load_file(path, *args)
+          return super if args.size > 1
+          if kwargs = args.first
+            return super unless kwargs.is_a?(Hash)
+            return super unless (kwargs.keys - ::Bootsnap::CompileCache::JSON.supported_options).empty?
+          end
+
+          begin
+            ::Bootsnap::CompileCache::Native.fetch(
+              Bootsnap::CompileCache::JSON.cache_dir,
+              File.realpath(path),
+              ::Bootsnap::CompileCache::JSON,
+              kwargs,
+            )
+          rescue Errno::EACCES
+            ::Bootsnap::CompileCache.permission_error(path)
+          end
+        end
+
+        ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
+      end
+    end
+  end
+end

data/lib/bootsnap/compile_cache/yaml.rb

@@ -23,7 +23,11 @@ module Bootsnap
         end
 
         def input_to_output(data, kwargs)
-          ::YAML.unsafe_load(data, **(kwargs || {}))
+          if ::YAML.respond_to?(:unsafe_load)
+            ::YAML.unsafe_load(data, **(kwargs || {}))
+          else
+            ::YAML.load(data, **(kwargs || {}))
+          end
         end
 
         def strict_load(payload, *args)

data/lib/bootsnap/compile_cache.rb

@@ -4,7 +4,7 @@ module Bootsnap
     Error           = Class.new(StandardError)
     PermissionError = Class.new(Error)
 
-    def self.setup(cache_dir:, iseq:, yaml:)
+    def self.setup(cache_dir:, iseq:, yaml:, json:)
       if iseq
         if supported?
           require_relative('compile_cache/iseq')
@@ -22,6 +22,15 @@ module Bootsnap
           warn("[bootsnap/setup] YAML parsing caching is not supported on this implementation of Ruby")
         end
       end
+
+      if json
+        if supported?
+          require_relative('compile_cache/json')
+          Bootsnap::CompileCache::JSON.install!(cache_dir)
+        elsif $VERBOSE
+          warn("[bootsnap/setup] JSON parsing caching is not supported on this implementation of Ruby")
+        end
+      end
     end
 
     def self.permission_error(path)

data/lib/bootsnap/load_path_cache/cache.rb

@@ -44,14 +44,20 @@ module Bootsnap
 
       # Try to resolve this feature to an absolute path without traversing the
       # loadpath.
-      def find(feature)
+      def find(feature, try_extensions: true)
         reinitialize if (@has_relative_paths && dir_changed?) || stale?
         feature = feature.to_s.freeze
+
         return feature if absolute_path?(feature)
-        return expand_path(feature) if feature.start_with?('./')
+
+        if feature.start_with?('./', '../')
+          return try_extensions ? expand_path(feature) : File.expand_path(feature).freeze
+        end
+
         @mutex.synchronize do
-          x = search_index(feature)
+          x = search_index(feature, try_extensions: try_extensions)
           return x if x
+          return unless try_extensions
 
           # Ruby has some built-in features that require lies about.
           # For example, 'enumerator' is built in. If you require it, ruby
@@ -177,16 +183,24 @@ module Bootsnap
       end
 
       if DLEXT2
-        def search_index(f)
-          try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f + DLEXT2) || try_index(f)
+        def search_index(f, try_extensions: true)
+          if try_extensions
+            try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f + DLEXT2) || try_index(f)
+          else
+            try_index(f)
+          end
         end
 
         def maybe_append_extension(f)
           try_ext(f + DOT_RB) || try_ext(f + DLEXT) || try_ext(f + DLEXT2) || f
         end
       else
-        def search_index(f)
-          try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f)
+        def search_index(f, try_extensions: true)
+          if try_extensions
+            try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f)
+          else
+            try_index(f)
+          end
         end
 
         def maybe_append_extension(f)

data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb

@@ -56,25 +56,9 @@ module Kernel
 
   alias_method(:load_without_bootsnap, :load)
   def load(path, wrap = false)
-    if (resolved = Bootsnap::LoadPathCache.load_path_cache.find(path))
-      return load_without_bootsnap(resolved, wrap)
-    end
-
-    # load also allows relative paths from pwd even when not in $:
-    if File.exist?(relative = File.expand_path(path).freeze)
-      return load_without_bootsnap(relative, wrap)
-    end
-
-    raise(Bootsnap::LoadPathCache::CoreExt.make_load_error(path))
-  rescue LoadError => e
-    e.instance_variable_set(Bootsnap::LoadPathCache::ERROR_TAG_IVAR, true)
-    raise(e)
-  rescue Bootsnap::LoadPathCache::ReturnFalse
-    false
-  rescue Bootsnap::LoadPathCache::FallbackScan
-    fallback = true
-  ensure
-    if fallback
+    if (resolved = Bootsnap::LoadPathCache.load_path_cache.find(path, try_extensions: false))
+      load_without_bootsnap(resolved, wrap)
+    else
       load_without_bootsnap(path, wrap)
     end
   end

data/lib/bootsnap/load_path_cache/loaded_features_index.rb

@@ -84,10 +84,18 @@ module Bootsnap
       #    entry.
       def register(short, long = nil)
         if long.nil?
-          pat = %r{/#{Regexp.escape(short)}(\.[^/]+)?$}
           len = $LOADED_FEATURES.size
           ret = yield
-          long = $LOADED_FEATURES[len..-1].detect { |feat| feat =~ pat }
+          long = $LOADED_FEATURES[len..-1].detect do |feat|
+            offset = 0
+            while offset = feat.index(short, offset)
+              if feat.index(".", offset + 1) && !feat.index("/", offset + 2)
+                break true
+              else
+                offset += 1
+              end
+            end
+          end
         else
           ret = yield
         end

data/lib/bootsnap/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Bootsnap
-  VERSION = "1.8.0"
+  VERSION = "1.9.2"
 end

data/lib/bootsnap.rb

@@ -43,7 +43,8 @@ module Bootsnap
     autoload_paths_cache: nil,
     disable_trace: nil,
     compile_cache_iseq: true,
-    compile_cache_yaml: true
+    compile_cache_yaml: true,
+    compile_cache_json: true
   )
     unless autoload_paths_cache.nil?
       warn "[DEPRECATED] Bootsnap's `autoload_paths_cache:` option is deprecated and will be removed. " \
@@ -56,9 +57,15 @@ module Bootsnap
         "If you use Ruby 2.5 or newer this option is useless, if not upgrading is recommended."
     end
 
-    if compile_cache_iseq && !iseq_cache_supported?
-      warn "Ruby 2.5 has a bug that break code tracing when code is loaded from cache. It is recommened " \
-        "to turn `compile_cache_iseq` off on Ruby 2.5"
+    if compile_cache_iseq
+      if iseq_cache_tracing_bug?
+        warn "Ruby 2.5 has a bug that break code tracing when code is loaded from cache. It is recommended " \
+          "to turn `compile_cache_iseq` off on Ruby 2.5"
+      end
+
+      if iseq_cache_anonymous_params_bug?
+        warn "Your version of Ruby 3.1.0-dev has a bug that break bytecode caching (https://github.com/ruby/ruby/pull/4961)."
+      end
     end
 
     Bootsnap::LoadPathCache.setup(
@@ -69,15 +76,33 @@ module Bootsnap
     Bootsnap::CompileCache.setup(
       cache_dir: cache_dir + '/bootsnap/compile-cache',
       iseq: compile_cache_iseq,
-      yaml: compile_cache_yaml
+      yaml: compile_cache_yaml,
+      json: compile_cache_json,
     )
   end
 
-  def self.iseq_cache_supported?
-    return @iseq_cache_supported if defined? @iseq_cache_supported
+  def self.iseq_cache_tracing_bug?
+    return @iseq_cache_tracing_bug if defined? @iseq_cache_tracing_bug
 
     ruby_version = Gem::Version.new(RUBY_VERSION)
-    @iseq_cache_supported = ruby_version < Gem::Version.new('2.5.0') || ruby_version >= Gem::Version.new('2.6.0')
+    @iseq_cache_tracing_bug = ruby_version < Gem::Version.new('2.5.0') || ruby_version >= Gem::Version.new('2.6.0')
+  end
+
+  def self.iseq_cache_anonymous_params_bug?
+    return @iseq_cache_anonymous_params_bug if defined? @iseq_cache_anonymous_params_bug
+    begin
+      if defined? RubyVM::InstructionSequence
+        RubyVM::InstructionSequence.compile("def foo(*); ->{ super }; end").to_binary
+        RubyVM::InstructionSequence.compile("def foo(**); ->{ super }; end").to_binary
+      end
+      false
+    rescue TypeError
+      true
+    end
+  end
+
+  def self.iseq_cache_supported?
+    !(iseq_cache_tracing_bug? || iseq_cache_anonymous_params_bug?)
   end
 
   def self.default_setup
@@ -113,6 +138,7 @@ module Bootsnap
         load_path_cache:      !ENV['DISABLE_BOOTSNAP_LOAD_PATH_CACHE'],
         compile_cache_iseq:   !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'] && iseq_cache_supported?,
         compile_cache_yaml:   !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'],
+        compile_cache_json:   !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'],
       )
 
       if ENV['BOOTSNAP_LOG']

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bootsnap
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.9.2
 platform: ruby
 authors:
 - Burke Libbey
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-08-26 00:00:00.000000000 Z
+date: 2021-11-24 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -116,6 +116,7 @@ files:
 - lib/bootsnap/cli/worker_pool.rb
 - lib/bootsnap/compile_cache.rb
 - lib/bootsnap/compile_cache/iseq.rb
+- lib/bootsnap/compile_cache/json.rb
 - lib/bootsnap/compile_cache/yaml.rb
 - lib/bootsnap/explicit_require.rb
 - lib/bootsnap/load_path_cache.rb