bootsnap 1.7.2 → 1.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ea223ff0ae18f28970e2b80aa24970fe24f5eeefc7ad12b5b4b77dc921af4ca3
-  data.tar.gz: 5f9ffdfcbeac27b128aa314749dd27c6182247fe7b6aa6492a1d99d00586ac3e
+  metadata.gz: 00c1b5b2c37c6a351bb7656ef17ba7f7e6143fd773ceebd6ee09d9a55e25b75a
+  data.tar.gz: d6d6d1d68d97e4f2c50c61f0b928714cf94f29ecab6604bb0092517779212738
 SHA512:
-  metadata.gz: 0e3bcea4690954e6e15a3b0c10e3bf5b93b23fefff0d3d61354cb04d683bd8925d61ce32dcc9f7dd526b2f14f4d44a24196e72003fa75714bb0e6f99fbc94f62
-  data.tar.gz: 587454d2a3a5019c56c81068e9202905328e7c2918f1786f35ff988424e2a9222b36db35f7ef6e403fc730a0752508ab248b06d6b164dc0b6e147623e5f0912e
+  metadata.gz: e1ade83cee1e4a917317cecd34107493424a5174b8bf92cc74db47743d4d84931de24533979480a23aa4ddb496cae8ce1f3cf30a2fbed3996b885a68138d16f0
+  data.tar.gz: 23e4f6c77307b30526b880bfd12983b7e66c9539556acd3bc05a2fa3fd6884a142c83dfaa590846651afad243c0bf69c96d994ca8492e5539d706ba3e6cf713f

data/CHANGELOG.md

@@ -1,5 +1,45 @@
 # Unreleased
 
+# 1.9.1
+
+* Removed a forgotten debug statement in JSON precompilation.
+
+# 1.9.0
+
+* Added a compilation cache for `JSON.load_file`. (#370)
+
+# 1.8.1
+
+* Fixed support for older Psych. (#369)
+
+# 1.8.0
+
+* Improve support for Pysch 4. (#368)
+
+# 1.7.7
+
+* Fix `require_relative` in evaled code on latest ruby 3.1.0-dev. (#366)
+
+# 1.7.6
+
+* Fix reliance on `set` to be required.
+* Fix `Encoding::UndefinedConversionError` error for Rails applications when precompiling cache. (#364)
+
+# 1.7.5
+
+* Handle a regression of Ruby 2.7.3 causing Bootsnap to call the deprecated `untaint` method. (#360)
+* Gracefully handle read-only file system as well as other errors preventing to persist the load path cache. (#358)
+
+# 1.7.4
+
+* Stop raising errors when encoutering various file system errors. The cache is now best effort,
+  if somehow it can't be saved, bootsnapp will gracefully fallback to the original operation (e.g. `Kernel.require`).
+  (#353, #177, #262)
+
+# 1.7.3
+
+* Disable YAML precompilation when encountering YAML tags. (#351)
+
 # 1.7.2
 
 * Fix compatibility with msgpack < 1. (#349)

data/ext/bootsnap/bootsnap.c

@@ -464,8 +464,7 @@ open_cache_file(const char * path, struct bs_cache_key * key, const char ** errn
   fd = open(path, O_RDONLY);
   if (fd < 0) {
     *errno_provenance = "bs_fetch:open_cache_file:open";
-    if (errno == ENOENT) return CACHE_MISS;
-    return ERROR_WITH_ERRNO;
+    return CACHE_MISS;
   }
   #ifdef _WIN32
   setmode(fd, O_BINARY);
@@ -763,9 +762,11 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args
   /* If storage_data isn't a string, we can't cache it */
   if (!RB_TYPE_P(storage_data, T_STRING)) goto invalid_type_storage_data;
 
-  /* Write the cache key and storage_data to the cache directory */
-  res = atomic_write_cache_file(cache_path, &current_key, storage_data, &errno_provenance);
-  if (res < 0) goto fail_errno;
+  /* Attempt to write the cache key and storage_data to the cache directory.
+   * We do however ignore any failures to persist the cache, as it's better
+   * to move along, than to interrupt the process.
+   */
+  atomic_write_cache_file(cache_path, &current_key, storage_data, &errno_provenance);
 
   /* Having written the cache, now convert storage_data to output_data */
   exception_tag = bs_storage_to_output(handler, args, storage_data, &output_data);

data/lib/bootsnap/cli/worker_pool.rb

@@ -37,7 +37,11 @@ module Bootsnap
 
         def initialize(jobs)
           @jobs = jobs
-          @pipe_out, @to_io = IO.pipe
+          @pipe_out, @to_io = IO.pipe(binmode: true)
+          # Set the writer encoding to binary since IO.pipe only sets it for the reader.
+          # https://github.com/rails/rails/issues/16514#issuecomment-52313290
+          @to_io.set_encoding(Encoding::BINARY)
+
           @pid = nil
         end
 

data/lib/bootsnap/cli.rb

@@ -21,7 +21,7 @@ module Bootsnap
 
     attr_reader :cache_dir, :argv
 
-    attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :jobs
+    attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :json, :jobs
 
     def initialize(argv)
       @argv = argv
@@ -32,37 +32,44 @@ module Bootsnap
       self.jobs = Etc.nprocessors
       self.iseq = true
       self.yaml = true
+      self.json = true
     end
 
     def precompile_command(*sources)
       require 'bootsnap/compile_cache/iseq'
       require 'bootsnap/compile_cache/yaml'
+      require 'bootsnap/compile_cache/json'
 
       fix_default_encoding do
         Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
         Bootsnap::CompileCache::YAML.init!
         Bootsnap::CompileCache::YAML.cache_dir = self.cache_dir
+        Bootsnap::CompileCache::JSON.init!
+        Bootsnap::CompileCache::JSON.cache_dir = self.cache_dir
 
         @work_pool = WorkerPool.create(size: jobs, jobs: {
           ruby: method(:precompile_ruby),
           yaml: method(:precompile_yaml),
+          json: method(:precompile_json),
         })
         @work_pool.spawn
 
         main_sources = sources.map { |d| File.expand_path(d) }
         precompile_ruby_files(main_sources)
         precompile_yaml_files(main_sources)
+        precompile_json_files(main_sources)
 
         if compile_gemfile
           # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
           gem_exclude = Regexp.union([exclude, '/spec/', '/test/'].compact)
           precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
 
-          # Gems that include YAML files usually don't put them in `lib/`.
+          # Gems that include JSON or YAML files usually don't put them in `lib/`.
           # So we look at the gem root.
           gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems\/[^/]+}
           gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
           precompile_yaml_files(gem_paths, exclude: gem_exclude)
+          precompile_json_files(gem_paths, exclude: gem_exclude)
         end
 
         if exitstatus = @work_pool.shutdown
@@ -137,6 +144,29 @@ module Bootsnap
       end
     end
 
+    def precompile_json_files(load_paths, exclude: self.exclude)
+      return unless json
+
+      load_paths.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_files(path, '**/*.json').each do |json_file|
+            # We ignore hidden files to not match the various .config.json files
+            if !File.basename(json_file).start_with?('.') && (!exclude || !exclude.match?(json_file))
+              @work_pool.push(:json, json_file)
+            end
+          end
+        end
+      end
+    end
+
+    def precompile_json(*json_files)
+      Array(json_files).each do |json_file|
+        if CompileCache::JSON.precompile(json_file, cache_dir: cache_dir)
+          STDERR.puts(json_file) if verbose
+        end
+      end
+    end
+
     def precompile_ruby_files(load_paths, exclude: self.exclude)
       return unless iseq
 
@@ -240,6 +270,11 @@ module Bootsnap
           Disable YAML precompilation.
         EOS
         opts.on('--no-yaml', help) { self.yaml = false }
+
+        help = <<~EOS
+          Disable JSON precompilation.
+        EOS
+        opts.on('--no-json', help) { self.json = false }
       end
     end
   end

data/lib/bootsnap/compile_cache/json.rb

@@ -0,0 +1,79 @@
+# frozen_string_literal: true
+require('bootsnap/bootsnap')
+
+module Bootsnap
+  module CompileCache
+    module JSON
+      class << self
+        attr_accessor(:msgpack_factory, :cache_dir, :supported_options)
+
+        def input_to_storage(payload, _)
+          obj = ::JSON.parse(payload)
+          msgpack_factory.dump(obj)
+        end
+
+        def storage_to_output(data, kwargs)
+          if kwargs && kwargs.key?(:symbolize_names)
+            kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
+          end
+          msgpack_factory.load(data, kwargs)
+        end
+
+        def input_to_output(data, kwargs)
+          ::JSON.parse(data, **(kwargs || {}))
+        end
+
+        def precompile(path, cache_dir: self.cache_dir)
+          Bootsnap::CompileCache::Native.precompile(
+            cache_dir,
+            path.to_s,
+            self,
+          )
+        end
+
+        def install!(cache_dir)
+          self.cache_dir = cache_dir
+          init!
+          if ::JSON.respond_to?(:load_file)
+            ::JSON.singleton_class.prepend(Patch)
+          end
+        end
+
+        def init!
+          require('json')
+          require('msgpack')
+
+          self.msgpack_factory = MessagePack::Factory.new
+          self.supported_options = [:symbolize_names]
+          if ::JSON.parse('["foo"]', freeze: true).first.frozen?
+            self.supported_options = [:freeze]
+          end
+          self.supported_options.freeze
+        end
+      end
+
+      module Patch
+        def load_file(path, *args)
+          return super if args.size > 1
+          if kwargs = args.first
+            return super unless kwargs.is_a?(Hash)
+            return super unless (kwargs.keys - ::Bootsnap::CompileCache::JSON.supported_options).empty?
+          end
+
+          begin
+            ::Bootsnap::CompileCache::Native.fetch(
+              Bootsnap::CompileCache::JSON.cache_dir,
+              File.realpath(path),
+              ::Bootsnap::CompileCache::JSON,
+              kwargs,
+            )
+          rescue Errno::EACCES
+            ::Bootsnap::CompileCache.permission_error(path)
+          end
+        end
+
+        ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
+      end
+    end
+  end
+end

data/lib/bootsnap/compile_cache/yaml.rb

@@ -8,8 +8,7 @@ module Bootsnap
         attr_accessor(:msgpack_factory, :cache_dir, :supported_options)
 
         def input_to_storage(contents, _)
-          raise(Uncompilable) if contents.index("!ruby/object")
-          obj = ::YAML.load(contents)
+          obj = strict_load(contents)
           msgpack_factory.dump(obj)
         rescue NoMethodError, RangeError
           # The object included things that we can't serialize
@@ -24,8 +23,19 @@ module Bootsnap
         end
 
         def input_to_output(data, kwargs)
-          ::YAML.load(data, **(kwargs || {}))
+          if ::YAML.respond_to?(:unsafe_load)
+            ::YAML.unsafe_load(data, **(kwargs || {}))
+          else
+            ::YAML.load(data, **(kwargs || {}))
+          end
+        end
+
+        def strict_load(payload, *args)
+          ast = ::YAML.parse(payload)
+          return ast unless ast
+          strict_visitor.create(*args).visit(ast)
         end
+        ruby2_keywords :strict_load if respond_to?(:ruby2_keywords, true)
 
         def precompile(path, cache_dir: YAML.cache_dir)
           Bootsnap::CompileCache::Native.precompile(
@@ -46,6 +56,13 @@ module Bootsnap
           require('msgpack')
           require('date')
 
+          if Patch.method_defined?(:unsafe_load_file) && !::YAML.respond_to?(:unsafe_load_file)
+            Patch.send(:remove_method, :unsafe_load_file)
+          end
+          if Patch.method_defined?(:load_file) && ::YAML::VERSION >= '4'
+            Patch.send(:remove_method, :load_file)
+          end
+
           # MessagePack serializes symbols as strings by default.
           # We want them to roundtrip cleanly, so we use a custom factory.
           # see: https://github.com/msgpack/msgpack-ruby/pull/122
@@ -86,6 +103,17 @@ module Bootsnap
           end
           self.supported_options.freeze
         end
+
+        def strict_visitor
+          self::NoTagsVisitor ||= Class.new(Psych::Visitors::ToRuby) do
+            def visit(target)
+              if target.tag
+                raise Uncompilable, "YAML tags are not supported: #{target.tag}"
+              end
+              super
+            end
+          end
+        end
       end
 
       module Patch
@@ -109,6 +137,27 @@ module Bootsnap
         end
 
         ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
+
+        def unsafe_load_file(path, *args)
+          return super if args.size > 1
+          if kwargs = args.first
+            return super unless kwargs.is_a?(Hash)
+            return super unless (kwargs.keys - ::Bootsnap::CompileCache::YAML.supported_options).empty?
+          end
+
+          begin
+            ::Bootsnap::CompileCache::Native.fetch(
+              Bootsnap::CompileCache::YAML.cache_dir,
+              File.realpath(path),
+              ::Bootsnap::CompileCache::YAML,
+              kwargs,
+            )
+          rescue Errno::EACCES
+            ::Bootsnap::CompileCache.permission_error(path)
+          end
+        end
+
+        ruby2_keywords :unsafe_load_file if respond_to?(:ruby2_keywords, true)
       end
     end
   end

data/lib/bootsnap/compile_cache.rb

@@ -4,7 +4,7 @@ module Bootsnap
     Error           = Class.new(StandardError)
     PermissionError = Class.new(Error)
 
-    def self.setup(cache_dir:, iseq:, yaml:)
+    def self.setup(cache_dir:, iseq:, yaml:, json:)
       if iseq
         if supported?
           require_relative('compile_cache/iseq')
@@ -22,6 +22,15 @@ module Bootsnap
           warn("[bootsnap/setup] YAML parsing caching is not supported on this implementation of Ruby")
         end
       end
+
+      if json
+        if supported?
+          require_relative('compile_cache/json')
+          Bootsnap::CompileCache::JSON.install!(cache_dir)
+        elsif $VERBOSE
+          warn("[bootsnap/setup] JSON parsing caching is not supported on this implementation of Ruby")
+        end
+      end
     end
 
     def self.permission_error(path)

data/lib/bootsnap/load_path_cache/cache.rb

@@ -196,7 +196,7 @@ module Bootsnap
 
       s = rand.to_s.force_encoding(Encoding::US_ASCII).freeze
       if s.respond_to?(:-@)
-        if (-s).equal?(s) && (-s.dup).equal?(s)
+        if (-s).equal?(s) && (-s.dup).equal?(s) || RUBY_VERSION >= '2.7'
           def try_index(f)
             if (p = @index[f])
               -(File.join(p, f).freeze)

data/lib/bootsnap/load_path_cache/change_observer.rb

@@ -15,11 +15,13 @@ module Bootsnap
           @lpc_observer.push_paths(self, *entries.map(&:to_s))
           super
         end
+        alias_method :append, :push
 
         def unshift(*entries)
           @lpc_observer.unshift_paths(self, *entries.map(&:to_s))
           super
         end
+        alias_method :prepend, :unshift
 
         def concat(entries)
           @lpc_observer.push_paths(self, *entries.map(&:to_s))

data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb

@@ -47,8 +47,9 @@ module Kernel
 
   alias_method(:require_relative_without_bootsnap, :require_relative)
   def require_relative(path)
+    location = caller_locations(1..1).first
     realpath = Bootsnap::LoadPathCache.realpath_cache.call(
-      caller_locations(1..1).first.absolute_path, path
+      location.absolute_path || location.path, path
     )
     require(realpath)
   end

data/lib/bootsnap/load_path_cache/loaded_features_index.rb

@@ -58,9 +58,9 @@ module Bootsnap
       end
 
       def purge_multi(features)
-        rejected_hashes = features.map(&:hash).to_set
+        rejected_hashes = features.each_with_object({}) { |f, h| h[f.hash] = true }
         @mutex.synchronize do
-          @lfi.reject! { |_, hash| rejected_hashes.include?(hash) }
+          @lfi.reject! { |_, hash| rejected_hashes.key?(hash) }
         end
       end
 

data/lib/bootsnap/load_path_cache/store.rb

@@ -91,6 +91,7 @@ module Bootsnap
         FileUtils.mv(tmp, @store_path)
       rescue Errno::EEXIST
         retry
+      rescue SystemCallError
       end
     end
   end

data/lib/bootsnap/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Bootsnap
-  VERSION = "1.7.2"
+  VERSION = "1.9.1"
 end

data/lib/bootsnap.rb

@@ -27,7 +27,9 @@ module Bootsnap
 
   def self.instrumentation=(callback)
     @instrumentation = callback
-    self.instrumentation_enabled = !!callback
+    if respond_to?(:instrumentation_enabled=, true)
+      self.instrumentation_enabled = !!callback
+    end
   end
 
   def self._instrument(event, path)
@@ -41,7 +43,8 @@ module Bootsnap
     autoload_paths_cache: nil,
     disable_trace: nil,
     compile_cache_iseq: true,
-    compile_cache_yaml: true
+    compile_cache_yaml: true,
+    compile_cache_json: true
   )
     unless autoload_paths_cache.nil?
       warn "[DEPRECATED] Bootsnap's `autoload_paths_cache:` option is deprecated and will be removed. " \
@@ -67,7 +70,8 @@ module Bootsnap
     Bootsnap::CompileCache.setup(
       cache_dir: cache_dir + '/bootsnap/compile-cache',
       iseq: compile_cache_iseq,
-      yaml: compile_cache_yaml
+      yaml: compile_cache_yaml,
+      json: compile_cache_json,
     )
   end
 
@@ -111,6 +115,7 @@ module Bootsnap
         load_path_cache:      !ENV['DISABLE_BOOTSNAP_LOAD_PATH_CACHE'],
         compile_cache_iseq:   !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'] && iseq_cache_supported?,
         compile_cache_yaml:   !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'],
+        compile_cache_json:   !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'],
       )
 
       if ENV['BOOTSNAP_LOG']

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bootsnap
 version: !ruby/object:Gem::Version
-  version: 1.7.2
+  version: 1.9.1
 platform: ruby
 authors:
 - Burke Libbey
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-08 00:00:00.000000000 Z
+date: 2021-09-20 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -42,14 +42,14 @@ dependencies:
   name: rake-compiler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
@@ -116,6 +116,7 @@ files:
 - lib/bootsnap/cli/worker_pool.rb
 - lib/bootsnap/compile_cache.rb
 - lib/bootsnap/compile_cache/iseq.rb
+- lib/bootsnap/compile_cache/json.rb
 - lib/bootsnap/compile_cache/yaml.rb
 - lib/bootsnap/explicit_require.rb
 - lib/bootsnap/load_path_cache.rb
@@ -153,7 +154,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.0.3
+rubygems_version: 3.2.20
 signing_key: 
 specification_version: 4
 summary: Boot large ruby/rails apps faster