bootsnap 1.7.0 → 1.7.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ac5be912b9602b8b20f12255208a6a6d0373d64167092e3267b2494376f079f0
-  data.tar.gz: 5f18b74f0c0670b86d40d4f41df87869ce5f145cbc3d71b64eddcee59b63fe12
+  metadata.gz: d5f10210480d5f0c9574fe970766d9ef4799e9a8a212f6ccafab6a61c2189481
+  data.tar.gz: 3502a4a6f2ab2614f470d1aaa4d648ec86ab0aebe9390074a71fc1b7544a9d1c
 SHA512:
-  metadata.gz: e7abd18f5a495b63083d5b4ae9b105acc7f80c2813fa2c40594d2d34998037e6f433b91bf9ab24e021ca3d727368e5580972fc48b9382a86d30a45e03c8274f6
-  data.tar.gz: 87aaef6229c936321f8940a28f453446a75fd2b6935e13872c1aa7455990c579c1f3f1ebea3ec6416e9f1b3af18c0468d2a1cae2bc4fafb44f4a2c972e4f4688
+  metadata.gz: 53975108838cdfe27d2b8c9e7b0ef1a25100e85ad691063d2dfd35e6db600bf5c76591aeaf51117651fab7c96779c6877a86984bfa97eb7152afecbd42d77828
+  data.tar.gz: 294d0cf74421f374dfde4bc551cdcb3a4f6dd4a971c0d50f3f03eb59c73ae75554689077af0cdc85aa7765aa8944e8e3edd919380fcfea9742bf7391d54756a8

data/CHANGELOG.md

@@ -1,5 +1,25 @@
 # Unreleased
 
+# 1.7.4
+
+* Stop raising errors when encoutering various file system errors. The cache is now best effort,
+  if somehow it can't be saved, bootsnapp will gracefully fallback to the original operation (e.g. `Kernel.require`).
+  (#353, #177, #262)
+
+# 1.7.3
+
+* Disable YAML precompilation when encountering YAML tags. (#351)
+
+# 1.7.2
+
+* Fix compatibility with msgpack < 1. (#349)
+
+# 1.7.1
+
+* Warn Ruby 2.5 users if they turn ISeq caching on. (#327, #244)
+* Disable ISeq caching for the whole 2.5.x series again.
+* Better handle hashing of Ruby strings. (#318)
+
 # 1.7.0
 
 * Fix detection of YAML files in gems.

data/ext/bootsnap/bootsnap.c

@@ -105,8 +105,7 @@ static VALUE bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handl
 static VALUE bs_rb_precompile(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler);
 
 /* Helpers */
-static uint64_t fnv1a_64(const char *str);
-static void bs_cache_path(const char * cachedir, const char * path, char (* cache_path)[MAX_CACHEPATH_SIZE]);
+static void bs_cache_path(const char * cachedir, const VALUE path, char (* cache_path)[MAX_CACHEPATH_SIZE]);
 static int bs_read_key(int fd, struct bs_cache_key * key);
 static int cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2);
 static VALUE bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args);
@@ -211,7 +210,7 @@ bs_compile_option_crc32_set(VALUE self, VALUE crc32_v)
  *   - 32 bits doesn't feel collision-resistant enough; 64 is nice.
  */
 static uint64_t
-fnv1a_64_iter(uint64_t h, const char *str)
+fnv1a_64_iter_cstr(uint64_t h, const char *str)
 {
   unsigned char *s = (unsigned char *)str;
 
@@ -224,7 +223,21 @@ fnv1a_64_iter(uint64_t h, const char *str)
 }
 
 static uint64_t
-fnv1a_64(const char *str)
+fnv1a_64_iter(uint64_t h, const VALUE str)
+{
+  unsigned char *s = (unsigned char *)RSTRING_PTR(str);
+  unsigned char *str_end = (unsigned char *)RSTRING_PTR(str) + RSTRING_LEN(str);
+
+  while (s < str_end) {
+    h ^= (uint64_t)*s++;
+    h += (h << 1) + (h << 4) + (h << 5) + (h << 7) + (h << 8) + (h << 40);
+  }
+
+  return h;
+}
+
+static uint64_t
+fnv1a_64(const VALUE str)
 {
   uint64_t h = (uint64_t)0xcbf29ce484222325ULL;
   return fnv1a_64_iter(h, str);
@@ -245,7 +258,7 @@ get_ruby_revision(void)
   } else {
     uint64_t hash;
 
-    hash = fnv1a_64(StringValueCStr(ruby_revision));
+    hash = fnv1a_64(ruby_revision);
     return (uint32_t)(hash >> 32);
   }
 }
@@ -265,19 +278,19 @@ get_ruby_platform(void)
   VALUE ruby_platform;
 
   ruby_platform = rb_const_get(rb_cObject, rb_intern("RUBY_PLATFORM"));
-  hash = fnv1a_64(RSTRING_PTR(ruby_platform));
+  hash = fnv1a_64(ruby_platform);
 
 #ifdef _WIN32
   return (uint32_t)(hash >> 32) ^ (uint32_t)GetVersion();
 #elif defined(__GLIBC__)
-  hash = fnv1a_64_iter(hash, gnu_get_libc_version());
+  hash = fnv1a_64_iter_cstr(hash, gnu_get_libc_version());
   return (uint32_t)(hash >> 32);
 #else
   struct utsname utsname;
 
   /* Not worth crashing if this fails; lose extra cache invalidation potential */
   if (uname(&utsname) >= 0) {
-    hash = fnv1a_64_iter(hash, utsname.version);
+    hash = fnv1a_64_iter_cstr(hash, utsname.version);
   }
 
   return (uint32_t)(hash >> 32);
@@ -292,13 +305,13 @@ get_ruby_platform(void)
  * The path will look something like: <cachedir>/12/34567890abcdef
  */
 static void
-bs_cache_path(const char * cachedir, const char * path, char (* cache_path)[MAX_CACHEPATH_SIZE])
+bs_cache_path(const char * cachedir, const VALUE path, char (* cache_path)[MAX_CACHEPATH_SIZE])
 {
   uint64_t hash = fnv1a_64(path);
   uint8_t first_byte = (hash >> (64 - 8));
   uint64_t remainder = hash & 0x00ffffffffffffff;
 
-  sprintf(*cache_path, "%s/%02x/%014llx", cachedir, first_byte, remainder);
+  sprintf(*cache_path, "%s/%02"PRIx8"/%014"PRIx64, cachedir, first_byte, remainder);
 }
 
 /*
@@ -344,7 +357,7 @@ bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler, VALUE arg
   char cache_path[MAX_CACHEPATH_SIZE];
 
   /* generate cache path to cache_path */
-  bs_cache_path(cachedir, path, &cache_path);
+  bs_cache_path(cachedir, path_v, &cache_path);
 
   return bs_fetch(path, path_v, cache_path, handler, args);
 }
@@ -371,7 +384,7 @@ bs_rb_precompile(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
   char cache_path[MAX_CACHEPATH_SIZE];
 
   /* generate cache path to cache_path */
-  bs_cache_path(cachedir, path, &cache_path);
+  bs_cache_path(cachedir, path_v, &cache_path);
 
   return bs_precompile(path, path_v, cache_path, handler);
 }
@@ -451,8 +464,7 @@ open_cache_file(const char * path, struct bs_cache_key * key, const char ** errn
   fd = open(path, O_RDONLY);
   if (fd < 0) {
     *errno_provenance = "bs_fetch:open_cache_file:open";
-    if (errno == ENOENT) return CACHE_MISS;
-    return ERROR_WITH_ERRNO;
+    return CACHE_MISS;
   }
   #ifdef _WIN32
   setmode(fd, O_BINARY);
@@ -750,9 +762,11 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args
   /* If storage_data isn't a string, we can't cache it */
   if (!RB_TYPE_P(storage_data, T_STRING)) goto invalid_type_storage_data;
 
-  /* Write the cache key and storage_data to the cache directory */
-  res = atomic_write_cache_file(cache_path, &current_key, storage_data, &errno_provenance);
-  if (res < 0) goto fail_errno;
+  /* Attempt to write the cache key and storage_data to the cache directory.
+   * We do however ignore any failures to persist the cache, as it's better
+   * to move along, than to interrupt the process.
+   */
+  atomic_write_cache_file(cache_path, &current_key, storage_data, &errno_provenance);
 
   /* Having written the cache, now convert storage_data to output_data */
   exception_tag = bs_storage_to_output(handler, args, storage_data, &output_data);

data/ext/bootsnap/extconf.rb

@@ -1,19 +1,24 @@
 # frozen_string_literal: true
 require("mkmf")
-$CFLAGS << ' -O3 '
-$CFLAGS << ' -std=c99'
 
-# ruby.h has some -Wpedantic fails in some cases
-# (e.g. https://github.com/Shopify/bootsnap/issues/15)
-unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC'])
-  $CFLAGS << ' -Wall'
-  $CFLAGS << ' -Werror'
-  $CFLAGS << ' -Wextra'
-  $CFLAGS << ' -Wpedantic'
+if RUBY_ENGINE == 'ruby'
+  $CFLAGS << ' -O3 '
+  $CFLAGS << ' -std=c99'
 
-  $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
-  $CFLAGS << ' -Wno-keyword-macro' # hiding return
-  $CFLAGS << ' -Wno-gcc-compat' # ruby.h 2.6.0 on macos 10.14, dunno
-end
+  # ruby.h has some -Wpedantic fails in some cases
+  # (e.g. https://github.com/Shopify/bootsnap/issues/15)
+  unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC'])
+    $CFLAGS << ' -Wall'
+    $CFLAGS << ' -Werror'
+    $CFLAGS << ' -Wextra'
+    $CFLAGS << ' -Wpedantic'
+
+    $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
+    $CFLAGS << ' -Wno-keyword-macro' # hiding return
+    $CFLAGS << ' -Wno-gcc-compat' # ruby.h 2.6.0 on macos 10.14, dunno
+  end
 
-create_makefile("bootsnap/bootsnap")
+  create_makefile("bootsnap/bootsnap")
+else
+  File.write("Makefile", dummy_makefile($srcdir).join(""))
+end

data/lib/bootsnap.rb

@@ -54,6 +54,11 @@ module Bootsnap
         "If you use Ruby 2.5 or newer this option is useless, if not upgrading is recommended."
     end
 
+    if compile_cache_iseq && !iseq_cache_supported?
+      warn "Ruby 2.5 has a bug that break code tracing when code is loaded from cache. It is recommened " \
+        "to turn `compile_cache_iseq` off on Ruby 2.5"
+    end
+
     Bootsnap::LoadPathCache.setup(
       cache_path:       cache_dir + '/bootsnap/load-path-cache',
       development_mode: development_mode,
@@ -66,6 +71,13 @@ module Bootsnap
     )
   end
 
+  def self.iseq_cache_supported?
+    return @iseq_cache_supported if defined? @iseq_cache_supported
+
+    ruby_version = Gem::Version.new(RUBY_VERSION)
+    @iseq_cache_supported = ruby_version < Gem::Version.new('2.5.0') || ruby_version >= Gem::Version.new('2.6.0')
+  end
+
   def self.default_setup
     env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || ENV['ENV']
     development_mode = ['', nil, 'development'].include?(env)
@@ -92,14 +104,12 @@ module Bootsnap
         cache_dir = File.join(app_root, 'tmp', 'cache')
       end
 
-      ruby_version = Gem::Version.new(RUBY_VERSION)
-      iseq_cache_enabled = ruby_version < Gem::Version.new('2.5.0') || ruby_version >= Gem::Version.new('2.5.4')
 
       setup(
         cache_dir:            cache_dir,
         development_mode:     development_mode,
         load_path_cache:      !ENV['DISABLE_BOOTSNAP_LOAD_PATH_CACHE'],
-        compile_cache_iseq:   !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'] && iseq_cache_enabled,
+        compile_cache_iseq:   !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'] && iseq_cache_supported?,
         compile_cache_yaml:   !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'],
       )
 

data/lib/bootsnap/compile_cache/yaml.rb

@@ -8,8 +8,7 @@ module Bootsnap
         attr_accessor(:msgpack_factory, :cache_dir, :supported_options)
 
         def input_to_storage(contents, _)
-          raise(Uncompilable) if contents.index("!ruby/object")
-          obj = ::YAML.load(contents)
+          obj = strict_load(contents)
           msgpack_factory.dump(obj)
         rescue NoMethodError, RangeError
           # The object included things that we can't serialize
@@ -27,6 +26,13 @@ module Bootsnap
           ::YAML.load(data, **(kwargs || {}))
         end
 
+        def strict_load(payload, *args)
+          ast = ::YAML.parse(payload)
+          return ast unless ast
+          strict_visitor.create(*args).visit(ast)
+        end
+        ruby2_keywords :strict_load if respond_to?(:ruby2_keywords, true)
+
         def precompile(path, cache_dir: YAML.cache_dir)
           Bootsnap::CompileCache::Native.precompile(
             cache_dir,
@@ -51,22 +57,25 @@ module Bootsnap
           # see: https://github.com/msgpack/msgpack-ruby/pull/122
           factory = MessagePack::Factory.new
           factory.register_type(0x00, Symbol)
-          factory.register_type(
-            MessagePack::Timestamp::TYPE, # or just -1
-            Time,
-            packer: MessagePack::Time::Packer,
-            unpacker: MessagePack::Time::Unpacker
-          )
 
-          marshal_fallback = {
-            packer: ->(value) { Marshal.dump(value) },
-            unpacker: ->(payload) { Marshal.load(payload) },
-          }
-          {
-            Date => 0x01,
-            Regexp => 0x02,
-          }.each do |type, code|
-            factory.register_type(code, type, marshal_fallback)
+          if defined? MessagePack::Timestamp
+            factory.register_type(
+              MessagePack::Timestamp::TYPE, # or just -1
+              Time,
+              packer: MessagePack::Time::Packer,
+              unpacker: MessagePack::Time::Unpacker
+            )
+
+            marshal_fallback = {
+              packer: ->(value) { Marshal.dump(value) },
+              unpacker: ->(payload) { Marshal.load(payload) },
+            }
+            {
+              Date => 0x01,
+              Regexp => 0x02,
+            }.each do |type, code|
+              factory.register_type(code, type, marshal_fallback)
+            end
           end
 
           self.msgpack_factory = factory
@@ -83,6 +92,17 @@ module Bootsnap
           end
           self.supported_options.freeze
         end
+
+        def strict_visitor
+          self::NoTagsVisitor ||= Class.new(Psych::Visitors::ToRuby) do
+            def visit(target)
+              if target.tag
+                raise Uncompilable, "YAML tags are not supported: #{target.tag}"
+              end
+              super
+            end
+          end
+        end
       end
 
       module Patch

data/lib/bootsnap/load_path_cache/cache.rb

@@ -10,8 +10,8 @@ module Bootsnap
       def initialize(store, path_obj, development_mode: false)
         @development_mode = development_mode
         @store = store
-        @mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new # TODO: Remove once Ruby 2.2 support is dropped.
-        @path_obj = path_obj.map! { |f| File.exist?(f) ? File.realpath(f) : f }
+        @mutex = Mutex.new
+        @path_obj = path_obj.map! { |f| PathScanner.os_path(File.exist?(f) ? File.realpath(f) : f.dup) }
         @has_relative_paths = nil
         reinitialize
       end
@@ -178,25 +178,42 @@ module Bootsnap
 
       if DLEXT2
         def search_index(f)
-          try_index("#{f}#{DOT_RB}") || try_index("#{f}#{DLEXT}") || try_index("#{f}#{DLEXT2}") || try_index(f)
+          try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f + DLEXT2) || try_index(f)
         end
 
         def maybe_append_extension(f)
-          try_ext("#{f}#{DOT_RB}") || try_ext("#{f}#{DLEXT}") || try_ext("#{f}#{DLEXT2}") || f
+          try_ext(f + DOT_RB) || try_ext(f + DLEXT) || try_ext(f + DLEXT2) || f
         end
       else
         def search_index(f)
-          try_index("#{f}#{DOT_RB}") || try_index("#{f}#{DLEXT}") || try_index(f)
+          try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f)
         end
 
         def maybe_append_extension(f)
-          try_ext("#{f}#{DOT_RB}") || try_ext("#{f}#{DLEXT}") || f
+          try_ext(f + DOT_RB) || try_ext(f + DLEXT) || f
         end
       end
 
-      def try_index(f)
-        if (p = @index[f])
-          "#{p}/#{f}"
+      s = rand.to_s.force_encoding(Encoding::US_ASCII).freeze
+      if s.respond_to?(:-@)
+        if (-s).equal?(s) && (-s.dup).equal?(s)
+          def try_index(f)
+            if (p = @index[f])
+              -(File.join(p, f).freeze)
+            end
+          end
+        else
+          def try_index(f)
+            if (p = @index[f])
+              -File.join(p, f).untaint
+            end
+          end
+        end
+      else
+        def try_index(f)
+          if (p = @index[f])
+            File.join(p, f)
+          end
         end
       end
 

data/lib/bootsnap/load_path_cache/loaded_features_index.rb

@@ -26,7 +26,7 @@ module Bootsnap
     class LoadedFeaturesIndex
       def initialize
         @lfi = {}
-        @mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new # TODO: Remove once Ruby 2.2 support is dropped.
+        @mutex = Mutex.new
 
         # In theory the user could mutate $LOADED_FEATURES and invalidate our
         # cache. If this ever comes up in practice — or if you, the

data/lib/bootsnap/load_path_cache/path_scanner.rb

@@ -33,10 +33,10 @@ module Bootsnap
           requirables = []
           walk(path, nil) do |relative_path, absolute_path, is_directory|
             if is_directory
-              dirs << relative_path
+              dirs << os_path(relative_path)
               !contains_bundle_path || !absolute_path.start_with?(BUNDLE_PATH)
             elsif relative_path.end_with?(*REQUIRABLE_EXTENSIONS)
-              requirables << relative_path
+              requirables << os_path(relative_path)
             end
           end
           [requirables, dirs]
@@ -45,7 +45,7 @@ module Bootsnap
         def walk(absolute_dir_path, relative_dir_path, &block)
           Dir.foreach(absolute_dir_path) do |name|
             next if name.start_with?('.')
-            relative_path = relative_dir_path ? "#{relative_dir_path}/#{name}" : name.freeze
+            relative_path = relative_dir_path ? File.join(relative_dir_path, name) : name
 
             absolute_path = "#{absolute_dir_path}/#{name}"
             if File.directory?(absolute_path)
@@ -57,6 +57,17 @@ module Bootsnap
             end
           end
         end
+
+        if RUBY_VERSION >= '3.1'
+          def os_path(path)
+            path.freeze
+          end
+        else
+          def os_path(path)
+            path.force_encoding(Encoding::US_ASCII) if path.ascii_only?
+            path.freeze
+          end
+        end
       end
     end
   end

data/lib/bootsnap/load_path_cache/store.rb

@@ -12,8 +12,7 @@ module Bootsnap
 
       def initialize(store_path)
         @store_path = store_path
-        # TODO: Remove conditional once Ruby 2.2 support is dropped.
-        @txn_mutex =  defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new
+        @txn_mutex = Mutex.new
         @dirty = false
         load_data
       end
@@ -63,12 +62,18 @@ module Bootsnap
 
       def load_data
         @data = begin
-          MessagePack.load(File.binread(@store_path))
-                # handle malformed data due to upgrade incompatibility
-                rescue Errno::ENOENT, MessagePack::MalformedFormatError, MessagePack::UnknownExtTypeError, EOFError
-                  {}
-                rescue ArgumentError => e
-                  e.message =~ /negative array size/ ? {} : raise
+          File.open(@store_path, encoding: Encoding::BINARY) do |io|
+            MessagePack.load(io)
+          end
+        # handle malformed data due to upgrade incompatibility
+        rescue Errno::ENOENT, MessagePack::MalformedFormatError, MessagePack::UnknownExtTypeError, EOFError
+          {}
+        rescue ArgumentError => error
+          if error.message =~ /negative array size/
+            {}
+          else
+            raise
+          end
         end
       end
 
@@ -80,7 +85,9 @@ module Bootsnap
         exclusive_write = File::Constants::CREAT | File::Constants::EXCL | File::Constants::WRONLY
         # `encoding:` looks redundant wrt `binwrite`, but necessary on windows
         # because binary is part of mode.
-        File.binwrite(tmp, MessagePack.dump(@data), mode: exclusive_write, encoding: Encoding::BINARY)
+        File.open(tmp, mode: exclusive_write, encoding: Encoding::BINARY) do |io|
+          MessagePack.dump(@data, io, freeze: true)
+        end
         FileUtils.mv(tmp, @store_path)
       rescue Errno::EEXIST
         retry

data/lib/bootsnap/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 module Bootsnap
-  VERSION = "1.7.0"
+  VERSION = "1.7.4"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bootsnap
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.7.4
 platform: ruby
 authors:
 - Burke Libbey
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2021-02-01 00:00:00.000000000 Z
+date: 2021-04-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler