bootsnap 1.24.1 → 1.25.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 86fd14b9f3a5920de24b3faf51c01e36ee19b20fc810ee6780c2ba8447871db9
-  data.tar.gz: 4df2ffc0617e1957f264e7fb72002cec56ffb8bfe6f84534343c1b852c37e5e4
+  metadata.gz: e46c28f258eb76b8c90fa22a7fa2c83887695b7c12d3aff8dbbefda5051dca87
+  data.tar.gz: 33aaa301627c53d2f4db1b8e98db0b2f52dc8d20aa533c8de8dc9448bf0c0b8e
 SHA512:
-  metadata.gz: 32879429b4f7473a5ed1ea6e32799a3d45d2203a46e867282d0e503396b40acce0a69faf2e83895271f80a369247fb4eeeae63b0022c25f805e5ed9fa073a581
-  data.tar.gz: f354b078188dfb893c6690a80aa830498bce489e868c132b29317020d73b455528914ce0880f748b8c5f2da2845df61fd2d986b142c58a0d8082b42e62329392
+  metadata.gz: 77f885b700f2d249da6afd6ed4a70e4b37bf27303ddabefdfbed915fea6e61587d3e0fa56ee5159a95416072d58d4994498ee29bf69a3ab70c3b9e98471c3a0a
+  data.tar.gz: 7044ce0c6a30d7312d947da11b20755dbd496ee7b5f460293bf9188c83493ac951e6e683f75009b17860c45a7ff31e3264abffd69dcaf5a0e8c6b759b7631332

data/CHANGELOG.md

@@ -1,5 +1,46 @@
 # Unreleased
 
+# 1.25.0
+
+* Improve YAML parsing cache to more efficiently handle `Time`, `Date` and `DateTime`.
+
+* Don't invalidate the compile cache when YJIT is toggled. YJIT is a runtime JIT and doesn't change the
+  serialized instruction sequences that are cached, but enabling it (via `--yjit`, `RUBYOPT`, or
+  `RubyVM::YJIT.enable`) adds a ` +YJIT` marker to `RUBY_DESCRIPTION` (` +YJIT <token>` on `YJIT_SUPPORT`
+  builds), which is part of the cache key. This previously discarded the entire compile cache whenever YJIT
+  was enabled at runtime but not at precompile time (or vice versa). The marker is now stripped before hashing.
+
+* Fix `CompileCache::Native.fetch` and `.precompile` reading a non-`String` path argument (e.g. a `Pathname`)
+  with `RSTRING_PTR`. Regression from 1.24.0.
+
+# 1.24.6
+
+* Fix detection of Ruby bug #22023 on some patch versions of Ruby 3.4, and properly apply the workaround.
+
+# 1.24.5
+
+* No longer load the config file by default when setup is done manually. This is so cli applications like homebrew
+  don't mistakenly load another app's boostnap config.
+
+# 1.24.4
+
+* Fix several compatibility issues with Ruby `4.0.4`, particularly the `should not compile with coverage` error. See #547.
+* Fix `Bootsnap.enable_frozen_string_literal` to work even when coverage is enabled. Unfortunately only possible on Ruby `4.0.4+`.
+  On older rubies if coverage is enabled a warning will be issued and the feature won't work.
+* Reduced cache files header size from 64 to 32 bytes, and got rid of the random padding element.
+* Avoid leaking a private method in `Object` when testing for Parse.y bugs.
+
+# 1.24.3
+
+* Fix the `1.24.2` workaround to parse Ruby files with UTF-8 even when the `LANG` environment variable
+  is unset or set to `C`.
+
+# 1.24.2
+
+* Workaround two Ruby bugs in `RubyVM::InstructionSequence.compile_file`, that were causing
+  files to be loaded with the old Ruby parser instead of Prism, causing issues with some pattern matching syntax.
+  Ref: https://bugs.ruby-lang.org/issues/22023
+
 # 1.24.1
 
 * Fix encoding of Ruby source files loaded when `BOOTSNAP_READONLY` is set.

data/README.md

@@ -237,14 +237,15 @@ This may look worse at a glance, but underlies a large performance difference.
 useful. [This ruby patch](https://bugs.ruby-lang.org/issues/13378) optimizes them out when coupled
 with bootsnap.)*
 
-Bootsnap writes a cache file containing a 64 byte header followed by the cache contents. The header
+Bootsnap writes a cache file containing a 32 byte header followed by the cache contents. The header
 is a cache key including several fields:
 
-* `version`, hardcoded in bootsnap. Essentially a schema version;
-* `ruby_platform`, A hash of `RUBY_PLATFORM` (e.g. x86_64-linux-gnu) variable.
-* `compile_option`, which changes with `RubyVM::InstructionSequence.compile_option` does;
-* `ruby_revision`, A hash of `RUBY_REVISION`, the exact version of Ruby;
+* `ruby_version_digest`, a digest of:
+  * The `RUBY_DESCRIPTION` constant e.g. `"ruby 4.0.2 (2026-03-17 revision d3da9fec82) +PRISM [arm64-darwin25]"`.
+  * Bootsnap's cache version. Hardcoded in bootsnap. Essentially a schema version;
+  * The content of `RubyVM::InstructionSequence.compile_option`.
 * `size`, the size of the source file;
+* `digest`, a fnv1a_64 hash of the source file;
 * `mtime`, the last-modification timestamp of the source file when it was compiled; and
 * `data_size`, the number of bytes following the header, which we need to read it into a buffer.
 
@@ -349,6 +350,10 @@ Bootsnap::CompileCache::ISeq.compiler_selector = ->(path) do
 end
 ```
 
+*Important note*: This feature is only fully supported by Ruby `4.0.4` and newer.
+On older rubies the feature work except if the `Coverage` module is enabled, in which case a warning will be emitted
+and the default Ruby compiler will be used.
+
 ## Precompilation
 
 In development environments the bootsnap compilation cache is generated on the fly when source files are loaded.

data/ext/bootsnap/bootsnap.c

@@ -40,7 +40,7 @@
 #define MAX_CACHEPATH_SIZE 1000
 #define MAX_CACHEDIR_SIZE  981
 
-#define KEY_SIZE 64
+#define KEY_SIZE 32
 
 #define MAX_CREATE_TEMPFILE_ATTEMPT 3
 
@@ -49,34 +49,27 @@
 #endif
 
 /*
- * An instance of this key is written as the first 64 bytes of each cache file.
+ * An instance of this key is written as the first `KEY_SIZE` bytes of each cache file.
  * The mtime and size members track whether the file contents have changed, and
- * the version, ruby_platform, compile_option, and ruby_revision members track
- * changes to the environment that could invalidate compile results without
+ * the ruby_version_digest (bootsnap_cache_version + RUBY_DESCRIPTION) and compile_option
+ * members track changes to the environment that could invalidate compile results without
  * file contents having changed. The data_size member is not truly part of the
  * "key". Really, this could be called a "header" with the first six members
  * being an embedded "key" struct and an additional data_size member.
  *
  * The data_size indicates the remaining number of bytes in the cache file
  * after the header (the size of the cached artifact).
- *
- * After data_size, the struct is padded to 64 bytes.
  */
 struct bs_cache_key {
-  uint32_t version;
-  uint32_t ruby_platform;
-  uint32_t compile_option;
-  uint32_t ruby_revision;
-  uint64_t size;
+  uint64_t ruby_version_digest;
   uint64_t mtime;
-  uint64_t data_size; //
   uint64_t digest;
-  uint8_t digest_set;
-  uint8_t pad[15];
+  uint32_t size;
+  uint32_t data_size;
 } __attribute__((packed));
 
 /*
- * If the struct padding isn't correct to pad the key to 64 bytes, refuse to
+ * If the struct padding isn't correct to pad the key to `KEY_SIZE` bytes, refuse to
  * compile.
  */
 #define STATIC_ASSERT(X)            STATIC_ASSERT2(X,__LINE__)
@@ -86,15 +79,15 @@ struct bs_cache_key {
 STATIC_ASSERT(sizeof(struct bs_cache_key) == KEY_SIZE);
 
 /* Effectively a schema version. Bumping invalidates all previous caches */
-static const uint32_t current_version = 6;
-
-/* hash of e.g. "x86_64-darwin17", invalidating when ruby is recompiled on a
- * new OS ABI, etc. */
-static uint32_t current_ruby_platform;
-/* Invalidates cache when switching ruby versions */
-static uint32_t current_ruby_revision;
-/* Invalidates cache when RubyVM::InstructionSequence.compile_option changes */
-static uint32_t current_compile_option_crc32 = 0;
+static const uint32_t bootsnap_cache_version = 8;
+
+/* Invalidates cache when switching ruby version, platform or ABI */
+static uint64_t base_ruby_version_digest = 0;
+
+// `base_ruby_version_digest` combined with RubyVM::InstructionSequence.compile_option
+// Invalidates cache when RubyVM::InstructionSequence.compile_option changes
+static uint64_t current_ruby_version_digest = 0;
+
 /* Current umask */
 static mode_t current_umask;
 
@@ -136,8 +129,7 @@ static VALUE bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handle
 static VALUE bs_precompile(char * path, VALUE path_v, char * cache_path, VALUE handler);
 static int open_current_file(const char * path, struct bs_cache_key * key, const char ** errno_provenance);
 static int fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE args, VALUE * output_data, int * exception_tag, const char ** errno_provenance);
-static uint32_t get_ruby_revision(void);
-static uint32_t get_ruby_platform(void);
+static uint64_t get_ruby_version_digest(void);
 
 /*
  * Helper functions to call ruby methods on handler object without crashing on
@@ -295,8 +287,7 @@ Init_bootsnap(void)
   rb_cBootsnap_CompileCache_UNCOMPILABLE = rb_const_get(rb_mBootsnap_CompileCache, rb_intern("UNCOMPILABLE"));
   rb_global_variable(&rb_cBootsnap_CompileCache_UNCOMPILABLE);
 
-  current_ruby_revision = get_ruby_revision();
-  current_ruby_platform = get_ruby_platform();
+  base_ruby_version_digest = get_ruby_version_digest();
 
   instrumentation_method = rb_intern("_instrument");
 
@@ -345,28 +336,10 @@ bs_revalidation_set(VALUE self, VALUE enabled)
   return enabled;
 }
 
-/*
- * Bootsnap's ruby code registers a hook that notifies us via this function
- * when compile_option changes. These changes invalidate all existing caches.
- *
- * Note that on 32-bit platforms, a CRC32 can't be represented in a Fixnum, but
- * can be represented by a uint.
- */
-static VALUE
-bs_compile_option_crc32_set(VALUE self, VALUE crc32_v)
-{
-  if (!RB_TYPE_P(crc32_v, T_BIGNUM) && !RB_TYPE_P(crc32_v, T_FIXNUM)) {
-    Check_Type(crc32_v, T_FIXNUM);
-  }
-  current_compile_option_crc32 = NUM2UINT(crc32_v);
-  return Qnil;
-}
-
 static uint64_t
-fnv1a_64_iter(uint64_t h, const VALUE str)
+fnv1a_64_iter(uint64_t h, const unsigned char *s, size_t len)
 {
-  unsigned char *s = (unsigned char *)RSTRING_PTR(str);
-  unsigned char *str_end = (unsigned char *)RSTRING_PTR(str) + RSTRING_LEN(str);
+  const unsigned char *str_end = s + len;
 
   while (s < str_end) {
     h ^= (uint64_t)*s++;
@@ -377,45 +350,43 @@ fnv1a_64_iter(uint64_t h, const VALUE str)
 }
 
 static uint64_t
-fnv1a_64(const VALUE str)
+fnv1a_64_iter_str(uint64_t h, const VALUE str)
+{
+  Check_Type(str, T_STRING);
+  return fnv1a_64_iter(h, (unsigned char *)RSTRING_PTR(str), RSTRING_LEN(str));
+}
+
+static uint64_t
+fnv1a_64_str(const VALUE str)
 {
   uint64_t h = (uint64_t)0xcbf29ce484222325ULL;
-  return fnv1a_64_iter(h, str);
+  return fnv1a_64_iter_str(h, str);
 }
 
 /*
- * Ruby's revision may be Integer or String. CRuby 2.7 or later uses
- * Git commit ID as revision. It's String.
+ * Bootsnap's ruby code registers a hook that notifies us via this function
+ * when compile_option changes. These changes invalidate all existing caches.
+ *
+ * Note that on 32-bit platforms, a CRC32 can't be represented in a Fixnum, but
+ * can be represented by a uint.
  */
-static uint32_t
-get_ruby_revision(void)
+static VALUE
+bs_compile_option_crc32_set(VALUE self, VALUE crc32_v)
 {
-  VALUE ruby_revision;
-
-  ruby_revision = rb_const_get(rb_cObject, rb_intern("RUBY_REVISION"));
-  if (RB_TYPE_P(ruby_revision, RUBY_T_FIXNUM)) {
-    return FIX2INT(ruby_revision);
-  } else {
-    uint64_t hash;
-
-    hash = fnv1a_64(ruby_revision);
-    return (uint32_t)(hash >> 32);
+  if (!RB_TYPE_P(crc32_v, T_BIGNUM) && !RB_TYPE_P(crc32_v, T_FIXNUM)) {
+    Check_Type(crc32_v, T_FIXNUM);
   }
+  uint32_t crc32 = (uint32_t)NUM2UINT(crc32_v);
+  current_ruby_version_digest = fnv1a_64_iter(base_ruby_version_digest, (unsigned char *)&crc32, sizeof(crc32));
+  return Qnil;
 }
 
-/*
- * When ruby's version doesn't change, but it's recompiled on a different OS
- * (or OS version), we need to invalidate the cache.
- */
-static uint32_t
-get_ruby_platform(void)
+static uint64_t
+get_ruby_version_digest(void)
 {
-  uint64_t hash;
-  VALUE ruby_platform;
-
-  ruby_platform = rb_const_get(rb_cObject, rb_intern("RUBY_PLATFORM"));
-  hash = fnv1a_64(ruby_platform);
-  return (uint32_t)(hash >> 32);
+  uint64_t hash = fnv1a_64_str(rb_const_get(rb_mBootsnap, rb_intern("RUBY_CACHE_KEY")));
+  hash = fnv1a_64_iter(hash, (unsigned char *)&bootsnap_cache_version, sizeof(bootsnap_cache_version));
+  return hash;
 }
 
 /*
@@ -424,26 +395,31 @@ get_ruby_platform(void)
  * be stored.
  *
  * The path will look something like: <cachedir>/12/34567890abcdef
+ *
+ * `path_v` must already have been through FilePathValue(): the callers read it
+ * with RSTRING_PTR() too, so the conversion has to happen in the frame that
+ * owns the variable.
  */
 static void
 bs_cache_path(VALUE cachedir_v, VALUE namespace_v, VALUE path_v, char (* cache_path)[MAX_CACHEPATH_SIZE])
 {
-  FilePathValue(path_v);
-
   Check_Type(cachedir_v, T_STRING);
   Check_Type(path_v, T_STRING);
+
+  long namespace_len = 0;
   if (!NIL_P(namespace_v)) {
     Check_Type(namespace_v, T_STRING);
+    namespace_len = RSTRING_LEN(namespace_v);
   }
 
-  if (RSTRING_LEN(cachedir_v) > MAX_CACHEDIR_SIZE) {
+  if (RSTRING_LEN(cachedir_v) + namespace_len > MAX_CACHEDIR_SIZE) {
     rb_raise(rb_eArgError, "cachedir too long");
   }
 
   const char * cachedir = RSTRING_PTR(cachedir_v);
   const char * namespace = NIL_P(namespace_v) ? "" : RSTRING_PTR(namespace_v);
 
-  uint64_t hash = fnv1a_64(path_v);
+  uint64_t hash = fnv1a_64_str(path_v);
   uint8_t first_byte = (hash >> (64 - 8));
   uint64_t remainder = hash & 0x00ffffffffffffff;
 
@@ -460,10 +436,8 @@ bs_cache_path(VALUE cachedir_v, VALUE namespace_v, VALUE path_v, char (* cache_p
  */
 static enum cache_status cache_key_equal_fast_path(struct bs_cache_key *k1,
                                      struct bs_cache_key *k2) {
-  if (k1->version == k2->version &&
-          k1->ruby_platform == k2->ruby_platform &&
-          k1->compile_option == k2->compile_option &&
-          k1->ruby_revision == k2->ruby_revision && k1->size == k2->size) {
+  if (k1->ruby_version_digest == k2->ruby_version_digest &&
+          k1->size == k2->size) {
       if (k1->mtime == k2->mtime) {
         return hit;
       }
@@ -507,10 +481,9 @@ static int update_cache_key(struct bs_cache_key *current_key, struct bs_cache_ke
  */
 static void bs_cache_key_digest(struct bs_cache_key *key,
                                 const VALUE input_data) {
-  if (key->digest_set)
+  if (key->digest)
     return;
-  key->digest = fnv1a_64(input_data);
-  key->digest_set = 1;
+  key->digest = fnv1a_64_str(input_data);
 }
 
 /*
@@ -521,6 +494,8 @@ static void bs_cache_key_digest(struct bs_cache_key *key,
 static VALUE
 bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE namespace_v, VALUE path_v, VALUE handler, VALUE args)
 {
+  FilePathValue(path_v);
+
   char cache_path[MAX_CACHEPATH_SIZE];
 
   /* generate cache path to cache_path */
@@ -537,6 +512,8 @@ bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE namespace_v, VALUE path_v, VALUE
 static VALUE
 bs_rb_precompile(VALUE self, VALUE cachedir_v, VALUE namespace_v, VALUE path_v, VALUE handler)
 {
+  FilePathValue(path_v);
+
   char cache_path[MAX_CACHEPATH_SIZE];
   /* generate cache path to cache_path */
   bs_cache_path(cachedir_v, namespace_v, path_v, &cache_path);
@@ -587,13 +564,19 @@ open_current_file(const char * path, struct bs_cache_key * key, const char ** er
     return -1;
   }
 
-  key->version        = current_version;
-  key->ruby_platform  = current_ruby_platform;
-  key->compile_option = current_compile_option_crc32;
-  key->ruby_revision  = current_ruby_revision;
-  key->size           = (uint64_t)statbuf.st_size;
-  key->mtime          = (uint64_t)statbuf.st_mtime;
-  key->digest_set     = false;
+  key->ruby_version_digest = current_ruby_version_digest;
+
+  // We're limited to file of 4GiB or less. Hopefully that's enough for everyone.
+  if (statbuf.st_size > (uint32_t)-1) {
+    *errno_provenance = "bs_fetch:open_current_file:file_too_big";
+    close(fd);
+    errno = EFBIG;
+    return -1;
+  }
+
+  key->size = (uint32_t)statbuf.st_size;
+  key->mtime = (uint64_t)statbuf.st_mtime;
+  key->digest = 0;
 
   return fd;
 }
@@ -663,10 +646,10 @@ open_cache_file(const char * path, struct bs_cache_key * key, const char ** errn
 
 /*
  * The cache file is laid out like:
- *   0...64 : bs_cache_key
- *   64..-1 : cached artifact
+ *   0...`KEY_SIZE` : bs_cache_key
+ *   `KEY_SIZE`..-1 : cached artifact
  *
- * This function takes a file descriptor whose position is pre-set to 64, and
+ * This function takes a file descriptor whose position is pre-set to `KEY_SIZE`, and
  * the data_size (corresponding to the remaining number of bytes) listed in the
  * cache header.
  *
@@ -782,7 +765,12 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, cons
   setmode(fd, O_BINARY);
   #endif
 
-  key->data_size = RSTRING_LEN(data);
+  uint64_t data_size = RSTRING_LEN(data);
+  if (data_size > (uint32_t)-1) {
+    return 0; // Don't cache.
+  }
+
+  key->data_size = (uint32_t)data_size;
   nwrite = write(fd, key, KEY_SIZE);
   if (nwrite < 0) {
     *errno_provenance = "bs_fetch:atomic_write_cache_file:write";

data/lib/bootsnap/cli/worker_pool.rb

@@ -62,7 +62,7 @@ module Bootsnap
           # Ref: https://github.com/rails/bootsnap/issues/495
           # The second forked process will hang on some QEMU environments
           r, w = IO.pipe
-          pids = 2.times.map do
+          pids = Array.new(2) do
             ::Process.fork do
               exit!(true)
             end
@@ -161,7 +161,7 @@ module Bootsnap
       end
 
       def spawn
-        @workers = @size.times.map { Worker.new(@jobs) }
+        @workers = Array.new(@size) { Worker.new(@jobs) }
         @workers.each(&:spawn)
         @dispatcher_thread = Thread.new { dispatch_loop }
         @dispatcher_thread.abort_on_exception = true

data/lib/bootsnap/cli.rb

@@ -43,7 +43,7 @@ module Bootsnap
           yaml: yaml,
           revalidation: true,
         )
-        Bootsnap.load_config
+        Bootsnap.load_config(ENV["BOOTSNAP_CONFIG"] || "config/bootsnap.rb")
 
         @work_pool = WorkerPool.create(size: jobs, jobs: {
           ruby: method(:precompile_ruby),

data/lib/bootsnap/compile_cache/iseq.rb

@@ -19,124 +19,209 @@ module Bootsnap
         end
       end
 
-      class Compiler
-        attr_reader :namespace, :compile_options
+      if supported?
+        class Compiler
+          attr_reader :namespace
+
+          def initialize(namespace = nil, compile_options = nil)
+            @namespace = namespace
+            @options = compile_options.freeze
+            update_options
+          end
 
-        def initialize(namespace = nil, compile_options = nil)
-          @namespace = namespace
-          @compile_options = compile_options
-        end
+          def update_options
+            @compile_options = if @options.nil? || @options < RubyVM::InstructionSequence.compile_option
+              nil
+            else
+              RubyVM::InstructionSequence.compile_option.merge(@options).freeze
+            end
+          end
 
-        has_ruby_bug_18250 = begin # https://bugs.ruby-lang.org/issues/18250
-          if defined? RubyVM::InstructionSequence
-            RubyVM::InstructionSequence.compile("def foo(*); ->{ super }; end; def foo(**); ->{ super }; end").to_binary
+          has_ruby_bug_18250 = RUBY_VERSION.start_with?("3.0.") && begin # https://bugs.ruby-lang.org/issues/18250
+            if defined? RubyVM::InstructionSequence
+              RubyVM::InstructionSequence.compile(<<~RUBY).to_binary
+                def foo(*); ->{ super }; end; def foo(**); ->{ super }; end
+              RUBY
+            end
+            false
+          rescue TypeError
+            true
           end
-          false
-        rescue TypeError
-          true
-        end
 
-        if has_ruby_bug_18250
-          def input_to_storage(_, path)
-            iseq = RubyVM::InstructionSequence.compile_file(path, @compile_options)
+          has_ruby_bug_22023 = case RUBY_VERSION
+          when /^3\.3\./
+            true
+          when /^3\.4\.(\d+)/
+            $1.to_i < 10
+          when /^4\.0\.(\d+)/
+            $1.to_i < 4
+          else
+            false
+          end
+
+          if has_ruby_bug_22023 && RUBY_DESCRIPTION.include?("+PRISM")
+            module PatchRubyBug22023
+              def compile_file(path, options = nil)
+                compile_file_prism(path, options)
+              end
+
+              has_ruby_bug_22023_bis = !RubyVM::InstructionSequence.compile_file_prism(
+                File.expand_path("../ruby_bug_22023_canary.rb", __FILE__),
+                {frozen_string_literal: true},
+              ).eval.frozen?
+
+              if has_ruby_bug_22023_bis
+                def compile_file_prism(path, options = nil)
+                  compile_prism(::File.read(path, encoding: Encoding::UTF_8), path, path, nil, options)
+                end
+              end
+            end
+            RubyVM::InstructionSequence.singleton_class.prepend(PatchRubyBug22023)
+          end
 
-            begin
+          if has_ruby_bug_18250
+            def input_to_storage(_, path)
+              iseq = RubyVM::InstructionSequence.compile_file(path, @compile_options)
               iseq.to_binary
-            rescue TypeError
-              UNCOMPILABLE # ruby bug #18250
+            rescue TypeError, SyntaxError # Ruby [Bug #18250] & [Bug #22023]
+              UNCOMPILABLE
+            end
+          else
+            def input_to_storage(_, path)
+              RubyVM::InstructionSequence.compile_file(path, @compile_options).to_binary
+            rescue SyntaxError # Ruby [Bug #22023]
+              UNCOMPILABLE
             end
           end
-        else
-          def input_to_storage(_, path)
-            RubyVM::InstructionSequence.compile_file(path, @compile_options).to_binary
+
+          def storage_to_output(binary, _args)
+            iseq = RubyVM::InstructionSequence.load_from_binary(binary)
+            binary.clear
+            iseq
+          rescue RuntimeError => error
+            if error.message == "broken binary format"
+              $stderr.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
+              nil
+            else
+              raise
+            end
           end
-        end
 
-        def storage_to_output(binary, _args)
-          iseq = RubyVM::InstructionSequence.load_from_binary(binary)
-          binary.clear
-          iseq
-        rescue RuntimeError => error
-          if error.message == "broken binary format"
-            $stderr.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
-            nil
-          else
-            raise
+          def input_to_output(source, path, _kwargs)
+            if @compile_options
+              if source
+                RubyVM::InstructionSequence.compile(
+                  source.force_encoding(Encoding.default_external),
+                  path,
+                  path,
+                  nil,
+                  @compile_options,
+                )
+              else
+                RubyVM::InstructionSequence.compile_file(path, @compile_options)
+              end
+            end
           end
         end
 
-        def input_to_output(source, path, _kwargs)
-          RubyVM::InstructionSequence.compile(
-            source.force_encoding(Encoding.default_external),
-            path,
-            path,
+        DEFAULT = Compiler.new
+        FROZEN_STRING_LITERAL = Compiler.new("-fstr", {frozen_string_literal: true}.freeze)
+        COVERAGE_SUPPORTED = RUBY_VERSION >= "4.0.4"
+
+        @default_compiler = DEFAULT
+        @coverage_support_warning_emitted = false
+
+        def self.fetch(path, cache_dir: ISeq.cache_dir)
+          compiler = compiler_selector&.call(path) || default_compiler
+
+          # Having coverage enabled prevents iseq dumping/loading.
+          if coverage_on?
+            return nil if compiler.equal?(DEFAULT)
+
+            if COVERAGE_SUPPORTED
+              return compiler.input_to_output(nil, path.to_s, nil)
+            elsif !@coverage_support_warning_emitted
+              @coverage_support_warning_emitted = true
+              warn(<<~MSG)
+                Using `Bootsnap.enable_frozen_string_literal` with code coverage enabled is only supported on Ruby 4.0.4+.
+                Files loaded while coverage is on, will have mutable string literals.
+              MSG
+            end
+
+            return nil
+          end
+
+          Bootsnap::CompileCache::Native.fetch(
+            cache_dir,
+            compiler.namespace,
+            path.to_s,
+            compiler,
             nil,
-            @compile_options,
           )
         end
-      end
 
-      DEFAULT = Compiler.new
-      FROZEN_STRING_LITERAL = Compiler.new("-fstr", {frozen_string_literal: true}.freeze)
-      MUTABLE_STRING_LITERAL = Compiler.new("-no-fstr", {frozen_string_literal: false}.freeze)
-      @default_compiler = DEFAULT
-
-      def self.fetch(path, cache_dir: ISeq.cache_dir)
-        compiler = compiler_selector&.call(path) || default_compiler
-        Bootsnap::CompileCache::Native.fetch(
-          cache_dir,
-          compiler.namespace,
-          path.to_s,
-          compiler,
-          nil,
-        )
-      end
+        def self.precompile(path)
+          compiler = compiler_selector&.call(path) || default_compiler
+          Bootsnap::CompileCache::Native.precompile(
+            cache_dir,
+            compiler.namespace,
+            path.to_s,
+            compiler,
+          )
+        end
 
-      def self.precompile(path)
-        compiler = compiler_selector&.call(path) || default_compiler
-        Bootsnap::CompileCache::Native.precompile(
-          cache_dir,
-          compiler.namespace,
-          path.to_s,
-          compiler,
-        )
-      end
+        if RUBY_VERSION < "3.1."
+          def self.coverage_on?
+            defined?(Coverage) && Coverage.running?
+          end
+        else
+          def self.coverage_on?
+            defined?(Coverage) && Coverage.state != :idle
+          end
+        end
 
-      module InstructionSequenceMixin
-        def load_iseq(path)
-          # Having coverage enabled prevents iseq dumping/loading.
-          return nil if defined?(Coverage) && Coverage.running?
+        module InstructionSequenceMixin
+          def load_iseq(path)
+            Bootsnap::CompileCache::ISeq.fetch(path.to_s)
+          rescue RuntimeError => error
+            if error.message.include?("unmatched platform")
+              puts("unmatched platform for file #{path}")
+            end
+            raise
+          end
 
-          Bootsnap::CompileCache::ISeq.fetch(path.to_s)
-        rescue RuntimeError => error
-          if error.message =~ /unmatched platform/
-            puts("unmatched platform for file #{path}")
+          def compile_option=(hash)
+            super
+            Bootsnap::CompileCache::ISeq.compile_option_updated
           end
-          raise
         end
 
-        def compile_option=(hash)
-          super
-          Bootsnap::CompileCache::ISeq.compile_option_updated
+        def self.compile_option_updated
+          option = RubyVM::InstructionSequence.compile_option
+          crc = Zlib.crc32(option.inspect)
+          Bootsnap::CompileCache::Native.compile_option_crc32 = crc
+          FROZEN_STRING_LITERAL.update_options
         end
-      end
+        compile_option_updated if supported?
 
-      def self.compile_option_updated
-        option = RubyVM::InstructionSequence.compile_option
-        crc = Zlib.crc32(option.inspect)
-        Bootsnap::CompileCache::Native.compile_option_crc32 = crc
-      end
-      compile_option_updated if supported?
+        def self.install!(cache_dir)
+          Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
 
-      def self.install!(cache_dir)
-        Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
+          return unless supported?
 
-        return unless supported?
+          Bootsnap::CompileCache::ISeq.compile_option_updated
 
-        Bootsnap::CompileCache::ISeq.compile_option_updated
+          class << RubyVM::InstructionSequence
+            prepend(InstructionSequenceMixin)
+          end
+        end
+      else
+        def self.install!(...)
+          # noop
+        end
 
-        class << RubyVM::InstructionSequence
-          prepend(InstructionSequenceMixin)
+        def self.precompile(...)
+          # noop
         end
       end
     end

data/lib/bootsnap/compile_cache/ruby_bug_22023_canary.rb

@@ -0,0 +1 @@
+_ = "test"

data/lib/bootsnap/compile_cache/yaml.rb

@@ -47,14 +47,6 @@ module Bootsnap
           SUPPORTED_INTERNAL_ENCODINGS.include?(Encoding.default_internal)
         end
 
-        module EncodingAwareSymbols
-          extend self
-
-          def unpack(payload)
-            (+payload).force_encoding(Encoding::UTF_8).to_sym
-          end
-        end
-
         def init!
           require "yaml"
           require "msgpack"
@@ -86,28 +78,78 @@ module Bootsnap
             0x00,
             Symbol,
             packer: :to_msgpack_ext,
-            unpacker: EncodingAwareSymbols.method(:unpack).to_proc,
+            unpacker: :from_msgpack_ext,
+            optimized_symbol_parsing: true,
           )
 
-          if defined? MessagePack::Timestamp
-            factory.register_type(
-              MessagePack::Timestamp::TYPE, # or just -1
-              Time,
-              packer: MessagePack::Time::Packer,
-              unpacker: MessagePack::Time::Unpacker,
-            )
+          factory.register_type(
+            MessagePack::Timestamp::TYPE, # or just -1
+            Time,
+            packer: MessagePack::Time::Packer,
+            unpacker: MessagePack::Time::Unpacker,
+          )
 
-            marshal_fallback = {
-              packer: ->(value) { Marshal.dump(value) },
-              unpacker: ->(payload) { Marshal.load(payload) },
-            }
-            {
-              Date => 0x01,
-              Regexp => 0x02,
-            }.each do |type, code|
-              factory.register_type(code, type, marshal_fallback)
-            end
-          end
+          factory.register_type(
+            0x01,
+            Date,
+            packer: lambda { |date, packer|
+              packer.write(date.year)
+              packer.write(date.month)
+              packer.write(date.day)
+            },
+            unpacker: lambda { |unpacker|
+              ::Date.new(unpacker.read, unpacker.read, unpacker.read)
+            },
+            recursive: true,
+          )
+
+          factory.register_type(
+            0x02,
+            Regexp,
+            packer: ->(value) { Marshal.dump(value) },
+            unpacker: ->(payload) { Marshal.load(payload) },
+          )
+
+          factory.register_type(
+            0x03,
+            DateTime,
+            packer: lambda { |dt, packer|
+              packer.write(dt.year)
+              packer.write(dt.month)
+              packer.write(dt.day)
+              packer.write(dt.hour)
+              packer.write(dt.minute)
+
+              sec = dt.sec + dt.sec_fraction
+              packer.write(sec.numerator)
+              packer.write(sec.denominator)
+
+              offset = dt.offset
+              packer.write(offset.numerator)
+              packer.write(offset.denominator)
+            },
+            unpacker: lambda { |unpacker|
+              ::DateTime.new(
+                unpacker.read, # year
+                unpacker.read, # month
+                unpacker.read, # day
+                unpacker.read, # hour
+                unpacker.read, # minute
+                Rational(unpacker.read, unpacker.read), # sec fraction
+                Rational(unpacker.read, unpacker.read), # offset fraction
+              )
+            },
+            recursive: true,
+          )
+
+          require "msgpack/bigint"
+          factory.register_type(
+            0x04,
+            Integer,
+            packer: MessagePack::Bigint.method(:to_msgpack_ext),
+            unpacker: MessagePack::Bigint.method(:from_msgpack_ext),
+            oversized_integer_extension: true,
+          )
 
           self.msgpack_factory = factory
 
@@ -116,7 +158,7 @@ module Bootsnap
           if params.include?([:key, :symbolize_names])
             supported_options << :symbolize_names
           end
-          if params.include?([:key, :freeze]) && factory.load(factory.dump("yaml"), freeze: true).frozen?
+          if params.include?([:key, :freeze])
             supported_options << :freeze
           end
           supported_options.freeze
@@ -135,6 +177,14 @@ module Bootsnap
 
           NoTagsVisitor.new(scanner, loader).visit(ast)
         end
+
+        def msgpack_unpacker_options(kwargs)
+          return kwargs unless kwargs&.key?(:symbolize_names)
+
+          kwargs = kwargs.dup
+          kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
+          kwargs
+        end
       end
 
       module Psych4
@@ -164,11 +214,9 @@ module Bootsnap
           end
 
           def storage_to_output(data, kwargs)
-            if kwargs&.key?(:symbolize_names)
-              kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
-            end
-
-            unpacker = CompileCache::YAML.msgpack_factory.unpacker(kwargs)
+            unpacker = CompileCache::YAML.msgpack_factory.unpacker(
+              CompileCache::YAML.msgpack_unpacker_options(kwargs),
+            )
             unpacker.feed(data)
             _safe_loaded = unpacker.unpack
             result = unpacker.unpack
@@ -202,11 +250,9 @@ module Bootsnap
           end
 
           def storage_to_output(data, kwargs)
-            if kwargs&.key?(:symbolize_names)
-              kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
-            end
-
-            unpacker = CompileCache::YAML.msgpack_factory.unpacker(kwargs)
+            unpacker = CompileCache::YAML.msgpack_factory.unpacker(
+              CompileCache::YAML.msgpack_unpacker_options(kwargs),
+            )
             unpacker.feed(data)
             safe_loaded = unpacker.unpack
             if safe_loaded
@@ -282,10 +328,9 @@ module Bootsnap
         end
 
         def storage_to_output(data, kwargs)
-          if kwargs&.key?(:symbolize_names)
-            kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
-          end
-          unpacker = CompileCache::YAML.msgpack_factory.unpacker(kwargs)
+          unpacker = CompileCache::YAML.msgpack_factory.unpacker(
+            CompileCache::YAML.msgpack_unpacker_options(kwargs),
+          )
           unpacker.feed(data)
           _safe_loaded = unpacker.unpack
           unpacker.unpack

data/lib/bootsnap/load_path_cache/store.rb

@@ -84,7 +84,7 @@ module Bootsnap
         rescue Errno::ENOENT, MessagePack::MalformedFormatError, MessagePack::UnknownExtTypeError, EOFError
           default_data
         rescue ArgumentError => error
-          if error.message =~ /negative array size/
+          if error.message.include?("negative array size")
             default_data
           else
             raise

data/lib/bootsnap/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Bootsnap
-  VERSION = "1.24.1"
+  VERSION = "1.25.0"
 end

data/lib/bootsnap.rb

@@ -6,6 +6,9 @@ require_relative "bootsnap/bundler"
 module Bootsnap
   InvalidConfiguration = Class.new(StandardError)
 
+  # JITs shouldn't change the cache key as they have no impact on iseq generation
+  RUBY_CACHE_KEY = RUBY_DESCRIPTION.gsub(/\s\+\wJIT/, "").freeze
+
   class << self
     attr_reader :cache_dir, :logger
 
@@ -43,13 +46,6 @@ module Bootsnap
       @instrumentation.call(event, path)
     end
 
-    def load_config
-      config_path = File.expand_path(ENV["BOOTSNAP_CONFIG"] || "config/bootsnap.rb")
-      if File.exist?(config_path)
-        require(config_path)
-      end
-    end
-
     def setup(
       cache_dir:,
       development_mode: true,
@@ -59,7 +55,8 @@ module Bootsnap
       revalidation: false,
       compile_cache_iseq: true,
       compile_cache_yaml: true,
-      compile_cache_json: (compile_cache_json_unset = true)
+      compile_cache_json: (compile_cache_json_unset = true),
+      config_path: nil
     )
       unless compile_cache_json_unset
         warn("Bootsnap.setup `compile_cache_json` argument is deprecated and has no effect")
@@ -84,7 +81,16 @@ module Bootsnap
         revalidation: revalidation,
       )
 
-      load_config
+      load_config(config_path)
+    end
+
+    def load_config(config_path)
+      if config_path
+        config_path = File.expand_path(config_path)
+        if File.exist?(config_path)
+          require(config_path)
+        end
+      end
     end
 
     def enable_frozen_string_literal(app_only: false)
@@ -102,8 +108,8 @@ module Bootsnap
           end
         }
       else
-        Bootsnap::CompileCache::ISeq.compiler_selector = nil
-        Bootsnap::CompileCache::ISeq.default_compiler = Bootsnap::CompileCache::ISeq::FROZEN_STRING_LITERAL
+        options = RubyVM::InstructionSequence.compile_option.merge(frozen_string_literal: true)
+        RubyVM::InstructionSequence.compile_option = options
       end
     end
 
@@ -150,6 +156,7 @@ module Bootsnap
           readonly: bool_env("BOOTSNAP_READONLY"),
           revalidation: bool_env("BOOTSNAP_REVALIDATE"),
           ignore_directories: ignore_directories,
+          config_path: ENV["BOOTSNAP_CONFIG"] || "config/bootsnap.rb",
         )
 
         if ENV["BOOTSNAP_LOG"]
@@ -160,7 +167,7 @@ module Bootsnap
       end
     end
 
-    if RbConfig::CONFIG["host_os"] =~ /mswin|mingw|cygwin/
+    if /mswin|mingw|cygwin/.match?(RbConfig::CONFIG["host_os"])
       def absolute_path?(path)
         path[1] == ":"
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: bootsnap
 version: !ruby/object:Gem::Version
-  version: 1.24.1
+  version: 1.25.0
 platform: ruby
 authors:
 - Burke Libbey
@@ -15,14 +15,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.5'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.2'
+        version: '1.5'
 description: Boot large ruby/rails apps faster
 email:
 - burke.libbey@shopify.com
@@ -44,6 +44,7 @@ files:
 - lib/bootsnap/cli/worker_pool.rb
 - lib/bootsnap/compile_cache.rb
 - lib/bootsnap/compile_cache/iseq.rb
+- lib/bootsnap/compile_cache/ruby_bug_22023_canary.rb
 - lib/bootsnap/compile_cache/yaml.rb
 - lib/bootsnap/explicit_require.rb
 - lib/bootsnap/load_path_cache.rb
@@ -80,7 +81,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Boot large ruby/rails apps faster
 test_files: []