bootsnap 1.1.8 → 1.7.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a6f66d96dbadca08e5c50985c45319350f1e85ca
-  data.tar.gz: c1227f8f8c6ed542abe582c221ccf45318655e87
+SHA256:
+  metadata.gz: ea223ff0ae18f28970e2b80aa24970fe24f5eeefc7ad12b5b4b77dc921af4ca3
+  data.tar.gz: 5f9ffdfcbeac27b128aa314749dd27c6182247fe7b6aa6492a1d99d00586ac3e
 SHA512:
-  metadata.gz: fa366476bf324d6eda9415e7170d67e31eb4a01d08758ff51d9e83c7661db8160cfa334ef9df67777ef059887de24a3f8a7da49b1675f3a1b0bf7704ecf42620
-  data.tar.gz: '06758fb4f8445d041816330983799684a3b79e6ec82ac1e21cf087544d377744b0bb8bc20012146d529876fac6398e1053821eea2468be60c8194c6c07855312'
+  metadata.gz: 0e3bcea4690954e6e15a3b0c10e3bf5b93b23fefff0d3d61354cb04d683bd8925d61ce32dcc9f7dd526b2f14f4d44a24196e72003fa75714bb0e6f99fbc94f62
+  data.tar.gz: 587454d2a3a5019c56c81068e9202905328e7c2918f1786f35ff988424e2a9222b36db35f7ef6e403fc730a0752508ab248b06d6b164dc0b6e147623e5f0912e

data/CHANGELOG.md

@@ -1,3 +1,124 @@
+# Unreleased
+
+# 1.7.2
+
+* Fix compatibility with msgpack < 1. (#349)
+
+# 1.7.1
+
+* Warn Ruby 2.5 users if they turn ISeq caching on. (#327, #244)
+* Disable ISeq caching for the whole 2.5.x series again.
+* Better handle hashing of Ruby strings. (#318)
+
+# 1.7.0
+
+* Fix detection of YAML files in gems.
+* Adds an instrumentation API to monitor cache misses.
+* Allow to control the behavior of `require 'bootsnap/setup'` using environment variables.
+* Deprecate the `disable_trace` option.
+* Deprecate the `ActiveSupport::Dependencies` (AKA Classic autoloader) integration. (#344) 
+
+# 1.6.0
+
+* Fix a Ruby 2.7/3.0 issue with `YAML.load_file` keyword arguments. (#342)
+* `bootsnap precompile` CLI use multiple processes to complete faster. (#341)
+* `bootsnap precompile` CLI also precompile YAML files. (#340)
+* Changed the load path cache directory from `$BOOTSNAP_CACHE_DIR/bootsnap-load-path-cache` to `$BOOTSNAP_CACHE_DIR/bootsnap/load-path-cache` for ease of use. (#334)
+* Changed the compile cache directory from `$BOOTSNAP_CACHE_DIR/bootsnap-compile-cache` to `$BOOTSNAP_CACHE_DIR/bootsnap/compile-cache` for ease of use. (#334)
+
+# 1.5.1
+
+* Workaround a Ruby bug in InstructionSequence.compile_file. (#332)
+
+# 1.5.0
+
+* Add a command line to statically precompile the ISeq cache. (#326)
+
+# 1.4.9
+
+* [Windows support](https://github.com/Shopify/bootsnap/pull/319)
+* [Fix potential crash](https://github.com/Shopify/bootsnap/pull/322)
+
+# 1.4.8
+
+* [Prevent FallbackScan from polluting exception cause](https://github.com/Shopify/bootsnap/pull/314)
+
+# 1.4.7
+
+* Various performance enhancements
+* Fix race condition in heavy concurrent load scenarios that would cause bootsnap to raise
+
+# 1.4.6
+
+* Fix bug that was erroneously considering that files containing `.` in the names were being
+  required if a different file with the same name was already being required
+
+  Example:
+  
+      require 'foo'
+      require 'foo.en'
+
+  Before bootsnap was considering `foo.en` to be the same file as `foo`
+
+* Use glibc as part of the ruby_platform cache key
+
+# 1.4.5
+
+* MRI 2.7 support
+* Fixed concurrency bugs
+
+# 1.4.4
+
+* Disable ISeq cache in `bootsnap/setup` by default in Ruby 2.5
+
+# 1.4.3
+
+* Fix some cache permissions and umask issues after switch to mkstemp
+
+# 1.4.2
+
+* Fix bug when removing features loaded by relative path from `$LOADED_FEATURES`
+* Fix bug with propagation of `NameError` up from nested calls to `require`
+
+# 1.4.1
+
+* Don't register change observers to frozen objects.
+
+# 1.4.0
+
+* When running in development mode, always fall back to a full path scan on LoadError, making
+  bootsnap more able to detect newly-created files. (#230)
+* Respect `$LOADED_FEATURES.delete` in order to support code reloading, for integration with
+  Zeitwerk. (#230)
+* Minor performance improvement: flow-control exceptions no longer generate backtraces.
+* Better support for requiring from environments where some features are not supported (especially
+  JRuby). (#226)k
+* More robust handling of OS errors when creating files. (#225)
+
+# 1.3.2
+
+* Fix Spring + Bootsnap incompatibility when there are files with similar names.
+* Fix `YAML.load_file` monkey patch to keep accepting File objects as arguments.
+* Fix the API for `ActiveSupport::Dependencies#autoloadable_module?`.
+* Some performance improvements.
+
+# 1.3.1
+
+* Change load path scanning to more correctly follow symlinks.
+
+# 1.3.0
+
+* Handle cases where load path entries are symlinked (https://github.com/Shopify/bootsnap/pull/136)
+
+# 1.2.1
+
+* Fix method visibility of `Kernel#require`.
+
+# 1.2.0
+
+* Add `LoadedFeaturesIndex` to preserve fix a common bug related to `LOAD_PATH` modifications after
+  loading bootsnap.
+
 # 1.1.8
 
 * Don't cache YAML documents with `!ruby/object`

data/README.md

@@ -1,16 +1,21 @@
-# Bootsnap [![Build Status](https://travis-ci.org/Shopify/bootsnap.svg?branch=master)](https://travis-ci.org/Shopify/bootsnap)
+# Bootsnap [![Actions Status](https://github.com/Shopify/bootsnap/workflows/ci/badge.svg)](https://github.com/Shopify/bootsnap/actions)
 
-Bootsnap is a library that plugs into Ruby, with optional support for `ActiveSupport` and `YAML`,
+Bootsnap is a library that plugs into Ruby, with optional support for `YAML`,
 to optimize and cache expensive computations. See [How Does This Work](#how-does-this-work).
 
 #### Performance
-- [Discourse](https://github.com/discourse/discourse) reports a boot time reduction of approximately 50%, from roughly 6 to 3 seconds on one machine;
+
+- [Discourse](https://github.com/discourse/discourse) reports a boot time reduction of approximately
+  50%, from roughly 6 to 3 seconds on one machine;
 - One of our smaller internal apps also sees a reduction of 50%, from 3.6 to 1.8 seconds;
-- The core Shopify platform -- a rather large monolithic application -- boots about 75% faster, dropping from around 25s to 6.5s.
+- The core Shopify platform -- a rather large monolithic application -- boots about 75% faster,
+  dropping from around 25s to 6.5s.
+* In Shopify core (a large app), about 25% of this gain can be attributed to `compile_cache_*`
+  features; 75% to path caching. This is fairly representative.
 
 ## Usage
 
-This gem works on MacOS and Linux.
+This gem works on macOS and Linux.
 
 Add `bootsnap` to your `Gemfile`:
 
@@ -24,6 +29,18 @@ If you are using Rails, add this to `config/boot.rb` immediately after `require
 require 'bootsnap/setup'
 ```
 
+Note that bootsnap writes to `tmp/cache` (or the path specified by `ENV['BOOTSNAP_CACHE_DIR']`),
+and that directory *must* be writable. Rails will fail to
+boot if it is not. If this is unacceptable (e.g. you are running in a read-only container and
+unwilling to mount in a writable tmpdir), you should remove this line or wrap it in a conditional.
+
+**Note also that bootsnap will never clean up its own cache: this is left up to you. Depending on your
+deployment strategy, you may need to periodically purge `tmp/cache/bootsnap*`. If you notice deploys
+getting progressively slower, this is almost certainly the cause.**
+
+It's technically possible to simply specify `gem 'bootsnap', require: 'bootsnap/setup'`, but it's
+important to load Bootsnap as early as possible to get maximum performance improvement.
+
 You can see how this require works [here](https://github.com/Shopify/bootsnap/blob/master/lib/bootsnap/setup.rb).
 
 If you are not using Rails, or if you are but want more control over things, add this to your
@@ -37,8 +54,6 @@ Bootsnap.setup(
   cache_dir:            'tmp/cache',          # Path to your cache
   development_mode:     env == 'development', # Current working environment, e.g. RACK_ENV, RAILS_ENV, etc
   load_path_cache:      true,                 # Optimize the LOAD_PATH with a cache
-  autoload_paths_cache: true,                 # Optimize ActiveSupport autoloads with cache
-  disable_trace:        true,                 # (Alpha) Set `RubyVM::InstructionSequence.compile_option = { trace_instruction: false }`
   compile_cache_iseq:   true,                 # Compile Ruby code into ISeq cache, breaks coverage reporting.
   compile_cache_yaml:   true                  # Compile YAML into a cache
 )
@@ -53,12 +68,39 @@ speeds up the loading of individual source files, Spring keeps a copy of a pre-b
 on hand to completely skip parts of the boot process the next time it's needed. The two tools work
 well together, and are both included in a newly-generated Rails applications by default.
 
+### Environment variables
+
+`require 'bootsnap/setup'` behavior can be changed using environment variables:
+
+- `BOOTSNAP_CACHE_DIR` allows to define the cache location.
+- `DISABLE_BOOTSNAP` allows to entirely disable bootsnap.
+- `DISABLE_BOOTSNAP_LOAD_PATH_CACHE` allows to disable load path caching.
+- `DISABLE_BOOTSNAP_COMPILE_CACHE` allows to disable ISeq and YAML caches.
+- `BOOTSNAP_LOG` configure bootsnap to log all caches misses to STDERR.
+
 ### Environments
 
 All Bootsnap features are enabled in development, test, production, and all other environments according to the configuration in the setup. At Shopify, we use this gem safely in all environments without issue.
 
 If you would like to disable any feature for a certain environment, we suggest changing the configuration to take into account the appropriate ENV var or configuration according to your needs.
 
+### Instrumentation
+
+Bootsnap cache misses can be monitored though a callback:
+
+```ruby
+Bootsnap.instrumentation = ->(event, path) { puts "#{event} #{path}" }
+```
+
+`event` is either `:miss` or `:stale`. You can also call `Bootsnap.log!` as a shortcut to
+log all events to STDERR.
+
+To turn instrumentation back off you can set it to nil:
+
+```ruby
+Bootsnap.instrumentation = nil
+```
+
 ## How does this work?
 
 Bootsnap optimizes methods to cache results of expensive computations, and can be grouped
@@ -66,8 +108,6 @@ into two broad categories:
 
 * [Path Pre-Scanning](#path-pre-scanning)
     * `Kernel#require` and `Kernel#load` are modified to eliminate `$LOAD_PATH` scans.
-    * `ActiveSupport::Dependencies.{autoloadable_module?,load_missing_constant,depend_on}` are
-      overridden to eliminate scans of `ActiveSupport::Dependencies.autoload_paths`.
 * [Compilation caching](#compilation-caching)
     * `RubyVM::InstructionSequence.load_iseq` is implemented to cache the result of ruby bytecode
       compilation.
@@ -106,10 +146,6 @@ open y/foo.rb
 ...
 ```
 
-Exactly the same strategy is employed for methods that traverse
-`ActiveSupport::Dependencies.autoload_paths` if the `autoload_paths_cache` option is given to
-`Bootsnap.setup`.
-
 The following diagram flowcharts the overrides that make the `*_path_cache` features work.
 
 ![Flowchart explaining
@@ -196,7 +232,7 @@ Bootsnap writes a cache file containing a 64 byte header followed by the cache c
 is a cache key including several fields:
 
 * `version`, hardcoded in bootsnap. Essentially a schema version;
-* `os_version`, A hash of the current kernel version (on macOS, BSD) or glibc version (on Linux);
+* `ruby_platform`, A hash of `RUBY_PLATFORM` (e.g. x86_64-linux-gnu) variable and glibc version (on Linux) or OS version (`uname -v` on BSD, macOS)
 * `compile_option`, which changes with `RubyVM::InstructionSequence.compile_option` does;
 * `ruby_revision`, the version of Ruby this was compiled with;
 * `size`, the size of the source file;
@@ -275,3 +311,25 @@ open    /c/nope.bundle -> -1
 ```
 # (nothing!)
 ```
+
+## Precompilation
+
+In development environments the bootsnap compilation cache is generated on the fly when source files are loaded.
+But in production environments, such as docker images, you might need to precompile the cache.
+
+To do so you can use the `bootsnap precompile` command.
+
+Example:
+
+```bash
+$ bundle exec bootsnap precompile --gemfile app/ lib/
+```
+
+## When not to use Bootsnap
+
+*Alternative engines*: Bootsnap is pretty reliant on MRI features, and parts are disabled entirely on alternative ruby
+engines.
+
+*Non-local filesystems*: Bootsnap depends on `tmp/cache` (or whatever you set its cache directory
+to) being on a relatively fast filesystem. If you put it on a network mount, bootsnap is very likely
+to slow your application down quite a lot.

data/exe/bootsnap

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bootsnap/cli'
+exit Bootsnap::CLI.new(ARGV).run

data/ext/bootsnap/bootsnap.c

@@ -14,6 +14,7 @@
 #include "bootsnap.h"
 #include "ruby.h"
 #include <stdint.h>
+#include <stdbool.h>
 #include <sys/types.h>
 #include <errno.h>
 #include <fcntl.h>
@@ -21,6 +22,9 @@
 #ifndef _WIN32
 #include <sys/utsname.h>
 #endif
+#ifdef __GLIBC__
+#include <gnu/libc-version.h>
+#endif
 
 /* 1000 is an arbitrary limit; FNV64 plus some slashes brings the cap down to
  * 981 for the cache dir */
@@ -29,6 +33,12 @@
 
 #define KEY_SIZE 64
 
+#define MAX_CREATE_TEMPFILE_ATTEMPT 3
+
+#ifndef RB_UNLIKELY
+  #define RB_UNLIKELY(x) (x)
+#endif
+
 /*
  * An instance of this key is written as the first 64 bytes of each cache file.
  * The mtime and size members track whether the file contents have changed, and
@@ -65,7 +75,7 @@ struct bs_cache_key {
 STATIC_ASSERT(sizeof(struct bs_cache_key) == KEY_SIZE);
 
 /* Effectively a schema version. Bumping invalidates all previous caches */
-static const uint32_t current_version = 2;
+static const uint32_t current_version = 3;
 
 /* hash of e.g. "x86_64-darwin17", invalidating when ruby is recompiled on a
  * new OS ABI, etc. */
@@ -74,6 +84,8 @@ static uint32_t current_ruby_platform;
 static uint32_t current_ruby_revision;
 /* Invalidates cache when RubyVM::InstructionSequence.compile_option changes */
 static uint32_t current_compile_option_crc32 = 0;
+/* Current umask */
+static mode_t current_umask;
 
 /* Bootsnap::CompileCache::{Native, Uncompilable} */
 static VALUE rb_mBootsnap;
@@ -81,32 +93,38 @@ static VALUE rb_mBootsnap_CompileCache;
 static VALUE rb_mBootsnap_CompileCache_Native;
 static VALUE rb_eBootsnap_CompileCache_Uncompilable;
 static ID uncompilable;
+static ID instrumentation_method;
+static VALUE sym_miss;
+static VALUE sym_stale;
+static bool instrumentation_enabled = false;
 
 /* Functions exposed as module functions on Bootsnap::CompileCache::Native */
+static VALUE bs_instrumentation_enabled_set(VALUE self, VALUE enabled);
 static VALUE bs_compile_option_crc32_set(VALUE self, VALUE crc32_v);
-static VALUE bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler);
+static VALUE bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler, VALUE args);
+static VALUE bs_rb_precompile(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler);
 
 /* Helpers */
-static uint64_t fnv1a_64(const char *str);
-static void bs_cache_path(const char * cachedir, const char * path, char ** cache_path);
+static void bs_cache_path(const char * cachedir, const VALUE path, char (* cache_path)[MAX_CACHEPATH_SIZE]);
 static int bs_read_key(int fd, struct bs_cache_key * key);
 static int cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2);
-static VALUE bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler);
-static int open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenance);
-static int fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data, int * exception_tag, char ** errno_provenance);
-static VALUE prot_exception_for_errno(VALUE err);
+static VALUE bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args);
+static VALUE bs_precompile(char * path, VALUE path_v, char * cache_path, VALUE handler);
+static int open_current_file(char * path, struct bs_cache_key * key, const char ** errno_provenance);
+static int fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE args, VALUE * output_data, int * exception_tag, const char ** errno_provenance);
+static uint32_t get_ruby_revision(void);
 static uint32_t get_ruby_platform(void);
 
 /*
  * Helper functions to call ruby methods on handler object without crashing on
  * exception.
  */
-static int bs_storage_to_output(VALUE handler, VALUE storage_data, VALUE * output_data);
+static int bs_storage_to_output(VALUE handler, VALUE args, VALUE storage_data, VALUE * output_data);
 static VALUE prot_storage_to_output(VALUE arg);
 static VALUE prot_input_to_output(VALUE arg);
-static void bs_input_to_output(VALUE handler, VALUE input_data, VALUE * output_data, int * exception_tag);
+static void bs_input_to_output(VALUE handler, VALUE args, VALUE input_data, VALUE * output_data, int * exception_tag);
 static VALUE prot_input_to_storage(VALUE arg);
-static int bs_input_to_storage(VALUE handler, VALUE input_data, VALUE pathval, VALUE * storage_data);
+static int bs_input_to_storage(VALUE handler, VALUE args, VALUE input_data, VALUE pathval, VALUE * storage_data);
 struct s2o_data;
 struct i2o_data;
 struct i2s_data;
@@ -135,14 +153,33 @@ Init_bootsnap(void)
   rb_mBootsnap_CompileCache_Native = rb_define_module_under(rb_mBootsnap_CompileCache, "Native");
   rb_eBootsnap_CompileCache_Uncompilable = rb_define_class_under(rb_mBootsnap_CompileCache, "Uncompilable", rb_eStandardError);
 
-  current_ruby_revision = FIX2INT(rb_const_get(rb_cObject, rb_intern("RUBY_REVISION")));
+  current_ruby_revision = get_ruby_revision();
   current_ruby_platform = get_ruby_platform();
 
   uncompilable = rb_intern("__bootsnap_uncompilable__");
+  instrumentation_method = rb_intern("_instrument");
+
+  sym_miss = ID2SYM(rb_intern("miss"));
+  rb_global_variable(&sym_miss);
 
+  sym_stale = ID2SYM(rb_intern("stale"));
+  rb_global_variable(&sym_stale);
+
+  rb_define_module_function(rb_mBootsnap, "instrumentation_enabled=", bs_instrumentation_enabled_set, 1);
   rb_define_module_function(rb_mBootsnap_CompileCache_Native, "coverage_running?", bs_rb_coverage_running, 0);
-  rb_define_module_function(rb_mBootsnap_CompileCache_Native, "fetch", bs_rb_fetch, 3);
+  rb_define_module_function(rb_mBootsnap_CompileCache_Native, "fetch", bs_rb_fetch, 4);
+  rb_define_module_function(rb_mBootsnap_CompileCache_Native, "precompile", bs_rb_precompile, 3);
   rb_define_module_function(rb_mBootsnap_CompileCache_Native, "compile_option_crc32=", bs_compile_option_crc32_set, 1);
+
+  current_umask = umask(0777);
+  umask(current_umask);
+}
+
+static VALUE
+bs_instrumentation_enabled_set(VALUE self, VALUE enabled)
+{
+  instrumentation_enabled = RTEST(enabled);
+  return enabled;
 }
 
 /*
@@ -173,7 +210,7 @@ bs_compile_option_crc32_set(VALUE self, VALUE crc32_v)
  *   - 32 bits doesn't feel collision-resistant enough; 64 is nice.
  */
 static uint64_t
-fnv1a_64_iter(uint64_t h, const char *str)
+fnv1a_64_iter_cstr(uint64_t h, const char *str)
 {
   unsigned char *s = (unsigned char *)str;
 
@@ -186,12 +223,46 @@ fnv1a_64_iter(uint64_t h, const char *str)
 }
 
 static uint64_t
-fnv1a_64(const char *str)
+fnv1a_64_iter(uint64_t h, const VALUE str)
+{
+  unsigned char *s = (unsigned char *)RSTRING_PTR(str);
+  unsigned char *str_end = (unsigned char *)RSTRING_PTR(str) + RSTRING_LEN(str);
+
+  while (s < str_end) {
+    h ^= (uint64_t)*s++;
+    h += (h << 1) + (h << 4) + (h << 5) + (h << 7) + (h << 8) + (h << 40);
+  }
+
+  return h;
+}
+
+static uint64_t
+fnv1a_64(const VALUE str)
 {
   uint64_t h = (uint64_t)0xcbf29ce484222325ULL;
   return fnv1a_64_iter(h, str);
 }
 
+/*
+ * Ruby's revision may be Integer or String. CRuby 2.7 or later uses
+ * Git commit ID as revision. It's String.
+ */
+static uint32_t
+get_ruby_revision(void)
+{
+  VALUE ruby_revision;
+
+  ruby_revision = rb_const_get(rb_cObject, rb_intern("RUBY_REVISION"));
+  if (RB_TYPE_P(ruby_revision, RUBY_T_FIXNUM)) {
+    return FIX2INT(ruby_revision);
+  } else {
+    uint64_t hash;
+
+    hash = fnv1a_64(ruby_revision);
+    return (uint32_t)(hash >> 32);
+  }
+}
+
 /*
  * When ruby's version doesn't change, but it's recompiled on a different OS
  * (or OS version), we need to invalidate the cache.
@@ -207,16 +278,19 @@ get_ruby_platform(void)
   VALUE ruby_platform;
 
   ruby_platform = rb_const_get(rb_cObject, rb_intern("RUBY_PLATFORM"));
-  hash = fnv1a_64(RSTRING_PTR(ruby_platform));
+  hash = fnv1a_64(ruby_platform);
 
 #ifdef _WIN32
   return (uint32_t)(hash >> 32) ^ (uint32_t)GetVersion();
+#elif defined(__GLIBC__)
+  hash = fnv1a_64_iter_cstr(hash, gnu_get_libc_version());
+  return (uint32_t)(hash >> 32);
 #else
   struct utsname utsname;
 
   /* Not worth crashing if this fails; lose extra cache invalidation potential */
   if (uname(&utsname) >= 0) {
-    hash = fnv1a_64_iter(hash, utsname.version);
+    hash = fnv1a_64_iter_cstr(hash, utsname.version);
   }
 
   return (uint32_t)(hash >> 32);
@@ -231,14 +305,13 @@ get_ruby_platform(void)
  * The path will look something like: <cachedir>/12/34567890abcdef
  */
 static void
-bs_cache_path(const char * cachedir, const char * path, char ** cache_path)
+bs_cache_path(const char * cachedir, const VALUE path, char (* cache_path)[MAX_CACHEPATH_SIZE])
 {
   uint64_t hash = fnv1a_64(path);
-
   uint8_t first_byte = (hash >> (64 - 8));
   uint64_t remainder = hash & 0x00ffffffffffffff;
 
-  sprintf(*cache_path, "%s/%02x/%014llx", cachedir, first_byte, remainder);
+  sprintf(*cache_path, "%s/%02"PRIx8"/%014"PRIx64, cachedir, first_byte, remainder);
 }
 
 /*
@@ -268,8 +341,10 @@ cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2)
  * conversions on the ruby VALUE arguments before passing them along.
  */
 static VALUE
-bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
+bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler, VALUE args)
 {
+  FilePathValue(path_v);
+
   Check_Type(cachedir_v, T_STRING);
   Check_Type(path_v, T_STRING);
 
@@ -281,27 +356,51 @@ bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
   char * path     = RSTRING_PTR(path_v);
   char cache_path[MAX_CACHEPATH_SIZE];
 
-  { /* generate cache path to cache_path */
-    char * tmp = (char *)&cache_path;
-    bs_cache_path(cachedir, path, &tmp);
-  }
+  /* generate cache path to cache_path */
+  bs_cache_path(cachedir, path_v, &cache_path);
 
-  return bs_fetch(path, path_v, cache_path, handler);
+  return bs_fetch(path, path_v, cache_path, handler, args);
 }
 
+/*
+ * Entrypoint for Bootsnap::CompileCache::Native.precompile.
+ * Similar to fetch, but it only generate the cache if missing
+ * and doesn't return the content.
+ */
+static VALUE
+bs_rb_precompile(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
+{
+  FilePathValue(path_v);
+
+  Check_Type(cachedir_v, T_STRING);
+  Check_Type(path_v, T_STRING);
+
+  if (RSTRING_LEN(cachedir_v) > MAX_CACHEDIR_SIZE) {
+    rb_raise(rb_eArgError, "cachedir too long");
+  }
+
+  char * cachedir = RSTRING_PTR(cachedir_v);
+  char * path     = RSTRING_PTR(path_v);
+  char cache_path[MAX_CACHEPATH_SIZE];
+
+  /* generate cache path to cache_path */
+  bs_cache_path(cachedir, path_v, &cache_path);
+
+  return bs_precompile(path, path_v, cache_path, handler);
+}
 /*
  * Open the file we want to load/cache and generate a cache key for it if it
  * was loaded.
  */
 static int
-open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenance)
+open_current_file(char * path, struct bs_cache_key * key, const char ** errno_provenance)
 {
   struct stat statbuf;
   int fd;
 
   fd = open(path, O_RDONLY);
   if (fd < 0) {
-    *errno_provenance = (char *)"bs_fetch:open_current_file:open";
+    *errno_provenance = "bs_fetch:open_current_file:open";
     return fd;
   }
   #ifdef _WIN32
@@ -309,7 +408,7 @@ open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenan
   #endif
 
   if (fstat(fd, &statbuf) < 0) {
-    *errno_provenance = (char *)"bs_fetch:open_current_file:fstat";
+    *errno_provenance = "bs_fetch:open_current_file:fstat";
     close(fd);
     return -1;
   }
@@ -325,7 +424,8 @@ open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenan
 }
 
 #define ERROR_WITH_ERRNO -1
-#define CACHE_MISSING_OR_INVALID -2
+#define CACHE_MISS -2
+#define CACHE_STALE -3
 
 /*
  * Read the cache key from the given fd, which must have position 0 (e.g.
@@ -333,15 +433,16 @@ open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenan
  *
  * Possible return values:
  *   - 0 (OK, key was loaded)
- *   - CACHE_MISSING_OR_INVALID (-2)
  *   - ERROR_WITH_ERRNO (-1, errno is set)
+ *   - CACHE_MISS (-2)
+ *   - CACHE_STALE (-3)
  */
 static int
 bs_read_key(int fd, struct bs_cache_key * key)
 {
   ssize_t nread = read(fd, key, KEY_SIZE);
   if (nread < 0)        return ERROR_WITH_ERRNO;
-  if (nread < KEY_SIZE) return CACHE_MISSING_OR_INVALID;
+  if (nread < KEY_SIZE) return CACHE_STALE;
   return 0;
 }
 
@@ -351,18 +452,19 @@ bs_read_key(int fd, struct bs_cache_key * key)
  *
  * Possible return values:
  *   - 0 (OK, key was loaded)
- *   - CACHE_MISSING_OR_INVALID (-2)
+ *   - CACHE_MISS (-2)
+ *   - CACHE_STALE (-3)
  *   - ERROR_WITH_ERRNO (-1, errno is set)
  */
 static int
-open_cache_file(const char * path, struct bs_cache_key * key, char ** errno_provenance)
+open_cache_file(const char * path, struct bs_cache_key * key, const char ** errno_provenance)
 {
   int fd, res;
 
   fd = open(path, O_RDONLY);
   if (fd < 0) {
-    *errno_provenance = (char *)"bs_fetch:open_cache_file:open";
-    if (errno == ENOENT) return CACHE_MISSING_OR_INVALID;
+    *errno_provenance = "bs_fetch:open_cache_file:open";
+    if (errno == ENOENT) return CACHE_MISS;
     return ERROR_WITH_ERRNO;
   }
   #ifdef _WIN32
@@ -371,7 +473,7 @@ open_cache_file(const char * path, struct bs_cache_key * key, char ** errno_prov
 
   res = bs_read_key(fd, key);
   if (res < 0) {
-    *errno_provenance = (char *)"bs_fetch:open_cache_file:read";
+    *errno_provenance = "bs_fetch:open_cache_file:read";
     close(fd);
     return res;
   }
@@ -395,7 +497,7 @@ open_cache_file(const char * path, struct bs_cache_key * key, char ** errno_prov
  * or exception, will be the final data returnable to the user.
  */
 static int
-fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data, int * exception_tag, char ** errno_provenance)
+fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE args, VALUE * output_data, int * exception_tag, const char ** errno_provenance)
 {
   char * data = NULL;
   ssize_t nread;
@@ -404,7 +506,7 @@ fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data,
   VALUE storage_data;
 
   if (data_size > 100000000000) {
-    *errno_provenance = (char *)"bs_fetch:fetch_cached_data:datasize";
+    *errno_provenance = "bs_fetch:fetch_cached_data:datasize";
     errno = EINVAL; /* because wtf? */
     ret = -1;
     goto done;
@@ -412,18 +514,18 @@ fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data,
   data = ALLOC_N(char, data_size);
   nread = read(fd, data, data_size);
   if (nread < 0) {
-    *errno_provenance = (char *)"bs_fetch:fetch_cached_data:read";
+    *errno_provenance = "bs_fetch:fetch_cached_data:read";
     ret = -1;
     goto done;
   }
   if (nread != data_size) {
-    ret = CACHE_MISSING_OR_INVALID;
+    ret = CACHE_STALE;
     goto done;
   }
 
-  storage_data = rb_str_new_static(data, data_size);
+  storage_data = rb_str_new(data, data_size);
 
-  *exception_tag = bs_storage_to_output(handler, storage_data, output_data);
+  *exception_tag = bs_storage_to_output(handler, args, storage_data, output_data);
   ret = 0;
 done:
   if (data != NULL) xfree(data);
@@ -464,30 +566,36 @@ mkpath(char * file_path, mode_t mode)
  * path.
  */
 static int
-atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char ** errno_provenance)
+atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, const char ** errno_provenance)
 {
   char template[MAX_CACHEPATH_SIZE + 20];
-  char * dest;
   char * tmp_path;
-  int fd, ret;
+  int fd, ret, attempt;
   ssize_t nwrite;
 
-  dest = strncpy(template, path, MAX_CACHEPATH_SIZE);
-  strcat(dest, ".tmp.XXXXXX");
+  for (attempt = 0; attempt < MAX_CREATE_TEMPFILE_ATTEMPT; ++attempt) {
+    tmp_path = strncpy(template, path, MAX_CACHEPATH_SIZE);
+    strcat(tmp_path, ".tmp.XXXXXX");
 
-  tmp_path = mktemp(template);
-  fd = open(tmp_path, O_WRONLY | O_CREAT, 0664);
-  if (fd < 0) {
-    if (mkpath(path, 0775) < 0) {
-      *errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:mkpath";
-      return -1;
-    }
-    fd = open(tmp_path, O_WRONLY | O_CREAT, 0664);
-    if (fd < 0) {
-      *errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:open";
+    // mkstemp modifies the template to be the actual created path
+    fd = mkstemp(tmp_path);
+    if (fd > 0) break;
+
+    if (attempt == 0 && mkpath(tmp_path, 0775) < 0) {
+      *errno_provenance = "bs_fetch:atomic_write_cache_file:mkpath";
       return -1;
     }
   }
+  if (fd < 0) {
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:mkstemp";
+    return -1;
+  }
+
+  if (chmod(tmp_path, 0644) < 0) {
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:chmod";
+    return -1;
+  }
+
   #ifdef _WIN32
   setmode(fd, O_BINARY);
   #endif
@@ -495,11 +603,11 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char
   key->data_size = RSTRING_LEN(data);
   nwrite = write(fd, key, KEY_SIZE);
   if (nwrite < 0) {
-    *errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:write";
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:write";
     return -1;
   }
   if (nwrite != KEY_SIZE) {
-    *errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:keysize";
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:keysize";
     errno = EIO; /* Lies but whatever */
     return -1;
   }
@@ -507,7 +615,7 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char
   nwrite = write(fd, RSTRING_PTR(data), RSTRING_LEN(data));
   if (nwrite < 0) return -1;
   if (nwrite != RSTRING_LEN(data)) {
-    *errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:writelength";
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:writelength";
     errno = EIO; /* Lies but whatever */
     return -1;
   }
@@ -515,38 +623,27 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char
   close(fd);
   ret = rename(tmp_path, path);
   if (ret < 0) {
-    *errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:rename";
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:rename";
+    return -1;
   }
-  return ret;
-}
-
-/*
- * Given an errno value (converted to a ruby Fixnum), return the corresponding
- * Errno::* constant. If none is found, return StandardError instead.
- */
-static VALUE
-prot_exception_for_errno(VALUE err)
-{
-  if (err != INT2FIX(0)) {
-    VALUE mErrno = rb_const_get(rb_cObject, rb_intern("Errno"));
-    VALUE constants = rb_funcall(mErrno, rb_intern("constants"), 0);
-    VALUE which = rb_funcall(constants, rb_intern("[]"), 1, err);
-    return rb_funcall(mErrno, rb_intern("const_get"), 1, which);
+  ret = chmod(path, 0664 & ~current_umask);
+  if (ret < 0) {
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:chmod";
   }
-  return rb_eStandardError;
+  return ret;
 }
 
 
 /* Read contents from an fd, whose contents are asserted to be +size+ bytes
  * long, into a buffer */
 static ssize_t
-bs_read_contents(int fd, size_t size, char ** contents, char ** errno_provenance)
+bs_read_contents(int fd, size_t size, char ** contents, const char ** errno_provenance)
 {
   ssize_t nread;
   *contents = ALLOC_N(char, size);
   nread = read(fd, *contents, size);
   if (nread < 0) {
-    *errno_provenance = (char *)"bs_fetch:bs_read_contents:read";
+    *errno_provenance = "bs_fetch:bs_read_contents:read";
   }
   return nread;
 }
@@ -596,13 +693,13 @@ bs_read_contents(int fd, size_t size, char ** contents, char ** errno_provenance
  *   - Return storage_to_output(storage_data)
  */
 static VALUE
-bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
+bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args)
 {
   struct bs_cache_key cached_key, current_key;
   char * contents = NULL;
   int cache_fd = -1, current_fd = -1;
   int res, valid_cache = 0, exception_tag = 0;
-  char * errno_provenance = NULL;
+  const char * errno_provenance = NULL;
 
   VALUE input_data;   /* data read from source file, e.g. YAML or ruby source */
   VALUE storage_data; /* compiled data, e.g. msgpack / binary iseq */
@@ -616,26 +713,34 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
 
   /* Open the cache key if it exists, and read its cache key in */
   cache_fd = open_cache_file(cache_path, &cached_key, &errno_provenance);
-  if (cache_fd == CACHE_MISSING_OR_INVALID) {
+  if (cache_fd == CACHE_MISS || cache_fd == CACHE_STALE) {
     /* This is ok: valid_cache remains false, we re-populate it. */
+    if (RB_UNLIKELY(instrumentation_enabled)) {
+      rb_funcall(rb_mBootsnap, instrumentation_method, 2, cache_fd == CACHE_MISS ? sym_miss : sym_stale, path_v);
+    }
   } else if (cache_fd < 0) {
     goto fail_errno;
   } else {
     /* True if the cache existed and no invalidating changes have occurred since
      * it was generated. */
     valid_cache = cache_key_equal(&current_key, &cached_key);
+    if (RB_UNLIKELY(instrumentation_enabled)) {
+      if (!valid_cache) {
+        rb_funcall(rb_mBootsnap, instrumentation_method, 2, sym_stale, path_v);
+      }
+    }
   }
 
   if (valid_cache) {
     /* Fetch the cache data and return it if we're able to load it successfully */
     res = fetch_cached_data(
-      cache_fd, (ssize_t)cached_key.data_size, handler,
+      cache_fd, (ssize_t)cached_key.data_size, handler, args,
       &output_data, &exception_tag, &errno_provenance
     );
-    if (exception_tag != 0)                   goto raise;
-    else if (res == CACHE_MISSING_OR_INVALID) valid_cache = 0;
-    else if (res == ERROR_WITH_ERRNO)         goto fail_errno;
-    else if (!NIL_P(output_data))             goto succeed; /* fast-path, goal */
+    if (exception_tag != 0) goto raise;
+    else if (res == CACHE_MISS || res == CACHE_STALE) valid_cache = 0;
+    else if (res == ERROR_WITH_ERRNO) goto fail_errno;
+    else if (!NIL_P(output_data)) goto succeed; /* fast-path, goal */
   }
   close(cache_fd);
   cache_fd = -1;
@@ -643,15 +748,15 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
 
   /* Read the contents of the source file into a buffer */
   if (bs_read_contents(current_fd, current_key.size, &contents, &errno_provenance) < 0) goto fail_errno;
-  input_data = rb_str_new_static(contents, current_key.size);
+  input_data = rb_str_new(contents, current_key.size);
 
   /* Try to compile the input_data using input_to_storage(input_data) */
-  exception_tag = bs_input_to_storage(handler, input_data, path_v, &storage_data);
+  exception_tag = bs_input_to_storage(handler, args, input_data, path_v, &storage_data);
   if (exception_tag != 0) goto raise;
   /* If input_to_storage raised Bootsnap::CompileCache::Uncompilable, don't try
    * to cache anything; just return input_to_output(input_data) */
   if (storage_data == uncompilable) {
-    bs_input_to_output(handler, input_data, &output_data, &exception_tag);
+    bs_input_to_output(handler, args, input_data, &output_data, &exception_tag);
     if (exception_tag != 0) goto raise;
     goto succeed;
   }
@@ -663,17 +768,17 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
   if (res < 0) goto fail_errno;
 
   /* Having written the cache, now convert storage_data to output_data */
-  exception_tag = bs_storage_to_output(handler, storage_data, &output_data);
+  exception_tag = bs_storage_to_output(handler, args, storage_data, &output_data);
   if (exception_tag != 0) goto raise;
 
   /* If output_data is nil, delete the cache entry and generate the output
    * using input_to_output */
   if (NIL_P(output_data)) {
     if (unlink(cache_path) < 0) {
-      errno_provenance = (char *)"bs_fetch:unlink";
+      errno_provenance = "bs_fetch:unlink";
       goto fail_errno;
     }
-    bs_input_to_output(handler, input_data, &output_data, &exception_tag);
+    bs_input_to_output(handler, args, input_data, &output_data, &exception_tag);
     if (exception_tag != 0) goto raise;
   }
 
@@ -689,11 +794,7 @@ succeed:
   return output_data;
 fail_errno:
   CLEANUP;
-  exception = rb_protect(prot_exception_for_errno, INT2FIX(errno), &res);
-  if (res) exception = rb_eStandardError;
-  if (errno_provenance != NULL) {
-    exception = rb_exc_new_str(exception, rb_str_new2(errno_provenance));
-  }
+  exception = rb_syserr_new(errno, errno_provenance);
   rb_exc_raise(exception);
   __builtin_unreachable();
 raise:
@@ -708,6 +809,79 @@ invalid_type_storage_data:
 #undef CLEANUP
 }
 
+static VALUE
+bs_precompile(char * path, VALUE path_v, char * cache_path, VALUE handler)
+{
+  struct bs_cache_key cached_key, current_key;
+  char * contents = NULL;
+  int cache_fd = -1, current_fd = -1;
+  int res, valid_cache = 0, exception_tag = 0;
+  const char * errno_provenance = NULL;
+
+  VALUE input_data;   /* data read from source file, e.g. YAML or ruby source */
+  VALUE storage_data; /* compiled data, e.g. msgpack / binary iseq */
+
+  /* Open the source file and generate a cache key for it */
+  current_fd = open_current_file(path, &current_key, &errno_provenance);
+  if (current_fd < 0) goto fail;
+
+  /* Open the cache key if it exists, and read its cache key in */
+  cache_fd = open_cache_file(cache_path, &cached_key, &errno_provenance);
+  if (cache_fd == CACHE_MISS || cache_fd == CACHE_STALE) {
+    /* This is ok: valid_cache remains false, we re-populate it. */
+  } else if (cache_fd < 0) {
+    goto fail;
+  } else {
+    /* True if the cache existed and no invalidating changes have occurred since
+     * it was generated. */
+    valid_cache = cache_key_equal(&current_key, &cached_key);
+  }
+
+  if (valid_cache) {
+    goto succeed;
+  }
+
+  close(cache_fd);
+  cache_fd = -1;
+  /* Cache is stale, invalid, or missing. Regenerate and write it out. */
+
+  /* Read the contents of the source file into a buffer */
+  if (bs_read_contents(current_fd, current_key.size, &contents, &errno_provenance) < 0) goto fail;
+  input_data = rb_str_new(contents, current_key.size);
+
+  /* Try to compile the input_data using input_to_storage(input_data) */
+  exception_tag = bs_input_to_storage(handler, Qnil, input_data, path_v, &storage_data);
+  if (exception_tag != 0) goto fail;
+
+  /* If input_to_storage raised Bootsnap::CompileCache::Uncompilable, don't try
+   * to cache anything; just return false */
+  if (storage_data == uncompilable) {
+    goto fail;
+  }
+  /* If storage_data isn't a string, we can't cache it */
+  if (!RB_TYPE_P(storage_data, T_STRING)) goto fail;
+
+  /* Write the cache key and storage_data to the cache directory */
+  res = atomic_write_cache_file(cache_path, &current_key, storage_data, &errno_provenance);
+  if (res < 0) goto fail;
+
+  goto succeed;
+
+#define CLEANUP \
+  if (contents != NULL) xfree(contents);   \
+  if (current_fd >= 0)  close(current_fd); \
+  if (cache_fd >= 0)    close(cache_fd);
+
+succeed:
+  CLEANUP;
+  return Qtrue;
+fail:
+  CLEANUP;
+  return Qfalse;
+#undef CLEANUP
+}
+
+
 /*****************************************************************************/
 /********************* Handler Wrappers **************************************/
 /*****************************************************************************
@@ -727,11 +901,13 @@ invalid_type_storage_data:
 
 struct s2o_data {
   VALUE handler;
+  VALUE args;
   VALUE storage_data;
 };
 
 struct i2o_data {
   VALUE handler;
+  VALUE args;
   VALUE input_data;
 };
 
@@ -745,15 +921,16 @@ static VALUE
 prot_storage_to_output(VALUE arg)
 {
   struct s2o_data * data = (struct s2o_data *)arg;
-  return rb_funcall(data->handler, rb_intern("storage_to_output"), 1, data->storage_data);
+  return rb_funcall(data->handler, rb_intern("storage_to_output"), 2, data->storage_data, data->args);
 }
 
 static int
-bs_storage_to_output(VALUE handler, VALUE storage_data, VALUE * output_data)
+bs_storage_to_output(VALUE handler, VALUE args, VALUE storage_data, VALUE * output_data)
 {
   int state;
   struct s2o_data s2o_data = {
     .handler      = handler,
+    .args         = args,
     .storage_data = storage_data,
   };
   *output_data = rb_protect(prot_storage_to_output, (VALUE)&s2o_data, &state);
@@ -761,10 +938,11 @@ bs_storage_to_output(VALUE handler, VALUE storage_data, VALUE * output_data)
 }
 
 static void
-bs_input_to_output(VALUE handler, VALUE input_data, VALUE * output_data, int * exception_tag)
+bs_input_to_output(VALUE handler, VALUE args, VALUE input_data, VALUE * output_data, int * exception_tag)
 {
   struct i2o_data i2o_data = {
     .handler    = handler,
+    .args       = args,
     .input_data = input_data,
   };
   *output_data = rb_protect(prot_input_to_output, (VALUE)&i2o_data, exception_tag);
@@ -774,7 +952,7 @@ static VALUE
 prot_input_to_output(VALUE arg)
 {
   struct i2o_data * data = (struct i2o_data *)arg;
-  return rb_funcall(data->handler, rb_intern("input_to_output"), 1, data->input_data);
+  return rb_funcall(data->handler, rb_intern("input_to_output"), 2, data->input_data, data->args);
 }
 
 static VALUE
@@ -785,7 +963,7 @@ try_input_to_storage(VALUE arg)
 }
 
 static VALUE
-rescue_input_to_storage(VALUE arg)
+rescue_input_to_storage(VALUE arg, VALUE e)
 {
   return uncompilable;
 }
@@ -801,7 +979,7 @@ prot_input_to_storage(VALUE arg)
 }
 
 static int
-bs_input_to_storage(VALUE handler, VALUE input_data, VALUE pathval, VALUE * storage_data)
+bs_input_to_storage(VALUE handler, VALUE args, VALUE input_data, VALUE pathval, VALUE * storage_data)
 {
   int state;
   struct i2s_data i2s_data = {