bootsnap 1.4.5 → 1.16.0

data/ext/bootsnap/bootsnap.c

@@ -14,13 +14,11 @@
 #include "bootsnap.h"
 #include "ruby.h"
 #include <stdint.h>
+#include <stdbool.h>
 #include <sys/types.h>
 #include <errno.h>
 #include <fcntl.h>
 #include <sys/stat.h>
-#ifndef _WIN32
-#include <sys/utsname.h>
-#endif
 
 /* 1000 is an arbitrary limit; FNV64 plus some slashes brings the cap down to
  * 981 for the cache dir */
@@ -29,6 +27,12 @@
 
 #define KEY_SIZE 64
 
+#define MAX_CREATE_TEMPFILE_ATTEMPT 3
+
+#ifndef RB_UNLIKELY
+  #define RB_UNLIKELY(x) (x)
+#endif
+
 /*
  * An instance of this key is written as the first 64 bytes of each cache file.
  * The mtime and size members track whether the file contents have changed, and
@@ -65,7 +69,7 @@ struct bs_cache_key {
 STATIC_ASSERT(sizeof(struct bs_cache_key) == KEY_SIZE);
 
 /* Effectively a schema version. Bumping invalidates all previous caches */
-static const uint32_t current_version = 2;
+static const uint32_t current_version = 4;
 
 /* hash of e.g. "x86_64-darwin17", invalidating when ruby is recompiled on a
  * new OS ABI, etc. */
@@ -81,21 +85,28 @@ static mode_t current_umask;
 static VALUE rb_mBootsnap;
 static VALUE rb_mBootsnap_CompileCache;
 static VALUE rb_mBootsnap_CompileCache_Native;
-static VALUE rb_eBootsnap_CompileCache_Uncompilable;
-static ID uncompilable;
+static VALUE rb_cBootsnap_CompileCache_UNCOMPILABLE;
+static ID instrumentation_method;
+static VALUE sym_miss;
+static VALUE sym_stale;
+static bool instrumentation_enabled = false;
+static bool readonly = false;
 
 /* Functions exposed as module functions on Bootsnap::CompileCache::Native */
+static VALUE bs_instrumentation_enabled_set(VALUE self, VALUE enabled);
+static VALUE bs_readonly_set(VALUE self, VALUE enabled);
 static VALUE bs_compile_option_crc32_set(VALUE self, VALUE crc32_v);
-static VALUE bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler);
+static VALUE bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler, VALUE args);
+static VALUE bs_rb_precompile(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler);
 
 /* Helpers */
-static uint64_t fnv1a_64(const char *str);
-static void bs_cache_path(const char * cachedir, const char * path, char ** cache_path);
+static void bs_cache_path(const char * cachedir, const VALUE path, char (* cache_path)[MAX_CACHEPATH_SIZE]);
 static int bs_read_key(int fd, struct bs_cache_key * key);
 static int cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2);
-static VALUE bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler);
-static int open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenance);
-static int fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data, int * exception_tag, char ** errno_provenance);
+static VALUE bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args);
+static VALUE bs_precompile(char * path, VALUE path_v, char * cache_path, VALUE handler);
+static int open_current_file(char * path, struct bs_cache_key * key, const char ** errno_provenance);
+static int fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE args, VALUE * output_data, int * exception_tag, const char ** errno_provenance);
 static uint32_t get_ruby_revision(void);
 static uint32_t get_ruby_platform(void);
 
@@ -103,12 +114,10 @@ static uint32_t get_ruby_platform(void);
  * Helper functions to call ruby methods on handler object without crashing on
  * exception.
  */
-static int bs_storage_to_output(VALUE handler, VALUE storage_data, VALUE * output_data);
-static VALUE prot_storage_to_output(VALUE arg);
+static int bs_storage_to_output(VALUE handler, VALUE args, VALUE storage_data, VALUE * output_data);
 static VALUE prot_input_to_output(VALUE arg);
-static void bs_input_to_output(VALUE handler, VALUE input_data, VALUE * output_data, int * exception_tag);
-static VALUE prot_input_to_storage(VALUE arg);
-static int bs_input_to_storage(VALUE handler, VALUE input_data, VALUE pathval, VALUE * storage_data);
+static void bs_input_to_output(VALUE handler, VALUE args, VALUE input_data, VALUE * output_data, int * exception_tag);
+static int bs_input_to_storage(VALUE handler, VALUE args, VALUE input_data, VALUE pathval, VALUE * storage_data);
 struct s2o_data;
 struct i2o_data;
 struct i2s_data;
@@ -122,6 +131,12 @@ bs_rb_coverage_running(VALUE self)
   return RTEST(cov) ? Qtrue : Qfalse;
 }
 
+static VALUE
+bs_rb_get_path(VALUE self, VALUE fname)
+{
+    return rb_get_path(fname);
+}
+
 /*
  * Ruby C extensions are initialized by calling Init_<extname>.
  *
@@ -133,23 +148,50 @@ void
 Init_bootsnap(void)
 {
   rb_mBootsnap = rb_define_module("Bootsnap");
+
+  rb_define_singleton_method(rb_mBootsnap, "rb_get_path", bs_rb_get_path, 1);
+
   rb_mBootsnap_CompileCache = rb_define_module_under(rb_mBootsnap, "CompileCache");
   rb_mBootsnap_CompileCache_Native = rb_define_module_under(rb_mBootsnap_CompileCache, "Native");
-  rb_eBootsnap_CompileCache_Uncompilable = rb_define_class_under(rb_mBootsnap_CompileCache, "Uncompilable", rb_eStandardError);
+  rb_cBootsnap_CompileCache_UNCOMPILABLE = rb_const_get(rb_mBootsnap_CompileCache, rb_intern("UNCOMPILABLE"));
+  rb_global_variable(&rb_cBootsnap_CompileCache_UNCOMPILABLE);
 
   current_ruby_revision = get_ruby_revision();
   current_ruby_platform = get_ruby_platform();
 
-  uncompilable = rb_intern("__bootsnap_uncompilable__");
+  instrumentation_method = rb_intern("_instrument");
+
+  sym_miss = ID2SYM(rb_intern("miss"));
+  rb_global_variable(&sym_miss);
+
+  sym_stale = ID2SYM(rb_intern("stale"));
+  rb_global_variable(&sym_stale);
 
+  rb_define_module_function(rb_mBootsnap, "instrumentation_enabled=", bs_instrumentation_enabled_set, 1);
+  rb_define_module_function(rb_mBootsnap_CompileCache_Native, "readonly=", bs_readonly_set, 1);
   rb_define_module_function(rb_mBootsnap_CompileCache_Native, "coverage_running?", bs_rb_coverage_running, 0);
-  rb_define_module_function(rb_mBootsnap_CompileCache_Native, "fetch", bs_rb_fetch, 3);
+  rb_define_module_function(rb_mBootsnap_CompileCache_Native, "fetch", bs_rb_fetch, 4);
+  rb_define_module_function(rb_mBootsnap_CompileCache_Native, "precompile", bs_rb_precompile, 3);
   rb_define_module_function(rb_mBootsnap_CompileCache_Native, "compile_option_crc32=", bs_compile_option_crc32_set, 1);
 
   current_umask = umask(0777);
   umask(current_umask);
 }
 
+static VALUE
+bs_instrumentation_enabled_set(VALUE self, VALUE enabled)
+{
+  instrumentation_enabled = RTEST(enabled);
+  return enabled;
+}
+
+static VALUE
+bs_readonly_set(VALUE self, VALUE enabled)
+{
+  readonly = RTEST(enabled);
+  return enabled;
+}
+
 /*
  * Bootsnap's ruby code registers a hook that notifies us via this function
  * when compile_option changes. These changes invalidate all existing caches.
@@ -167,22 +209,13 @@ bs_compile_option_crc32_set(VALUE self, VALUE crc32_v)
   return Qnil;
 }
 
-/*
- * We use FNV1a-64 to derive cache paths. The choice is somewhat arbitrary but
- * it has several nice properties:
- *
- *   - Tiny implementation
- *   - No external dependency
- *   - Solid performance
- *   - Solid randomness
- *   - 32 bits doesn't feel collision-resistant enough; 64 is nice.
- */
 static uint64_t
-fnv1a_64_iter(uint64_t h, const char *str)
+fnv1a_64_iter(uint64_t h, const VALUE str)
 {
-  unsigned char *s = (unsigned char *)str;
+  unsigned char *s = (unsigned char *)RSTRING_PTR(str);
+  unsigned char *str_end = (unsigned char *)RSTRING_PTR(str) + RSTRING_LEN(str);
 
-  while (*s) {
+  while (s < str_end) {
     h ^= (uint64_t)*s++;
     h += (h << 1) + (h << 4) + (h << 5) + (h << 7) + (h << 8) + (h << 40);
   }
@@ -191,7 +224,7 @@ fnv1a_64_iter(uint64_t h, const char *str)
 }
 
 static uint64_t
-fnv1a_64(const char *str)
+fnv1a_64(const VALUE str)
 {
   uint64_t h = (uint64_t)0xcbf29ce484222325ULL;
   return fnv1a_64_iter(h, str);
@@ -212,7 +245,7 @@ get_ruby_revision(void)
   } else {
     uint64_t hash;
 
-    hash = fnv1a_64(StringValueCStr(ruby_revision));
+    hash = fnv1a_64(ruby_revision);
     return (uint32_t)(hash >> 32);
   }
 }
@@ -220,10 +253,6 @@ get_ruby_revision(void)
 /*
  * When ruby's version doesn't change, but it's recompiled on a different OS
  * (or OS version), we need to invalidate the cache.
- *
- * We actually factor in some extra information here, to be extra confident
- * that we don't try to re-use caches that will not be compatible, by factoring
- * in utsname.version.
  */
 static uint32_t
 get_ruby_platform(void)
@@ -232,20 +261,8 @@ get_ruby_platform(void)
   VALUE ruby_platform;
 
   ruby_platform = rb_const_get(rb_cObject, rb_intern("RUBY_PLATFORM"));
-  hash = fnv1a_64(RSTRING_PTR(ruby_platform));
-
-#ifdef _WIN32
-  return (uint32_t)(hash >> 32) ^ (uint32_t)GetVersion();
-#else
-  struct utsname utsname;
-
-  /* Not worth crashing if this fails; lose extra cache invalidation potential */
-  if (uname(&utsname) >= 0) {
-    hash = fnv1a_64_iter(hash, utsname.version);
-  }
-
+  hash = fnv1a_64(ruby_platform);
   return (uint32_t)(hash >> 32);
-#endif
 }
 
 /*
@@ -256,14 +273,13 @@ get_ruby_platform(void)
  * The path will look something like: <cachedir>/12/34567890abcdef
  */
 static void
-bs_cache_path(const char * cachedir, const char * path, char ** cache_path)
+bs_cache_path(const char * cachedir, const VALUE path, char (* cache_path)[MAX_CACHEPATH_SIZE])
 {
   uint64_t hash = fnv1a_64(path);
-
   uint8_t first_byte = (hash >> (64 - 8));
   uint64_t remainder = hash & 0x00ffffffffffffff;
 
-  sprintf(*cache_path, "%s/%02x/%014llx", cachedir, first_byte, remainder);
+  sprintf(*cache_path, "%s/%02"PRIx8"/%014"PRIx64, cachedir, first_byte, remainder);
 }
 
 /*
@@ -293,7 +309,7 @@ cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2)
  * conversions on the ruby VALUE arguments before passing them along.
  */
 static VALUE
-bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
+bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler, VALUE args)
 {
   FilePathValue(path_v);
 
@@ -308,27 +324,51 @@ bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
   char * path     = RSTRING_PTR(path_v);
   char cache_path[MAX_CACHEPATH_SIZE];
 
-  { /* generate cache path to cache_path */
-    char * tmp = (char *)&cache_path;
-    bs_cache_path(cachedir, path, &tmp);
-  }
+  /* generate cache path to cache_path */
+  bs_cache_path(cachedir, path_v, &cache_path);
 
-  return bs_fetch(path, path_v, cache_path, handler);
+  return bs_fetch(path, path_v, cache_path, handler, args);
 }
 
+/*
+ * Entrypoint for Bootsnap::CompileCache::Native.precompile.
+ * Similar to fetch, but it only generate the cache if missing
+ * and doesn't return the content.
+ */
+static VALUE
+bs_rb_precompile(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
+{
+  FilePathValue(path_v);
+
+  Check_Type(cachedir_v, T_STRING);
+  Check_Type(path_v, T_STRING);
+
+  if (RSTRING_LEN(cachedir_v) > MAX_CACHEDIR_SIZE) {
+    rb_raise(rb_eArgError, "cachedir too long");
+  }
+
+  char * cachedir = RSTRING_PTR(cachedir_v);
+  char * path     = RSTRING_PTR(path_v);
+  char cache_path[MAX_CACHEPATH_SIZE];
+
+  /* generate cache path to cache_path */
+  bs_cache_path(cachedir, path_v, &cache_path);
+
+  return bs_precompile(path, path_v, cache_path, handler);
+}
 /*
  * Open the file we want to load/cache and generate a cache key for it if it
  * was loaded.
  */
 static int
-open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenance)
+open_current_file(char * path, struct bs_cache_key * key, const char ** errno_provenance)
 {
   struct stat statbuf;
   int fd;
 
   fd = open(path, O_RDONLY);
   if (fd < 0) {
-    *errno_provenance = (char *)"bs_fetch:open_current_file:open";
+    *errno_provenance = "bs_fetch:open_current_file:open";
     return fd;
   }
   #ifdef _WIN32
@@ -336,7 +376,7 @@ open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenan
   #endif
 
   if (fstat(fd, &statbuf) < 0) {
-    *errno_provenance = (char *)"bs_fetch:open_current_file:fstat";
+    *errno_provenance = "bs_fetch:open_current_file:fstat";
     close(fd);
     return -1;
   }
@@ -352,7 +392,9 @@ open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenan
 }
 
 #define ERROR_WITH_ERRNO -1
-#define CACHE_MISSING_OR_INVALID -2
+#define CACHE_MISS -2
+#define CACHE_STALE -3
+#define CACHE_UNCOMPILABLE -4
 
 /*
  * Read the cache key from the given fd, which must have position 0 (e.g.
@@ -360,15 +402,16 @@ open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenan
  *
  * Possible return values:
  *   - 0 (OK, key was loaded)
- *   - CACHE_MISSING_OR_INVALID (-2)
  *   - ERROR_WITH_ERRNO (-1, errno is set)
+ *   - CACHE_MISS (-2)
+ *   - CACHE_STALE (-3)
  */
 static int
 bs_read_key(int fd, struct bs_cache_key * key)
 {
   ssize_t nread = read(fd, key, KEY_SIZE);
   if (nread < 0)        return ERROR_WITH_ERRNO;
-  if (nread < KEY_SIZE) return CACHE_MISSING_OR_INVALID;
+  if (nread < KEY_SIZE) return CACHE_STALE;
   return 0;
 }
 
@@ -378,19 +421,19 @@ bs_read_key(int fd, struct bs_cache_key * key)
  *
  * Possible return values:
  *   - 0 (OK, key was loaded)
- *   - CACHE_MISSING_OR_INVALID (-2)
+ *   - CACHE_MISS (-2)
+ *   - CACHE_STALE (-3)
  *   - ERROR_WITH_ERRNO (-1, errno is set)
  */
 static int
-open_cache_file(const char * path, struct bs_cache_key * key, char ** errno_provenance)
+open_cache_file(const char * path, struct bs_cache_key * key, const char ** errno_provenance)
 {
   int fd, res;
 
   fd = open(path, O_RDONLY);
   if (fd < 0) {
-    *errno_provenance = (char *)"bs_fetch:open_cache_file:open";
-    if (errno == ENOENT) return CACHE_MISSING_OR_INVALID;
-    return ERROR_WITH_ERRNO;
+    *errno_provenance = "bs_fetch:open_cache_file:open";
+    return CACHE_MISS;
   }
   #ifdef _WIN32
   setmode(fd, O_BINARY);
@@ -398,7 +441,7 @@ open_cache_file(const char * path, struct bs_cache_key * key, char ** errno_prov
 
   res = bs_read_key(fd, key);
   if (res < 0) {
-    *errno_provenance = (char *)"bs_fetch:open_cache_file:read";
+    *errno_provenance = "bs_fetch:open_cache_file:read";
     close(fd);
     return res;
   }
@@ -422,7 +465,7 @@ open_cache_file(const char * path, struct bs_cache_key * key, char ** errno_prov
  * or exception, will be the final data returnable to the user.
  */
 static int
-fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data, int * exception_tag, char ** errno_provenance)
+fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE args, VALUE * output_data, int * exception_tag, const char ** errno_provenance)
 {
   char * data = NULL;
   ssize_t nread;
@@ -431,26 +474,30 @@ fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data,
   VALUE storage_data;
 
   if (data_size > 100000000000) {
-    *errno_provenance = (char *)"bs_fetch:fetch_cached_data:datasize";
+    *errno_provenance = "bs_fetch:fetch_cached_data:datasize";
     errno = EINVAL; /* because wtf? */
-    ret = -1;
+    ret = ERROR_WITH_ERRNO;
     goto done;
   }
   data = ALLOC_N(char, data_size);
   nread = read(fd, data, data_size);
   if (nread < 0) {
-    *errno_provenance = (char *)"bs_fetch:fetch_cached_data:read";
-    ret = -1;
+    *errno_provenance = "bs_fetch:fetch_cached_data:read";
+    ret = ERROR_WITH_ERRNO;
     goto done;
   }
   if (nread != data_size) {
-    ret = CACHE_MISSING_OR_INVALID;
+    ret = CACHE_STALE;
     goto done;
   }
 
-  storage_data = rb_str_new_static(data, data_size);
+  storage_data = rb_str_new(data, data_size);
 
-  *exception_tag = bs_storage_to_output(handler, storage_data, output_data);
+  *exception_tag = bs_storage_to_output(handler, args, storage_data, output_data);
+  if (*output_data == rb_cBootsnap_CompileCache_UNCOMPILABLE) {
+    ret = CACHE_UNCOMPILABLE;
+    goto done;
+  }
   ret = 0;
 done:
   if (data != NULL) xfree(data);
@@ -491,29 +538,36 @@ mkpath(char * file_path, mode_t mode)
  * path.
  */
 static int
-atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char ** errno_provenance)
+atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, const char ** errno_provenance)
 {
   char template[MAX_CACHEPATH_SIZE + 20];
   char * tmp_path;
-  int fd, ret;
+  int fd, ret, attempt;
   ssize_t nwrite;
 
-  tmp_path = strncpy(template, path, MAX_CACHEPATH_SIZE);
-  strcat(tmp_path, ".tmp.XXXXXX");
+  for (attempt = 0; attempt < MAX_CREATE_TEMPFILE_ATTEMPT; ++attempt) {
+    tmp_path = strncpy(template, path, MAX_CACHEPATH_SIZE);
+    strcat(tmp_path, ".tmp.XXXXXX");
 
-  // mkstemp modifies the template to be the actual created path
-  fd = mkstemp(tmp_path);
-  if (fd < 0) {
-    if (mkpath(tmp_path, 0775) < 0) {
-      *errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:mkpath";
-      return -1;
-    }
-    fd = open(tmp_path, O_WRONLY | O_CREAT, 0664);
-    if (fd < 0) {
-      *errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:open";
+    // mkstemp modifies the template to be the actual created path
+    fd = mkstemp(tmp_path);
+    if (fd > 0) break;
+
+    if (attempt == 0 && mkpath(tmp_path, 0775) < 0) {
+      *errno_provenance = "bs_fetch:atomic_write_cache_file:mkpath";
       return -1;
     }
   }
+  if (fd < 0) {
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:mkstemp";
+    return -1;
+  }
+
+  if (chmod(tmp_path, 0644) < 0) {
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:chmod";
+    return -1;
+  }
+
   #ifdef _WIN32
   setmode(fd, O_BINARY);
   #endif
@@ -521,11 +575,11 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char
   key->data_size = RSTRING_LEN(data);
   nwrite = write(fd, key, KEY_SIZE);
   if (nwrite < 0) {
-    *errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:write";
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:write";
     return -1;
   }
   if (nwrite != KEY_SIZE) {
-    *errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:keysize";
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:keysize";
     errno = EIO; /* Lies but whatever */
     return -1;
   }
@@ -533,7 +587,7 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char
   nwrite = write(fd, RSTRING_PTR(data), RSTRING_LEN(data));
   if (nwrite < 0) return -1;
   if (nwrite != RSTRING_LEN(data)) {
-    *errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:writelength";
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:writelength";
     errno = EIO; /* Lies but whatever */
     return -1;
   }
@@ -541,12 +595,12 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char
   close(fd);
   ret = rename(tmp_path, path);
   if (ret < 0) {
-    *errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:rename";
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:rename";
     return -1;
   }
   ret = chmod(path, 0664 & ~current_umask);
   if (ret < 0) {
-    *errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:chmod";
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:chmod";
   }
   return ret;
 }
@@ -555,13 +609,13 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char
 /* Read contents from an fd, whose contents are asserted to be +size+ bytes
  * long, into a buffer */
 static ssize_t
-bs_read_contents(int fd, size_t size, char ** contents, char ** errno_provenance)
+bs_read_contents(int fd, size_t size, char ** contents, const char ** errno_provenance)
 {
   ssize_t nread;
   *contents = ALLOC_N(char, size);
   nread = read(fd, *contents, size);
   if (nread < 0) {
-    *errno_provenance = (char *)"bs_fetch:bs_read_contents:read";
+    *errno_provenance = "bs_fetch:bs_read_contents:read";
   }
   return nread;
 }
@@ -611,13 +665,13 @@ bs_read_contents(int fd, size_t size, char ** contents, char ** errno_provenance
  *   - Return storage_to_output(storage_data)
  */
 static VALUE
-bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
+bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args)
 {
   struct bs_cache_key cached_key, current_key;
   char * contents = NULL;
   int cache_fd = -1, current_fd = -1;
   int res, valid_cache = 0, exception_tag = 0;
-  char * errno_provenance = NULL;
+  const char * errno_provenance = NULL;
 
   VALUE input_data;   /* data read from source file, e.g. YAML or ruby source */
   VALUE storage_data; /* compiled data, e.g. msgpack / binary iseq */
@@ -631,26 +685,42 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
 
   /* Open the cache key if it exists, and read its cache key in */
   cache_fd = open_cache_file(cache_path, &cached_key, &errno_provenance);
-  if (cache_fd == CACHE_MISSING_OR_INVALID) {
+  if (cache_fd == CACHE_MISS || cache_fd == CACHE_STALE) {
     /* This is ok: valid_cache remains false, we re-populate it. */
+    if (RB_UNLIKELY(instrumentation_enabled)) {
+      rb_funcall(rb_mBootsnap, instrumentation_method, 2, cache_fd == CACHE_MISS ? sym_miss : sym_stale, path_v);
+    }
   } else if (cache_fd < 0) {
     goto fail_errno;
   } else {
     /* True if the cache existed and no invalidating changes have occurred since
      * it was generated. */
     valid_cache = cache_key_equal(&current_key, &cached_key);
+    if (RB_UNLIKELY(instrumentation_enabled)) {
+      if (!valid_cache) {
+        rb_funcall(rb_mBootsnap, instrumentation_method, 2, sym_stale, path_v);
+      }
+    }
   }
 
   if (valid_cache) {
     /* Fetch the cache data and return it if we're able to load it successfully */
     res = fetch_cached_data(
-      cache_fd, (ssize_t)cached_key.data_size, handler,
+      cache_fd, (ssize_t)cached_key.data_size, handler, args,
       &output_data, &exception_tag, &errno_provenance
     );
-    if (exception_tag != 0)                   goto raise;
-    else if (res == CACHE_MISSING_OR_INVALID) valid_cache = 0;
-    else if (res == ERROR_WITH_ERRNO)         goto fail_errno;
-    else if (!NIL_P(output_data))             goto succeed; /* fast-path, goal */
+    if (exception_tag != 0) goto raise;
+    else if (res == CACHE_UNCOMPILABLE) {
+      /* If fetch_cached_data returned `Uncompilable` we fallback to `input_to_output`
+        This happens if we have say, an unsafe YAML cache, but try to load it in safe mode */
+      if (bs_read_contents(current_fd, current_key.size, &contents, &errno_provenance) < 0) goto fail_errno;
+      input_data = rb_str_new(contents, current_key.size);
+      bs_input_to_output(handler, args, input_data, &output_data, &exception_tag);
+      if (exception_tag != 0) goto raise;
+      goto succeed;
+    } else if (res == CACHE_MISS || res == CACHE_STALE) valid_cache = 0;
+    else if (res == ERROR_WITH_ERRNO) goto fail_errno;
+    else if (!NIL_P(output_data)) goto succeed; /* fast-path, goal */
   }
   close(cache_fd);
   cache_fd = -1;
@@ -658,37 +728,47 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
 
   /* Read the contents of the source file into a buffer */
   if (bs_read_contents(current_fd, current_key.size, &contents, &errno_provenance) < 0) goto fail_errno;
-  input_data = rb_str_new_static(contents, current_key.size);
+  input_data = rb_str_new(contents, current_key.size);
 
   /* Try to compile the input_data using input_to_storage(input_data) */
-  exception_tag = bs_input_to_storage(handler, input_data, path_v, &storage_data);
+  exception_tag = bs_input_to_storage(handler, args, input_data, path_v, &storage_data);
   if (exception_tag != 0) goto raise;
   /* If input_to_storage raised Bootsnap::CompileCache::Uncompilable, don't try
    * to cache anything; just return input_to_output(input_data) */
-  if (storage_data == uncompilable) {
-    bs_input_to_output(handler, input_data, &output_data, &exception_tag);
+  if (storage_data == rb_cBootsnap_CompileCache_UNCOMPILABLE) {
+    bs_input_to_output(handler, args, input_data, &output_data, &exception_tag);
     if (exception_tag != 0) goto raise;
     goto succeed;
   }
   /* If storage_data isn't a string, we can't cache it */
   if (!RB_TYPE_P(storage_data, T_STRING)) goto invalid_type_storage_data;
 
-  /* Write the cache key and storage_data to the cache directory */
-  res = atomic_write_cache_file(cache_path, &current_key, storage_data, &errno_provenance);
-  if (res < 0) goto fail_errno;
+  /* Attempt to write the cache key and storage_data to the cache directory.
+   * We do however ignore any failures to persist the cache, as it's better
+   * to move along, than to interrupt the process.
+   */
+  atomic_write_cache_file(cache_path, &current_key, storage_data, &errno_provenance);
 
   /* Having written the cache, now convert storage_data to output_data */
-  exception_tag = bs_storage_to_output(handler, storage_data, &output_data);
+  exception_tag = bs_storage_to_output(handler, args, storage_data, &output_data);
   if (exception_tag != 0) goto raise;
 
-  /* If output_data is nil, delete the cache entry and generate the output
-   * using input_to_output */
-  if (NIL_P(output_data)) {
+  if (output_data == rb_cBootsnap_CompileCache_UNCOMPILABLE) {
+    /* If storage_to_output returned `Uncompilable` we fallback to `input_to_output` */
+    bs_input_to_output(handler, args, input_data, &output_data, &exception_tag);
+    if (exception_tag != 0) goto raise;
+  } else if (NIL_P(output_data)) {
+    /* If output_data is nil, delete the cache entry and generate the output
+     * using input_to_output */
     if (unlink(cache_path) < 0) {
-      errno_provenance = (char *)"bs_fetch:unlink";
-      goto fail_errno;
+      /* If the cache was already deleted, it might be that another process did it before us.
+      * No point raising an error */
+      if (errno != ENOENT) {
+        errno_provenance = "bs_fetch:unlink";
+        goto fail_errno;
+      }
     }
-    bs_input_to_output(handler, input_data, &output_data, &exception_tag);
+    bs_input_to_output(handler, args, input_data, &output_data, &exception_tag);
     if (exception_tag != 0) goto raise;
   }
 
@@ -719,6 +799,79 @@ invalid_type_storage_data:
 #undef CLEANUP
 }
 
+static VALUE
+bs_precompile(char * path, VALUE path_v, char * cache_path, VALUE handler)
+{
+  struct bs_cache_key cached_key, current_key;
+  char * contents = NULL;
+  int cache_fd = -1, current_fd = -1;
+  int res, valid_cache = 0, exception_tag = 0;
+  const char * errno_provenance = NULL;
+
+  VALUE input_data;   /* data read from source file, e.g. YAML or ruby source */
+  VALUE storage_data; /* compiled data, e.g. msgpack / binary iseq */
+
+  /* Open the source file and generate a cache key for it */
+  current_fd = open_current_file(path, &current_key, &errno_provenance);
+  if (current_fd < 0) goto fail;
+
+  /* Open the cache key if it exists, and read its cache key in */
+  cache_fd = open_cache_file(cache_path, &cached_key, &errno_provenance);
+  if (cache_fd == CACHE_MISS || cache_fd == CACHE_STALE) {
+    /* This is ok: valid_cache remains false, we re-populate it. */
+  } else if (cache_fd < 0) {
+    goto fail;
+  } else {
+    /* True if the cache existed and no invalidating changes have occurred since
+     * it was generated. */
+    valid_cache = cache_key_equal(&current_key, &cached_key);
+  }
+
+  if (valid_cache) {
+    goto succeed;
+  }
+
+  close(cache_fd);
+  cache_fd = -1;
+  /* Cache is stale, invalid, or missing. Regenerate and write it out. */
+
+  /* Read the contents of the source file into a buffer */
+  if (bs_read_contents(current_fd, current_key.size, &contents, &errno_provenance) < 0) goto fail;
+  input_data = rb_str_new(contents, current_key.size);
+
+  /* Try to compile the input_data using input_to_storage(input_data) */
+  exception_tag = bs_input_to_storage(handler, Qnil, input_data, path_v, &storage_data);
+  if (exception_tag != 0) goto fail;
+
+  /* If input_to_storage raised Bootsnap::CompileCache::Uncompilable, don't try
+   * to cache anything; just return false */
+  if (storage_data == rb_cBootsnap_CompileCache_UNCOMPILABLE) {
+    goto fail;
+  }
+  /* If storage_data isn't a string, we can't cache it */
+  if (!RB_TYPE_P(storage_data, T_STRING)) goto fail;
+
+  /* Write the cache key and storage_data to the cache directory */
+  res = atomic_write_cache_file(cache_path, &current_key, storage_data, &errno_provenance);
+  if (res < 0) goto fail;
+
+  goto succeed;
+
+#define CLEANUP \
+  if (contents != NULL) xfree(contents);   \
+  if (current_fd >= 0)  close(current_fd); \
+  if (cache_fd >= 0)    close(cache_fd);
+
+succeed:
+  CLEANUP;
+  return Qtrue;
+fail:
+  CLEANUP;
+  return Qfalse;
+#undef CLEANUP
+}
+
+
 /*****************************************************************************/
 /********************* Handler Wrappers **************************************/
 /*****************************************************************************
@@ -738,11 +891,13 @@ invalid_type_storage_data:
 
 struct s2o_data {
   VALUE handler;
+  VALUE args;
   VALUE storage_data;
 };
 
 struct i2o_data {
   VALUE handler;
+  VALUE args;
   VALUE input_data;
 };
 
@@ -753,29 +908,31 @@ struct i2s_data {
 };
 
 static VALUE
-prot_storage_to_output(VALUE arg)
+try_storage_to_output(VALUE arg)
 {
   struct s2o_data * data = (struct s2o_data *)arg;
-  return rb_funcall(data->handler, rb_intern("storage_to_output"), 1, data->storage_data);
+  return rb_funcall(data->handler, rb_intern("storage_to_output"), 2, data->storage_data, data->args);
 }
 
 static int
-bs_storage_to_output(VALUE handler, VALUE storage_data, VALUE * output_data)
+bs_storage_to_output(VALUE handler, VALUE args, VALUE storage_data, VALUE * output_data)
 {
   int state;
   struct s2o_data s2o_data = {
     .handler      = handler,
+    .args         = args,
     .storage_data = storage_data,
   };
-  *output_data = rb_protect(prot_storage_to_output, (VALUE)&s2o_data, &state);
+  *output_data = rb_protect(try_storage_to_output, (VALUE)&s2o_data, &state);
   return state;
 }
 
 static void
-bs_input_to_output(VALUE handler, VALUE input_data, VALUE * output_data, int * exception_tag)
+bs_input_to_output(VALUE handler, VALUE args, VALUE input_data, VALUE * output_data, int * exception_tag)
 {
   struct i2o_data i2o_data = {
     .handler    = handler,
+    .args       = args,
     .input_data = input_data,
   };
   *output_data = rb_protect(prot_input_to_output, (VALUE)&i2o_data, exception_tag);
@@ -785,7 +942,7 @@ static VALUE
 prot_input_to_output(VALUE arg)
 {
   struct i2o_data * data = (struct i2o_data *)arg;
-  return rb_funcall(data->handler, rb_intern("input_to_output"), 1, data->input_data);
+  return rb_funcall(data->handler, rb_intern("input_to_output"), 2, data->input_data, data->args);
 }
 
 static VALUE
@@ -795,31 +952,20 @@ try_input_to_storage(VALUE arg)
   return rb_funcall(data->handler, rb_intern("input_to_storage"), 2, data->input_data, data->pathval);
 }
 
-static VALUE
-rescue_input_to_storage(VALUE arg)
-{
-  return uncompilable;
-}
-
-static VALUE
-prot_input_to_storage(VALUE arg)
-{
-  struct i2s_data * data = (struct i2s_data *)arg;
-  return rb_rescue2(
-      try_input_to_storage, (VALUE)data,
-      rescue_input_to_storage, Qnil,
-      rb_eBootsnap_CompileCache_Uncompilable, 0);
-}
-
 static int
-bs_input_to_storage(VALUE handler, VALUE input_data, VALUE pathval, VALUE * storage_data)
+bs_input_to_storage(VALUE handler, VALUE args, VALUE input_data, VALUE pathval, VALUE * storage_data)
 {
-  int state;
-  struct i2s_data i2s_data = {
-    .handler    = handler,
-    .input_data = input_data,
-    .pathval    = pathval,
-  };
-  *storage_data = rb_protect(prot_input_to_storage, (VALUE)&i2s_data, &state);
-  return state;
+  if (readonly) {
+    *storage_data = rb_cBootsnap_CompileCache_UNCOMPILABLE;
+    return 0;
+  } else {
+    int state;
+    struct i2s_data i2s_data = {
+      .handler    = handler,
+      .input_data = input_data,
+      .pathval    = pathval,
+    };
+    *storage_data = rb_protect(try_input_to_storage, (VALUE)&i2s_data, &state);
+    return state;
+  }
 }