bootsnap 0.3.0.pre → 0.3.0.pre2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: '082f8dd8be53486b79ff5248ad426cc28263778b'
-  data.tar.gz: 5bc6a79f2054aa8b869704cdb9aeab685eabb400
+  metadata.gz: aa9caa4faf08302d5050a4d6f41e90920732beed
+  data.tar.gz: 60d4a539b9065e0771be2c330a614d4bd5fb8efe
 SHA512:
-  metadata.gz: ad062964bbc28d997901a1f6016289a4c6671071f9e1c4640820fe9545427136e1268f6588b350e7d058d85641604b0004225a744792fcce9b24bfcb9e5a3172
-  data.tar.gz: 33c77f2e085ba965e1a9fccb65b5afd42a4db0485dbfca7aa5d3e3920497ff41d0a3ca0ca4b6706df55814d29f015056e6d862568fe8b48ad5194f6eb80e26dc
+  metadata.gz: 383af9b805278c8d682c368b52b1ec14dd8c1d391692422c269fc9f86d9199aeee3aecdf8e33e1f369ba028316633e97d58d20da176eae74d2d0879dec14a801
+  data.tar.gz: 307d8dda6d264b1179ee2a955f3fa45c9c01add22233934a333a9a50b122e3d5aa7cfb80199611d35e0e0518091c474c5f3b35d5f0c88912e5fe78e036df0977

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+# 0.3.0 (pre)
+
+* Migrate CompileCache from xattr as a cache backend to a cache directory
+    * Adds support for Linux and FreeBSD
+
 # 0.2.15
 
 * Support more versions of ActiveSupport (`depend_on`'s signature varies; don't reiterate it)

data/ext/bootsnap/bootsnap.c

@@ -1,3 +1,16 @@
+/*
+ * Suggested reading order:
+ * 1. Skim Init_bootsnap
+ * 2. Skim bs_fetch
+ * 3. The rest of everything
+ *
+ * Init_bootsnap sets up the ruby objects and binds bs_fetch to
+ * Bootsnap::CompileCache::Native.fetch.
+ *
+ * bs_fetch is the ultimate caller for for just about every other function in
+ * here.
+ */
+
 #include "bootsnap.h"
 #include "ruby.h"
 #include <stdint.h>
@@ -5,18 +18,29 @@
 #include <errno.h>
 #include <fcntl.h>
 #include <sys/stat.h>
+#include <sys/utsname.h>
 
-#ifdef __linux__
-#include <gnu/libc-version.h>
-#else
-#include <sys/sysctl.h>
-#endif
-
+/* 1000 is an arbitrary limit; FNV64 plus some slashes brings the cap down to
+ * 981 for the cache dir */
 #define MAX_CACHEPATH_SIZE 1000
 #define MAX_CACHEDIR_SIZE  981
 
 #define KEY_SIZE 64
 
+/*
+ * An instance of this key is written as the first 64 bytes of each cache file.
+ * The mtime and size members track whether the file contents have changed, and
+ * the version, os_version, compile_option, and ruby_revision members track
+ * changes to the environment that could invalidate compile results without
+ * file contents having changed. The data_size member is not truly part of the
+ * "key". Really, this could be called a "header" with the first six members
+ * being an embedded "key" struct and an additional data_size member.
+ *
+ * The data_size indicates the remaining number of bytes in the cache file
+ * after the header (the size of the cached artifact).
+ *
+ * After data_size, the struct is padded to 64 bytes.
+ */
 struct bs_cache_key {
   uint32_t version;
   uint32_t os_version;
@@ -28,31 +52,40 @@ struct bs_cache_key {
   uint8_t  pad[24];
 } __attribute__((packed));
 
+/*
+ * If the struct padding isn't correct to pad the key to 64 bytes, refuse to
+ * compile.
+ */
 #define STATIC_ASSERT(X)            STATIC_ASSERT2(X,__LINE__)
 #define STATIC_ASSERT2(X,L)         STATIC_ASSERT3(X,L)
 #define STATIC_ASSERT3(X,L)         STATIC_ASSERT_MSG(X,at_line_##L)
 #define STATIC_ASSERT_MSG(COND,MSG) typedef char static_assertion_##MSG[(!!(COND))*2-1]
-
 STATIC_ASSERT(sizeof(struct bs_cache_key) == KEY_SIZE);
 
+/* Effectively a schema version. Bumping invalidates all previous caches */
 static const uint32_t current_version = 2;
 
+/* Derived from kernel or libc version; intended to roughly correspond to when
+ * ABIs have changed, requiring recompilation of native gems. */
 static uint32_t current_os_version;
+/* Invalidates cache when switching ruby versions */
 static uint32_t current_ruby_revision;
+/* Invalidates cache when RubyVM::InstructionSequence.compile_option changes */
 static uint32_t current_compile_option_crc32 = 0;
 
+/* Bootsnap::CompileCache::{Native, Uncompilable} */
 static VALUE rb_mBootsnap;
 static VALUE rb_mBootsnap_CompileCache;
 static VALUE rb_mBootsnap_CompileCache_Native;
 static VALUE rb_eBootsnap_CompileCache_Uncompilable;
 static ID uncompilable;
 
-/* Real API */
+/* Functions exposed as module functions on Bootsnap::CompileCache::Native */
 static VALUE bs_compile_option_crc32_set(VALUE self, VALUE crc32_v);
 static VALUE bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler);
 
 /* Helpers */
-static uint64_t fnv_64a(const char *str);
+static uint64_t fnv1a_64(const char *str);
 static void bs_cache_path(const char * cachedir, const char * path, char ** cache_path);
 static int bs_read_key(int fd, struct bs_cache_key * key);
 static int cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2);
@@ -62,29 +95,27 @@ static int fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * o
 static VALUE prot_exception_for_errno(VALUE err);
 static uint32_t get_os_version(void);
 
+/*
+ * Helper functions to call ruby methods on handler object without crashing on
+ * exception.
+ */
 static int bs_storage_to_output(VALUE handler, VALUE storage_data, VALUE * output_data);
 static VALUE prot_storage_to_output(VALUE arg);
 static VALUE prot_input_to_output(VALUE arg);
 static void bs_input_to_output(VALUE handler, VALUE input_data, VALUE * output_data, int * exception_tag);
 static VALUE prot_input_to_storage(VALUE arg);
 static int bs_input_to_storage(VALUE handler, VALUE input_data, VALUE pathval, VALUE * storage_data);
+struct s2o_data;
+struct i2o_data;
+struct i2s_data;
 
-struct s2o_data {
-  VALUE handler;
-  VALUE storage_data;
-};
-
-struct i2o_data {
-  VALUE handler;
-  VALUE input_data;
-};
-
-struct i2s_data {
-  VALUE handler;
-  VALUE input_data;
-  VALUE pathval;
-};
-
+/*
+ * Ruby C extensions are initialized by calling Init_<extname>.
+ *
+ * This sets up the module hierarchy and attaches functions as methods.
+ *
+ * We also populate some semi-static information about the current OS and so on.
+ */
 void
 Init_bootsnap(void)
 {
@@ -102,6 +133,10 @@ Init_bootsnap(void)
   rb_define_module_function(rb_mBootsnap_CompileCache_Native, "compile_option_crc32=", bs_compile_option_crc32_set, 1);
 }
 
+/*
+ * Bootsnap's ruby code registers a hook that notifies us via this function
+ * when compile_option changes. These changes invalidate all existing caches.
+ */
 static VALUE
 bs_compile_option_crc32_set(VALUE self, VALUE crc32_v)
 {
@@ -110,11 +145,21 @@ bs_compile_option_crc32_set(VALUE self, VALUE crc32_v)
   return Qnil;
 }
 
+/*
+ * We use FNV1a-64 to derive cache paths. The choice is somewhat arbitrary but
+ * it has several nice properties:
+ *
+ *   - Tiny implementation
+ *   - No external dependency
+ *   - Solid performance
+ *   - Solid randomness
+ *   - 32 bits doesn't feel collision-resistant enough; 64 is nice.
+ */
 static uint64_t
-fnv_64a(const char *str)
+fnv1a_64(const char *str)
 {
   unsigned char *s = (unsigned char *)str;
-  uint64_t h = ((uint64_t)0xcbf29ce484222325ULL);
+  uint64_t h = (uint64_t)0xcbf29ce484222325ULL;
 
   while (*s) {
     h ^= (uint64_t)*s++;
@@ -124,36 +169,44 @@ fnv_64a(const char *str)
   return h;
 }
 
-/* On Darwin and FreeBSD, this is the kernel version, since that tends to vary
+/*
+ * The idea here is that we want a cache key member that changes when the OS
+ * changes in such a way as to make existing compiled ISeqs unloadable.
+ *
+ * On Darwin and FreeBSD, this is the kernel version, since that tends to vary
  * with whole-system upgrades. What we probably care about more is the libc
  * version, which is what we explicitly ask for on linux.
  * (and KERN_OSRELEASE came back empty for me one one linux box, so...?)
+ *
+ * I'm kind of guessing about the important factors here. We could probably do
+ * this better.
  */
 static uint32_t
 get_os_version(void)
 {
   size_t len;
   uint64_t hash;
-#ifdef __linux__
-  const char * version;
-  version = gnu_get_libc_version();
-  hash = fnv_64a(version);
-#else
-  char * version;
-  int mib[2] = {CTL_KERN, KERN_OSRELEASE};
-  if (sysctl(mib, 2, NULL, &len, NULL, 0) < 0) return 0;
-  version = malloc(sizeof(char) * len);
-  if (sysctl(mib, 2, version, &len, NULL, 0) < 0) return 0;
-  hash = fnv_64a(version);
-  free(version);
-#endif
+  struct utsname utsname;
+
+  /* Not worth crashing if this fails; lose cache invalidation potential */
+  if (uname(&utsname) < 0) return 0;
+
+  hash = fnv1a_64(utsname.version);
+
   return (uint32_t)(hash >> 32);
 }
 
+/*
+ * Given a cache root directory and the full path to a file being cached,
+ * generate a path under the cache directory at which the cached artifact will
+ * be stored.
+ *
+ * The path will look something like: <cachedir>/12/34567890abcdef
+ */
 static void
 bs_cache_path(const char * cachedir, const char * path, char ** cache_path)
 {
-  uint64_t hash = fnv_64a(path);
+  uint64_t hash = fnv1a_64(path);
 
   uint8_t first_byte = (hash >> (64 - 8));
   uint64_t remainder = hash & 0x00ffffffffffffff;
@@ -161,6 +214,14 @@ bs_cache_path(const char * cachedir, const char * path, char ** cache_path)
   sprintf(*cache_path, "%s/%02x/%014llx", cachedir, first_byte, remainder);
 }
 
+/*
+ * Test whether a newly-generated cache key based on the file as it exists on
+ * disk matches the one that was generated when the file was cached (or really
+ * compare any two keys).
+ *
+ * The data_size member is not compared, as it serves more of a "header"
+ * function.
+ */
 static int
 cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2)
 {
@@ -174,6 +235,11 @@ cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2)
   );
 }
 
+/*
+ * Entrypoint for Bootsnap::CompileCache::Native.fetch. The real work is done
+ * in bs_fetch; this function just performs some basic typechecks and
+ * conversions on the ruby VALUE arguments before passing them along.
+ */
 static VALUE
 bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
 {
@@ -196,6 +262,10 @@ bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
   return bs_fetch(path, path_v, cache_path, handler);
 }
 
+/*
+ * Open the file we want to load/cache and generate a cache key for it if it
+ * was loaded.
+ */
 static int
 open_current_file(char * path, struct bs_cache_key * key)
 {
@@ -224,8 +294,13 @@ open_current_file(char * path, struct bs_cache_key * key)
 #define CACHE_MISSING_OR_INVALID -2
 
 /*
- * @return CACHE_MISSING_OR_INVALID
- * @return ERROR_WITH_ERRNO(-1) and errno
+ * Read the cache key from the given fd, which must have position 0 (e.g.
+ * freshly opened file).
+ *
+ * Possible return values:
+ *   - 0 (OK, key was loaded)
+ *   - CACHE_MISSING_OR_INVALID (-2)
+ *   - ERROR_WITH_ERRNO (-1, errno is set)
  */
 static int
 bs_read_key(int fd, struct bs_cache_key * key)
@@ -237,8 +312,13 @@ bs_read_key(int fd, struct bs_cache_key * key)
 }
 
 /*
- * @return CACHE_MISSING_OR_INVALID
- * @return ERROR_WITH_ERRNO(-1) and errno
+ * Open the cache file at a given path, if it exists, and read its key into the
+ * struct.
+ *
+ * Possible return values:
+ *   - 0 (OK, key was loaded)
+ *   - CACHE_MISSING_OR_INVALID (-2)
+ *   - ERROR_WITH_ERRNO (-1, errno is set)
  */
 static int
 open_cache_file(const char * path, struct bs_cache_key * key)
@@ -260,10 +340,25 @@ open_cache_file(const char * path, struct bs_cache_key * key)
   return fd;
 }
 
+/*
+ * The cache file is laid out like:
+ *   0...64 : bs_cache_key
+ *   64..-1 : cached artifact
+ *
+ * This function takes a file descriptor whose position is pre-set to 64, and
+ * the data_size (corresponding to the remaining number of bytes) listed in the
+ * cache header.
+ *
+ * We load the text from this file into a buffer, and pass it to the ruby-land
+ * handler with exception handling via the exception_tag param.
+ *
+ * Data is returned via the output_data parameter, which, if there's no error
+ * or exception, will be the final data returnable to the user.
+ */
 static int
 fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data, int * exception_tag)
 {
-  char * data;
+  char * data = NULL;
   ssize_t nread;
   int ret;
 
@@ -271,7 +366,8 @@ fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data,
 
   if (data_size > 100000000000) {
     errno = EINVAL; /* because wtf? */
-    return -1;
+    ret = -1;
+    goto done;
   }
   data = ALLOC_N(char, data_size);
   nread = read(fd, data, data_size);
@@ -289,17 +385,14 @@ fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data,
   *exception_tag = bs_storage_to_output(handler, storage_data, output_data);
   ret = 0;
 done:
-  xfree(data);
+  if (data != NULL) xfree(data);
   return ret;
 }
 
-static ssize_t
-bs_read_contents(int fd, size_t size, char ** contents)
-{
-  *contents = ALLOC_N(char, size);
-  return read(fd, *contents, size);
-}
-
+/*
+ * Like mkdir -p, this recursively creates directory parents of a file. e.g.
+ * given /a/b/c, creates /a and /a/b.
+ */
 static int
 mkpath(char * file_path, mode_t mode)
 {
@@ -320,6 +413,11 @@ mkpath(char * file_path, mode_t mode)
   return 0;
 }
 
+/*
+ * Write a cache header/key and a compiled artifact to a given cache path by
+ * writing to a tmpfile and then renaming the tmpfile over top of the final
+ * path.
+ */
 static int
 atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data)
 {
@@ -359,6 +457,76 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data)
   return rename(tmp_path, path);
 }
 
+/*
+ * Given an errno value (converted to a ruby Fixnum), return the corresponding
+ * Errno::* constant. If none is found, return StandardError instead.
+ */
+static VALUE
+prot_exception_for_errno(VALUE err)
+{
+  if (err != INT2FIX(0)) {
+    VALUE mErrno = rb_const_get(rb_cObject, rb_intern("Errno"));
+    VALUE constants = rb_funcall(mErrno, rb_intern("constants"), 0);
+    VALUE which = rb_funcall(constants, rb_intern("[]"), 1, err);
+    return rb_funcall(mErrno, rb_intern("const_get"), 1, which);
+  }
+  return rb_eStandardError;
+}
+
+
+/* Read contents from an fd, whose contents are asserted to be +size+ bytes
+ * long, into a buffer */
+static ssize_t
+bs_read_contents(int fd, size_t size, char ** contents)
+{
+  *contents = ALLOC_N(char, size);
+  return read(fd, *contents, size);
+}
+
+/*
+ * This is the meat of the extension. bs_fetch is
+ * Bootsnap::CompileCache::Native.fetch.
+ *
+ * There are three "formats" in use here:
+ *   1. "input" fomat, which is what we load from the source file;
+ *   2. "storage" format, which we write to the cache;
+ *   3. "output" format, which is what we return.
+ *
+ * E.g., For ISeq compilation:
+ *   input: ruby source, as text
+ *   storage: binary string (RubyVM::InstructionSequence#to_binary)
+ *   output: Instance of RubyVM::InstructionSequence
+ *
+ * And for YAML:
+ *   input: yaml as text
+ *   storage: MessagePack or Marshal text
+ *   output: ruby object, loaded from yaml/messagepack/marshal
+ *
+ * The handler passed in must support three messages:
+ *   * storage_to_output(s) -> o
+ *   * input_to_output(i)   -> o
+ *   * input_to_storage(i)  -> s
+ *     (input_to_storage may raise Bootsnap::CompileCache::Uncompilable, which
+ *     will prevent caching and cause output to be generated with
+ *     input_to_output)
+ *
+ * The semantics of this function are basically:
+ *
+ *   return storage_to_output(cache[path]) if cache[path]
+ *   storage = input_to_storage(input)
+ *   cache[path] = storage
+ *   return storage_to_output(storage)
+ *
+ * Or expanded a bit:
+ *
+ *   - Check if the cache file exists and is up to date.
+ *   - If it is, load this data to storage_data.
+ *   - return storage_to_output(storage_data)
+ *   - Read the file to input_data
+ *   - Generate storage_data using input_to_storage(input_data)
+ *   - Write storage_data data, with a cache key, to the cache file.
+ *   - Return storage_to_output(storage_data)
+ */
 static VALUE
 bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
 {
@@ -371,14 +539,18 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
   VALUE storage_data; /* compiled data, e.g. msgpack / binary iseq */
   VALUE output_data;  /* return data, e.g. ruby hash or loaded iseq */
 
-  VALUE exception;
+  VALUE exception; /* ruby exception object to raise instead of returning */
 
+  /* Open the source file and generate a cache key for it */
   current_fd = open_current_file(path, &current_key);
   if (current_fd < 0) goto fail_errno;
 
+  /* Open the cache key if it exists, and read its cache key in */
   cache_fd = open_cache_file(cache_path, &cached_key);
   if (cache_fd < 0 && cache_fd != CACHE_MISSING_OR_INVALID) goto fail_errno;
 
+  /* True if the cache existed and no invalidating changes have occurred since
+   * it was generated. */
   valid_cache = cache_key_equal(&current_key, &cached_key);
 
   if (valid_cache) {
@@ -393,30 +565,40 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
   cache_fd = -1;
   /* Cache is stale, invalid, or missing. Regenerate and write it out. */
 
+  /* Read the contents of the source file into a buffer */
   if (bs_read_contents(current_fd, current_key.size, &contents) < 0) goto fail_errno;
   input_data = rb_str_new_static(contents, current_key.size);
 
+  /* Try to compile the input_data using input_to_storage(input_data) */
   exception_tag = bs_input_to_storage(handler, input_data, path_v, &storage_data);
   if (exception_tag != 0) goto raise;
+  /* If input_to_storage raised Bootsnap::CompileCache::Uncompilable, don't try
+   * to cache anything; just return input_to_output(input_data) */
   if (storage_data == uncompilable) {
     bs_input_to_output(handler, input_data, &output_data, &exception_tag);
     if (exception_tag != 0) goto raise;
     goto succeed;
   }
+  /* If storage_data isn't a string, we can't cache it */
   if (!RB_TYPE_P(storage_data, T_STRING)) goto invalid_type_storage_data;
 
+  /* Write the cache key and storage_data to the cache directory */
   res = atomic_write_cache_file(cache_path, &current_key, storage_data);
   if (res < 0) goto fail_errno;
 
+  /* Having written the cache, now convert storage_data to output_data */
   exception_tag = bs_storage_to_output(handler, storage_data, &output_data);
   if (exception_tag != 0) goto raise;
 
+  /* If output_data is nil, delete the cache entry and generate the output
+   * using input_to_output */
   if (NIL_P(output_data)) {
     if (unlink(cache_path) < 0) goto fail_errno;
     bs_input_to_output(handler, input_data, &output_data, &exception_tag);
     if (exception_tag != 0) goto raise;
   }
-  // output_data is now the correct return; cascade into succeed:
+
+  goto succeed; /* output_data is now the correct return. */
 
 #define CLEANUP \
   if (contents != NULL) xfree(contents);   \
@@ -444,22 +626,28 @@ invalid_type_storage_data:
 #undef CLEANUP
 }
 
-static VALUE
-prot_exception_for_errno(VALUE err)
-{
-  if (err != INT2FIX(0)) {
-    VALUE mErrno = rb_const_get(rb_cObject, rb_intern("Errno"));
-    VALUE constants = rb_funcall(mErrno, rb_intern("constants"), 0);
-    VALUE which = rb_funcall(constants, rb_intern("[]"), 1, err);
-    return rb_funcall(mErrno, rb_intern("const_get"), 1, which);
-  }
-  return rb_eStandardError;
-}
-
-
 /*****************************************************************************/
 /********************* Handler Wrappers **************************************/
-/*****************************************************************************/
+/*****************************************************************************
+ * Everything after this point in the file is just wrappers to deal with ruby's
+ * clunky method of handling exceptions from ruby methods invoked from C.
+ */
+
+struct s2o_data {
+  VALUE handler;
+  VALUE storage_data;
+};
+
+struct i2o_data {
+  VALUE handler;
+  VALUE input_data;
+};
+
+struct i2s_data {
+  VALUE handler;
+  VALUE input_data;
+  VALUE pathval;
+};
 
 static VALUE
 prot_storage_to_output(VALUE arg)

data/lib/bootsnap/version.rb

@@ -1,3 +1,3 @@
 module Bootsnap
-  VERSION = "0.3.0.pre"
+  VERSION = "0.3.0.pre2"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bootsnap
 version: !ruby/object:Gem::Version
-  version: 0.3.0.pre
+  version: 0.3.0.pre2
 platform: ruby
 authors:
 - Burke Libbey
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-05-25 00:00:00.000000000 Z
+date: 2017-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler