bootsnap 1.4.6 → 1.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d4f38db9a609c2adb0a0ede991bd993dff7ae59885cb1722eb699658211fd96
-  data.tar.gz: 9f363c21a154e123693f18e48073451c6cfe6c05ec378c980e6ef770f01e658c
+  metadata.gz: 2540d5dccef7466e75321e3c503b9c6d10aa6195ea905ed75336ad56f6eea2d0
+  data.tar.gz: 2d36002c885f981b1dd77e211cdc5583a97328995a15c14ca4a127fa74043214
 SHA512:
-  metadata.gz: 925f595e21911c61ff7cf3a86cb055d25e56bb65a8a6437c513f25bfea3aec6c086259808b5aed3289e5d82f66706f98314f31d2ff3886d85edeb47085d6a918
-  data.tar.gz: 31507ba8393d47361f8332064a9a39220392a4242e1f3f3c3a88c4de032b51eb8aab8d3769869fec7f9da55400ce773c42aecc52604d5293c9ff7ea3f9f40e54
+  metadata.gz: 672246870422714d082b8233c9bd51d45cf96db5ff6f37f609aafeee208e74822619059df66af56562650c61c655b73c550767b24d76ee9018c088283838bd2e
+  data.tar.gz: 6e6aa8e96453eda1f8c46ffa0aedf7e39a59168d31ef1dc54aaf1de1d86d8bcd67324f668406b068a6f8a42b13178d91c301a9bbe8512058a4907de9233787b7

data/CHANGELOG.md

@@ -1,3 +1,35 @@
+# 1.5.0
+
+* Add a command line to statically precompile the ISeq cache. (#326)
+
+# 1.4.9
+
+* [Windows support](https://github.com/Shopify/bootsnap/pull/319)
+* [Fix potential crash](https://github.com/Shopify/bootsnap/pull/322)
+
+# 1.4.8
+
+* [Prevent FallbackScan from polluting exception cause](https://github.com/Shopify/bootsnap/pull/314)
+
+# 1.4.7
+
+* Various performance enhancements
+* Fix race condition in heavy concurrent load scenarios that would cause bootsnap to raise
+
+# 1.4.6
+
+* Fix bug that was erroneously considering that files containing `.` in the names were being
+  required if a different file with the same name was already being required
+
+  Example:
+  
+      require 'foo'
+      require 'foo.en'
+
+  Before bootsnap was considering `foo.en` to be the same file as `foo`
+
+* Use glibc as part of the ruby_platform cache key
+
 # 1.4.5
 
 * MRI 2.7 support

data/README.md

@@ -1,4 +1,4 @@
-# Bootsnap [![Build Status](https://travis-ci.org/Shopify/bootsnap.svg?branch=master)](https://travis-ci.org/Shopify/bootsnap)
+# Bootsnap [![Actions Status](https://github.com/Shopify/bootsnap/workflows/ci/badge.svg)](https://github.com/Shopify/bootsnap/actions)
 
 Bootsnap is a library that plugs into Ruby, with optional support for `ActiveSupport` and `YAML`,
 to optimize and cache expensive computations. See [How Does This Work](#how-does-this-work).
@@ -294,6 +294,19 @@ open    /c/nope.bundle -> -1
 # (nothing!)
 ```
 
+## Precompilation
+
+In development environments the bootsnap compilation cache is generated on the fly when source files are loaded.
+But in production environments, such as docker images, you might need to precompile the cache.
+
+To do so you can use the `bootsnap precompile` command.
+
+Example:
+
+```bash
+$ bundle exec bootsnap precompile --gemfile app/ lib/
+```
+
 ## When not to use Bootsnap
 
 *Alternative engines*: Bootsnap is pretty reliant on MRI features, and parts are disabled entirely on alternative ruby

data/exe/bootsnap

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+require 'bootsnap/cli'
+exit Bootsnap::CLI.new(ARGV).run

data/ext/bootsnap/bootsnap.c

@@ -32,6 +32,8 @@
 
 #define KEY_SIZE 64
 
+#define MAX_CREATE_TEMPFILE_ATTEMPT 3
+
 /*
  * An instance of this key is written as the first 64 bytes of each cache file.
  * The mtime and size members track whether the file contents have changed, and
@@ -89,16 +91,16 @@ static ID uncompilable;
 
 /* Functions exposed as module functions on Bootsnap::CompileCache::Native */
 static VALUE bs_compile_option_crc32_set(VALUE self, VALUE crc32_v);
-static VALUE bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler);
+static VALUE bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler, VALUE args);
 
 /* Helpers */
 static uint64_t fnv1a_64(const char *str);
-static void bs_cache_path(const char * cachedir, const char * path, char (* cache_path)[MAX_CACHEPATH_SIZE]);
+static void bs_cache_path(const char * cachedir, const char * path, const char * extra, char (* cache_path)[MAX_CACHEPATH_SIZE]);
 static int bs_read_key(int fd, struct bs_cache_key * key);
 static int cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2);
-static VALUE bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler);
+static VALUE bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args);
 static int open_current_file(char * path, struct bs_cache_key * key, const char ** errno_provenance);
-static int fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data, int * exception_tag, const char ** errno_provenance);
+static int fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE args, VALUE * output_data, int * exception_tag, const char ** errno_provenance);
 static uint32_t get_ruby_revision(void);
 static uint32_t get_ruby_platform(void);
 
@@ -106,12 +108,12 @@ static uint32_t get_ruby_platform(void);
  * Helper functions to call ruby methods on handler object without crashing on
  * exception.
  */
-static int bs_storage_to_output(VALUE handler, VALUE storage_data, VALUE * output_data);
+static int bs_storage_to_output(VALUE handler, VALUE args, VALUE storage_data, VALUE * output_data);
 static VALUE prot_storage_to_output(VALUE arg);
 static VALUE prot_input_to_output(VALUE arg);
-static void bs_input_to_output(VALUE handler, VALUE input_data, VALUE * output_data, int * exception_tag);
+static void bs_input_to_output(VALUE handler, VALUE args, VALUE input_data, VALUE * output_data, int * exception_tag);
 static VALUE prot_input_to_storage(VALUE arg);
-static int bs_input_to_storage(VALUE handler, VALUE input_data, VALUE pathval, VALUE * storage_data);
+static int bs_input_to_storage(VALUE handler, VALUE args, VALUE input_data, VALUE pathval, VALUE * storage_data);
 struct s2o_data;
 struct i2o_data;
 struct i2s_data;
@@ -146,7 +148,7 @@ Init_bootsnap(void)
   uncompilable = rb_intern("__bootsnap_uncompilable__");
 
   rb_define_module_function(rb_mBootsnap_CompileCache_Native, "coverage_running?", bs_rb_coverage_running, 0);
-  rb_define_module_function(rb_mBootsnap_CompileCache_Native, "fetch", bs_rb_fetch, 3);
+  rb_define_module_function(rb_mBootsnap_CompileCache_Native, "fetch", bs_rb_fetch, 4);
   rb_define_module_function(rb_mBootsnap_CompileCache_Native, "compile_option_crc32=", bs_compile_option_crc32_set, 1);
 
   current_umask = umask(0777);
@@ -262,9 +264,12 @@ get_ruby_platform(void)
  * The path will look something like: <cachedir>/12/34567890abcdef
  */
 static void
-bs_cache_path(const char * cachedir, const char * path, char (* cache_path)[MAX_CACHEPATH_SIZE])
+bs_cache_path(const char * cachedir, const char * path, const char * extra, char (* cache_path)[MAX_CACHEPATH_SIZE])
 {
   uint64_t hash = fnv1a_64(path);
+  if (extra) {
+    hash ^= fnv1a_64(extra);
+  }
 
   uint8_t first_byte = (hash >> (64 - 8));
   uint64_t remainder = hash & 0x00ffffffffffffff;
@@ -299,7 +304,7 @@ cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2)
  * conversions on the ruby VALUE arguments before passing them along.
  */
 static VALUE
-bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
+bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler, VALUE args)
 {
   FilePathValue(path_v);
 
@@ -313,11 +318,16 @@ bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
   char * cachedir = RSTRING_PTR(cachedir_v);
   char * path     = RSTRING_PTR(path_v);
   char cache_path[MAX_CACHEPATH_SIZE];
+  char * extra    = NULL;
+  if (!NIL_P(args)) {
+    VALUE args_serial = rb_marshal_dump(args, Qnil);
+    extra = RSTRING_PTR(args_serial);
+  }
 
   /* generate cache path to cache_path */
-  bs_cache_path(cachedir, path, &cache_path);
+  bs_cache_path(cachedir, path, extra, &cache_path);
 
-  return bs_fetch(path, path_v, cache_path, handler);
+  return bs_fetch(path, path_v, cache_path, handler, args);
 }
 
 /*
@@ -426,7 +436,7 @@ open_cache_file(const char * path, struct bs_cache_key * key, const char ** errn
  * or exception, will be the final data returnable to the user.
  */
 static int
-fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data, int * exception_tag, const char ** errno_provenance)
+fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE args, VALUE * output_data, int * exception_tag, const char ** errno_provenance)
 {
   char * data = NULL;
   ssize_t nread;
@@ -452,9 +462,9 @@ fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data,
     goto done;
   }
 
-  storage_data = rb_str_new_static(data, data_size);
+  storage_data = rb_str_new(data, data_size);
 
-  *exception_tag = bs_storage_to_output(handler, storage_data, output_data);
+  *exception_tag = bs_storage_to_output(handler, args, storage_data, output_data);
   ret = 0;
 done:
   if (data != NULL) xfree(data);
@@ -499,25 +509,32 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, cons
 {
   char template[MAX_CACHEPATH_SIZE + 20];
   char * tmp_path;
-  int fd, ret;
+  int fd, ret, attempt;
   ssize_t nwrite;
 
-  tmp_path = strncpy(template, path, MAX_CACHEPATH_SIZE);
-  strcat(tmp_path, ".tmp.XXXXXX");
+  for (attempt = 0; attempt < MAX_CREATE_TEMPFILE_ATTEMPT; ++attempt) {
+    tmp_path = strncpy(template, path, MAX_CACHEPATH_SIZE);
+    strcat(tmp_path, ".tmp.XXXXXX");
 
-  // mkstemp modifies the template to be the actual created path
-  fd = mkstemp(tmp_path);
-  if (fd < 0) {
-    if (mkpath(tmp_path, 0775) < 0) {
+    // mkstemp modifies the template to be the actual created path
+    fd = mkstemp(tmp_path);
+    if (fd > 0) break;
+
+    if (attempt == 0 && mkpath(tmp_path, 0775) < 0) {
       *errno_provenance = "bs_fetch:atomic_write_cache_file:mkpath";
       return -1;
     }
-    fd = open(tmp_path, O_WRONLY | O_CREAT, 0664);
-    if (fd < 0) {
-      *errno_provenance = "bs_fetch:atomic_write_cache_file:open";
-      return -1;
-    }
   }
+  if (fd < 0) {
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:mkstemp";
+    return -1;
+  }
+
+  if (chmod(tmp_path, 0644) < 0) {
+    *errno_provenance = "bs_fetch:atomic_write_cache_file:chmod";
+    return -1;
+  }
+
   #ifdef _WIN32
   setmode(fd, O_BINARY);
   #endif
@@ -615,7 +632,7 @@ bs_read_contents(int fd, size_t size, char ** contents, const char ** errno_prov
  *   - Return storage_to_output(storage_data)
  */
 static VALUE
-bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
+bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args)
 {
   struct bs_cache_key cached_key, current_key;
   char * contents = NULL;
@@ -648,7 +665,7 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
   if (valid_cache) {
     /* Fetch the cache data and return it if we're able to load it successfully */
     res = fetch_cached_data(
-      cache_fd, (ssize_t)cached_key.data_size, handler,
+      cache_fd, (ssize_t)cached_key.data_size, handler, args,
       &output_data, &exception_tag, &errno_provenance
     );
     if (exception_tag != 0)                   goto raise;
@@ -662,15 +679,15 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
 
   /* Read the contents of the source file into a buffer */
   if (bs_read_contents(current_fd, current_key.size, &contents, &errno_provenance) < 0) goto fail_errno;
-  input_data = rb_str_new_static(contents, current_key.size);
+  input_data = rb_str_new(contents, current_key.size);
 
   /* Try to compile the input_data using input_to_storage(input_data) */
-  exception_tag = bs_input_to_storage(handler, input_data, path_v, &storage_data);
+  exception_tag = bs_input_to_storage(handler, args, input_data, path_v, &storage_data);
   if (exception_tag != 0) goto raise;
   /* If input_to_storage raised Bootsnap::CompileCache::Uncompilable, don't try
    * to cache anything; just return input_to_output(input_data) */
   if (storage_data == uncompilable) {
-    bs_input_to_output(handler, input_data, &output_data, &exception_tag);
+    bs_input_to_output(handler, args, input_data, &output_data, &exception_tag);
     if (exception_tag != 0) goto raise;
     goto succeed;
   }
@@ -682,7 +699,7 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
   if (res < 0) goto fail_errno;
 
   /* Having written the cache, now convert storage_data to output_data */
-  exception_tag = bs_storage_to_output(handler, storage_data, &output_data);
+  exception_tag = bs_storage_to_output(handler, args, storage_data, &output_data);
   if (exception_tag != 0) goto raise;
 
   /* If output_data is nil, delete the cache entry and generate the output
@@ -692,7 +709,7 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
       errno_provenance = "bs_fetch:unlink";
       goto fail_errno;
     }
-    bs_input_to_output(handler, input_data, &output_data, &exception_tag);
+    bs_input_to_output(handler, args, input_data, &output_data, &exception_tag);
     if (exception_tag != 0) goto raise;
   }
 
@@ -742,16 +759,19 @@ invalid_type_storage_data:
 
 struct s2o_data {
   VALUE handler;
+  VALUE args;
   VALUE storage_data;
 };
 
 struct i2o_data {
   VALUE handler;
+  VALUE args;
   VALUE input_data;
 };
 
 struct i2s_data {
   VALUE handler;
+  VALUE args;
   VALUE input_data;
   VALUE pathval;
 };
@@ -760,15 +780,16 @@ static VALUE
 prot_storage_to_output(VALUE arg)
 {
   struct s2o_data * data = (struct s2o_data *)arg;
-  return rb_funcall(data->handler, rb_intern("storage_to_output"), 1, data->storage_data);
+  return rb_funcall(data->handler, rb_intern("storage_to_output"), 2, data->storage_data, data->args);
 }
 
 static int
-bs_storage_to_output(VALUE handler, VALUE storage_data, VALUE * output_data)
+bs_storage_to_output(VALUE handler, VALUE args, VALUE storage_data, VALUE * output_data)
 {
   int state;
   struct s2o_data s2o_data = {
     .handler      = handler,
+    .args       = args,
     .storage_data = storage_data,
   };
   *output_data = rb_protect(prot_storage_to_output, (VALUE)&s2o_data, &state);
@@ -776,10 +797,11 @@ bs_storage_to_output(VALUE handler, VALUE storage_data, VALUE * output_data)
 }
 
 static void
-bs_input_to_output(VALUE handler, VALUE input_data, VALUE * output_data, int * exception_tag)
+bs_input_to_output(VALUE handler, VALUE args, VALUE input_data, VALUE * output_data, int * exception_tag)
 {
   struct i2o_data i2o_data = {
     .handler    = handler,
+    .args       = args,
     .input_data = input_data,
   };
   *output_data = rb_protect(prot_input_to_output, (VALUE)&i2o_data, exception_tag);
@@ -789,18 +811,18 @@ static VALUE
 prot_input_to_output(VALUE arg)
 {
   struct i2o_data * data = (struct i2o_data *)arg;
-  return rb_funcall(data->handler, rb_intern("input_to_output"), 1, data->input_data);
+  return rb_funcall(data->handler, rb_intern("input_to_output"), 2, data->input_data, data->args);
 }
 
 static VALUE
 try_input_to_storage(VALUE arg)
 {
   struct i2s_data * data = (struct i2s_data *)arg;
-  return rb_funcall(data->handler, rb_intern("input_to_storage"), 2, data->input_data, data->pathval);
+  return rb_funcall(data->handler, rb_intern("input_to_storage"), 3, data->input_data, data->pathval, data->args);
 }
 
 static VALUE
-rescue_input_to_storage(VALUE arg)
+rescue_input_to_storage(VALUE arg, VALUE e)
 {
   return uncompilable;
 }
@@ -816,11 +838,12 @@ prot_input_to_storage(VALUE arg)
 }
 
 static int
-bs_input_to_storage(VALUE handler, VALUE input_data, VALUE pathval, VALUE * storage_data)
+bs_input_to_storage(VALUE handler, VALUE args, VALUE input_data, VALUE pathval, VALUE * storage_data)
 {
   int state;
   struct i2s_data i2s_data = {
     .handler    = handler,
+    .args       = args,
     .input_data = input_data,
     .pathval    = pathval,
   };

data/lib/bootsnap.rb

@@ -1,4 +1,5 @@
 # frozen_string_literal: true
+
 require_relative('bootsnap/version')
 require_relative('bootsnap/bundler')
 require_relative('bootsnap/load_path_cache')

data/lib/bootsnap/cli.rb

@@ -0,0 +1,136 @@
+# frozen_string_literal: true
+
+require 'bootsnap'
+require 'optparse'
+require 'fileutils'
+
+module Bootsnap
+  class CLI
+    unless Regexp.method_defined?(:match?)
+      module RegexpMatchBackport
+        refine Regepx do
+          def match?(string)
+            !!match(string)
+          end
+        end
+      end
+      using RegexpMatchBackport
+    end
+
+    attr_reader :cache_dir, :argv
+
+    attr_accessor :compile_gemfile, :exclude
+
+    def initialize(argv)
+      @argv = argv
+      self.cache_dir = 'tmp/cache'
+      self.compile_gemfile = false
+      self.exclude = nil
+    end
+
+    def precompile_command(*sources)
+      require 'bootsnap/compile_cache/iseq'
+
+      Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
+
+      if compile_gemfile
+        sources += $LOAD_PATH
+      end
+
+      sources.map { |d| File.expand_path(d) }.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_ruby_files(path).each do |ruby_file|
+            if !exclude || !exclude.match?(ruby_file)
+              CompileCache::ISeq.fetch(ruby_file, cache_dir: cache_dir)
+            end
+          end
+        end
+      end
+      0
+    end
+
+    dir_sort = begin
+      Dir['.', sort: false]
+      true
+    rescue ArgumentError, TypeError
+      false
+    end
+
+    if dir_sort
+      def list_ruby_files(path)
+        if File.directory?(path)
+          Dir[File.join(path, '**/*.rb'), sort: false]
+        elsif File.exist?(path)
+          [path]
+        else
+          []
+        end
+      end
+    else
+      def list_ruby_files(path)
+        if File.directory?(path)
+          Dir[File.join(path, '**/*.rb')]
+        elsif File.exist?(path)
+          [path]
+        else
+          []
+        end
+      end
+    end
+
+    def run
+      parser.parse!(argv)
+      command = argv.shift
+      method = "#{command}_command"
+      if respond_to?(method)
+        public_send(method, *argv)
+      else
+        invalid_usage!("Unknown command: #{command}")
+      end
+    end
+
+    private
+
+    def invalid_usage!(message)
+      STDERR.puts message
+      STDERR.puts
+      STDERR.puts parser
+      1
+    end
+
+    def cache_dir=(dir)
+      @cache_dir = File.expand_path(File.join(dir, 'bootsnap-compile-cache'))
+    end
+
+    def parser
+      @parser ||= OptionParser.new do |opts|
+        opts.banner = "Usage: bootsnap COMMAND [ARGS]"
+        opts.separator ""
+        opts.separator "GLOBAL OPTIONS"
+        opts.separator ""
+
+        help = <<~EOS
+          Path to the bootsnap cache directory. Defaults to tmp/cache
+        EOS
+        opts.on('--cache-dir DIR', help.strip) do |dir|
+          self.cache_dir = dir
+        end
+
+        opts.separator ""
+        opts.separator "COMMANDS"
+        opts.separator ""
+        opts.separator "    precompile [DIRECTORIES...]: Precompile all .rb files in the passed directories"
+
+        help = <<~EOS
+          Precompile the gems in Gemfile
+        EOS
+        opts.on('--gemfile', help) { self.compile_gemfile = true }
+
+        help = <<~EOS
+          Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api'
+        EOS
+        opts.on('--exclude PATTERN', help) { |pattern| self.exclude = Regexp.new(pattern) }
+      end
+    end
+  end
+end