bootsnap 0.2.0

data/ext/bootsnap/bootsnap.h

@@ -0,0 +1,10 @@
+#ifndef BOOTSNAP_H
+#define BOOTSNAP_H 1
+
+#include <stdint.h>
+#include <sys/types.h>
+#include "ruby.h"
+
+uint32_t crc32(const char *bytes, size_t size);
+
+#endif /* BOOTSNAP_H */

data/ext/bootsnap/crc32.c

@@ -0,0 +1,16 @@
+#include "bootsnap.h"
+
+#include <x86intrin.h>
+
+uint32_t
+crc32(const char *bytes, size_t size)
+{
+  size_t i;
+  uint32_t hash = 0;
+
+  for (i = 0; i < size; i++) {
+    hash = _mm_crc32_u8(hash, bytes[i]);
+  }
+  return hash;
+}
+

data/ext/bootsnap/extconf.rb

@@ -0,0 +1,7 @@
+require "mkmf"
+$CFLAGS << ' -O3 -msse4.2 -std=c99'
+have_header('x86intrin.h')
+$CFLAGS << ' -Wall -Wextra -Wpedantic -Werror'
+$CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
+$CFLAGS << ' -Wno-keyword-macro' # hiding return
+create_makefile("bootsnap/bootsnap")

data/lib/bootsnap.rb

@@ -0,0 +1,38 @@
+require_relative 'bootsnap/version'
+require_relative 'bootsnap/load_path_cache'
+require_relative 'bootsnap/compile_cache'
+
+module Bootsnap
+  InvalidConfiguration = Class.new(StandardError)
+
+  def self.setup(
+    cache_dir:,
+    development_mode: true,
+    load_path_cache: true,
+    autoload_paths_cache: true,
+    disable_trace: false,
+    compile_cache_iseq: true,
+    compile_cache_yaml: true
+  )
+    if autoload_paths_cache && !load_path_cache
+      raise InvalidConfiguration, "feature 'autoload_paths_cache' depends on feature 'load_path_cache'"
+    end
+
+    setup_disable_trace if disable_trace
+
+    Bootsnap::LoadPathCache.setup(
+      cache_path:       cache_dir + '/bootsnap-load-path-cache',
+      development_mode: development_mode,
+      active_support:   autoload_paths_cache
+    ) if load_path_cache
+
+    Bootsnap::CompileCache.setup(
+      iseq: compile_cache_iseq,
+      yaml: compile_cache_yaml
+    )
+  end
+
+  def self.setup_disable_trace
+    RubyVM::InstructionSequence.compile_option = { trace_instruction: false }
+  end
+end

data/lib/bootsnap/compile_cache.rb

@@ -0,0 +1,16 @@
+require_relative 'compile_cache/iseq'
+require_relative 'compile_cache/yaml'
+
+module Bootsnap
+  module CompileCache
+    def self.setup(iseq:, yaml:)
+      if iseq
+        Bootsnap::CompileCache::ISeq.install!
+      end
+
+      if yaml
+        Bootsnap::CompileCache::YAML.install!
+      end
+    end
+  end
+end

data/lib/bootsnap/compile_cache/iseq.rb

@@ -0,0 +1,74 @@
+require 'bootsnap/bootsnap'
+require 'zlib'
+
+module Bootsnap
+  module CompileCache
+    module ISeq
+      def self.input_to_storage(_, path)
+        RubyVM::InstructionSequence.compile_file(path).to_binary
+      rescue SyntaxError
+        raise Uncompilable, 'syntax error'
+      end
+
+      def self.storage_to_output(binary)
+        RubyVM::InstructionSequence.load_from_binary(binary)
+      rescue RuntimeError => e
+        if e.message == 'broken binary format'
+          STDERR.puts "[Bootsnap::CompileCache] warning: rejecting broken binary"
+          return nil
+        else
+          raise
+        end
+      end
+
+      def self.input_to_output(_)
+        nil # ruby handles this
+      end
+
+      module InstructionSequenceMixin
+        def load_iseq(path)
+          Bootsnap::CompileCache::Native.fetch(
+            path.to_s,
+            Bootsnap::CompileCache::ISeq
+          )
+        rescue RuntimeError => e
+          if e.message =~ /unmatched platform/
+            puts "unmatched platform for file #{path}"
+          end
+          raise
+        rescue Errno::ERANGE
+          STDERR.puts <<~EOF
+            \x1b[31mError loading ISeq from cache for \x1b[1;34m#{path}\x1b[0;31m!
+            You can likely fix this by running:
+              \x1b[1;32mxattr -c #{path}
+            \x1b[0;31m...but, first, please make sure \x1b[1;34m@burke\x1b[0;31m knows you ran into this bug!
+            He will want to see the results of:
+              \x1b[1;32m/bin/ls -l@ #{path}
+            \x1b[0;31mand:
+              \x1b[1;32mxattr -p user.aotcc.key #{path}\x1b[0m
+          EOF
+          raise
+        end
+
+        def compile_option=(hash)
+          super(hash)
+          Bootsnap::CompileCache::ISeq.compile_option_updated
+        end
+      end
+
+      def self.compile_option_updated
+        option = RubyVM::InstructionSequence.compile_option
+        crc = Zlib.crc32(option.inspect)
+        Bootsnap::CompileCache::Native.compile_option_crc32 = crc
+      end
+
+      def self.install!
+        Bootsnap::CompileCache::ISeq.compile_option_updated
+        class << RubyVM::InstructionSequence
+          prepend InstructionSequenceMixin
+        end
+      end
+    end
+  end
+end
+

data/lib/bootsnap/compile_cache/yaml.rb

@@ -0,0 +1,54 @@
+module Bootsnap
+  module CompileCache
+    module YAML
+      class << self
+        attr_accessor :msgpack_factory
+      end
+
+      def self.input_to_storage(contents, _)
+        obj = ::YAML.load(contents)
+        msgpack_factory.packer.write(obj).to_s
+      rescue NoMethodError, RangeError
+        # if the object included things that we can't serialize, fall back to
+        # Marshal. It's a bit slower, but can encode anything yaml can.
+        # NoMethodError is unexpected types; RangeError is Bignums
+        return Marshal.dump(obj)
+      end
+
+      def self.storage_to_output(data)
+        # This could have a meaning in messagepack, and we're being a little lazy
+        # about it. -- but a leading 0x04 would indicate the contents of the YAML
+        # is a positive integer, which is rare, to say the least.
+        if data[0] == 0x04.chr && data[1] == 0x08.chr
+          Marshal.load(data)
+        else
+          msgpack_factory.unpacker.feed(data).read
+        end
+      end
+
+      def self.input_to_output(data)
+        ::YAML.load(data)
+      end
+
+      def self.install!
+        require 'yaml'
+        require 'msgpack'
+
+        # MessagePack serializes symbols as strings by default.
+        # We want them to roundtrip cleanly, so we use a custom factory.
+        # see: https://github.com/msgpack/msgpack-ruby/pull/122
+        factory = MessagePack::Factory.new
+        factory.register_type(0x00, Symbol)
+        Bootsnap::CompileCache::YAML.msgpack_factory = factory
+
+        klass = class << ::YAML; self; end
+        klass.send(:define_method, :load_file) do |path|
+          Bootsnap::CompileCache::Native.fetch(
+            path.to_s,
+            Bootsnap::CompileCache::YAML
+          )
+        end
+      end
+    end
+  end
+end

data/lib/bootsnap/explicit_require.rb

@@ -0,0 +1,44 @@
+module Bootsnap
+  module ExplicitRequire
+    ARCHDIR    = RbConfig::CONFIG['archdir']
+    RUBYLIBDIR = RbConfig::CONFIG['rubylibdir']
+    DLEXT      = RbConfig::CONFIG['DLEXT']
+
+    def self.from_self(feature)
+      require_relative "../#{feature}"
+    end
+
+    def self.from_rubylibdir(feature)
+      require(File.join(RUBYLIBDIR, "#{feature}.rb"))
+    end
+
+    def self.from_archdir(feature)
+      require(File.join(ARCHDIR, "#{feature}.#{DLEXT}"))
+    end
+
+    # Given a set of gems, run a block with the LOAD_PATH narrowed to include
+    # only core ruby source paths and these gems -- that is, roughly,
+    # temporarily remove all gems not listed in this call from the LOAD_PATH.
+    #
+    # This is useful before bootsnap is fully-initialized to load gems that it
+    # depends on, without forcing full LOAD_PATH traversals.
+    def self.with_gems(*gems)
+      orig = $LOAD_PATH.dup
+      $LOAD_PATH.clear
+      gems.each do |gem|
+        pat = %r{
+          /
+          (gems|extensions/[^/]+/[^/]+)          # "gems" or "extensions/x64_64-darwin16/2.3.0"
+          /
+          #{Regexp.escape(gem)}-(\h{12}|(\d+\.)) # msgpack-1.2.3 or msgpack-1234567890ab
+        }x
+        $LOAD_PATH.concat(orig.grep(pat))
+      end
+      $LOAD_PATH << ARCHDIR
+      $LOAD_PATH << RUBYLIBDIR
+      yield
+    ensure
+      $LOAD_PATH.replace(orig)
+    end
+  end
+end

data/lib/bootsnap/load_path_cache.rb

@@ -0,0 +1,52 @@
+module Bootsnap
+  module LoadPathCache
+    ReturnFalse = Class.new(StandardError)
+    FallbackScan = Class.new(StandardError)
+
+    DOT_RB = '.rb'
+    DOT_SO = '.so'
+    SLASH  = '/'
+
+    DL_EXTENSIONS = ::RbConfig::CONFIG
+      .values_at('DLEXT', 'DLEXT2')
+      .reject { |ext| !ext || ext.empty? }
+      .map    { |ext| ".#{ext}" }
+      .freeze
+    DLEXT = DL_EXTENSIONS[0]
+    # This is nil on linux and darwin, but I think it's '.o' on some other
+    # platform.  I'm not really sure which, but it seems better to replicate
+    # ruby's semantics as faithfully as possible.
+    DLEXT2 = DL_EXTENSIONS[1]
+
+    CACHED_EXTENSIONS = DLEXT2 ? [DOT_RB, DLEXT, DLEXT2] : [DOT_RB, DLEXT]
+
+    class << self
+      attr_reader :load_path_cache, :autoload_paths_cache
+
+      def setup(cache_path:, development_mode:, active_support: true)
+        store = Store.new(cache_path)
+
+        @load_path_cache = Cache.new(store, $LOAD_PATH, development_mode: development_mode)
+        require_relative 'load_path_cache/core_ext/kernel_require'
+
+        if active_support
+          # this should happen after setting up the initial cache because it
+          # loads a lot of code. It's better to do after +require+ is optimized.
+          require 'active_support/dependencies'
+          @autoload_paths_cache = Cache.new(
+            store,
+            ::ActiveSupport::Dependencies.autoload_paths,
+            development_mode: development_mode
+          )
+          require_relative 'load_path_cache/core_ext/active_support'
+        end
+      end
+    end
+  end
+end
+
+require_relative 'load_path_cache/path_scanner'
+require_relative 'load_path_cache/path'
+require_relative 'load_path_cache/cache'
+require_relative 'load_path_cache/store'
+require_relative 'load_path_cache/change_observer'

data/lib/bootsnap/load_path_cache/cache.rb

@@ -0,0 +1,150 @@
+require_relative '../load_path_cache'
+require_relative '../explicit_require'
+Bootsnap::ExplicitRequire.from_archdir('thread')
+
+module Bootsnap
+  module LoadPathCache
+    class Cache
+      AGE_THRESHOLD = 30 # seconds
+
+      def initialize(store, path_obj, development_mode: false)
+        @development_mode = development_mode
+        @store = store
+        @mutex = ::Thread::Mutex.new
+        @path_obj = path_obj
+        reinitialize
+      end
+
+      # Does this directory exist as a child of one of the path items?
+      # e.g. given "/a/b/c/d" exists, and the path is ["/a/b"], has_dir?("c/d")
+      # is true.
+      def has_dir?(dir)
+        reinitialize if stale?
+        @mutex.synchronize { @dirs[dir] }
+      end
+
+      # Try to resolve this feature to an absolute path without traversing the
+      # loadpath.
+      def find(feature)
+        reinitialize if stale?
+        feature = feature.to_s
+        return feature if feature.start_with?(SLASH)
+        return File.expand_path(feature) if feature.start_with?('./')
+        @mutex.synchronize do
+          x = search_index(feature)
+          return x if x
+
+          # The feature wasn't found on our preliminary search through the index.
+          # We resolve this differently depending on what the extension was.
+          case File.extname(feature)
+          # If the extension was one of the ones we explicitly cache (.rb and the
+          # native dynamic extension, e.g. .bundle or .so), we know it was a
+          # failure and there's nothing more we can do to find the file.
+          when *CACHED_EXTENSIONS # .rb, .bundle or .so
+            nil
+          # If no extension was specified, it's the same situation, since we
+          # try appending both cachable extensions in search_index. However,
+          # there's a special-case for 'enumerator'. Before ruby 1.9, you had
+          # to `require 'enumerator'` to use it. In 1.9+, it's pre-loaded, but
+          # doesn't correspond to any entry on the filesystem. Ruby lies. So we
+          # lie too, forcing our monkeypatch to return false like ruby would.
+          when ""
+            raise LoadPathCache::ReturnFalse if feature == 'enumerator'
+            nil
+          # Ruby allows specifying native extensions as '.so' even when DLEXT
+          # is '.bundle'. This is where we handle that case.
+          when DOT_SO
+            x = search_index(feature[0..-4] + DLEXT)
+            return x if x
+            if DLEXT2
+              search_index(feature[0..-4] + DLEXT2)
+            end
+          else
+            # other, unknown extension. For example, `.rake`. Since we haven't
+            # cached these, we legitimately need to run the load path search.
+            raise LoadPathCache::FallbackScan
+          end
+        end
+      end
+
+      def unshift_paths(sender, *paths)
+        return unless sender == @path_obj
+        @mutex.synchronize { unshift_paths_locked(*paths) }
+      end
+
+      def push_paths(sender, *paths)
+        return unless sender == @path_obj
+        @mutex.synchronize { push_paths_locked(*paths) }
+      end
+
+      def each_requirable
+        @mutex.synchronize do
+          @index.each do |rel, entry|
+            yield "#{entry}/#{rel}"
+          end
+        end
+      end
+
+      def reinitialize(path_obj = @path_obj)
+        @mutex.synchronize do
+          @path_obj = path_obj
+          ChangeObserver.register(self, @path_obj)
+          @index = {}
+          @dirs = Hash.new(false)
+          @generated_at = now
+          push_paths_locked(*@path_obj)
+        end
+      end
+
+      private
+
+      def push_paths_locked(*paths)
+        @store.transaction do
+          paths.each do |path|
+            p = Path.new(path)
+            entries, dirs = p.entries_and_dirs(@store)
+            # push -> low precedence -> set only if unset
+            dirs.each    { |dir| @dirs[dir]  ||= true }
+            entries.each { |rel| @index[rel] ||= path }
+          end
+        end
+      end
+
+      def unshift_paths_locked(*paths)
+        @store.transaction do
+          paths.reverse.each do |path|
+            p = Path.new(path)
+            entries, dirs = p.entries_and_dirs(@store)
+            # unshift -> high precedence -> unconditional set
+            dirs.each    { |dir| @dirs[dir]  = true }
+            entries.each { |rel| @index[rel] = path }
+          end
+        end
+      end
+
+      def stale?
+        @development_mode && @generated_at + AGE_THRESHOLD < now
+      end
+
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC).to_i
+      end
+
+      if DLEXT2
+        def search_index(f)
+          try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f + DLEXT2) || try_index(f)
+        end
+      else
+        def search_index(f)
+          try_index(f + DOT_RB) || try_index(f + DLEXT) || try_index(f)
+        end
+      end
+
+      def try_index(f)
+        if p = @index[f]
+          p + '/' + f
+        end
+      end
+    end
+  end
+end