bootsnap 0.2.2 → 0.3.0

data/ext/bootsnap/bootsnap.h

@@ -1,10 +1,6 @@
 #ifndef BOOTSNAP_H
 #define BOOTSNAP_H 1
 
-#include <stdint.h>
-#include <sys/types.h>
-#include "ruby.h"
-
-uint32_t crc32(const char *bytes, size_t size);
+/* doesn't expose anything */
 
 #endif /* BOOTSNAP_H */

data/ext/bootsnap/extconf.rb

@@ -1,7 +1,17 @@
 require "mkmf"
-$CFLAGS << ' -O3 -msse4.2 -std=c99'
-have_header('x86intrin.h')
-$CFLAGS << ' -Wall -Wextra -Wpedantic -Werror'
-$CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
-$CFLAGS << ' -Wno-keyword-macro' # hiding return
+$CFLAGS << ' -O3 '
+$CFLAGS << ' -std=c99'
+
+# ruby.h has some -Wpedantic fails in some cases
+# (e.g. https://github.com/Shopify/bootsnap/issues/15)
+unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC'])
+  $CFLAGS << ' -Wall'
+  $CFLAGS << ' -Werror'
+  $CFLAGS << ' -Wextra'
+  $CFLAGS << ' -Wpedantic'
+
+  $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
+  $CFLAGS << ' -Wno-keyword-macro' # hiding return
+end
+
 create_makefile("bootsnap/bootsnap")

data/lib/bootsnap/compile_cache/iseq.rb

@@ -4,6 +4,10 @@ require 'zlib'
 module Bootsnap
   module CompileCache
     module ISeq
+      class << self
+        attr_accessor :cache_dir
+      end
+
       def self.input_to_storage(_, path)
         RubyVM::InstructionSequence.compile_file(path).to_binary
       rescue SyntaxError
@@ -28,6 +32,7 @@ module Bootsnap
       module InstructionSequenceMixin
         def load_iseq(path)
           Bootsnap::CompileCache::Native.fetch(
+            Bootsnap::CompileCache::ISeq.cache_dir,
             path.to_s,
             Bootsnap::CompileCache::ISeq
           )
@@ -62,7 +67,8 @@ module Bootsnap
         Bootsnap::CompileCache::Native.compile_option_crc32 = crc
       end
 
-      def self.install!
+      def self.install!(cache_dir)
+        Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
         Bootsnap::CompileCache::ISeq.compile_option_updated
         class << RubyVM::InstructionSequence
           prepend InstructionSequenceMixin

data/lib/bootsnap/compile_cache/yaml.rb

@@ -30,7 +30,7 @@ module Bootsnap
         ::YAML.load(data)
       end
 
-      def self.install!
+      def self.install!(cache_dir)
         require 'yaml'
         require 'msgpack'
 
@@ -44,6 +44,7 @@ module Bootsnap
         klass = class << ::YAML; self; end
         klass.send(:define_method, :load_file) do |path|
           Bootsnap::CompileCache::Native.fetch(
+            cache_dir,
             path.to_s,
             Bootsnap::CompileCache::YAML
           )

data/lib/bootsnap/compile_cache.rb

@@ -3,13 +3,13 @@ require_relative 'compile_cache/yaml'
 
 module Bootsnap
   module CompileCache
-    def self.setup(iseq:, yaml:)
+    def self.setup(cache_dir:, iseq:, yaml:)
       if iseq
-        Bootsnap::CompileCache::ISeq.install!
+        Bootsnap::CompileCache::ISeq.install!(cache_dir)
       end
 
       if yaml
-        Bootsnap::CompileCache::YAML.install!
+        Bootsnap::CompileCache::YAML.install!(cache_dir)
       end
     end
   end

data/lib/bootsnap/load_path_cache/cache.rb

@@ -22,6 +22,27 @@ module Bootsnap
         @mutex.synchronize { @dirs[dir] }
       end
 
+      # { 'enumerator' => nil, 'enumerator.so' => nil, ... }
+      BUILTIN_FEATURES = $LOADED_FEATURES.reduce({}) do |acc, feat|
+        # Builtin features are of the form 'enumerator.so'.
+        # All others include paths.
+        next acc unless feat.size < 20 && !feat.include?('/')
+
+        base = File.basename(feat, '.*') # enumerator.so -> enumerator
+        ext  = File.extname(feat) # .so
+
+        acc[feat] = nil # enumerator.so
+        acc[base] = nil # enumerator
+
+        if [DOT_SO, *DL_EXTENSIONS].include?(ext)
+          DL_EXTENSIONS.each do |ext|
+            acc["#{base}#{ext}"] = nil # enumerator.bundle
+          end
+        end
+
+        acc
+      end.freeze
+
       # Try to resolve this feature to an absolute path without traversing the
       # loadpath.
       def find(feature)
@@ -33,22 +54,20 @@ module Bootsnap
           x = search_index(feature)
           return x if x
 
+          # Ruby has some built-in features that require lies about.
+          # For example, 'enumerator' is built in. If you require it, ruby
+          # returns false as if it were already loaded; however, there is no
+          # file to find on disk. We've pre-built a list of these, and we
+          # return false if any of them is loaded.
+          raise LoadPathCache::ReturnFalse if BUILTIN_FEATURES.key?(feature)
+
           # The feature wasn't found on our preliminary search through the index.
           # We resolve this differently depending on what the extension was.
           case File.extname(feature)
           # If the extension was one of the ones we explicitly cache (.rb and the
           # native dynamic extension, e.g. .bundle or .so), we know it was a
           # failure and there's nothing more we can do to find the file.
-          when *CACHED_EXTENSIONS # .rb, .bundle or .so
-            nil
-          # If no extension was specified, it's the same situation, since we
-          # try appending both cachable extensions in search_index. However,
-          # there's a special-case for 'enumerator'. Before ruby 1.9, you had
-          # to `require 'enumerator'` to use it. In 1.9+, it's pre-loaded, but
-          # doesn't correspond to any entry on the filesystem. Ruby lies. So we
-          # lie too, forcing our monkeypatch to return false like ruby would.
-          when ""
-            raise LoadPathCache::ReturnFalse if feature == 'enumerator' || feature == 'thread'
+          when '', *CACHED_EXTENSIONS # no extension, .rb, (.bundle or .so)
             nil
           # Ruby allows specifying native extensions as '.so' even when DLEXT
           # is '.bundle'. This is where we handle that case.
@@ -99,8 +118,9 @@ module Bootsnap
 
       def push_paths_locked(*paths)
         @store.transaction do
-          paths.each do |path|
+          paths.map(&:to_s).each do |path|
             p = Path.new(path)
+            next if p.non_directory?
             entries, dirs = p.entries_and_dirs(@store)
             # push -> low precedence -> set only if unset
             dirs.each    { |dir| @dirs[dir]  ||= true }
@@ -111,8 +131,9 @@ module Bootsnap
 
       def unshift_paths_locked(*paths)
         @store.transaction do
-          paths.reverse.each do |path|
+          paths.map(&:to_s).reverse.each do |path|
             p = Path.new(path)
+            next if p.non_directory?
             entries, dirs = p.entries_and_dirs(@store)
             # unshift -> high precedence -> unconditional set
             dirs.each    { |dir| @dirs[dir]  = true }

data/lib/bootsnap/load_path_cache/change_observer.rb

@@ -12,25 +12,25 @@ module Bootsnap
         sc = arr.singleton_class
         sc.send(:alias_method, :shovel_without_lpc, :<<)
         arr.define_singleton_method(:<<) do |entry|
-          observer.push_paths(self, entry)
+          observer.push_paths(self, entry.to_s)
           shovel_without_lpc(entry)
         end
 
         sc.send(:alias_method, :push_without_lpc, :push)
         arr.define_singleton_method(:push) do |*entries|
-          observer.push_paths(self, *entries)
+          observer.push_paths(self, *entries.map(&:to_s))
           push_without_lpc(*entries)
         end
 
         sc.send(:alias_method, :unshift_without_lpc, :unshift)
         arr.define_singleton_method(:unshift) do |*entries|
-          observer.unshift_paths(self, *entries)
+          observer.unshift_paths(self, *entries.map(&:to_s))
           unshift_without_lpc(*entries)
         end
 
         sc.send(:alias_method, :concat_without_lpc, :concat)
         arr.define_singleton_method(:concat) do |entries|
-          observer.push_paths(self, *entries)
+          observer.push_paths(self, *entries.map(&:to_s))
           concat_without_lpc(entries)
         end
 

data/lib/bootsnap/load_path_cache/core_ext/active_support.rb

@@ -4,7 +4,10 @@ module Bootsnap
       module ActiveSupport
         def self.with_bootsnap_fallback(error)
           yield
-        rescue error
+        rescue error => e
+          # NoMethodError is a NameError, but we only want to handle actual
+          # NameError instances.
+          raise unless e.class == error
           without_bootsnap_cache { yield }
         end
 
@@ -50,7 +53,9 @@ module Bootsnap
             CoreExt::ActiveSupport.with_bootsnap_fallback(NameError) { super }
           end
 
-          def depend_on(file_name, message = "No such file to load -- %s.rb")
+          # Signature has changed a few times over the years; easiest to not
+          # reiterate it with version polymorphism here...
+          def depend_on(*)
             CoreExt::ActiveSupport.with_bootsnap_fallback(LoadError) { super }
           end
         end

data/lib/bootsnap/load_path_cache/path.rb

@@ -20,7 +20,14 @@ module Bootsnap
       attr_reader :path
 
       def initialize(path)
-        @path = path
+        @path = path.to_s
+      end
+
+      # True if the path exists, but represents a non-directory object
+      def non_directory?
+        !File.stat(path).directory?
+      rescue Errno::ENOENT
+        false
       end
 
       # Return a list of all the requirable files and all of the subdirectories
@@ -75,13 +82,16 @@ module Bootsnap
       STABLE   = :stable
       VOLATILE = :volatile
 
+      # Built-in ruby lib stuff doesn't change, but things can occasionally be
+      # installed into sitedir, which often lives under prefix.
       RUBY_PREFIX = RbConfig::CONFIG['prefix']
+      SITE_DIR    = RbConfig::CONFIG['sitedir']
 
       def stability
         @stability ||= begin
-          if Gem.path.detect { |p| path.start_with?(p) }
+          if Gem.path.detect { |p| path.start_with?(p.to_s) }
             STABLE
-          elsif path.start_with?(RUBY_PREFIX)
+          elsif path.start_with?(RUBY_PREFIX) && !path.start_with?(SITE_DIR)
             STABLE
           else
             VOLATILE

data/lib/bootsnap/load_path_cache/path_scanner.rb

@@ -12,6 +12,7 @@ module Bootsnap
       BUNDLE_PATH = (Bundler.bundle_path.cleanpath.to_s << LoadPathCache::SLASH).freeze
 
       def self.call(path)
+        path = path.to_s
         raise RelativePathNotSupported unless path.start_with?(SLASH)
 
         relative_slice = (path.size + 1)..-1

data/lib/bootsnap/load_path_cache/store.rb

@@ -1,6 +1,5 @@
 require_relative '../explicit_require'
 
-Bootsnap::ExplicitRequire.with_gems('snappy')  { require 'snappy' }
 Bootsnap::ExplicitRequire.with_gems('msgpack') { require 'msgpack' }
 Bootsnap::ExplicitRequire.from_rubylibdir('fileutils')
 
@@ -58,8 +57,9 @@ module Bootsnap
 
       def load_data
         @data = begin
-          MessagePack.load(Snappy.inflate(File.binread(@store_path)))
-        rescue Errno::ENOENT, Snappy::Error
+          MessagePack.load(File.binread(@store_path))
+        # handle malformed data due to upgrade incompatability
+        rescue Errno::ENOENT, MessagePack::MalformedFormatError, MessagePack::UnknownExtTypeError
           {}
         end
       end
@@ -69,7 +69,7 @@ module Bootsnap
         # caches if they read at an inopportune time.
         tmp = "#{@store_path}.#{(rand * 100000).to_i}.tmp"
         FileUtils.mkpath(File.dirname(tmp))
-        File.binwrite(tmp, Snappy.deflate(MessagePack.dump(@data)))
+        File.binwrite(tmp, MessagePack.dump(@data))
         FileUtils.mv(tmp, @store_path)
       end
     end

data/lib/bootsnap/version.rb

@@ -1,3 +1,3 @@
 module Bootsnap
-  VERSION = "0.2.2"
+  VERSION = "0.3.0"
 end

data/lib/bootsnap.rb

@@ -27,6 +27,7 @@ module Bootsnap
     ) if load_path_cache
 
     Bootsnap::CompileCache.setup(
+      cache_dir: cache_dir + '/bootsnap-compile-cache',
       iseq: compile_cache_iseq,
       yaml: compile_cache_yaml
     )

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bootsnap
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.3.0
 platform: ruby
 authors:
 - Burke Libbey
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2017-04-05 00:00:00.000000000 Z
+date: 2017-05-26 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -81,33 +81,33 @@ dependencies:
       - !ruby/object:Gem::Version
         version: '1.2'
 - !ruby/object:Gem::Dependency
-  name: msgpack
+  name: ffi-xattr
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
-  type: :runtime
+        version: 0.1.2
+  type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: 0.1.2
 - !ruby/object:Gem::Dependency
-  name: snappy
+  name: msgpack
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.15
+        version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: 0.0.15
+        version: '1.0'
 description: wip.
 email:
 - burke.libbey@shopify.com
@@ -118,6 +118,8 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".rubocop.yml"
+- ".travis.yml"
+- CHANGELOG.md
 - CONTRIBUTING.md
 - Gemfile
 - LICENSE
@@ -130,7 +132,6 @@ files:
 - dev.yml
 - ext/bootsnap/bootsnap.c
 - ext/bootsnap/bootsnap.h
-- ext/bootsnap/crc32.c
 - ext/bootsnap/extconf.rb
 - lib/bootsnap.rb
 - lib/bootsnap/compile_cache.rb
@@ -158,7 +159,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '0'
+      version: 2.3.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="

data/ext/bootsnap/crc32.c

@@ -1,16 +0,0 @@
-#include "bootsnap.h"
-
-#include <x86intrin.h>
-
-uint32_t
-crc32(const char *bytes, size_t size)
-{
-  size_t i;
-  uint32_t hash = 0;
-
-  for (i = 0; i < size; i++) {
-    hash = _mm_crc32_u8(hash, bytes[i]);
-  }
-  return hash;
-}
-