bootsnap 1.1.0 → 1.2.0

data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb

@@ -11,78 +11,104 @@ module Bootsnap
 end
 
 module Kernel
-  alias_method :require_without_cache, :require
+  alias_method :require_without_bootsnap, :require
+
+  # Note that require registers to $LOADED_FEATURES while load does not.
+  def require_with_bootsnap_lfi(path, resolved = nil)
+    Bootsnap::LoadPathCache.loaded_features_index.register(path, resolved) do
+      require_without_bootsnap(resolved || path)
+    end
+  end
+
   def require(path)
+    return false if Bootsnap::LoadPathCache.loaded_features_index.key?(path)
+
     if resolved = Bootsnap::LoadPathCache.load_path_cache.find(path)
-      require_without_cache(resolved)
-    else
-      raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
+      return require_with_bootsnap_lfi(path, resolved)
     end
+
+    raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
   rescue Bootsnap::LoadPathCache::ReturnFalse
     return false
   rescue Bootsnap::LoadPathCache::FallbackScan
-    require_without_cache(path)
+    require_with_bootsnap_lfi(path)
   end
 
-  alias_method :load_without_cache, :load
+  alias_method :load_without_bootsnap, :load
   def load(path, wrap = false)
     if resolved = Bootsnap::LoadPathCache.load_path_cache.find(path)
-      load_without_cache(resolved, wrap)
-    else
-      # load also allows relative paths from pwd even when not in $:
-      relative = File.expand_path(path)
-      if File.exist?(File.expand_path(path))
-        return load_without_cache(relative, wrap)
-      end
-      raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
+      return load_without_bootsnap(resolved, wrap)
     end
+
+    # load also allows relative paths from pwd even when not in $:
+    if File.exist?(relative = File.expand_path(path))
+      return load_without_bootsnap(relative, wrap)
+    end
+
+    raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
   rescue Bootsnap::LoadPathCache::ReturnFalse
     return false
   rescue Bootsnap::LoadPathCache::FallbackScan
-    load_without_cache(path, wrap)
+    load_without_bootsnap(path, wrap)
   end
 end
 
 class << Kernel
-  alias_method :require_without_cache, :require
+  alias_method :require_without_bootsnap, :require
+
+  def require_with_bootsnap_lfi(path, resolved = nil)
+    Bootsnap::LoadPathCache.loaded_features_index.register(path, resolved) do
+      require_without_bootsnap(resolved || path)
+    end
+  end
+
   def require(path)
+    return false if Bootsnap::LoadPathCache.loaded_features_index.key?(path)
+
     if resolved = Bootsnap::LoadPathCache.load_path_cache.find(path)
-      require_without_cache(resolved)
-    else
-      raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
+      return require_with_bootsnap_lfi(path, resolved)
     end
+
+    raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
   rescue Bootsnap::LoadPathCache::ReturnFalse
     return false
   rescue Bootsnap::LoadPathCache::FallbackScan
-    require_without_cache(path)
+    require_with_bootsnap_lfi(path)
   end
 
-  alias_method :load_without_cache, :load
+  alias_method :load_without_bootsnap, :load
   def load(path, wrap = false)
     if resolved = Bootsnap::LoadPathCache.load_path_cache.find(path)
-      load_without_cache(resolved, wrap)
-    else
-      # load also allows relative paths from pwd even when not in $:
-      relative = File.expand_path(path)
-      if File.exist?(relative)
-        return load_without_cache(relative, wrap)
-      end
-      raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
+      return load_without_bootsnap(resolved, wrap)
     end
+
+    # load also allows relative paths from pwd even when not in $:
+    if File.exist?(relative = File.expand_path(path))
+      return load_without_bootsnap(relative, wrap)
+    end
+
+    raise Bootsnap::LoadPathCache::CoreExt.make_load_error(path)
   rescue Bootsnap::LoadPathCache::ReturnFalse
     return false
   rescue Bootsnap::LoadPathCache::FallbackScan
-    load_without_cache(path, wrap)
+    load_without_bootsnap(path, wrap)
   end
 end
 
 class Module
-  alias_method :autoload_without_cache, :autoload
+  alias_method :autoload_without_bootsnap, :autoload
   def autoload(const, path)
-    autoload_without_cache(const, Bootsnap::LoadPathCache.load_path_cache.find(path) || path)
+    # NOTE: This may defeat LoadedFeaturesIndex, but it's not immediately
+    # obvious how to make it work. This feels like a pretty niche case, unclear
+    # if it will ever burn anyone.
+    #
+    # The challenge is that we don't control the point at which the entry gets
+    # added to $LOADED_FEATURES and won't be able to hook that modification
+    # since it's done in C-land.
+    autoload_without_bootsnap(const, Bootsnap::LoadPathCache.load_path_cache.find(path) || path)
   rescue Bootsnap::LoadPathCache::ReturnFalse
     return false
   rescue Bootsnap::LoadPathCache::FallbackScan
-    autoload_without_cache(const, path)
+    autoload_without_bootsnap(const, path)
   end
 end

data/lib/bootsnap/load_path_cache/loaded_features_index.rb

@@ -0,0 +1,95 @@
+module Bootsnap
+  module LoadPathCache
+    # LoadedFeaturesIndex partially mirrors an internal structure in ruby that
+    # we can't easily obtain an interface to.
+    #
+    # This works around an issue where, without bootsnap, *ruby* knows that it
+    # has already required a file by its short name (e.g. require 'bundler') if
+    # a new instance of bundler is added to the $LOAD_PATH which resolves to a
+    # different absolute path. This class makes bootsnap smart enough to
+    # realize that it has already loaded 'bundler', and not just
+    # '/path/to/bundler'.
+    #
+    # If you disable LoadedFeaturesIndex, you can see the problem this solves by:
+    #
+    # 1. `require 'a'`
+    # 2. Prepend a new $LOAD_PATH element containing an `a.rb`
+    # 3. `require 'a'`
+    #
+    # Ruby returns false from step 3.
+    # With bootsnap but with no LoadedFeaturesIndex, this loads two different
+    #   `a.rb`s.
+    # With bootsnap and with LoadedFeaturesIndex, this skips the second load,
+    #   returning false like ruby.
+    class LoadedFeaturesIndex
+      def initialize
+        @lfi = {}
+        @mutex = defined?(::Mutex) ? ::Mutex.new : ::Thread::Mutex.new # TODO: Remove once Ruby 2.2 support is dropped.
+
+        # In theory the user could mutate $LOADED_FEATURES and invalidate our
+        # cache. If this ever comes up in practice — or if you, the
+        # enterprising reader, feels inclined to solve this problem — we could
+        # parallel the work done with ChangeObserver on $LOAD_PATH to mirror
+        # updates to our @lfi.
+        $LOADED_FEATURES.each do |feat|
+          $LOAD_PATH.each do |lpe|
+            next unless feat.start_with?(lpe)
+            # /a/b/lib/my/foo.rb
+            #          ^^^^^^^^^
+            short = feat[(lpe.length + 1)..-1]
+            @lfi[short] = true
+            @lfi[strip_extension(short)] = true
+          end
+        end
+      end
+
+      def key?(feature)
+        @mutex.synchronize { @lfi.key?(feature) }
+      end
+
+      # There is a relatively uncommon case where we could miss adding an
+      # entry:
+      #
+      # If the user asked for e.g. `require 'bundler'`, and we went through the
+      # `FallbackScan` pathway in `kernel_require.rb` and therefore did not
+      # pass `long` (the full expanded absolute path), then we did are not able
+      # to confidently add the `bundler.rb` form to @lfi.
+      #
+      # We could either:
+      #
+      # 1. Just add `bundler.rb`, `bundler.so`, and so on, which is close but
+      #    not quite right; or
+      # 2. Inspect $LOADED_FEATURES upon return from yield to find the matching
+      #    entry.
+      def register(short, long = nil)
+        ret = yield
+
+        # do we have 'bundler' or 'bundler.rb'?
+        altname = if File.extname(short) != ''
+          # strip the path from 'bundler.rb' -> 'bundler'
+          strip_extension(short)
+        elsif long && ext = File.extname(long)
+          # get the extension from the expanded path if given
+          # 'bundler' + '.rb'
+          short + ext
+        end
+
+        @mutex.synchronize do
+          @lfi[short] = true
+          (@lfi[altname] = true) if altname
+        end
+
+        ret
+      end
+
+      private
+
+      STRIP_EXTENSION = /\..*?$/
+      private_constant :STRIP_EXTENSION
+
+      def strip_extension(f)
+        f.sub(STRIP_EXTENSION, '')
+      end
+    end
+  end
+end

data/lib/bootsnap/load_path_cache/path.rb

@@ -99,7 +99,7 @@ module Bootsnap
         @stability ||= begin
           if Gem.path.detect { |p| expanded_path.start_with?(p.to_s) }
             STABLE
-          elsif expanded_path.start_with?(Bundler.bundle_path.to_s)
+          elsif Bootsnap.bundler? && expanded_path.start_with?(Bundler.bundle_path.to_s)
             STABLE
           elsif expanded_path.start_with?(RUBY_LIBDIR) && !expanded_path.start_with?(RUBY_SITEDIR)
             STABLE

data/lib/bootsnap/load_path_cache/path_scanner.rb

@@ -1,13 +1,22 @@
-require_relative '../load_path_cache'
+require_relative '../explicit_require'
 
 module Bootsnap
   module LoadPathCache
     module PathScanner
-      REQUIRABLES_AND_DIRS = "/**/*{#{DOT_RB},#{DL_EXTENSIONS.join(',')},/}"
-      IS_DIR = %r{(.*)/\z}
+      # Glob pattern to find requirable files and subdirectories in given path.
+      # It expands to:
+      #
+      #   * `/*{.rb,.so,/}` - It matches requirable files, directories and
+      #     symlinks to directories at given path.
+      #   * `/*/**/*{.rb,.so,/}` - It matches requirable files and
+      #     subdirectories in any (even symlinked) directory at given path at
+      #     any directory tree depth.
+      #
+      REQUIRABLES_AND_DIRS = "/{,*/**/}*{#{DOT_RB},#{DL_EXTENSIONS.join(',')},/}"
       NORMALIZE_NATIVE_EXTENSIONS = !DL_EXTENSIONS.include?(LoadPathCache::DOT_SO)
       ALTERNATIVE_NATIVE_EXTENSIONS_PATTERN = /\.(o|bundle|dylib)\z/
-      BUNDLE_PATH = (Bundler.bundle_path.cleanpath.to_s << LoadPathCache::SLASH).freeze
+      BUNDLE_PATH = Bootsnap.bundler? ?
+        (Bundler.bundle_path.cleanpath.to_s << LoadPathCache::SLASH).freeze : ''.freeze
 
       def self.call(path)
         path = path.to_s
@@ -29,12 +38,13 @@ module Bootsnap
           next if contains_bundle_path && absolute_path.start_with?(BUNDLE_PATH)
           relative_path = absolute_path.slice!(relative_slice)
 
-          if md = relative_path.match(IS_DIR)
-            dirs << md[1]
+          if relative_path.end_with?('/')
+            dirs << relative_path[0..-2]
           else
             requirables << relative_path
           end
         end
+
         [requirables, dirs]
       end
     end

data/lib/bootsnap/load_path_cache/store.rb

@@ -11,6 +11,8 @@ module Bootsnap
 
       def initialize(store_path)
         @store_path = store_path
+        @in_txn = false
+        @dirty = false
         load_data
       end
 
@@ -67,10 +69,15 @@ module Bootsnap
       def dump_data
         # Change contents atomically so other processes can't get invalid
         # caches if they read at an inopportune time.
-        tmp = "#{@store_path}.#{(rand * 100000).to_i}.tmp"
+        tmp = "#{@store_path}.#{Process.pid}.#{(rand * 100000).to_i}.tmp"
         FileUtils.mkpath(File.dirname(tmp))
-        File.binwrite(tmp, MessagePack.dump(@data))
+        exclusive_write = File::Constants::CREAT | File::Constants::EXCL | File::Constants::WRONLY
+        # `encoding:` looks redundant wrt `binwrite`, but necessary on windows
+        # because binary is part of mode.
+        File.binwrite(tmp, MessagePack.dump(@data), mode: exclusive_write, encoding: Encoding::BINARY)
         FileUtils.mv(tmp, @store_path)
+      rescue Errno::EEXIST
+        retry
       end
     end
   end

data/lib/bootsnap/load_path_cache.rb

@@ -21,11 +21,13 @@ module Bootsnap
     CACHED_EXTENSIONS = DLEXT2 ? [DOT_RB, DLEXT, DLEXT2] : [DOT_RB, DLEXT]
 
     class << self
-      attr_reader :load_path_cache, :autoload_paths_cache
+      attr_reader :load_path_cache, :autoload_paths_cache, :loaded_features_index
 
       def setup(cache_path:, development_mode:, active_support: true)
         store = Store.new(cache_path)
 
+        @loaded_features_index = LoadedFeaturesIndex.new
+
         @load_path_cache = Cache.new(store, $LOAD_PATH, development_mode: development_mode)
         require_relative 'load_path_cache/core_ext/kernel_require'
 
@@ -50,3 +52,4 @@ require_relative 'load_path_cache/path'
 require_relative 'load_path_cache/cache'
 require_relative 'load_path_cache/store'
 require_relative 'load_path_cache/change_observer'
+require_relative 'load_path_cache/loaded_features_index'

data/lib/bootsnap/setup.rb

@@ -4,16 +4,10 @@ env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || ENV['ENV']
 development_mode = ['', nil, 'development'].include?(env)
 
 # only enable on 'ruby' (MRI), POSIX (darin, linux, *bsd), and >= 2.3.0
-enable_cc = \
-  RUBY_ENGINE == 'ruby' && \
-  RUBY_PLATFORM =~ /darwin|linux|bsd/ && \
-  RUBY_VERSION                    # "1.9.3"
-    .split('.')                   # ["1", "9", "3"]
-    .map(&:to_i)                  # [1, 9, 3]
-    .zip([2, 3, -1])              # [[1, 2], [9, 3], [3, -1]]
-    .map { |a, b| a <=> b }       # [-1, 1, 1]
-    .detect { |e| !e.zero? }      # -1
-    .==(1)                        # false
+enable_cc =
+  RUBY_ENGINE == 'ruby' &&
+  RUBY_PLATFORM =~ /darwin|linux|bsd/ &&
+  Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("2.3.0")
 
 cache_dir = ENV['BOOTSNAP_CACHE_DIR']
 unless cache_dir

data/lib/bootsnap/version.rb

@@ -1,3 +1,3 @@
 module Bootsnap
-  VERSION = "1.1.0"
+  VERSION = "1.2.0"
 end

data/lib/bootsnap.rb

@@ -1,4 +1,5 @@
 require_relative 'bootsnap/version'
+require_relative 'bootsnap/bundler'
 require_relative 'bootsnap/load_path_cache'
 require_relative 'bootsnap/compile_cache'
 

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: bootsnap
 version: !ruby/object:Gem::Version
-  version: 1.1.0
+  version: 1.2.0
 platform: ruby
 authors:
 - Burke Libbey
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2017-06-19 00:00:00.000000000 Z
+date: 2018-03-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -94,7 +94,7 @@ dependencies:
     - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
-description: wip.
+description: Boot large ruby/rails apps faster
 email:
 - burke.libbey@shopify.com
 executables: []
@@ -106,9 +106,11 @@ files:
 - ".rubocop.yml"
 - ".travis.yml"
 - CHANGELOG.md
+- CODE_OF_CONDUCT.md
 - CONTRIBUTING.md
 - Gemfile
-- LICENSE
+- LICENSE.txt
+- README.jp.md
 - README.md
 - Rakefile
 - bin/console
@@ -120,6 +122,7 @@ files:
 - ext/bootsnap/bootsnap.h
 - ext/bootsnap/extconf.rb
 - lib/bootsnap.rb
+- lib/bootsnap/bundler.rb
 - lib/bootsnap/compile_cache.rb
 - lib/bootsnap/compile_cache/iseq.rb
 - lib/bootsnap/compile_cache/yaml.rb
@@ -129,6 +132,7 @@ files:
 - lib/bootsnap/load_path_cache/change_observer.rb
 - lib/bootsnap/load_path_cache/core_ext/active_support.rb
 - lib/bootsnap/load_path_cache/core_ext/kernel_require.rb
+- lib/bootsnap/load_path_cache/loaded_features_index.rb
 - lib/bootsnap/load_path_cache/path.rb
 - lib/bootsnap/load_path_cache/path_scanner.rb
 - lib/bootsnap/load_path_cache/store.rb
@@ -154,8 +158,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.6.10
+rubygems_version: 2.7.6
 signing_key: 
 specification_version: 4
-summary: wip
+summary: Boot large ruby/rails apps faster
 test_files: []

data/LICENSE

@@ -1,20 +0,0 @@
-Copyright (c) 2017 Shopify
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.