bootsnap 1.5.0 → 1.7.1

data/ext/bootsnap/extconf.rb

@@ -1,19 +1,24 @@
 # frozen_string_literal: true
 require("mkmf")
-$CFLAGS << ' -O3 '
-$CFLAGS << ' -std=c99'
 
-# ruby.h has some -Wpedantic fails in some cases
-# (e.g. https://github.com/Shopify/bootsnap/issues/15)
-unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC'])
-  $CFLAGS << ' -Wall'
-  $CFLAGS << ' -Werror'
-  $CFLAGS << ' -Wextra'
-  $CFLAGS << ' -Wpedantic'
+if RUBY_ENGINE == 'ruby'
+  $CFLAGS << ' -O3 '
+  $CFLAGS << ' -std=c99'
 
-  $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
-  $CFLAGS << ' -Wno-keyword-macro' # hiding return
-  $CFLAGS << ' -Wno-gcc-compat' # ruby.h 2.6.0 on macos 10.14, dunno
-end
+  # ruby.h has some -Wpedantic fails in some cases
+  # (e.g. https://github.com/Shopify/bootsnap/issues/15)
+  unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC'])
+    $CFLAGS << ' -Wall'
+    $CFLAGS << ' -Werror'
+    $CFLAGS << ' -Wextra'
+    $CFLAGS << ' -Wpedantic'
+
+    $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
+    $CFLAGS << ' -Wno-keyword-macro' # hiding return
+    $CFLAGS << ' -Wno-gcc-compat' # ruby.h 2.6.0 on macos 10.14, dunno
+  end
 
-create_makefile("bootsnap/bootsnap")
+  create_makefile("bootsnap/bootsnap")
+else
+  File.write("Makefile", dummy_makefile($srcdir).join(""))
+end

data/lib/bootsnap.rb

@@ -8,42 +8,114 @@ require_relative('bootsnap/compile_cache')
 module Bootsnap
   InvalidConfiguration = Class.new(StandardError)
 
+  class << self
+    attr_reader :logger
+  end
+
+  def self.log!
+    self.logger = $stderr.method(:puts)
+  end
+
+  def self.logger=(logger)
+    @logger = logger
+    if logger.respond_to?(:debug)
+      self.instrumentation = ->(event, path) { @logger.debug("[Bootsnap] #{event} #{path}") }
+    else
+      self.instrumentation = ->(event, path) { @logger.call("[Bootsnap] #{event} #{path}") }
+    end
+  end
+
+  def self.instrumentation=(callback)
+    @instrumentation = callback
+    self.instrumentation_enabled = !!callback
+  end
+
+  def self._instrument(event, path)
+    @instrumentation.call(event, path)
+  end
+
   def self.setup(
     cache_dir:,
     development_mode: true,
     load_path_cache: true,
-    autoload_paths_cache: true,
-    disable_trace: false,
+    autoload_paths_cache: nil,
+    disable_trace: nil,
     compile_cache_iseq: true,
     compile_cache_yaml: true
   )
-    if autoload_paths_cache && !load_path_cache
-      raise(InvalidConfiguration, "feature 'autoload_paths_cache' depends on feature 'load_path_cache'")
+    unless autoload_paths_cache.nil?
+      warn "[DEPRECATED] Bootsnap's `autoload_paths_cache:` option is deprecated and will be removed. " \
+        "If you use Zeitwerk this option is useless, and if you are still using the classic autoloader " \
+        "upgrading is recommended."
     end
 
-    setup_disable_trace if disable_trace
+    unless disable_trace.nil?
+      warn "[DEPRECATED] Bootsnap's `disable_trace:` option is deprecated and will be removed. " \
+        "If you use Ruby 2.5 or newer this option is useless, if not upgrading is recommended."
+    end
+
+    if compile_cache_iseq && !iseq_cache_supported?
+      warn "Ruby 2.5 has a bug that break code tracing when code is loaded from cache. It is recommened " \
+        "to turn `compile_cache_iseq` off on Ruby 2.5"
+    end
 
     Bootsnap::LoadPathCache.setup(
-      cache_path:       cache_dir + '/bootsnap-load-path-cache',
+      cache_path:       cache_dir + '/bootsnap/load-path-cache',
       development_mode: development_mode,
-      active_support:   autoload_paths_cache
     ) if load_path_cache
 
     Bootsnap::CompileCache.setup(
-      cache_dir: cache_dir + '/bootsnap-compile-cache',
+      cache_dir: cache_dir + '/bootsnap/compile-cache',
       iseq: compile_cache_iseq,
       yaml: compile_cache_yaml
     )
   end
 
-  def self.setup_disable_trace
-    if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.5.0')
-      warn(
-        "from #{caller_locations(1, 1)[0]}: The 'disable_trace' method is not allowed with this Ruby version. " \
-        "current: #{RUBY_VERSION}, allowed version: < 2.5.0",
+  def self.iseq_cache_supported?
+    return @iseq_cache_supported if defined? @iseq_cache_supported
+
+    ruby_version = Gem::Version.new(RUBY_VERSION)
+    @iseq_cache_supported = ruby_version < Gem::Version.new('2.5.0') || ruby_version >= Gem::Version.new('2.6.0')
+  end
+
+  def self.default_setup
+    env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || ENV['ENV']
+    development_mode = ['', nil, 'development'].include?(env)
+
+    unless ENV['DISABLE_BOOTSNAP']
+      cache_dir = ENV['BOOTSNAP_CACHE_DIR']
+      unless cache_dir
+        config_dir_frame = caller.detect do |line|
+          line.include?('/config/')
+        end
+
+        unless config_dir_frame
+          $stderr.puts("[bootsnap/setup] couldn't infer cache directory! Either:")
+          $stderr.puts("[bootsnap/setup]   1. require bootsnap/setup from your application's config directory; or")
+          $stderr.puts("[bootsnap/setup]   2. Define the environment variable BOOTSNAP_CACHE_DIR")
+
+          raise("couldn't infer bootsnap cache directory")
+        end
+
+        path = config_dir_frame.split(/:\d+:/).first
+        path = File.dirname(path) until File.basename(path) == 'config'
+        app_root = File.dirname(path)
+
+        cache_dir = File.join(app_root, 'tmp', 'cache')
+      end
+
+
+      setup(
+        cache_dir:            cache_dir,
+        development_mode:     development_mode,
+        load_path_cache:      !ENV['DISABLE_BOOTSNAP_LOAD_PATH_CACHE'],
+        compile_cache_iseq:   !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'] && iseq_cache_supported?,
+        compile_cache_yaml:   !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'],
       )
-    else
-      RubyVM::InstructionSequence.compile_option = { trace_instruction: false }
+
+      if ENV['BOOTSNAP_LOG']
+        log!
+      end
     end
   end
 end

data/lib/bootsnap/cli.rb

@@ -1,14 +1,16 @@
 # frozen_string_literal: true
 
 require 'bootsnap'
+require 'bootsnap/cli/worker_pool'
 require 'optparse'
 require 'fileutils'
+require 'etc'
 
 module Bootsnap
   class CLI
     unless Regexp.method_defined?(:match?)
       module RegexpMatchBackport
-        refine Regepx do
+        refine Regexp do
           def match?(string)
             !!match(string)
           end
@@ -19,47 +21,68 @@ module Bootsnap
 
     attr_reader :cache_dir, :argv
 
-    attr_accessor :compile_gemfile, :exclude
+    attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :jobs
 
     def initialize(argv)
       @argv = argv
-      self.cache_dir = 'tmp/cache'
+      self.cache_dir = ENV.fetch('BOOTSNAP_CACHE_DIR', 'tmp/cache')
       self.compile_gemfile = false
       self.exclude = nil
+      self.verbose = false
+      self.jobs = Etc.nprocessors
+      self.iseq = true
+      self.yaml = true
     end
 
     def precompile_command(*sources)
       require 'bootsnap/compile_cache/iseq'
+      require 'bootsnap/compile_cache/yaml'
+
+      fix_default_encoding do
+        Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
+        Bootsnap::CompileCache::YAML.init!
+        Bootsnap::CompileCache::YAML.cache_dir = self.cache_dir
+
+        @work_pool = WorkerPool.create(size: jobs, jobs: {
+          ruby: method(:precompile_ruby),
+          yaml: method(:precompile_yaml),
+        })
+        @work_pool.spawn
+
+        main_sources = sources.map { |d| File.expand_path(d) }
+        precompile_ruby_files(main_sources)
+        precompile_yaml_files(main_sources)
+
+        if compile_gemfile
+          # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
+          gem_exclude = Regexp.union([exclude, '/spec/', '/test/'].compact)
+          precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
+
+          # Gems that include YAML files usually don't put them in `lib/`.
+          # So we look at the gem root.
+          gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems\/[^/]+}
+          gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
+          precompile_yaml_files(gem_paths, exclude: gem_exclude)
+        end
 
-      Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
-
-      if compile_gemfile
-        sources += $LOAD_PATH
-      end
-
-      sources.map { |d| File.expand_path(d) }.each do |path|
-        if !exclude || !exclude.match?(path)
-          list_ruby_files(path).each do |ruby_file|
-            if !exclude || !exclude.match?(ruby_file)
-              CompileCache::ISeq.fetch(ruby_file, cache_dir: cache_dir)
-            end
-          end
+        if exitstatus = @work_pool.shutdown
+          exit(exitstatus)
         end
       end
       0
     end
 
     dir_sort = begin
-      Dir['.', sort: false]
+      Dir[__FILE__, sort: false]
       true
     rescue ArgumentError, TypeError
       false
     end
 
     if dir_sort
-      def list_ruby_files(path)
+      def list_files(path, pattern)
         if File.directory?(path)
-          Dir[File.join(path, '**/*.rb'), sort: false]
+          Dir[File.join(path,  pattern), sort: false]
         elsif File.exist?(path)
           [path]
         else
@@ -67,9 +90,9 @@ module Bootsnap
         end
       end
     else
-      def list_ruby_files(path)
+      def list_files(path, pattern)
         if File.directory?(path)
-          Dir[File.join(path, '**/*.rb')]
+          Dir[File.join(path,  pattern)]
         elsif File.exist?(path)
           [path]
         else
@@ -91,6 +114,64 @@ module Bootsnap
 
     private
 
+    def precompile_yaml_files(load_paths, exclude: self.exclude)
+      return unless yaml
+
+      load_paths.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_files(path, '**/*.{yml,yaml}').each do |yaml_file|
+            # We ignore hidden files to not match the various .ci.yml files
+            if !File.basename(yaml_file).start_with?('.') && (!exclude || !exclude.match?(yaml_file))
+              @work_pool.push(:yaml, yaml_file)
+            end
+          end
+        end
+      end
+    end
+
+    def precompile_yaml(*yaml_files)
+      Array(yaml_files).each do |yaml_file|
+        if CompileCache::YAML.precompile(yaml_file, cache_dir: cache_dir)
+          STDERR.puts(yaml_file) if verbose
+        end
+      end
+    end
+
+    def precompile_ruby_files(load_paths, exclude: self.exclude)
+      return unless iseq
+
+      load_paths.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_files(path, '**/*.rb').each do |ruby_file|
+            if !exclude || !exclude.match?(ruby_file)
+              @work_pool.push(:ruby, ruby_file)
+            end
+          end
+        end
+      end
+    end
+
+    def precompile_ruby(*ruby_files)
+      Array(ruby_files).each do |ruby_file|
+        if CompileCache::ISeq.precompile(ruby_file, cache_dir: cache_dir)
+          STDERR.puts(ruby_file) if verbose
+        end
+      end
+    end
+
+    def fix_default_encoding
+      if Encoding.default_external == Encoding::US_ASCII
+        Encoding.default_external = Encoding::UTF_8
+        begin
+          yield
+        ensure
+          Encoding.default_external = Encoding::US_ASCII
+        end
+      else
+        yield
+      end
+    end
+
     def invalid_usage!(message)
       STDERR.puts message
       STDERR.puts
@@ -99,7 +180,12 @@ module Bootsnap
     end
 
     def cache_dir=(dir)
-      @cache_dir = File.expand_path(File.join(dir, 'bootsnap-compile-cache'))
+      @cache_dir = File.expand_path(File.join(dir, 'bootsnap/compile-cache'))
+    end
+
+    def exclude_pattern(pattern)
+      (@exclude_patterns ||= []) << Regexp.new(pattern)
+      self.exclude = Regexp.union(@exclude_patterns)
     end
 
     def parser
@@ -116,6 +202,20 @@ module Bootsnap
           self.cache_dir = dir
         end
 
+        help = <<~EOS
+          Print precompiled paths.
+        EOS
+        opts.on('--verbose', '-v', help.strip) do
+          self.verbose = true
+        end
+
+        help = <<~EOS
+          Number of workers to use. Default to number of processors, set to 0 to disable multi-processing.
+        EOS
+        opts.on('--jobs JOBS', '-j', help.strip) do |jobs|
+          self.jobs = Integer(jobs)
+        end
+
         opts.separator ""
         opts.separator "COMMANDS"
         opts.separator ""
@@ -129,7 +229,17 @@ module Bootsnap
         help = <<~EOS
           Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api'
         EOS
-        opts.on('--exclude PATTERN', help) { |pattern| self.exclude = Regexp.new(pattern) }
+        opts.on('--exclude PATTERN', help) { |pattern| exclude_pattern(pattern) }
+
+        help = <<~EOS
+          Disable ISeq (.rb) precompilation.
+        EOS
+        opts.on('--no-iseq', help) { self.iseq = false }
+
+        help = <<~EOS
+          Disable YAML precompilation.
+        EOS
+        opts.on('--no-yaml', help) { self.yaml = false }
       end
     end
   end

data/lib/bootsnap/cli/worker_pool.rb

@@ -0,0 +1,131 @@
+# frozen_string_literal: true
+
+module Bootsnap
+  class CLI
+    class WorkerPool
+      class << self
+        def create(size:, jobs:)
+          if size > 0 && Process.respond_to?(:fork)
+            new(size: size, jobs: jobs)
+          else
+            Inline.new(jobs: jobs)
+          end
+        end
+      end
+
+      class Inline
+        def initialize(jobs: {})
+          @jobs = jobs
+        end
+
+        def push(job, *args)
+          @jobs.fetch(job).call(*args)
+          nil
+        end
+
+        def spawn
+          # noop
+        end
+
+        def shutdown
+          # noop
+        end
+      end
+
+      class Worker
+        attr_reader :to_io, :pid
+
+        def initialize(jobs)
+          @jobs = jobs
+          @pipe_out, @to_io = IO.pipe
+          @pid = nil
+        end
+
+        def write(message, block: true)
+          payload = Marshal.dump(message)
+          if block
+            to_io.write(payload)
+            true
+          else
+            to_io.write_nonblock(payload, exception: false) != :wait_writable
+          end
+        end
+
+        def close
+          to_io.close
+        end
+
+        def work_loop
+          loop do
+            job, *args = Marshal.load(@pipe_out)
+            return if job == :exit
+            @jobs.fetch(job).call(*args)
+          end
+        rescue IOError
+          nil
+        end
+
+        def spawn
+          @pid = Process.fork do
+            to_io.close
+            work_loop
+            exit!(0)
+          end
+          @pipe_out.close
+          true
+        end
+      end
+
+      def initialize(size:, jobs: {})
+        @size = size
+        @jobs = jobs
+        @queue = Queue.new
+        @pids = []
+      end
+
+      def spawn
+        @workers = @size.times.map { Worker.new(@jobs) }
+        @workers.each(&:spawn)
+        @dispatcher_thread = Thread.new { dispatch_loop }
+        @dispatcher_thread.abort_on_exception = true
+        true
+      end
+
+      def dispatch_loop
+        loop do
+          case job = @queue.pop
+          when nil
+            @workers.each do |worker|
+              worker.write([:exit])
+              worker.close
+            end
+            return true
+          else
+            unless @workers.sample.write(job, block: false)
+              free_worker.write(job)
+            end
+          end
+        end
+      end
+
+      def free_worker
+        IO.select(nil, @workers)[1].sample
+      end
+
+      def push(*args)
+        @queue.push(args)
+        nil
+      end
+
+      def shutdown
+        @queue.close
+        @dispatcher_thread.join
+        @workers.each do |worker|
+          _pid, status = Process.wait2(worker.pid)
+          return status.exitstatus unless status.success?
+        end
+        nil
+      end
+    end
+  end
+end