bootsnap 1.6.0 → 1.15.0

data/lib/bootsnap/cli.rb

@@ -1,10 +1,10 @@
 # frozen_string_literal: true
 
-require 'bootsnap'
-require 'bootsnap/cli/worker_pool'
-require 'optparse'
-require 'fileutils'
-require 'etc'
+require "bootsnap"
+require "bootsnap/cli/worker_pool"
+require "optparse"
+require "fileutils"
+require "etc"
 
 module Bootsnap
   class CLI
@@ -21,51 +21,58 @@ module Bootsnap
 
     attr_reader :cache_dir, :argv
 
-    attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :jobs
+    attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :json, :jobs
 
     def initialize(argv)
       @argv = argv
-      self.cache_dir = ENV.fetch('BOOTSNAP_CACHE_DIR', 'tmp/cache')
+      self.cache_dir = ENV.fetch("BOOTSNAP_CACHE_DIR", "tmp/cache")
       self.compile_gemfile = false
       self.exclude = nil
       self.verbose = false
       self.jobs = Etc.nprocessors
       self.iseq = true
       self.yaml = true
+      self.json = true
     end
 
     def precompile_command(*sources)
-      require 'bootsnap/compile_cache/iseq'
-      require 'bootsnap/compile_cache/yaml'
+      require "bootsnap/compile_cache/iseq"
+      require "bootsnap/compile_cache/yaml"
+      require "bootsnap/compile_cache/json"
 
       fix_default_encoding do
-        Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
+        Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
         Bootsnap::CompileCache::YAML.init!
-        Bootsnap::CompileCache::YAML.cache_dir = self.cache_dir
+        Bootsnap::CompileCache::YAML.cache_dir = cache_dir
+        Bootsnap::CompileCache::JSON.init!
+        Bootsnap::CompileCache::JSON.cache_dir = cache_dir
 
         @work_pool = WorkerPool.create(size: jobs, jobs: {
           ruby: method(:precompile_ruby),
           yaml: method(:precompile_yaml),
+          json: method(:precompile_json),
         })
         @work_pool.spawn
 
         main_sources = sources.map { |d| File.expand_path(d) }
         precompile_ruby_files(main_sources)
         precompile_yaml_files(main_sources)
+        precompile_json_files(main_sources)
 
         if compile_gemfile
           # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
-          gem_exclude = Regexp.union([exclude, '/spec/', '/test/'].compact)
+          gem_exclude = Regexp.union([exclude, "/spec/", "/test/"].compact)
           precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
 
-          # Gems that include YAML files usually don't put them in `lib/`.
+          # Gems that include JSON or YAML files usually don't put them in `lib/`.
           # So we look at the gem root.
-          gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gem\/[^/]+}
+          gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems/[^/]+}
           gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
           precompile_yaml_files(gem_paths, exclude: gem_exclude)
+          precompile_json_files(gem_paths, exclude: gem_exclude)
         end
 
-        if exitstatus = @work_pool.shutdown
+        if (exitstatus = @work_pool.shutdown)
           exit(exitstatus)
         end
       end
@@ -82,7 +89,7 @@ module Bootsnap
     if dir_sort
       def list_files(path, pattern)
         if File.directory?(path)
-          Dir[File.join(path,  pattern), sort: false]
+          Dir[File.join(path, pattern), sort: false]
         elsif File.exist?(path)
           [path]
         else
@@ -92,7 +99,7 @@ module Bootsnap
     else
       def list_files(path, pattern)
         if File.directory?(path)
-          Dir[File.join(path,  pattern)]
+          Dir[File.join(path, pattern)]
         elsif File.exist?(path)
           [path]
         else
@@ -119,9 +126,9 @@ module Bootsnap
 
       load_paths.each do |path|
         if !exclude || !exclude.match?(path)
-          list_files(path, '**/*.{yml,yaml}').each do |yaml_file|
+          list_files(path, "**/*.{yml,yaml}").each do |yaml_file|
             # We ignore hidden files to not match the various .ci.yml files
-            if !yaml_file.include?('/.') && (!exclude || !exclude.match?(yaml_file))
+            if !File.basename(yaml_file).start_with?(".") && (!exclude || !exclude.match?(yaml_file))
               @work_pool.push(:yaml, yaml_file)
             end
           end
@@ -131,8 +138,31 @@ module Bootsnap
 
     def precompile_yaml(*yaml_files)
       Array(yaml_files).each do |yaml_file|
-        if CompileCache::YAML.precompile(yaml_file, cache_dir: cache_dir)
-          STDERR.puts(yaml_file) if verbose
+        if CompileCache::YAML.precompile(yaml_file) && verbose
+          $stderr.puts(yaml_file)
+        end
+      end
+    end
+
+    def precompile_json_files(load_paths, exclude: self.exclude)
+      return unless json
+
+      load_paths.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_files(path, "**/*.json").each do |json_file|
+            # We ignore hidden files to not match the various .config.json files
+            if !File.basename(json_file).start_with?(".") && (!exclude || !exclude.match?(json_file))
+              @work_pool.push(:json, json_file)
+            end
+          end
+        end
+      end
+    end
+
+    def precompile_json(*json_files)
+      Array(json_files).each do |json_file|
+        if CompileCache::JSON.precompile(json_file) && verbose
+          $stderr.puts(json_file)
         end
       end
     end
@@ -142,7 +172,7 @@ module Bootsnap
 
       load_paths.each do |path|
         if !exclude || !exclude.match?(path)
-          list_files(path, '**/*.rb').each do |ruby_file|
+          list_files(path, "**/{*.rb,*.rake,Rakefile}").each do |ruby_file|
             if !exclude || !exclude.match?(ruby_file)
               @work_pool.push(:ruby, ruby_file)
             end
@@ -153,8 +183,8 @@ module Bootsnap
 
     def precompile_ruby(*ruby_files)
       Array(ruby_files).each do |ruby_file|
-        if CompileCache::ISeq.precompile(ruby_file, cache_dir: cache_dir)
-          STDERR.puts(ruby_file) if verbose
+        if CompileCache::ISeq.precompile(ruby_file) && verbose
+          $stderr.puts(ruby_file)
         end
       end
     end
@@ -173,14 +203,14 @@ module Bootsnap
     end
 
     def invalid_usage!(message)
-      STDERR.puts message
-      STDERR.puts
-      STDERR.puts parser
+      $stderr.puts message
+      $stderr.puts
+      $stderr.puts parser
       1
     end
 
     def cache_dir=(dir)
-      @cache_dir = File.expand_path(File.join(dir, 'bootsnap/compile-cache'))
+      @cache_dir = File.expand_path(File.join(dir, "bootsnap/compile-cache"))
     end
 
     def exclude_pattern(pattern)
@@ -195,24 +225,24 @@ module Bootsnap
         opts.separator "GLOBAL OPTIONS"
         opts.separator ""
 
-        help = <<~EOS
+        help = <<~HELP
           Path to the bootsnap cache directory. Defaults to tmp/cache
-        EOS
-        opts.on('--cache-dir DIR', help.strip) do |dir|
+        HELP
+        opts.on("--cache-dir DIR", help.strip) do |dir|
           self.cache_dir = dir
         end
 
-        help = <<~EOS
+        help = <<~HELP
           Print precompiled paths.
-        EOS
-        opts.on('--verbose', '-v', help.strip) do
+        HELP
+        opts.on("--verbose", "-v", help.strip) do
           self.verbose = true
         end
 
-        help = <<~EOS
+        help = <<~HELP
           Number of workers to use. Default to number of processors, set to 0 to disable multi-processing.
-        EOS
-        opts.on('--jobs JOBS', '-j', help.strip) do |jobs|
+        HELP
+        opts.on("--jobs JOBS", "-j", help.strip) do |jobs|
           self.jobs = Integer(jobs)
         end
 
@@ -221,25 +251,30 @@ module Bootsnap
         opts.separator ""
         opts.separator "    precompile [DIRECTORIES...]: Precompile all .rb files in the passed directories"
 
-        help = <<~EOS
+        help = <<~HELP
           Precompile the gems in Gemfile
-        EOS
-        opts.on('--gemfile', help) { self.compile_gemfile = true }
+        HELP
+        opts.on("--gemfile", help) { self.compile_gemfile = true }
 
-        help = <<~EOS
+        help = <<~HELP
           Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api'
-        EOS
-        opts.on('--exclude PATTERN', help) { |pattern| exclude_pattern(pattern) }
+        HELP
+        opts.on("--exclude PATTERN", help) { |pattern| exclude_pattern(pattern) }
 
-        help = <<~EOS
+        help = <<~HELP
           Disable ISeq (.rb) precompilation.
-        EOS
-        opts.on('--no-iseq', help) { self.iseq = false }
+        HELP
+        opts.on("--no-iseq", help) { self.iseq = false }
 
-        help = <<~EOS
+        help = <<~HELP
           Disable YAML precompilation.
-        EOS
-        opts.on('--no-yaml', help) { self.yaml = false }
+        HELP
+        opts.on("--no-yaml", help) { self.yaml = false }
+
+        help = <<~HELP
+          Disable JSON precompilation.
+        HELP
+        opts.on("--no-json", help) { self.json = false }
       end
     end
   end

data/lib/bootsnap/compile_cache/iseq.rb

@@ -1,25 +1,55 @@
 # frozen_string_literal: true
-require('bootsnap/bootsnap')
-require('zlib')
+
+require("bootsnap/bootsnap")
+require("zlib")
 
 module Bootsnap
   module CompileCache
     module ISeq
       class << self
-        attr_accessor(:cache_dir)
+        attr_reader(:cache_dir)
+
+        def cache_dir=(cache_dir)
+          @cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}iseq" : "#{cache_dir}-iseq"
+        end
+      end
+
+      has_ruby_bug_18250 = begin # https://bugs.ruby-lang.org/issues/18250
+        if defined? RubyVM::InstructionSequence
+          RubyVM::InstructionSequence.compile("def foo(*); ->{ super }; end; def foo(**); ->{ super }; end").to_binary
+        end
+        false
+      rescue TypeError
+        true
       end
 
-      def self.input_to_storage(_, path)
-        RubyVM::InstructionSequence.compile_file(path).to_binary
-      rescue SyntaxError
-        raise(Uncompilable, 'syntax error')
+      if has_ruby_bug_18250
+        def self.input_to_storage(_, path)
+          iseq = begin
+            RubyVM::InstructionSequence.compile_file(path)
+          rescue SyntaxError
+            return UNCOMPILABLE # syntax error
+          end
+
+          begin
+            iseq.to_binary
+          rescue TypeError
+            UNCOMPILABLE # ruby bug #18250
+          end
+        end
+      else
+        def self.input_to_storage(_, path)
+          RubyVM::InstructionSequence.compile_file(path).to_binary
+        rescue SyntaxError
+          UNCOMPILABLE # syntax error
+        end
       end
 
       def self.storage_to_output(binary, _args)
         RubyVM::InstructionSequence.load_from_binary(binary)
-      rescue RuntimeError => e
-        if e.message == 'broken binary format'
-          STDERR.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
+      rescue RuntimeError => error
+        if error.message == "broken binary format"
+          $stderr.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
           nil
         else
           raise
@@ -35,7 +65,7 @@ module Bootsnap
         )
       end
 
-      def self.precompile(path, cache_dir: ISeq.cache_dir)
+      def self.precompile(path)
         Bootsnap::CompileCache::Native.precompile(
           cache_dir,
           path.to_s,
@@ -55,8 +85,8 @@ module Bootsnap
           Bootsnap::CompileCache::ISeq.fetch(path.to_s)
         rescue Errno::EACCES
           Bootsnap::CompileCache.permission_error(path)
-        rescue RuntimeError => e
-          if e.message =~ /unmatched platform/
+        rescue RuntimeError => error
+          if error.message =~ /unmatched platform/
             puts("unmatched platform for file #{path}")
           end
           raise

data/lib/bootsnap/compile_cache/json.rb

@@ -0,0 +1,93 @@
+# frozen_string_literal: true
+
+require("bootsnap/bootsnap")
+
+module Bootsnap
+  module CompileCache
+    module JSON
+      class << self
+        attr_accessor(:msgpack_factory, :supported_options)
+        attr_reader(:cache_dir)
+
+        def cache_dir=(cache_dir)
+          @cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}json" : "#{cache_dir}-json"
+        end
+
+        def input_to_storage(payload, _)
+          obj = ::JSON.parse(payload)
+          msgpack_factory.dump(obj)
+        end
+
+        def storage_to_output(data, kwargs)
+          if kwargs&.key?(:symbolize_names)
+            kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
+          end
+          msgpack_factory.load(data, kwargs)
+        end
+
+        def input_to_output(data, kwargs)
+          ::JSON.parse(data, **(kwargs || {}))
+        end
+
+        def precompile(path)
+          Bootsnap::CompileCache::Native.precompile(
+            cache_dir,
+            path.to_s,
+            self,
+          )
+        end
+
+        def install!(cache_dir)
+          self.cache_dir = cache_dir
+          init!
+          if ::JSON.respond_to?(:load_file)
+            ::JSON.singleton_class.prepend(Patch)
+          end
+        end
+
+        def init!
+          require("json")
+          require("msgpack")
+
+          self.msgpack_factory = MessagePack::Factory.new
+          self.supported_options = [:symbolize_names]
+          if supports_freeze?
+            self.supported_options = [:freeze]
+          end
+          supported_options.freeze
+        end
+
+        private
+
+        def supports_freeze?
+          ::JSON.parse('["foo"]', freeze: true).first.frozen? &&
+            MessagePack.load(MessagePack.dump("foo"), freeze: true).frozen?
+        end
+      end
+
+      module Patch
+        def load_file(path, *args)
+          return super if args.size > 1
+
+          if (kwargs = args.first)
+            return super unless kwargs.is_a?(Hash)
+            return super unless (kwargs.keys - ::Bootsnap::CompileCache::JSON.supported_options).empty?
+          end
+
+          begin
+            ::Bootsnap::CompileCache::Native.fetch(
+              Bootsnap::CompileCache::JSON.cache_dir,
+              File.realpath(path),
+              ::Bootsnap::CompileCache::JSON,
+              kwargs,
+            )
+          rescue Errno::EACCES
+            ::Bootsnap::CompileCache.permission_error(path)
+          end
+        end
+
+        ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
+      end
+    end
+  end
+end