ed-precompiled_bootsnap 1.18.6-arm64-darwin

data/ext/bootsnap/bootsnap.h

@@ -0,0 +1,6 @@
+#ifndef BOOTSNAP_H
+#define BOOTSNAP_H 1
+
+/* doesn't expose anything */
+
+#endif /* BOOTSNAP_H */

data/ext/bootsnap/extconf.rb

@@ -0,0 +1,33 @@
+# frozen_string_literal: true
+
+require "mkmf"
+
+if %w[ruby truffleruby].include?(RUBY_ENGINE)
+  have_func "fdatasync", "unistd.h"
+
+  unless RUBY_PLATFORM.match?(/mswin|mingw|cygwin/)
+    append_cppflags ["-D_GNU_SOURCE"] # Needed of O_NOATIME
+  end
+
+  append_cflags ["-O3", "-std=c99"]
+
+  # ruby.h has some -Wpedantic fails in some cases
+  # (e.g. https://github.com/rails/bootsnap/issues/15)
+  unless ["0", "", nil].include?(ENV["BOOTSNAP_PEDANTIC"])
+    append_cflags([
+      "-Wall",
+      "-Werror",
+      "-Wextra",
+      "-Wpedantic",
+
+      "-Wno-unused-parameter", # VALUE self has to be there but we don't care what it is.
+      "-Wno-keyword-macro", # hiding return
+      "-Wno-gcc-compat", # ruby.h 2.6.0 on macos 10.14, dunno
+      "-Wno-compound-token-split-by-macro",
+    ])
+  end
+
+  create_makefile("bootsnap/bootsnap")
+else
+  File.write("Makefile", dummy_makefile($srcdir).join)
+end

data/lib/bootsnap/bootsnap_ext.rb

@@ -0,0 +1,7 @@
+ruby_version = /(\d+\.\d+)/.match(::RUBY_VERSION)
+
+begin
+  require "bootsnap/#{ruby_version}/bootsnap"
+rescue LoadError
+  require "bootsnap/bootsnap"
+end

data/lib/bootsnap/bundler.rb

@@ -0,0 +1,16 @@
+# frozen_string_literal: true
+
+module Bootsnap
+  extend self
+
+  def bundler?
+    return false unless defined?(::Bundler)
+
+    # Bundler environment variable
+    %w(BUNDLE_BIN_PATH BUNDLE_GEMFILE).each do |current|
+      return true if ENV.key?(current)
+    end
+
+    false
+  end
+end

data/lib/bootsnap/cli/worker_pool.rb

@@ -0,0 +1,208 @@
+# frozen_string_literal: true
+
+require "etc"
+require "rbconfig"
+require "io/wait" unless IO.method_defined?(:wait_readable)
+
+module Bootsnap
+  class CLI
+    class WorkerPool
+      class << self
+        def create(size:, jobs:)
+          size ||= default_size
+          if size > 0 && Process.respond_to?(:fork)
+            new(size: size, jobs: jobs)
+          else
+            Inline.new(jobs: jobs)
+          end
+        end
+
+        def default_size
+          nprocessors = Etc.nprocessors
+          size = [nprocessors, cpu_quota&.to_i || nprocessors].min
+          case size
+          when 0, 1
+            0
+          else
+            if fork_defunct?
+              $stderr.puts "warning: faulty fork(2) detected, probably in cross platform docker builds. " \
+                           "Disabling parallel compilation."
+              0
+            else
+              size
+            end
+          end
+        end
+
+        def cpu_quota
+          if RbConfig::CONFIG["target_os"].include?("linux")
+            if File.exist?("/sys/fs/cgroup/cpu.max")
+              # cgroups v2: https://docs.kernel.org/admin-guide/cgroup-v2.html#cpu-interface-files
+              cpu_max = File.read("/sys/fs/cgroup/cpu.max")
+              return nil if cpu_max.start_with?("max ") # no limit
+
+              max, period = cpu_max.split.map(&:to_f)
+              max / period
+            elsif File.exist?("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us")
+              # cgroups v1: https://kernel.googlesource.com/pub/scm/linux/kernel/git/glommer/memcg/+/cpu_stat/Documentation/cgroups/cpu.txt
+              max = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_quota_us").to_i
+              # If the cpu.cfs_quota_us is -1, cgroup does not adhere to any CPU time restrictions
+              # https://docs.kernel.org/scheduler/sched-bwc.html#management
+              return nil if max <= 0
+
+              period = File.read("/sys/fs/cgroup/cpu,cpuacct/cpu.cfs_period_us").to_f
+              max / period
+            end
+          end
+        end
+
+        def fork_defunct?
+          return true unless ::Process.respond_to?(:fork)
+
+          # Ref: https://github.com/rails/bootsnap/issues/495
+          # The second forked process will hang on some QEMU environments
+          r, w = IO.pipe
+          pids = 2.times.map do
+            ::Process.fork do
+              exit!(true)
+            end
+          end
+          w.close
+          r.wait_readable(1) # Wait at most 1s
+
+          defunct = false
+
+          pids.each do |pid|
+            _pid, status = ::Process.wait2(pid, ::Process::WNOHANG)
+            if status.nil? # Didn't exit in 1s
+              defunct = true
+              Process.kill(:KILL, pid)
+              ::Process.wait2(pid)
+            end
+          end
+
+          defunct
+        end
+      end
+
+      class Inline
+        def initialize(jobs: {})
+          @jobs = jobs
+        end
+
+        def push(job, *args)
+          @jobs.fetch(job).call(*args)
+          nil
+        end
+
+        def spawn
+          # noop
+        end
+
+        def shutdown
+          # noop
+        end
+      end
+
+      class Worker
+        attr_reader :to_io, :pid
+
+        def initialize(jobs)
+          @jobs = jobs
+          @pipe_out, @to_io = IO.pipe(binmode: true)
+          # Set the writer encoding to binary since IO.pipe only sets it for the reader.
+          # https://github.com/rails/rails/issues/16514#issuecomment-52313290
+          @to_io.set_encoding(Encoding::BINARY)
+
+          @pid = nil
+        end
+
+        def write(message, block: true)
+          payload = Marshal.dump(message)
+          if block
+            to_io.write(payload)
+            true
+          else
+            to_io.write_nonblock(payload, exception: false) != :wait_writable
+          end
+        end
+
+        def close
+          to_io.close
+        end
+
+        def work_loop
+          loop do
+            job, *args = Marshal.load(@pipe_out)
+            return if job == :exit
+
+            @jobs.fetch(job).call(*args)
+          end
+        rescue IOError
+          nil
+        end
+
+        def spawn
+          @pid = Process.fork do
+            to_io.close
+            work_loop
+            exit!(0)
+          end
+          @pipe_out.close
+          true
+        end
+      end
+
+      def initialize(size:, jobs: {})
+        @size = size
+        @jobs = jobs
+        @queue = Queue.new
+        @pids = []
+      end
+
+      def spawn
+        @workers = @size.times.map { Worker.new(@jobs) }
+        @workers.each(&:spawn)
+        @dispatcher_thread = Thread.new { dispatch_loop }
+        @dispatcher_thread.abort_on_exception = true
+        true
+      end
+
+      def dispatch_loop
+        loop do
+          case job = @queue.pop
+          when nil
+            @workers.each do |worker|
+              worker.write([:exit])
+              worker.close
+            end
+            return true
+          else
+            unless @workers.sample.write(job, block: false)
+              free_worker.write(job)
+            end
+          end
+        end
+      end
+
+      def free_worker
+        IO.select(nil, @workers)[1].sample
+      end
+
+      def push(*args)
+        @queue.push(args)
+        nil
+      end
+
+      def shutdown
+        @queue.close
+        @dispatcher_thread.join
+        @workers.each do |worker|
+          _pid, status = Process.wait2(worker.pid)
+          return status.exitstatus unless status.success?
+        end
+        nil
+      end
+    end
+  end
+end

data/lib/bootsnap/cli.rb

@@ -0,0 +1,285 @@
+# frozen_string_literal: true
+
+require "bootsnap"
+require "bootsnap/cli/worker_pool"
+require "optparse"
+require "fileutils"
+
+module Bootsnap
+  class CLI
+    unless Regexp.method_defined?(:match?)
+      module RegexpMatchBackport
+        refine Regexp do
+          def match?(string)
+            !!match(string)
+          end
+        end
+      end
+      using RegexpMatchBackport
+    end
+
+    attr_reader :cache_dir, :argv
+
+    attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :json, :jobs
+
+    def initialize(argv)
+      @argv = argv
+      self.cache_dir = ENV.fetch("BOOTSNAP_CACHE_DIR", "tmp/cache")
+      self.compile_gemfile = false
+      self.exclude = nil
+      self.verbose = false
+      self.jobs = nil
+      self.iseq = true
+      self.yaml = true
+      self.json = true
+    end
+
+    def precompile_command(*sources)
+      require "bootsnap/compile_cache"
+
+      fix_default_encoding do
+        Bootsnap::CompileCache.setup(
+          cache_dir: cache_dir,
+          iseq: iseq,
+          yaml: yaml,
+          json: json,
+          revalidation: true,
+        )
+
+        @work_pool = WorkerPool.create(size: jobs, jobs: {
+          ruby: method(:precompile_ruby),
+          yaml: method(:precompile_yaml),
+          json: method(:precompile_json),
+        })
+        @work_pool.spawn
+
+        main_sources = sources.map { |d| File.expand_path(d) }
+        precompile_ruby_files(main_sources)
+        precompile_yaml_files(main_sources)
+        precompile_json_files(main_sources)
+
+        if compile_gemfile
+          # Gems that include JSON or YAML files usually don't put them in `lib/`.
+          # So we look at the gem root.
+          # Similarly, gems that include Rails engines generally file Ruby files in `app/`.
+          # However some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
+          gem_exclude = Regexp.union([exclude, "/spec/", "/test/", "/features/"].compact)
+
+          gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems/[^/]+}
+          gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] || p }.uniq
+
+          precompile_ruby_files(gem_paths, exclude: gem_exclude)
+          precompile_yaml_files(gem_paths, exclude: gem_exclude)
+          precompile_json_files(gem_paths, exclude: gem_exclude)
+        end
+
+        if (exitstatus = @work_pool.shutdown)
+          exit(exitstatus)
+        end
+      end
+      0
+    end
+
+    dir_sort = begin
+      Dir[__FILE__, sort: false]
+      true
+    rescue ArgumentError, TypeError
+      false
+    end
+
+    if dir_sort
+      def list_files(path, pattern)
+        if File.directory?(path)
+          Dir[File.join(path, pattern), sort: false]
+        elsif File.exist?(path)
+          [path]
+        else
+          []
+        end
+      end
+    else
+      def list_files(path, pattern)
+        if File.directory?(path)
+          Dir[File.join(path, pattern)]
+        elsif File.exist?(path)
+          [path]
+        else
+          []
+        end
+      end
+    end
+
+    def run
+      parser.parse!(argv)
+      command = argv.shift
+      method = "#{command}_command"
+      if respond_to?(method)
+        public_send(method, *argv)
+      else
+        invalid_usage!("Unknown command: #{command}")
+      end
+    end
+
+    private
+
+    def precompile_yaml_files(load_paths, exclude: self.exclude)
+      return unless yaml
+
+      load_paths.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_files(path, "**/*.{yml,yaml}").each do |yaml_file|
+            # We ignore hidden files to not match the various .ci.yml files
+            if !File.basename(yaml_file).start_with?(".") && (!exclude || !exclude.match?(yaml_file))
+              @work_pool.push(:yaml, yaml_file)
+            end
+          end
+        end
+      end
+    end
+
+    def precompile_yaml(*yaml_files)
+      Array(yaml_files).each do |yaml_file|
+        if CompileCache::YAML.precompile(yaml_file) && verbose
+          $stderr.puts(yaml_file)
+        end
+      end
+    end
+
+    def precompile_json_files(load_paths, exclude: self.exclude)
+      return unless json
+
+      load_paths.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_files(path, "**/*.json").each do |json_file|
+            # We ignore hidden files to not match the various .config.json files
+            if !File.basename(json_file).start_with?(".") && (!exclude || !exclude.match?(json_file))
+              @work_pool.push(:json, json_file)
+            end
+          end
+        end
+      end
+    end
+
+    def precompile_json(*json_files)
+      Array(json_files).each do |json_file|
+        if CompileCache::JSON.precompile(json_file) && verbose
+          $stderr.puts(json_file)
+        end
+      end
+    end
+
+    def precompile_ruby_files(load_paths, exclude: self.exclude)
+      return unless iseq
+
+      load_paths.each do |path|
+        if !exclude || !exclude.match?(path)
+          list_files(path, "**/{*.rb,*.rake,Rakefile}").each do |ruby_file|
+            if !exclude || !exclude.match?(ruby_file)
+              @work_pool.push(:ruby, ruby_file)
+            end
+          end
+        end
+      end
+    end
+
+    def precompile_ruby(*ruby_files)
+      Array(ruby_files).each do |ruby_file|
+        if CompileCache::ISeq.precompile(ruby_file) && verbose
+          $stderr.puts(ruby_file)
+        end
+      end
+    end
+
+    def fix_default_encoding
+      if Encoding.default_external == Encoding::US_ASCII
+        Encoding.default_external = Encoding::UTF_8
+        begin
+          yield
+        ensure
+          Encoding.default_external = Encoding::US_ASCII
+        end
+      else
+        yield
+      end
+    end
+
+    def invalid_usage!(message)
+      $stderr.puts message
+      $stderr.puts
+      $stderr.puts parser
+      1
+    end
+
+    def cache_dir=(dir)
+      @cache_dir = File.expand_path(File.join(dir, "bootsnap/compile-cache"))
+    end
+
+    def exclude_pattern(pattern)
+      (@exclude_patterns ||= []) << Regexp.new(pattern)
+      self.exclude = Regexp.union(@exclude_patterns)
+    end
+
+    def parser
+      @parser ||= OptionParser.new do |opts|
+        opts.version = Bootsnap::VERSION
+        opts.program_name = "bootsnap"
+
+        opts.banner = "Usage: bootsnap COMMAND [ARGS]"
+        opts.separator ""
+        opts.separator "GLOBAL OPTIONS"
+        opts.separator ""
+
+        help = <<~HELP
+          Path to the bootsnap cache directory. Defaults to tmp/cache
+        HELP
+        opts.on("--cache-dir DIR", help.strip) do |dir|
+          self.cache_dir = dir
+        end
+
+        help = <<~HELP
+          Print precompiled paths.
+        HELP
+        opts.on("--verbose", "-v", help.strip) do
+          self.verbose = true
+        end
+
+        help = <<~HELP
+          Number of workers to use. Default to number of processors, set to 0 to disable multi-processing.
+        HELP
+        opts.on("--jobs JOBS", "-j", help.strip) do |jobs|
+          self.jobs = Integer(jobs)
+        end
+
+        opts.separator ""
+        opts.separator "COMMANDS"
+        opts.separator ""
+        opts.separator "    precompile [DIRECTORIES...]: Precompile all .rb files in the passed directories"
+
+        help = <<~HELP
+          Precompile the gems in Gemfile
+        HELP
+        opts.on("--gemfile", help) { self.compile_gemfile = true }
+
+        help = <<~HELP
+          Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api'
+        HELP
+        opts.on("--exclude PATTERN", help) { |pattern| exclude_pattern(pattern) }
+
+        help = <<~HELP
+          Disable ISeq (.rb) precompilation.
+        HELP
+        opts.on("--no-iseq", help) { self.iseq = false }
+
+        help = <<~HELP
+          Disable YAML precompilation.
+        HELP
+        opts.on("--no-yaml", help) { self.yaml = false }
+
+        help = <<~HELP
+          Disable JSON precompilation.
+        HELP
+        opts.on("--no-json", help) { self.json = false }
+      end
+    end
+  end
+end

data/lib/bootsnap/compile_cache/iseq.rb

@@ -0,0 +1,123 @@
+# frozen_string_literal: true
+
+require "bootsnap/bootsnap_ext"
+require "zlib"
+
+module Bootsnap
+  module CompileCache
+    module ISeq
+      class << self
+        attr_reader(:cache_dir)
+
+        def cache_dir=(cache_dir)
+          @cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}iseq" : "#{cache_dir}-iseq"
+        end
+
+        def supported?
+          CompileCache.supported? && defined?(RubyVM)
+        end
+      end
+
+      has_ruby_bug_18250 = begin # https://bugs.ruby-lang.org/issues/18250
+        if defined? RubyVM::InstructionSequence
+          RubyVM::InstructionSequence.compile("def foo(*); ->{ super }; end; def foo(**); ->{ super }; end").to_binary
+        end
+        false
+      rescue TypeError
+        true
+      end
+
+      if has_ruby_bug_18250
+        def self.input_to_storage(_, path)
+          iseq = begin
+            RubyVM::InstructionSequence.compile_file(path)
+          rescue SyntaxError
+            return UNCOMPILABLE # syntax error
+          end
+
+          begin
+            iseq.to_binary
+          rescue TypeError
+            UNCOMPILABLE # ruby bug #18250
+          end
+        end
+      else
+        def self.input_to_storage(_, path)
+          RubyVM::InstructionSequence.compile_file(path).to_binary
+        rescue SyntaxError
+          UNCOMPILABLE # syntax error
+        end
+      end
+
+      def self.storage_to_output(binary, _args)
+        RubyVM::InstructionSequence.load_from_binary(binary)
+      rescue RuntimeError => error
+        if error.message == "broken binary format"
+          $stderr.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
+          nil
+        else
+          raise
+        end
+      end
+
+      def self.fetch(path, cache_dir: ISeq.cache_dir)
+        Bootsnap::CompileCache::Native.fetch(
+          cache_dir,
+          path.to_s,
+          Bootsnap::CompileCache::ISeq,
+          nil,
+        )
+      end
+
+      def self.precompile(path)
+        Bootsnap::CompileCache::Native.precompile(
+          cache_dir,
+          path.to_s,
+          Bootsnap::CompileCache::ISeq,
+        )
+      end
+
+      def self.input_to_output(_data, _kwargs)
+        nil # ruby handles this
+      end
+
+      module InstructionSequenceMixin
+        def load_iseq(path)
+          # Having coverage enabled prevents iseq dumping/loading.
+          return nil if defined?(Coverage) && Coverage.running?
+
+          Bootsnap::CompileCache::ISeq.fetch(path.to_s)
+        rescue RuntimeError => error
+          if error.message =~ /unmatched platform/
+            puts("unmatched platform for file #{path}")
+          end
+          raise
+        end
+
+        def compile_option=(hash)
+          super(hash)
+          Bootsnap::CompileCache::ISeq.compile_option_updated
+        end
+      end
+
+      def self.compile_option_updated
+        option = RubyVM::InstructionSequence.compile_option
+        crc = Zlib.crc32(option.inspect)
+        Bootsnap::CompileCache::Native.compile_option_crc32 = crc
+      end
+      compile_option_updated if supported?
+
+      def self.install!(cache_dir)
+        Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
+
+        return unless supported?
+
+        Bootsnap::CompileCache::ISeq.compile_option_updated
+
+        class << RubyVM::InstructionSequence
+          prepend(InstructionSequenceMixin)
+        end
+      end
+    end
+  end
+end