bootsnap 1.5.0 → 1.7.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,19 +1,24 @@
1
1
  # frozen_string_literal: true
2
2
  require("mkmf")
3
- $CFLAGS << ' -O3 '
4
- $CFLAGS << ' -std=c99'
5
3
 
6
- # ruby.h has some -Wpedantic fails in some cases
7
- # (e.g. https://github.com/Shopify/bootsnap/issues/15)
8
- unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC'])
9
- $CFLAGS << ' -Wall'
10
- $CFLAGS << ' -Werror'
11
- $CFLAGS << ' -Wextra'
12
- $CFLAGS << ' -Wpedantic'
4
+ if RUBY_ENGINE == 'ruby'
5
+ $CFLAGS << ' -O3 '
6
+ $CFLAGS << ' -std=c99'
13
7
 
14
- $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
15
- $CFLAGS << ' -Wno-keyword-macro' # hiding return
16
- $CFLAGS << ' -Wno-gcc-compat' # ruby.h 2.6.0 on macos 10.14, dunno
17
- end
8
+ # ruby.h has some -Wpedantic fails in some cases
9
+ # (e.g. https://github.com/Shopify/bootsnap/issues/15)
10
+ unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC'])
11
+ $CFLAGS << ' -Wall'
12
+ $CFLAGS << ' -Werror'
13
+ $CFLAGS << ' -Wextra'
14
+ $CFLAGS << ' -Wpedantic'
15
+
16
+ $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
17
+ $CFLAGS << ' -Wno-keyword-macro' # hiding return
18
+ $CFLAGS << ' -Wno-gcc-compat' # ruby.h 2.6.0 on macos 10.14, dunno
19
+ end
18
20
 
19
- create_makefile("bootsnap/bootsnap")
21
+ create_makefile("bootsnap/bootsnap")
22
+ else
23
+ File.write("Makefile", dummy_makefile($srcdir).join(""))
24
+ end
data/lib/bootsnap.rb CHANGED
@@ -8,42 +8,114 @@ require_relative('bootsnap/compile_cache')
8
8
  module Bootsnap
9
9
  InvalidConfiguration = Class.new(StandardError)
10
10
 
11
+ class << self
12
+ attr_reader :logger
13
+ end
14
+
15
+ def self.log!
16
+ self.logger = $stderr.method(:puts)
17
+ end
18
+
19
+ def self.logger=(logger)
20
+ @logger = logger
21
+ if logger.respond_to?(:debug)
22
+ self.instrumentation = ->(event, path) { @logger.debug("[Bootsnap] #{event} #{path}") }
23
+ else
24
+ self.instrumentation = ->(event, path) { @logger.call("[Bootsnap] #{event} #{path}") }
25
+ end
26
+ end
27
+
28
+ def self.instrumentation=(callback)
29
+ @instrumentation = callback
30
+ self.instrumentation_enabled = !!callback
31
+ end
32
+
33
+ def self._instrument(event, path)
34
+ @instrumentation.call(event, path)
35
+ end
36
+
11
37
  def self.setup(
12
38
  cache_dir:,
13
39
  development_mode: true,
14
40
  load_path_cache: true,
15
- autoload_paths_cache: true,
16
- disable_trace: false,
41
+ autoload_paths_cache: nil,
42
+ disable_trace: nil,
17
43
  compile_cache_iseq: true,
18
44
  compile_cache_yaml: true
19
45
  )
20
- if autoload_paths_cache && !load_path_cache
21
- raise(InvalidConfiguration, "feature 'autoload_paths_cache' depends on feature 'load_path_cache'")
46
+ unless autoload_paths_cache.nil?
47
+ warn "[DEPRECATED] Bootsnap's `autoload_paths_cache:` option is deprecated and will be removed. " \
48
+ "If you use Zeitwerk this option is useless, and if you are still using the classic autoloader " \
49
+ "upgrading is recommended."
22
50
  end
23
51
 
24
- setup_disable_trace if disable_trace
52
+ unless disable_trace.nil?
53
+ warn "[DEPRECATED] Bootsnap's `disable_trace:` option is deprecated and will be removed. " \
54
+ "If you use Ruby 2.5 or newer this option is useless, if not upgrading is recommended."
55
+ end
56
+
57
+ if compile_cache_iseq && !iseq_cache_supported?
58
+ warn "Ruby 2.5 has a bug that break code tracing when code is loaded from cache. It is recommened " \
59
+ "to turn `compile_cache_iseq` off on Ruby 2.5"
60
+ end
25
61
 
26
62
  Bootsnap::LoadPathCache.setup(
27
- cache_path: cache_dir + '/bootsnap-load-path-cache',
63
+ cache_path: cache_dir + '/bootsnap/load-path-cache',
28
64
  development_mode: development_mode,
29
- active_support: autoload_paths_cache
30
65
  ) if load_path_cache
31
66
 
32
67
  Bootsnap::CompileCache.setup(
33
- cache_dir: cache_dir + '/bootsnap-compile-cache',
68
+ cache_dir: cache_dir + '/bootsnap/compile-cache',
34
69
  iseq: compile_cache_iseq,
35
70
  yaml: compile_cache_yaml
36
71
  )
37
72
  end
38
73
 
39
- def self.setup_disable_trace
40
- if Gem::Version.new(RUBY_VERSION) >= Gem::Version.new('2.5.0')
41
- warn(
42
- "from #{caller_locations(1, 1)[0]}: The 'disable_trace' method is not allowed with this Ruby version. " \
43
- "current: #{RUBY_VERSION}, allowed version: < 2.5.0",
74
+ def self.iseq_cache_supported?
75
+ return @iseq_cache_supported if defined? @iseq_cache_supported
76
+
77
+ ruby_version = Gem::Version.new(RUBY_VERSION)
78
+ @iseq_cache_supported = ruby_version < Gem::Version.new('2.5.0') || ruby_version >= Gem::Version.new('2.6.0')
79
+ end
80
+
81
+ def self.default_setup
82
+ env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || ENV['ENV']
83
+ development_mode = ['', nil, 'development'].include?(env)
84
+
85
+ unless ENV['DISABLE_BOOTSNAP']
86
+ cache_dir = ENV['BOOTSNAP_CACHE_DIR']
87
+ unless cache_dir
88
+ config_dir_frame = caller.detect do |line|
89
+ line.include?('/config/')
90
+ end
91
+
92
+ unless config_dir_frame
93
+ $stderr.puts("[bootsnap/setup] couldn't infer cache directory! Either:")
94
+ $stderr.puts("[bootsnap/setup] 1. require bootsnap/setup from your application's config directory; or")
95
+ $stderr.puts("[bootsnap/setup] 2. Define the environment variable BOOTSNAP_CACHE_DIR")
96
+
97
+ raise("couldn't infer bootsnap cache directory")
98
+ end
99
+
100
+ path = config_dir_frame.split(/:\d+:/).first
101
+ path = File.dirname(path) until File.basename(path) == 'config'
102
+ app_root = File.dirname(path)
103
+
104
+ cache_dir = File.join(app_root, 'tmp', 'cache')
105
+ end
106
+
107
+
108
+ setup(
109
+ cache_dir: cache_dir,
110
+ development_mode: development_mode,
111
+ load_path_cache: !ENV['DISABLE_BOOTSNAP_LOAD_PATH_CACHE'],
112
+ compile_cache_iseq: !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'] && iseq_cache_supported?,
113
+ compile_cache_yaml: !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'],
44
114
  )
45
- else
46
- RubyVM::InstructionSequence.compile_option = { trace_instruction: false }
115
+
116
+ if ENV['BOOTSNAP_LOG']
117
+ log!
118
+ end
47
119
  end
48
120
  end
49
121
  end
data/lib/bootsnap/cli.rb CHANGED
@@ -1,14 +1,16 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'bootsnap'
4
+ require 'bootsnap/cli/worker_pool'
4
5
  require 'optparse'
5
6
  require 'fileutils'
7
+ require 'etc'
6
8
 
7
9
  module Bootsnap
8
10
  class CLI
9
11
  unless Regexp.method_defined?(:match?)
10
12
  module RegexpMatchBackport
11
- refine Regepx do
13
+ refine Regexp do
12
14
  def match?(string)
13
15
  !!match(string)
14
16
  end
@@ -19,47 +21,68 @@ module Bootsnap
19
21
 
20
22
  attr_reader :cache_dir, :argv
21
23
 
22
- attr_accessor :compile_gemfile, :exclude
24
+ attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :jobs
23
25
 
24
26
  def initialize(argv)
25
27
  @argv = argv
26
- self.cache_dir = 'tmp/cache'
28
+ self.cache_dir = ENV.fetch('BOOTSNAP_CACHE_DIR', 'tmp/cache')
27
29
  self.compile_gemfile = false
28
30
  self.exclude = nil
31
+ self.verbose = false
32
+ self.jobs = Etc.nprocessors
33
+ self.iseq = true
34
+ self.yaml = true
29
35
  end
30
36
 
31
37
  def precompile_command(*sources)
32
38
  require 'bootsnap/compile_cache/iseq'
39
+ require 'bootsnap/compile_cache/yaml'
40
+
41
+ fix_default_encoding do
42
+ Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
43
+ Bootsnap::CompileCache::YAML.init!
44
+ Bootsnap::CompileCache::YAML.cache_dir = self.cache_dir
45
+
46
+ @work_pool = WorkerPool.create(size: jobs, jobs: {
47
+ ruby: method(:precompile_ruby),
48
+ yaml: method(:precompile_yaml),
49
+ })
50
+ @work_pool.spawn
51
+
52
+ main_sources = sources.map { |d| File.expand_path(d) }
53
+ precompile_ruby_files(main_sources)
54
+ precompile_yaml_files(main_sources)
55
+
56
+ if compile_gemfile
57
+ # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
58
+ gem_exclude = Regexp.union([exclude, '/spec/', '/test/'].compact)
59
+ precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
60
+
61
+ # Gems that include YAML files usually don't put them in `lib/`.
62
+ # So we look at the gem root.
63
+ gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems\/[^/]+}
64
+ gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
65
+ precompile_yaml_files(gem_paths, exclude: gem_exclude)
66
+ end
33
67
 
34
- Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
35
-
36
- if compile_gemfile
37
- sources += $LOAD_PATH
38
- end
39
-
40
- sources.map { |d| File.expand_path(d) }.each do |path|
41
- if !exclude || !exclude.match?(path)
42
- list_ruby_files(path).each do |ruby_file|
43
- if !exclude || !exclude.match?(ruby_file)
44
- CompileCache::ISeq.fetch(ruby_file, cache_dir: cache_dir)
45
- end
46
- end
68
+ if exitstatus = @work_pool.shutdown
69
+ exit(exitstatus)
47
70
  end
48
71
  end
49
72
  0
50
73
  end
51
74
 
52
75
  dir_sort = begin
53
- Dir['.', sort: false]
76
+ Dir[__FILE__, sort: false]
54
77
  true
55
78
  rescue ArgumentError, TypeError
56
79
  false
57
80
  end
58
81
 
59
82
  if dir_sort
60
- def list_ruby_files(path)
83
+ def list_files(path, pattern)
61
84
  if File.directory?(path)
62
- Dir[File.join(path, '**/*.rb'), sort: false]
85
+ Dir[File.join(path, pattern), sort: false]
63
86
  elsif File.exist?(path)
64
87
  [path]
65
88
  else
@@ -67,9 +90,9 @@ module Bootsnap
67
90
  end
68
91
  end
69
92
  else
70
- def list_ruby_files(path)
93
+ def list_files(path, pattern)
71
94
  if File.directory?(path)
72
- Dir[File.join(path, '**/*.rb')]
95
+ Dir[File.join(path, pattern)]
73
96
  elsif File.exist?(path)
74
97
  [path]
75
98
  else
@@ -91,6 +114,64 @@ module Bootsnap
91
114
 
92
115
  private
93
116
 
117
+ def precompile_yaml_files(load_paths, exclude: self.exclude)
118
+ return unless yaml
119
+
120
+ load_paths.each do |path|
121
+ if !exclude || !exclude.match?(path)
122
+ list_files(path, '**/*.{yml,yaml}').each do |yaml_file|
123
+ # We ignore hidden files to not match the various .ci.yml files
124
+ if !File.basename(yaml_file).start_with?('.') && (!exclude || !exclude.match?(yaml_file))
125
+ @work_pool.push(:yaml, yaml_file)
126
+ end
127
+ end
128
+ end
129
+ end
130
+ end
131
+
132
+ def precompile_yaml(*yaml_files)
133
+ Array(yaml_files).each do |yaml_file|
134
+ if CompileCache::YAML.precompile(yaml_file, cache_dir: cache_dir)
135
+ STDERR.puts(yaml_file) if verbose
136
+ end
137
+ end
138
+ end
139
+
140
+ def precompile_ruby_files(load_paths, exclude: self.exclude)
141
+ return unless iseq
142
+
143
+ load_paths.each do |path|
144
+ if !exclude || !exclude.match?(path)
145
+ list_files(path, '**/*.rb').each do |ruby_file|
146
+ if !exclude || !exclude.match?(ruby_file)
147
+ @work_pool.push(:ruby, ruby_file)
148
+ end
149
+ end
150
+ end
151
+ end
152
+ end
153
+
154
+ def precompile_ruby(*ruby_files)
155
+ Array(ruby_files).each do |ruby_file|
156
+ if CompileCache::ISeq.precompile(ruby_file, cache_dir: cache_dir)
157
+ STDERR.puts(ruby_file) if verbose
158
+ end
159
+ end
160
+ end
161
+
162
+ def fix_default_encoding
163
+ if Encoding.default_external == Encoding::US_ASCII
164
+ Encoding.default_external = Encoding::UTF_8
165
+ begin
166
+ yield
167
+ ensure
168
+ Encoding.default_external = Encoding::US_ASCII
169
+ end
170
+ else
171
+ yield
172
+ end
173
+ end
174
+
94
175
  def invalid_usage!(message)
95
176
  STDERR.puts message
96
177
  STDERR.puts
@@ -99,7 +180,12 @@ module Bootsnap
99
180
  end
100
181
 
101
182
  def cache_dir=(dir)
102
- @cache_dir = File.expand_path(File.join(dir, 'bootsnap-compile-cache'))
183
+ @cache_dir = File.expand_path(File.join(dir, 'bootsnap/compile-cache'))
184
+ end
185
+
186
+ def exclude_pattern(pattern)
187
+ (@exclude_patterns ||= []) << Regexp.new(pattern)
188
+ self.exclude = Regexp.union(@exclude_patterns)
103
189
  end
104
190
 
105
191
  def parser
@@ -116,6 +202,20 @@ module Bootsnap
116
202
  self.cache_dir = dir
117
203
  end
118
204
 
205
+ help = <<~EOS
206
+ Print precompiled paths.
207
+ EOS
208
+ opts.on('--verbose', '-v', help.strip) do
209
+ self.verbose = true
210
+ end
211
+
212
+ help = <<~EOS
213
+ Number of workers to use. Default to number of processors, set to 0 to disable multi-processing.
214
+ EOS
215
+ opts.on('--jobs JOBS', '-j', help.strip) do |jobs|
216
+ self.jobs = Integer(jobs)
217
+ end
218
+
119
219
  opts.separator ""
120
220
  opts.separator "COMMANDS"
121
221
  opts.separator ""
@@ -129,7 +229,17 @@ module Bootsnap
129
229
  help = <<~EOS
130
230
  Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api'
131
231
  EOS
132
- opts.on('--exclude PATTERN', help) { |pattern| self.exclude = Regexp.new(pattern) }
232
+ opts.on('--exclude PATTERN', help) { |pattern| exclude_pattern(pattern) }
233
+
234
+ help = <<~EOS
235
+ Disable ISeq (.rb) precompilation.
236
+ EOS
237
+ opts.on('--no-iseq', help) { self.iseq = false }
238
+
239
+ help = <<~EOS
240
+ Disable YAML precompilation.
241
+ EOS
242
+ opts.on('--no-yaml', help) { self.yaml = false }
133
243
  end
134
244
  end
135
245
  end
@@ -0,0 +1,131 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Bootsnap
4
+ class CLI
5
+ class WorkerPool
6
+ class << self
7
+ def create(size:, jobs:)
8
+ if size > 0 && Process.respond_to?(:fork)
9
+ new(size: size, jobs: jobs)
10
+ else
11
+ Inline.new(jobs: jobs)
12
+ end
13
+ end
14
+ end
15
+
16
+ class Inline
17
+ def initialize(jobs: {})
18
+ @jobs = jobs
19
+ end
20
+
21
+ def push(job, *args)
22
+ @jobs.fetch(job).call(*args)
23
+ nil
24
+ end
25
+
26
+ def spawn
27
+ # noop
28
+ end
29
+
30
+ def shutdown
31
+ # noop
32
+ end
33
+ end
34
+
35
+ class Worker
36
+ attr_reader :to_io, :pid
37
+
38
+ def initialize(jobs)
39
+ @jobs = jobs
40
+ @pipe_out, @to_io = IO.pipe
41
+ @pid = nil
42
+ end
43
+
44
+ def write(message, block: true)
45
+ payload = Marshal.dump(message)
46
+ if block
47
+ to_io.write(payload)
48
+ true
49
+ else
50
+ to_io.write_nonblock(payload, exception: false) != :wait_writable
51
+ end
52
+ end
53
+
54
+ def close
55
+ to_io.close
56
+ end
57
+
58
+ def work_loop
59
+ loop do
60
+ job, *args = Marshal.load(@pipe_out)
61
+ return if job == :exit
62
+ @jobs.fetch(job).call(*args)
63
+ end
64
+ rescue IOError
65
+ nil
66
+ end
67
+
68
+ def spawn
69
+ @pid = Process.fork do
70
+ to_io.close
71
+ work_loop
72
+ exit!(0)
73
+ end
74
+ @pipe_out.close
75
+ true
76
+ end
77
+ end
78
+
79
+ def initialize(size:, jobs: {})
80
+ @size = size
81
+ @jobs = jobs
82
+ @queue = Queue.new
83
+ @pids = []
84
+ end
85
+
86
+ def spawn
87
+ @workers = @size.times.map { Worker.new(@jobs) }
88
+ @workers.each(&:spawn)
89
+ @dispatcher_thread = Thread.new { dispatch_loop }
90
+ @dispatcher_thread.abort_on_exception = true
91
+ true
92
+ end
93
+
94
+ def dispatch_loop
95
+ loop do
96
+ case job = @queue.pop
97
+ when nil
98
+ @workers.each do |worker|
99
+ worker.write([:exit])
100
+ worker.close
101
+ end
102
+ return true
103
+ else
104
+ unless @workers.sample.write(job, block: false)
105
+ free_worker.write(job)
106
+ end
107
+ end
108
+ end
109
+ end
110
+
111
+ def free_worker
112
+ IO.select(nil, @workers)[1].sample
113
+ end
114
+
115
+ def push(*args)
116
+ @queue.push(args)
117
+ nil
118
+ end
119
+
120
+ def shutdown
121
+ @queue.close
122
+ @dispatcher_thread.join
123
+ @workers.each do |worker|
124
+ _pid, status = Process.wait2(worker.pid)
125
+ return status.exitstatus unless status.success?
126
+ end
127
+ nil
128
+ end
129
+ end
130
+ end
131
+ end