bootsnap 1.1.8 → 1.7.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (40) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +121 -0
  3. data/README.md +72 -14
  4. data/exe/bootsnap +5 -0
  5. data/ext/bootsnap/bootsnap.c +283 -105
  6. data/ext/bootsnap/extconf.rb +21 -14
  7. data/lib/bootsnap.rb +95 -14
  8. data/lib/bootsnap/bundler.rb +6 -3
  9. data/lib/bootsnap/cli.rb +246 -0
  10. data/lib/bootsnap/cli/worker_pool.rb +131 -0
  11. data/lib/bootsnap/compile_cache.rb +32 -4
  12. data/lib/bootsnap/compile_cache/iseq.rb +32 -15
  13. data/lib/bootsnap/compile_cache/yaml.rb +97 -40
  14. data/lib/bootsnap/explicit_require.rb +2 -1
  15. data/lib/bootsnap/load_path_cache.rb +33 -20
  16. data/lib/bootsnap/load_path_cache/cache.rb +65 -29
  17. data/lib/bootsnap/load_path_cache/change_observer.rb +36 -29
  18. data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb +70 -53
  19. data/lib/bootsnap/load_path_cache/core_ext/loaded_features.rb +18 -0
  20. data/lib/bootsnap/load_path_cache/loaded_features_index.rb +148 -0
  21. data/lib/bootsnap/load_path_cache/path.rb +8 -7
  22. data/lib/bootsnap/load_path_cache/path_scanner.rb +61 -39
  23. data/lib/bootsnap/load_path_cache/realpath_cache.rb +32 -0
  24. data/lib/bootsnap/load_path_cache/store.rb +27 -14
  25. data/lib/bootsnap/setup.rb +3 -40
  26. data/lib/bootsnap/version.rb +2 -1
  27. metadata +26 -29
  28. data/.gitignore +0 -17
  29. data/.rubocop.yml +0 -20
  30. data/.travis.yml +0 -4
  31. data/CODE_OF_CONDUCT.md +0 -74
  32. data/CONTRIBUTING.md +0 -21
  33. data/Gemfile +0 -8
  34. data/Rakefile +0 -11
  35. data/bin/console +0 -14
  36. data/bin/setup +0 -8
  37. data/bin/testunit +0 -8
  38. data/bootsnap.gemspec +0 -39
  39. data/dev.yml +0 -10
  40. data/lib/bootsnap/load_path_cache/core_ext/active_support.rb +0 -75
@@ -1,17 +1,24 @@
1
- require "mkmf"
2
- $CFLAGS << ' -O3 '
3
- $CFLAGS << ' -std=c99'
1
+ # frozen_string_literal: true
2
+ require("mkmf")
4
3
 
5
- # ruby.h has some -Wpedantic fails in some cases
6
- # (e.g. https://github.com/Shopify/bootsnap/issues/15)
7
- unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC'])
8
- $CFLAGS << ' -Wall'
9
- $CFLAGS << ' -Werror'
10
- $CFLAGS << ' -Wextra'
11
- $CFLAGS << ' -Wpedantic'
4
+ if RUBY_ENGINE == 'ruby'
5
+ $CFLAGS << ' -O3 '
6
+ $CFLAGS << ' -std=c99'
12
7
 
13
- $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
14
- $CFLAGS << ' -Wno-keyword-macro' # hiding return
15
- end
8
+ # ruby.h has some -Wpedantic fails in some cases
9
+ # (e.g. https://github.com/Shopify/bootsnap/issues/15)
10
+ unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC'])
11
+ $CFLAGS << ' -Wall'
12
+ $CFLAGS << ' -Werror'
13
+ $CFLAGS << ' -Wextra'
14
+ $CFLAGS << ' -Wpedantic'
15
+
16
+ $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
17
+ $CFLAGS << ' -Wno-keyword-macro' # hiding return
18
+ $CFLAGS << ' -Wno-gcc-compat' # ruby.h 2.6.0 on macos 10.14, dunno
19
+ end
16
20
 
17
- create_makefile("bootsnap/bootsnap")
21
+ create_makefile("bootsnap/bootsnap")
22
+ else
23
+ File.write("Makefile", dummy_makefile($srcdir).join(""))
24
+ end
data/lib/bootsnap.rb CHANGED
@@ -1,40 +1,121 @@
1
- require_relative 'bootsnap/version'
2
- require_relative 'bootsnap/bundler'
3
- require_relative 'bootsnap/load_path_cache'
4
- require_relative 'bootsnap/compile_cache'
1
+ # frozen_string_literal: true
2
+
3
+ require_relative('bootsnap/version')
4
+ require_relative('bootsnap/bundler')
5
+ require_relative('bootsnap/load_path_cache')
6
+ require_relative('bootsnap/compile_cache')
5
7
 
6
8
  module Bootsnap
7
9
  InvalidConfiguration = Class.new(StandardError)
8
10
 
11
+ class << self
12
+ attr_reader :logger
13
+ end
14
+
15
+ def self.log!
16
+ self.logger = $stderr.method(:puts)
17
+ end
18
+
19
+ def self.logger=(logger)
20
+ @logger = logger
21
+ if logger.respond_to?(:debug)
22
+ self.instrumentation = ->(event, path) { @logger.debug("[Bootsnap] #{event} #{path}") }
23
+ else
24
+ self.instrumentation = ->(event, path) { @logger.call("[Bootsnap] #{event} #{path}") }
25
+ end
26
+ end
27
+
28
+ def self.instrumentation=(callback)
29
+ @instrumentation = callback
30
+ self.instrumentation_enabled = !!callback
31
+ end
32
+
33
+ def self._instrument(event, path)
34
+ @instrumentation.call(event, path)
35
+ end
36
+
9
37
  def self.setup(
10
38
  cache_dir:,
11
39
  development_mode: true,
12
40
  load_path_cache: true,
13
- autoload_paths_cache: true,
14
- disable_trace: false,
41
+ autoload_paths_cache: nil,
42
+ disable_trace: nil,
15
43
  compile_cache_iseq: true,
16
44
  compile_cache_yaml: true
17
45
  )
18
- if autoload_paths_cache && !load_path_cache
19
- raise InvalidConfiguration, "feature 'autoload_paths_cache' depends on feature 'load_path_cache'"
46
+ unless autoload_paths_cache.nil?
47
+ warn "[DEPRECATED] Bootsnap's `autoload_paths_cache:` option is deprecated and will be removed. " \
48
+ "If you use Zeitwerk this option is useless, and if you are still using the classic autoloader " \
49
+ "upgrading is recommended."
20
50
  end
21
51
 
22
- setup_disable_trace if disable_trace
52
+ unless disable_trace.nil?
53
+ warn "[DEPRECATED] Bootsnap's `disable_trace:` option is deprecated and will be removed. " \
54
+ "If you use Ruby 2.5 or newer this option is useless, if not upgrading is recommended."
55
+ end
56
+
57
+ if compile_cache_iseq && !iseq_cache_supported?
58
+ warn "Ruby 2.5 has a bug that break code tracing when code is loaded from cache. It is recommened " \
59
+ "to turn `compile_cache_iseq` off on Ruby 2.5"
60
+ end
23
61
 
24
62
  Bootsnap::LoadPathCache.setup(
25
- cache_path: cache_dir + '/bootsnap-load-path-cache',
63
+ cache_path: cache_dir + '/bootsnap/load-path-cache',
26
64
  development_mode: development_mode,
27
- active_support: autoload_paths_cache
28
65
  ) if load_path_cache
29
66
 
30
67
  Bootsnap::CompileCache.setup(
31
- cache_dir: cache_dir + '/bootsnap-compile-cache',
68
+ cache_dir: cache_dir + '/bootsnap/compile-cache',
32
69
  iseq: compile_cache_iseq,
33
70
  yaml: compile_cache_yaml
34
71
  )
35
72
  end
36
73
 
37
- def self.setup_disable_trace
38
- RubyVM::InstructionSequence.compile_option = { trace_instruction: false }
74
+ def self.iseq_cache_supported?
75
+ return @iseq_cache_supported if defined? @iseq_cache_supported
76
+
77
+ ruby_version = Gem::Version.new(RUBY_VERSION)
78
+ @iseq_cache_supported = ruby_version < Gem::Version.new('2.5.0') || ruby_version >= Gem::Version.new('2.6.0')
79
+ end
80
+
81
+ def self.default_setup
82
+ env = ENV['RAILS_ENV'] || ENV['RACK_ENV'] || ENV['ENV']
83
+ development_mode = ['', nil, 'development'].include?(env)
84
+
85
+ unless ENV['DISABLE_BOOTSNAP']
86
+ cache_dir = ENV['BOOTSNAP_CACHE_DIR']
87
+ unless cache_dir
88
+ config_dir_frame = caller.detect do |line|
89
+ line.include?('/config/')
90
+ end
91
+
92
+ unless config_dir_frame
93
+ $stderr.puts("[bootsnap/setup] couldn't infer cache directory! Either:")
94
+ $stderr.puts("[bootsnap/setup] 1. require bootsnap/setup from your application's config directory; or")
95
+ $stderr.puts("[bootsnap/setup] 2. Define the environment variable BOOTSNAP_CACHE_DIR")
96
+
97
+ raise("couldn't infer bootsnap cache directory")
98
+ end
99
+
100
+ path = config_dir_frame.split(/:\d+:/).first
101
+ path = File.dirname(path) until File.basename(path) == 'config'
102
+ app_root = File.dirname(path)
103
+
104
+ cache_dir = File.join(app_root, 'tmp', 'cache')
105
+ end
106
+
107
+
108
+ setup(
109
+ cache_dir: cache_dir,
110
+ development_mode: development_mode,
111
+ load_path_cache: !ENV['DISABLE_BOOTSNAP_LOAD_PATH_CACHE'],
112
+ compile_cache_iseq: !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'] && iseq_cache_supported?,
113
+ compile_cache_yaml: !ENV['DISABLE_BOOTSNAP_COMPILE_CACHE'],
114
+ )
115
+
116
+ if ENV['BOOTSNAP_LOG']
117
+ log!
118
+ end
119
+ end
39
120
  end
40
121
  end
@@ -1,12 +1,15 @@
1
+ # frozen_string_literal: true
1
2
  module Bootsnap
2
- module_function
3
+ extend(self)
3
4
 
4
5
  def bundler?
6
+ return false unless defined?(::Bundler)
7
+
5
8
  # Bundler environment variable
6
- ['BUNDLE_BIN_PATH', 'BUNDLE_GEMFILE'].each do |current|
9
+ %w(BUNDLE_BIN_PATH BUNDLE_GEMFILE).each do |current|
7
10
  return true if ENV.key?(current)
8
11
  end
9
-
12
+
10
13
  false
11
14
  end
12
15
  end
@@ -0,0 +1,246 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bootsnap'
4
+ require 'bootsnap/cli/worker_pool'
5
+ require 'optparse'
6
+ require 'fileutils'
7
+ require 'etc'
8
+
9
+ module Bootsnap
10
+ class CLI
11
+ unless Regexp.method_defined?(:match?)
12
+ module RegexpMatchBackport
13
+ refine Regexp do
14
+ def match?(string)
15
+ !!match(string)
16
+ end
17
+ end
18
+ end
19
+ using RegexpMatchBackport
20
+ end
21
+
22
+ attr_reader :cache_dir, :argv
23
+
24
+ attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :jobs
25
+
26
+ def initialize(argv)
27
+ @argv = argv
28
+ self.cache_dir = ENV.fetch('BOOTSNAP_CACHE_DIR', 'tmp/cache')
29
+ self.compile_gemfile = false
30
+ self.exclude = nil
31
+ self.verbose = false
32
+ self.jobs = Etc.nprocessors
33
+ self.iseq = true
34
+ self.yaml = true
35
+ end
36
+
37
+ def precompile_command(*sources)
38
+ require 'bootsnap/compile_cache/iseq'
39
+ require 'bootsnap/compile_cache/yaml'
40
+
41
+ fix_default_encoding do
42
+ Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
43
+ Bootsnap::CompileCache::YAML.init!
44
+ Bootsnap::CompileCache::YAML.cache_dir = self.cache_dir
45
+
46
+ @work_pool = WorkerPool.create(size: jobs, jobs: {
47
+ ruby: method(:precompile_ruby),
48
+ yaml: method(:precompile_yaml),
49
+ })
50
+ @work_pool.spawn
51
+
52
+ main_sources = sources.map { |d| File.expand_path(d) }
53
+ precompile_ruby_files(main_sources)
54
+ precompile_yaml_files(main_sources)
55
+
56
+ if compile_gemfile
57
+ # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
58
+ gem_exclude = Regexp.union([exclude, '/spec/', '/test/'].compact)
59
+ precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
60
+
61
+ # Gems that include YAML files usually don't put them in `lib/`.
62
+ # So we look at the gem root.
63
+ gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems\/[^/]+}
64
+ gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
65
+ precompile_yaml_files(gem_paths, exclude: gem_exclude)
66
+ end
67
+
68
+ if exitstatus = @work_pool.shutdown
69
+ exit(exitstatus)
70
+ end
71
+ end
72
+ 0
73
+ end
74
+
75
+ dir_sort = begin
76
+ Dir[__FILE__, sort: false]
77
+ true
78
+ rescue ArgumentError, TypeError
79
+ false
80
+ end
81
+
82
+ if dir_sort
83
+ def list_files(path, pattern)
84
+ if File.directory?(path)
85
+ Dir[File.join(path, pattern), sort: false]
86
+ elsif File.exist?(path)
87
+ [path]
88
+ else
89
+ []
90
+ end
91
+ end
92
+ else
93
+ def list_files(path, pattern)
94
+ if File.directory?(path)
95
+ Dir[File.join(path, pattern)]
96
+ elsif File.exist?(path)
97
+ [path]
98
+ else
99
+ []
100
+ end
101
+ end
102
+ end
103
+
104
+ def run
105
+ parser.parse!(argv)
106
+ command = argv.shift
107
+ method = "#{command}_command"
108
+ if respond_to?(method)
109
+ public_send(method, *argv)
110
+ else
111
+ invalid_usage!("Unknown command: #{command}")
112
+ end
113
+ end
114
+
115
+ private
116
+
117
+ def precompile_yaml_files(load_paths, exclude: self.exclude)
118
+ return unless yaml
119
+
120
+ load_paths.each do |path|
121
+ if !exclude || !exclude.match?(path)
122
+ list_files(path, '**/*.{yml,yaml}').each do |yaml_file|
123
+ # We ignore hidden files to not match the various .ci.yml files
124
+ if !File.basename(yaml_file).start_with?('.') && (!exclude || !exclude.match?(yaml_file))
125
+ @work_pool.push(:yaml, yaml_file)
126
+ end
127
+ end
128
+ end
129
+ end
130
+ end
131
+
132
+ def precompile_yaml(*yaml_files)
133
+ Array(yaml_files).each do |yaml_file|
134
+ if CompileCache::YAML.precompile(yaml_file, cache_dir: cache_dir)
135
+ STDERR.puts(yaml_file) if verbose
136
+ end
137
+ end
138
+ end
139
+
140
+ def precompile_ruby_files(load_paths, exclude: self.exclude)
141
+ return unless iseq
142
+
143
+ load_paths.each do |path|
144
+ if !exclude || !exclude.match?(path)
145
+ list_files(path, '**/*.rb').each do |ruby_file|
146
+ if !exclude || !exclude.match?(ruby_file)
147
+ @work_pool.push(:ruby, ruby_file)
148
+ end
149
+ end
150
+ end
151
+ end
152
+ end
153
+
154
+ def precompile_ruby(*ruby_files)
155
+ Array(ruby_files).each do |ruby_file|
156
+ if CompileCache::ISeq.precompile(ruby_file, cache_dir: cache_dir)
157
+ STDERR.puts(ruby_file) if verbose
158
+ end
159
+ end
160
+ end
161
+
162
+ def fix_default_encoding
163
+ if Encoding.default_external == Encoding::US_ASCII
164
+ Encoding.default_external = Encoding::UTF_8
165
+ begin
166
+ yield
167
+ ensure
168
+ Encoding.default_external = Encoding::US_ASCII
169
+ end
170
+ else
171
+ yield
172
+ end
173
+ end
174
+
175
+ def invalid_usage!(message)
176
+ STDERR.puts message
177
+ STDERR.puts
178
+ STDERR.puts parser
179
+ 1
180
+ end
181
+
182
+ def cache_dir=(dir)
183
+ @cache_dir = File.expand_path(File.join(dir, 'bootsnap/compile-cache'))
184
+ end
185
+
186
+ def exclude_pattern(pattern)
187
+ (@exclude_patterns ||= []) << Regexp.new(pattern)
188
+ self.exclude = Regexp.union(@exclude_patterns)
189
+ end
190
+
191
+ def parser
192
+ @parser ||= OptionParser.new do |opts|
193
+ opts.banner = "Usage: bootsnap COMMAND [ARGS]"
194
+ opts.separator ""
195
+ opts.separator "GLOBAL OPTIONS"
196
+ opts.separator ""
197
+
198
+ help = <<~EOS
199
+ Path to the bootsnap cache directory. Defaults to tmp/cache
200
+ EOS
201
+ opts.on('--cache-dir DIR', help.strip) do |dir|
202
+ self.cache_dir = dir
203
+ end
204
+
205
+ help = <<~EOS
206
+ Print precompiled paths.
207
+ EOS
208
+ opts.on('--verbose', '-v', help.strip) do
209
+ self.verbose = true
210
+ end
211
+
212
+ help = <<~EOS
213
+ Number of workers to use. Default to number of processors, set to 0 to disable multi-processing.
214
+ EOS
215
+ opts.on('--jobs JOBS', '-j', help.strip) do |jobs|
216
+ self.jobs = Integer(jobs)
217
+ end
218
+
219
+ opts.separator ""
220
+ opts.separator "COMMANDS"
221
+ opts.separator ""
222
+ opts.separator " precompile [DIRECTORIES...]: Precompile all .rb files in the passed directories"
223
+
224
+ help = <<~EOS
225
+ Precompile the gems in Gemfile
226
+ EOS
227
+ opts.on('--gemfile', help) { self.compile_gemfile = true }
228
+
229
+ help = <<~EOS
230
+ Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api'
231
+ EOS
232
+ opts.on('--exclude PATTERN', help) { |pattern| exclude_pattern(pattern) }
233
+
234
+ help = <<~EOS
235
+ Disable ISeq (.rb) precompilation.
236
+ EOS
237
+ opts.on('--no-iseq', help) { self.iseq = false }
238
+
239
+ help = <<~EOS
240
+ Disable YAML precompilation.
241
+ EOS
242
+ opts.on('--no-yaml', help) { self.yaml = false }
243
+ end
244
+ end
245
+ end
246
+ end
@@ -0,0 +1,131 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Bootsnap
4
+ class CLI
5
+ class WorkerPool
6
+ class << self
7
+ def create(size:, jobs:)
8
+ if size > 0 && Process.respond_to?(:fork)
9
+ new(size: size, jobs: jobs)
10
+ else
11
+ Inline.new(jobs: jobs)
12
+ end
13
+ end
14
+ end
15
+
16
+ class Inline
17
+ def initialize(jobs: {})
18
+ @jobs = jobs
19
+ end
20
+
21
+ def push(job, *args)
22
+ @jobs.fetch(job).call(*args)
23
+ nil
24
+ end
25
+
26
+ def spawn
27
+ # noop
28
+ end
29
+
30
+ def shutdown
31
+ # noop
32
+ end
33
+ end
34
+
35
+ class Worker
36
+ attr_reader :to_io, :pid
37
+
38
+ def initialize(jobs)
39
+ @jobs = jobs
40
+ @pipe_out, @to_io = IO.pipe
41
+ @pid = nil
42
+ end
43
+
44
+ def write(message, block: true)
45
+ payload = Marshal.dump(message)
46
+ if block
47
+ to_io.write(payload)
48
+ true
49
+ else
50
+ to_io.write_nonblock(payload, exception: false) != :wait_writable
51
+ end
52
+ end
53
+
54
+ def close
55
+ to_io.close
56
+ end
57
+
58
+ def work_loop
59
+ loop do
60
+ job, *args = Marshal.load(@pipe_out)
61
+ return if job == :exit
62
+ @jobs.fetch(job).call(*args)
63
+ end
64
+ rescue IOError
65
+ nil
66
+ end
67
+
68
+ def spawn
69
+ @pid = Process.fork do
70
+ to_io.close
71
+ work_loop
72
+ exit!(0)
73
+ end
74
+ @pipe_out.close
75
+ true
76
+ end
77
+ end
78
+
79
+ def initialize(size:, jobs: {})
80
+ @size = size
81
+ @jobs = jobs
82
+ @queue = Queue.new
83
+ @pids = []
84
+ end
85
+
86
+ def spawn
87
+ @workers = @size.times.map { Worker.new(@jobs) }
88
+ @workers.each(&:spawn)
89
+ @dispatcher_thread = Thread.new { dispatch_loop }
90
+ @dispatcher_thread.abort_on_exception = true
91
+ true
92
+ end
93
+
94
+ def dispatch_loop
95
+ loop do
96
+ case job = @queue.pop
97
+ when nil
98
+ @workers.each do |worker|
99
+ worker.write([:exit])
100
+ worker.close
101
+ end
102
+ return true
103
+ else
104
+ unless @workers.sample.write(job, block: false)
105
+ free_worker.write(job)
106
+ end
107
+ end
108
+ end
109
+ end
110
+
111
+ def free_worker
112
+ IO.select(nil, @workers)[1].sample
113
+ end
114
+
115
+ def push(*args)
116
+ @queue.push(args)
117
+ nil
118
+ end
119
+
120
+ def shutdown
121
+ @queue.close
122
+ @dispatcher_thread.join
123
+ @workers.each do |worker|
124
+ _pid, status = Process.wait2(worker.pid)
125
+ return status.exitstatus unless status.success?
126
+ end
127
+ nil
128
+ end
129
+ end
130
+ end
131
+ end