bootsnap 1.4.0 → 1.9.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (45) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +126 -0
  3. data/README.md +68 -13
  4. data/exe/bootsnap +5 -0
  5. data/ext/bootsnap/bootsnap.c +285 -87
  6. data/ext/bootsnap/extconf.rb +20 -14
  7. data/lib/bootsnap/bundler.rb +1 -0
  8. data/lib/bootsnap/cli/worker_pool.rb +135 -0
  9. data/lib/bootsnap/cli.rb +281 -0
  10. data/lib/bootsnap/compile_cache/iseq.rb +24 -7
  11. data/lib/bootsnap/compile_cache/json.rb +79 -0
  12. data/lib/bootsnap/compile_cache/yaml.rb +145 -39
  13. data/lib/bootsnap/compile_cache.rb +25 -3
  14. data/lib/bootsnap/explicit_require.rb +1 -0
  15. data/lib/bootsnap/load_path_cache/cache.rb +44 -9
  16. data/lib/bootsnap/load_path_cache/change_observer.rb +5 -1
  17. data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb +30 -6
  18. data/lib/bootsnap/load_path_cache/core_ext/loaded_features.rb +11 -0
  19. data/lib/bootsnap/load_path_cache/loaded_features_index.rb +43 -11
  20. data/lib/bootsnap/load_path_cache/path.rb +3 -2
  21. data/lib/bootsnap/load_path_cache/path_scanner.rb +53 -27
  22. data/lib/bootsnap/load_path_cache/realpath_cache.rb +5 -5
  23. data/lib/bootsnap/load_path_cache/store.rb +28 -14
  24. data/lib/bootsnap/load_path_cache.rb +10 -16
  25. data/lib/bootsnap/setup.rb +2 -33
  26. data/lib/bootsnap/version.rb +2 -1
  27. data/lib/bootsnap.rb +96 -17
  28. metadata +18 -29
  29. data/.gitignore +0 -17
  30. data/.rubocop.yml +0 -20
  31. data/.travis.yml +0 -24
  32. data/CODE_OF_CONDUCT.md +0 -74
  33. data/CONTRIBUTING.md +0 -21
  34. data/Gemfile +0 -8
  35. data/README.jp.md +0 -231
  36. data/Rakefile +0 -12
  37. data/bin/ci +0 -10
  38. data/bin/console +0 -14
  39. data/bin/setup +0 -8
  40. data/bin/test-minimal-support +0 -7
  41. data/bin/testunit +0 -8
  42. data/bootsnap.gemspec +0 -45
  43. data/dev.yml +0 -10
  44. data/lib/bootsnap/load_path_cache/core_ext/active_support.rb +0 -100
  45. data/shipit.rubygems.yml +0 -4
@@ -1,18 +1,24 @@
1
+ # frozen_string_literal: true
1
2
  require("mkmf")
2
- $CFLAGS << ' -O3 '
3
- $CFLAGS << ' -std=c99'
4
3
 
5
- # ruby.h has some -Wpedantic fails in some cases
6
- # (e.g. https://github.com/Shopify/bootsnap/issues/15)
7
- unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC'])
8
- $CFLAGS << ' -Wall'
9
- $CFLAGS << ' -Werror'
10
- $CFLAGS << ' -Wextra'
11
- $CFLAGS << ' -Wpedantic'
4
+ if RUBY_ENGINE == 'ruby'
5
+ $CFLAGS << ' -O3 '
6
+ $CFLAGS << ' -std=c99'
12
7
 
13
- $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
14
- $CFLAGS << ' -Wno-keyword-macro' # hiding return
15
- $CFLAGS << ' -Wno-gcc-compat' # ruby.h 2.6.0 on macos 10.14, dunno
16
- end
8
+ # ruby.h has some -Wpedantic fails in some cases
9
+ # (e.g. https://github.com/Shopify/bootsnap/issues/15)
10
+ unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC'])
11
+ $CFLAGS << ' -Wall'
12
+ $CFLAGS << ' -Werror'
13
+ $CFLAGS << ' -Wextra'
14
+ $CFLAGS << ' -Wpedantic'
15
+
16
+ $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
17
+ $CFLAGS << ' -Wno-keyword-macro' # hiding return
18
+ $CFLAGS << ' -Wno-gcc-compat' # ruby.h 2.6.0 on macos 10.14, dunno
19
+ end
17
20
 
18
- create_makefile("bootsnap/bootsnap")
21
+ create_makefile("bootsnap/bootsnap")
22
+ else
23
+ File.write("Makefile", dummy_makefile($srcdir).join(""))
24
+ end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  module Bootsnap
2
3
  extend(self)
3
4
 
@@ -0,0 +1,135 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Bootsnap
4
+ class CLI
5
+ class WorkerPool
6
+ class << self
7
+ def create(size:, jobs:)
8
+ if size > 0 && Process.respond_to?(:fork)
9
+ new(size: size, jobs: jobs)
10
+ else
11
+ Inline.new(jobs: jobs)
12
+ end
13
+ end
14
+ end
15
+
16
+ class Inline
17
+ def initialize(jobs: {})
18
+ @jobs = jobs
19
+ end
20
+
21
+ def push(job, *args)
22
+ @jobs.fetch(job).call(*args)
23
+ nil
24
+ end
25
+
26
+ def spawn
27
+ # noop
28
+ end
29
+
30
+ def shutdown
31
+ # noop
32
+ end
33
+ end
34
+
35
+ class Worker
36
+ attr_reader :to_io, :pid
37
+
38
+ def initialize(jobs)
39
+ @jobs = jobs
40
+ @pipe_out, @to_io = IO.pipe(binmode: true)
41
+ # Set the writer encoding to binary since IO.pipe only sets it for the reader.
42
+ # https://github.com/rails/rails/issues/16514#issuecomment-52313290
43
+ @to_io.set_encoding(Encoding::BINARY)
44
+
45
+ @pid = nil
46
+ end
47
+
48
+ def write(message, block: true)
49
+ payload = Marshal.dump(message)
50
+ if block
51
+ to_io.write(payload)
52
+ true
53
+ else
54
+ to_io.write_nonblock(payload, exception: false) != :wait_writable
55
+ end
56
+ end
57
+
58
+ def close
59
+ to_io.close
60
+ end
61
+
62
+ def work_loop
63
+ loop do
64
+ job, *args = Marshal.load(@pipe_out)
65
+ return if job == :exit
66
+ @jobs.fetch(job).call(*args)
67
+ end
68
+ rescue IOError
69
+ nil
70
+ end
71
+
72
+ def spawn
73
+ @pid = Process.fork do
74
+ to_io.close
75
+ work_loop
76
+ exit!(0)
77
+ end
78
+ @pipe_out.close
79
+ true
80
+ end
81
+ end
82
+
83
+ def initialize(size:, jobs: {})
84
+ @size = size
85
+ @jobs = jobs
86
+ @queue = Queue.new
87
+ @pids = []
88
+ end
89
+
90
+ def spawn
91
+ @workers = @size.times.map { Worker.new(@jobs) }
92
+ @workers.each(&:spawn)
93
+ @dispatcher_thread = Thread.new { dispatch_loop }
94
+ @dispatcher_thread.abort_on_exception = true
95
+ true
96
+ end
97
+
98
+ def dispatch_loop
99
+ loop do
100
+ case job = @queue.pop
101
+ when nil
102
+ @workers.each do |worker|
103
+ worker.write([:exit])
104
+ worker.close
105
+ end
106
+ return true
107
+ else
108
+ unless @workers.sample.write(job, block: false)
109
+ free_worker.write(job)
110
+ end
111
+ end
112
+ end
113
+ end
114
+
115
+ def free_worker
116
+ IO.select(nil, @workers)[1].sample
117
+ end
118
+
119
+ def push(*args)
120
+ @queue.push(args)
121
+ nil
122
+ end
123
+
124
+ def shutdown
125
+ @queue.close
126
+ @dispatcher_thread.join
127
+ @workers.each do |worker|
128
+ _pid, status = Process.wait2(worker.pid)
129
+ return status.exitstatus unless status.success?
130
+ end
131
+ nil
132
+ end
133
+ end
134
+ end
135
+ end
@@ -0,0 +1,281 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'bootsnap'
4
+ require 'bootsnap/cli/worker_pool'
5
+ require 'optparse'
6
+ require 'fileutils'
7
+ require 'etc'
8
+
9
+ module Bootsnap
10
+ class CLI
11
+ unless Regexp.method_defined?(:match?)
12
+ module RegexpMatchBackport
13
+ refine Regexp do
14
+ def match?(string)
15
+ !!match(string)
16
+ end
17
+ end
18
+ end
19
+ using RegexpMatchBackport
20
+ end
21
+
22
+ attr_reader :cache_dir, :argv
23
+
24
+ attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :json, :jobs
25
+
26
+ def initialize(argv)
27
+ @argv = argv
28
+ self.cache_dir = ENV.fetch('BOOTSNAP_CACHE_DIR', 'tmp/cache')
29
+ self.compile_gemfile = false
30
+ self.exclude = nil
31
+ self.verbose = false
32
+ self.jobs = Etc.nprocessors
33
+ self.iseq = true
34
+ self.yaml = true
35
+ self.json = true
36
+ end
37
+
38
+ def precompile_command(*sources)
39
+ require 'bootsnap/compile_cache/iseq'
40
+ require 'bootsnap/compile_cache/yaml'
41
+ require 'bootsnap/compile_cache/json'
42
+
43
+ fix_default_encoding do
44
+ Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
45
+ Bootsnap::CompileCache::YAML.init!
46
+ Bootsnap::CompileCache::YAML.cache_dir = self.cache_dir
47
+ Bootsnap::CompileCache::JSON.init!
48
+ Bootsnap::CompileCache::JSON.cache_dir = self.cache_dir
49
+
50
+ @work_pool = WorkerPool.create(size: jobs, jobs: {
51
+ ruby: method(:precompile_ruby),
52
+ yaml: method(:precompile_yaml),
53
+ json: method(:precompile_json),
54
+ })
55
+ @work_pool.spawn
56
+
57
+ main_sources = sources.map { |d| File.expand_path(d) }
58
+ precompile_ruby_files(main_sources)
59
+ precompile_yaml_files(main_sources)
60
+ precompile_json_files(main_sources)
61
+
62
+ if compile_gemfile
63
+ # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
64
+ gem_exclude = Regexp.union([exclude, '/spec/', '/test/'].compact)
65
+ precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
66
+
67
+ # Gems that include JSON or YAML files usually don't put them in `lib/`.
68
+ # So we look at the gem root.
69
+ gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems\/[^/]+}
70
+ gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
71
+ precompile_yaml_files(gem_paths, exclude: gem_exclude)
72
+ precompile_json_files(gem_paths, exclude: gem_exclude)
73
+ end
74
+
75
+ if exitstatus = @work_pool.shutdown
76
+ exit(exitstatus)
77
+ end
78
+ end
79
+ 0
80
+ end
81
+
82
+ dir_sort = begin
83
+ Dir[__FILE__, sort: false]
84
+ true
85
+ rescue ArgumentError, TypeError
86
+ false
87
+ end
88
+
89
+ if dir_sort
90
+ def list_files(path, pattern)
91
+ if File.directory?(path)
92
+ Dir[File.join(path, pattern), sort: false]
93
+ elsif File.exist?(path)
94
+ [path]
95
+ else
96
+ []
97
+ end
98
+ end
99
+ else
100
+ def list_files(path, pattern)
101
+ if File.directory?(path)
102
+ Dir[File.join(path, pattern)]
103
+ elsif File.exist?(path)
104
+ [path]
105
+ else
106
+ []
107
+ end
108
+ end
109
+ end
110
+
111
+ def run
112
+ parser.parse!(argv)
113
+ command = argv.shift
114
+ method = "#{command}_command"
115
+ if respond_to?(method)
116
+ public_send(method, *argv)
117
+ else
118
+ invalid_usage!("Unknown command: #{command}")
119
+ end
120
+ end
121
+
122
+ private
123
+
124
+ def precompile_yaml_files(load_paths, exclude: self.exclude)
125
+ return unless yaml
126
+
127
+ load_paths.each do |path|
128
+ if !exclude || !exclude.match?(path)
129
+ list_files(path, '**/*.{yml,yaml}').each do |yaml_file|
130
+ # We ignore hidden files to not match the various .ci.yml files
131
+ if !File.basename(yaml_file).start_with?('.') && (!exclude || !exclude.match?(yaml_file))
132
+ @work_pool.push(:yaml, yaml_file)
133
+ end
134
+ end
135
+ end
136
+ end
137
+ end
138
+
139
+ def precompile_yaml(*yaml_files)
140
+ Array(yaml_files).each do |yaml_file|
141
+ if CompileCache::YAML.precompile(yaml_file, cache_dir: cache_dir)
142
+ STDERR.puts(yaml_file) if verbose
143
+ end
144
+ end
145
+ end
146
+
147
+ def precompile_json_files(load_paths, exclude: self.exclude)
148
+ return unless json
149
+
150
+ load_paths.each do |path|
151
+ if !exclude || !exclude.match?(path)
152
+ list_files(path, '**/*.json').each do |json_file|
153
+ # We ignore hidden files to not match the various .config.json files
154
+ if !File.basename(json_file).start_with?('.') && (!exclude || !exclude.match?(json_file))
155
+ @work_pool.push(:json, json_file)
156
+ end
157
+ end
158
+ end
159
+ end
160
+ end
161
+
162
+ def precompile_json(*json_files)
163
+ Array(json_files).each do |json_file|
164
+ if CompileCache::JSON.precompile(json_file, cache_dir: cache_dir)
165
+ STDERR.puts(json_file) if verbose
166
+ end
167
+ end
168
+ end
169
+
170
+ def precompile_ruby_files(load_paths, exclude: self.exclude)
171
+ return unless iseq
172
+
173
+ load_paths.each do |path|
174
+ if !exclude || !exclude.match?(path)
175
+ list_files(path, '**/*.rb').each do |ruby_file|
176
+ if !exclude || !exclude.match?(ruby_file)
177
+ @work_pool.push(:ruby, ruby_file)
178
+ end
179
+ end
180
+ end
181
+ end
182
+ end
183
+
184
+ def precompile_ruby(*ruby_files)
185
+ Array(ruby_files).each do |ruby_file|
186
+ if CompileCache::ISeq.precompile(ruby_file, cache_dir: cache_dir)
187
+ STDERR.puts(ruby_file) if verbose
188
+ end
189
+ end
190
+ end
191
+
192
+ def fix_default_encoding
193
+ if Encoding.default_external == Encoding::US_ASCII
194
+ Encoding.default_external = Encoding::UTF_8
195
+ begin
196
+ yield
197
+ ensure
198
+ Encoding.default_external = Encoding::US_ASCII
199
+ end
200
+ else
201
+ yield
202
+ end
203
+ end
204
+
205
+ def invalid_usage!(message)
206
+ STDERR.puts message
207
+ STDERR.puts
208
+ STDERR.puts parser
209
+ 1
210
+ end
211
+
212
+ def cache_dir=(dir)
213
+ @cache_dir = File.expand_path(File.join(dir, 'bootsnap/compile-cache'))
214
+ end
215
+
216
+ def exclude_pattern(pattern)
217
+ (@exclude_patterns ||= []) << Regexp.new(pattern)
218
+ self.exclude = Regexp.union(@exclude_patterns)
219
+ end
220
+
221
+ def parser
222
+ @parser ||= OptionParser.new do |opts|
223
+ opts.banner = "Usage: bootsnap COMMAND [ARGS]"
224
+ opts.separator ""
225
+ opts.separator "GLOBAL OPTIONS"
226
+ opts.separator ""
227
+
228
+ help = <<~EOS
229
+ Path to the bootsnap cache directory. Defaults to tmp/cache
230
+ EOS
231
+ opts.on('--cache-dir DIR', help.strip) do |dir|
232
+ self.cache_dir = dir
233
+ end
234
+
235
+ help = <<~EOS
236
+ Print precompiled paths.
237
+ EOS
238
+ opts.on('--verbose', '-v', help.strip) do
239
+ self.verbose = true
240
+ end
241
+
242
+ help = <<~EOS
243
+ Number of workers to use. Default to number of processors, set to 0 to disable multi-processing.
244
+ EOS
245
+ opts.on('--jobs JOBS', '-j', help.strip) do |jobs|
246
+ self.jobs = Integer(jobs)
247
+ end
248
+
249
+ opts.separator ""
250
+ opts.separator "COMMANDS"
251
+ opts.separator ""
252
+ opts.separator " precompile [DIRECTORIES...]: Precompile all .rb files in the passed directories"
253
+
254
+ help = <<~EOS
255
+ Precompile the gems in Gemfile
256
+ EOS
257
+ opts.on('--gemfile', help) { self.compile_gemfile = true }
258
+
259
+ help = <<~EOS
260
+ Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api'
261
+ EOS
262
+ opts.on('--exclude PATTERN', help) { |pattern| exclude_pattern(pattern) }
263
+
264
+ help = <<~EOS
265
+ Disable ISeq (.rb) precompilation.
266
+ EOS
267
+ opts.on('--no-iseq', help) { self.iseq = false }
268
+
269
+ help = <<~EOS
270
+ Disable YAML precompilation.
271
+ EOS
272
+ opts.on('--no-yaml', help) { self.yaml = false }
273
+
274
+ help = <<~EOS
275
+ Disable JSON precompilation.
276
+ EOS
277
+ opts.on('--no-json', help) { self.json = false }
278
+ end
279
+ end
280
+ end
281
+ end
@@ -1,3 +1,4 @@
1
+ # frozen_string_literal: true
1
2
  require('bootsnap/bootsnap')
2
3
  require('zlib')
3
4
 
@@ -14,7 +15,7 @@ module Bootsnap
14
15
  raise(Uncompilable, 'syntax error')
15
16
  end
16
17
 
17
- def self.storage_to_output(binary)
18
+ def self.storage_to_output(binary, _args)
18
19
  RubyVM::InstructionSequence.load_from_binary(binary)
19
20
  rescue RuntimeError => e
20
21
  if e.message == 'broken binary format'
@@ -25,7 +26,24 @@ module Bootsnap
25
26
  end
26
27
  end
27
28
 
28
- def self.input_to_output(_)
29
+ def self.fetch(path, cache_dir: ISeq.cache_dir)
30
+ Bootsnap::CompileCache::Native.fetch(
31
+ cache_dir,
32
+ path.to_s,
33
+ Bootsnap::CompileCache::ISeq,
34
+ nil,
35
+ )
36
+ end
37
+
38
+ def self.precompile(path, cache_dir: ISeq.cache_dir)
39
+ Bootsnap::CompileCache::Native.precompile(
40
+ cache_dir,
41
+ path.to_s,
42
+ Bootsnap::CompileCache::ISeq,
43
+ )
44
+ end
45
+
46
+ def self.input_to_output(_data, _kwargs)
29
47
  nil # ruby handles this
30
48
  end
31
49
 
@@ -34,11 +52,9 @@ module Bootsnap
34
52
  # Having coverage enabled prevents iseq dumping/loading.
35
53
  return nil if defined?(Coverage) && Bootsnap::CompileCache::Native.coverage_running?
36
54
 
37
- Bootsnap::CompileCache::Native.fetch(
38
- Bootsnap::CompileCache::ISeq.cache_dir,
39
- path.to_s,
40
- Bootsnap::CompileCache::ISeq
41
- )
55
+ Bootsnap::CompileCache::ISeq.fetch(path.to_s)
56
+ rescue Errno::EACCES
57
+ Bootsnap::CompileCache.permission_error(path)
42
58
  rescue RuntimeError => e
43
59
  if e.message =~ /unmatched platform/
44
60
  puts("unmatched platform for file #{path}")
@@ -57,6 +73,7 @@ module Bootsnap
57
73
  crc = Zlib.crc32(option.inspect)
58
74
  Bootsnap::CompileCache::Native.compile_option_crc32 = crc
59
75
  end
76
+ compile_option_updated
60
77
 
61
78
  def self.install!(cache_dir)
62
79
  Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
@@ -0,0 +1,79 @@
1
+ # frozen_string_literal: true
2
+ require('bootsnap/bootsnap')
3
+
4
+ module Bootsnap
5
+ module CompileCache
6
+ module JSON
7
+ class << self
8
+ attr_accessor(:msgpack_factory, :cache_dir, :supported_options)
9
+
10
+ def input_to_storage(payload, _)
11
+ obj = ::JSON.parse(payload)
12
+ msgpack_factory.dump(obj)
13
+ end
14
+
15
+ def storage_to_output(data, kwargs)
16
+ if kwargs && kwargs.key?(:symbolize_names)
17
+ kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
18
+ end
19
+ msgpack_factory.load(data, kwargs)
20
+ end
21
+
22
+ def input_to_output(data, kwargs)
23
+ ::JSON.parse(data, **(kwargs || {}))
24
+ end
25
+
26
+ def precompile(path, cache_dir: self.cache_dir)
27
+ Bootsnap::CompileCache::Native.precompile(
28
+ cache_dir,
29
+ path.to_s,
30
+ self,
31
+ )
32
+ end
33
+
34
+ def install!(cache_dir)
35
+ self.cache_dir = cache_dir
36
+ init!
37
+ if ::JSON.respond_to?(:load_file)
38
+ ::JSON.singleton_class.prepend(Patch)
39
+ end
40
+ end
41
+
42
+ def init!
43
+ require('json')
44
+ require('msgpack')
45
+
46
+ self.msgpack_factory = MessagePack::Factory.new
47
+ self.supported_options = [:symbolize_names]
48
+ if ::JSON.parse('["foo"]', freeze: true).first.frozen?
49
+ self.supported_options = [:freeze]
50
+ end
51
+ self.supported_options.freeze
52
+ end
53
+ end
54
+
55
+ module Patch
56
+ def load_file(path, *args)
57
+ return super if args.size > 1
58
+ if kwargs = args.first
59
+ return super unless kwargs.is_a?(Hash)
60
+ return super unless (kwargs.keys - ::Bootsnap::CompileCache::JSON.supported_options).empty?
61
+ end
62
+
63
+ begin
64
+ ::Bootsnap::CompileCache::Native.fetch(
65
+ Bootsnap::CompileCache::JSON.cache_dir,
66
+ File.realpath(path),
67
+ ::Bootsnap::CompileCache::JSON,
68
+ kwargs,
69
+ )
70
+ rescue Errno::EACCES
71
+ ::Bootsnap::CompileCache.permission_error(path)
72
+ end
73
+ end
74
+
75
+ ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
76
+ end
77
+ end
78
+ end
79
+ end