bootsnap 1.6.0 → 1.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +174 -0
- data/LICENSE.txt +1 -1
- data/README.md +41 -18
- data/exe/bootsnap +1 -1
- data/ext/bootsnap/bootsnap.c +133 -105
- data/ext/bootsnap/extconf.rb +21 -14
- data/lib/bootsnap/bundler.rb +1 -0
- data/lib/bootsnap/cli/worker_pool.rb +6 -1
- data/lib/bootsnap/cli.rb +84 -49
- data/lib/bootsnap/compile_cache/iseq.rb +43 -13
- data/lib/bootsnap/compile_cache/json.rb +93 -0
- data/lib/bootsnap/compile_cache/yaml.rb +299 -61
- data/lib/bootsnap/compile_cache.rb +24 -7
- data/lib/bootsnap/explicit_require.rb +4 -3
- data/lib/bootsnap/load_path_cache/cache.rb +63 -35
- data/lib/bootsnap/load_path_cache/change_observer.rb +17 -2
- data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb +27 -95
- data/lib/bootsnap/load_path_cache/core_ext/loaded_features.rb +1 -0
- data/lib/bootsnap/load_path_cache/loaded_features_index.rb +36 -25
- data/lib/bootsnap/load_path_cache/path.rb +39 -17
- data/lib/bootsnap/load_path_cache/path_scanner.rb +25 -7
- data/lib/bootsnap/load_path_cache/store.rb +64 -24
- data/lib/bootsnap/load_path_cache.rb +31 -38
- data/lib/bootsnap/setup.rb +2 -36
- data/lib/bootsnap/version.rb +2 -1
- data/lib/bootsnap.rb +125 -36
- metadata +8 -79
- data/lib/bootsnap/load_path_cache/core_ext/active_support.rb +0 -107
- data/lib/bootsnap/load_path_cache/realpath_cache.rb +0 -32
data/lib/bootsnap/cli.rb
CHANGED
@@ -1,10 +1,10 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
|
-
require
|
4
|
-
require
|
5
|
-
require
|
6
|
-
require
|
7
|
-
require
|
3
|
+
require "bootsnap"
|
4
|
+
require "bootsnap/cli/worker_pool"
|
5
|
+
require "optparse"
|
6
|
+
require "fileutils"
|
7
|
+
require "etc"
|
8
8
|
|
9
9
|
module Bootsnap
|
10
10
|
class CLI
|
@@ -21,51 +21,58 @@ module Bootsnap
|
|
21
21
|
|
22
22
|
attr_reader :cache_dir, :argv
|
23
23
|
|
24
|
-
attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :jobs
|
24
|
+
attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :json, :jobs
|
25
25
|
|
26
26
|
def initialize(argv)
|
27
27
|
@argv = argv
|
28
|
-
self.cache_dir = ENV.fetch(
|
28
|
+
self.cache_dir = ENV.fetch("BOOTSNAP_CACHE_DIR", "tmp/cache")
|
29
29
|
self.compile_gemfile = false
|
30
30
|
self.exclude = nil
|
31
31
|
self.verbose = false
|
32
32
|
self.jobs = Etc.nprocessors
|
33
33
|
self.iseq = true
|
34
34
|
self.yaml = true
|
35
|
+
self.json = true
|
35
36
|
end
|
36
37
|
|
37
38
|
def precompile_command(*sources)
|
38
|
-
require
|
39
|
-
require
|
39
|
+
require "bootsnap/compile_cache/iseq"
|
40
|
+
require "bootsnap/compile_cache/yaml"
|
41
|
+
require "bootsnap/compile_cache/json"
|
40
42
|
|
41
43
|
fix_default_encoding do
|
42
|
-
Bootsnap::CompileCache::ISeq.cache_dir =
|
44
|
+
Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
|
43
45
|
Bootsnap::CompileCache::YAML.init!
|
44
|
-
Bootsnap::CompileCache::YAML.cache_dir =
|
46
|
+
Bootsnap::CompileCache::YAML.cache_dir = cache_dir
|
47
|
+
Bootsnap::CompileCache::JSON.init!
|
48
|
+
Bootsnap::CompileCache::JSON.cache_dir = cache_dir
|
45
49
|
|
46
50
|
@work_pool = WorkerPool.create(size: jobs, jobs: {
|
47
51
|
ruby: method(:precompile_ruby),
|
48
52
|
yaml: method(:precompile_yaml),
|
53
|
+
json: method(:precompile_json),
|
49
54
|
})
|
50
55
|
@work_pool.spawn
|
51
56
|
|
52
57
|
main_sources = sources.map { |d| File.expand_path(d) }
|
53
58
|
precompile_ruby_files(main_sources)
|
54
59
|
precompile_yaml_files(main_sources)
|
60
|
+
precompile_json_files(main_sources)
|
55
61
|
|
56
62
|
if compile_gemfile
|
57
63
|
# Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
|
58
|
-
gem_exclude = Regexp.union([exclude,
|
64
|
+
gem_exclude = Regexp.union([exclude, "/spec/", "/test/"].compact)
|
59
65
|
precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
|
60
66
|
|
61
|
-
# Gems that include YAML files usually don't put them in `lib/`.
|
67
|
+
# Gems that include JSON or YAML files usually don't put them in `lib/`.
|
62
68
|
# So we look at the gem root.
|
63
|
-
gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?
|
69
|
+
gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems/[^/]+}
|
64
70
|
gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
|
65
71
|
precompile_yaml_files(gem_paths, exclude: gem_exclude)
|
72
|
+
precompile_json_files(gem_paths, exclude: gem_exclude)
|
66
73
|
end
|
67
74
|
|
68
|
-
if exitstatus = @work_pool.shutdown
|
75
|
+
if (exitstatus = @work_pool.shutdown)
|
69
76
|
exit(exitstatus)
|
70
77
|
end
|
71
78
|
end
|
@@ -82,7 +89,7 @@ module Bootsnap
|
|
82
89
|
if dir_sort
|
83
90
|
def list_files(path, pattern)
|
84
91
|
if File.directory?(path)
|
85
|
-
Dir[File.join(path,
|
92
|
+
Dir[File.join(path, pattern), sort: false]
|
86
93
|
elsif File.exist?(path)
|
87
94
|
[path]
|
88
95
|
else
|
@@ -92,7 +99,7 @@ module Bootsnap
|
|
92
99
|
else
|
93
100
|
def list_files(path, pattern)
|
94
101
|
if File.directory?(path)
|
95
|
-
Dir[File.join(path,
|
102
|
+
Dir[File.join(path, pattern)]
|
96
103
|
elsif File.exist?(path)
|
97
104
|
[path]
|
98
105
|
else
|
@@ -119,9 +126,9 @@ module Bootsnap
|
|
119
126
|
|
120
127
|
load_paths.each do |path|
|
121
128
|
if !exclude || !exclude.match?(path)
|
122
|
-
list_files(path,
|
129
|
+
list_files(path, "**/*.{yml,yaml}").each do |yaml_file|
|
123
130
|
# We ignore hidden files to not match the various .ci.yml files
|
124
|
-
if !yaml_file.
|
131
|
+
if !File.basename(yaml_file).start_with?(".") && (!exclude || !exclude.match?(yaml_file))
|
125
132
|
@work_pool.push(:yaml, yaml_file)
|
126
133
|
end
|
127
134
|
end
|
@@ -131,8 +138,31 @@ module Bootsnap
|
|
131
138
|
|
132
139
|
def precompile_yaml(*yaml_files)
|
133
140
|
Array(yaml_files).each do |yaml_file|
|
134
|
-
if CompileCache::YAML.precompile(yaml_file
|
135
|
-
|
141
|
+
if CompileCache::YAML.precompile(yaml_file) && verbose
|
142
|
+
$stderr.puts(yaml_file)
|
143
|
+
end
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
def precompile_json_files(load_paths, exclude: self.exclude)
|
148
|
+
return unless json
|
149
|
+
|
150
|
+
load_paths.each do |path|
|
151
|
+
if !exclude || !exclude.match?(path)
|
152
|
+
list_files(path, "**/*.json").each do |json_file|
|
153
|
+
# We ignore hidden files to not match the various .config.json files
|
154
|
+
if !File.basename(json_file).start_with?(".") && (!exclude || !exclude.match?(json_file))
|
155
|
+
@work_pool.push(:json, json_file)
|
156
|
+
end
|
157
|
+
end
|
158
|
+
end
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
def precompile_json(*json_files)
|
163
|
+
Array(json_files).each do |json_file|
|
164
|
+
if CompileCache::JSON.precompile(json_file) && verbose
|
165
|
+
$stderr.puts(json_file)
|
136
166
|
end
|
137
167
|
end
|
138
168
|
end
|
@@ -142,7 +172,7 @@ module Bootsnap
|
|
142
172
|
|
143
173
|
load_paths.each do |path|
|
144
174
|
if !exclude || !exclude.match?(path)
|
145
|
-
list_files(path,
|
175
|
+
list_files(path, "**/{*.rb,*.rake,Rakefile}").each do |ruby_file|
|
146
176
|
if !exclude || !exclude.match?(ruby_file)
|
147
177
|
@work_pool.push(:ruby, ruby_file)
|
148
178
|
end
|
@@ -153,8 +183,8 @@ module Bootsnap
|
|
153
183
|
|
154
184
|
def precompile_ruby(*ruby_files)
|
155
185
|
Array(ruby_files).each do |ruby_file|
|
156
|
-
if CompileCache::ISeq.precompile(ruby_file
|
157
|
-
|
186
|
+
if CompileCache::ISeq.precompile(ruby_file) && verbose
|
187
|
+
$stderr.puts(ruby_file)
|
158
188
|
end
|
159
189
|
end
|
160
190
|
end
|
@@ -173,14 +203,14 @@ module Bootsnap
|
|
173
203
|
end
|
174
204
|
|
175
205
|
def invalid_usage!(message)
|
176
|
-
|
177
|
-
|
178
|
-
|
206
|
+
$stderr.puts message
|
207
|
+
$stderr.puts
|
208
|
+
$stderr.puts parser
|
179
209
|
1
|
180
210
|
end
|
181
211
|
|
182
212
|
def cache_dir=(dir)
|
183
|
-
@cache_dir = File.expand_path(File.join(dir,
|
213
|
+
@cache_dir = File.expand_path(File.join(dir, "bootsnap/compile-cache"))
|
184
214
|
end
|
185
215
|
|
186
216
|
def exclude_pattern(pattern)
|
@@ -195,24 +225,24 @@ module Bootsnap
|
|
195
225
|
opts.separator "GLOBAL OPTIONS"
|
196
226
|
opts.separator ""
|
197
227
|
|
198
|
-
help = <<~
|
228
|
+
help = <<~HELP
|
199
229
|
Path to the bootsnap cache directory. Defaults to tmp/cache
|
200
|
-
|
201
|
-
opts.on(
|
230
|
+
HELP
|
231
|
+
opts.on("--cache-dir DIR", help.strip) do |dir|
|
202
232
|
self.cache_dir = dir
|
203
233
|
end
|
204
234
|
|
205
|
-
help = <<~
|
235
|
+
help = <<~HELP
|
206
236
|
Print precompiled paths.
|
207
|
-
|
208
|
-
opts.on(
|
237
|
+
HELP
|
238
|
+
opts.on("--verbose", "-v", help.strip) do
|
209
239
|
self.verbose = true
|
210
240
|
end
|
211
241
|
|
212
|
-
help = <<~
|
242
|
+
help = <<~HELP
|
213
243
|
Number of workers to use. Default to number of processors, set to 0 to disable multi-processing.
|
214
|
-
|
215
|
-
opts.on(
|
244
|
+
HELP
|
245
|
+
opts.on("--jobs JOBS", "-j", help.strip) do |jobs|
|
216
246
|
self.jobs = Integer(jobs)
|
217
247
|
end
|
218
248
|
|
@@ -221,25 +251,30 @@ module Bootsnap
|
|
221
251
|
opts.separator ""
|
222
252
|
opts.separator " precompile [DIRECTORIES...]: Precompile all .rb files in the passed directories"
|
223
253
|
|
224
|
-
help = <<~
|
254
|
+
help = <<~HELP
|
225
255
|
Precompile the gems in Gemfile
|
226
|
-
|
227
|
-
opts.on(
|
256
|
+
HELP
|
257
|
+
opts.on("--gemfile", help) { self.compile_gemfile = true }
|
228
258
|
|
229
|
-
help = <<~
|
259
|
+
help = <<~HELP
|
230
260
|
Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api'
|
231
|
-
|
232
|
-
opts.on(
|
261
|
+
HELP
|
262
|
+
opts.on("--exclude PATTERN", help) { |pattern| exclude_pattern(pattern) }
|
233
263
|
|
234
|
-
help = <<~
|
264
|
+
help = <<~HELP
|
235
265
|
Disable ISeq (.rb) precompilation.
|
236
|
-
|
237
|
-
opts.on(
|
266
|
+
HELP
|
267
|
+
opts.on("--no-iseq", help) { self.iseq = false }
|
238
268
|
|
239
|
-
help = <<~
|
269
|
+
help = <<~HELP
|
240
270
|
Disable YAML precompilation.
|
241
|
-
|
242
|
-
opts.on(
|
271
|
+
HELP
|
272
|
+
opts.on("--no-yaml", help) { self.yaml = false }
|
273
|
+
|
274
|
+
help = <<~HELP
|
275
|
+
Disable JSON precompilation.
|
276
|
+
HELP
|
277
|
+
opts.on("--no-json", help) { self.json = false }
|
243
278
|
end
|
244
279
|
end
|
245
280
|
end
|
@@ -1,25 +1,55 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
|
-
|
3
|
-
require(
|
2
|
+
|
3
|
+
require("bootsnap/bootsnap")
|
4
|
+
require("zlib")
|
4
5
|
|
5
6
|
module Bootsnap
|
6
7
|
module CompileCache
|
7
8
|
module ISeq
|
8
9
|
class << self
|
9
|
-
|
10
|
+
attr_reader(:cache_dir)
|
11
|
+
|
12
|
+
def cache_dir=(cache_dir)
|
13
|
+
@cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}iseq" : "#{cache_dir}-iseq"
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
17
|
+
has_ruby_bug_18250 = begin # https://bugs.ruby-lang.org/issues/18250
|
18
|
+
if defined? RubyVM::InstructionSequence
|
19
|
+
RubyVM::InstructionSequence.compile("def foo(*); ->{ super }; end; def foo(**); ->{ super }; end").to_binary
|
20
|
+
end
|
21
|
+
false
|
22
|
+
rescue TypeError
|
23
|
+
true
|
10
24
|
end
|
11
25
|
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
26
|
+
if has_ruby_bug_18250
|
27
|
+
def self.input_to_storage(_, path)
|
28
|
+
iseq = begin
|
29
|
+
RubyVM::InstructionSequence.compile_file(path)
|
30
|
+
rescue SyntaxError
|
31
|
+
return UNCOMPILABLE # syntax error
|
32
|
+
end
|
33
|
+
|
34
|
+
begin
|
35
|
+
iseq.to_binary
|
36
|
+
rescue TypeError
|
37
|
+
UNCOMPILABLE # ruby bug #18250
|
38
|
+
end
|
39
|
+
end
|
40
|
+
else
|
41
|
+
def self.input_to_storage(_, path)
|
42
|
+
RubyVM::InstructionSequence.compile_file(path).to_binary
|
43
|
+
rescue SyntaxError
|
44
|
+
UNCOMPILABLE # syntax error
|
45
|
+
end
|
16
46
|
end
|
17
47
|
|
18
48
|
def self.storage_to_output(binary, _args)
|
19
49
|
RubyVM::InstructionSequence.load_from_binary(binary)
|
20
|
-
rescue RuntimeError =>
|
21
|
-
if
|
22
|
-
|
50
|
+
rescue RuntimeError => error
|
51
|
+
if error.message == "broken binary format"
|
52
|
+
$stderr.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
|
23
53
|
nil
|
24
54
|
else
|
25
55
|
raise
|
@@ -35,7 +65,7 @@ module Bootsnap
|
|
35
65
|
)
|
36
66
|
end
|
37
67
|
|
38
|
-
def self.precompile(path
|
68
|
+
def self.precompile(path)
|
39
69
|
Bootsnap::CompileCache::Native.precompile(
|
40
70
|
cache_dir,
|
41
71
|
path.to_s,
|
@@ -55,8 +85,8 @@ module Bootsnap
|
|
55
85
|
Bootsnap::CompileCache::ISeq.fetch(path.to_s)
|
56
86
|
rescue Errno::EACCES
|
57
87
|
Bootsnap::CompileCache.permission_error(path)
|
58
|
-
rescue RuntimeError =>
|
59
|
-
if
|
88
|
+
rescue RuntimeError => error
|
89
|
+
if error.message =~ /unmatched platform/
|
60
90
|
puts("unmatched platform for file #{path}")
|
61
91
|
end
|
62
92
|
raise
|
@@ -0,0 +1,93 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require("bootsnap/bootsnap")
|
4
|
+
|
5
|
+
module Bootsnap
|
6
|
+
module CompileCache
|
7
|
+
module JSON
|
8
|
+
class << self
|
9
|
+
attr_accessor(:msgpack_factory, :supported_options)
|
10
|
+
attr_reader(:cache_dir)
|
11
|
+
|
12
|
+
def cache_dir=(cache_dir)
|
13
|
+
@cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}json" : "#{cache_dir}-json"
|
14
|
+
end
|
15
|
+
|
16
|
+
def input_to_storage(payload, _)
|
17
|
+
obj = ::JSON.parse(payload)
|
18
|
+
msgpack_factory.dump(obj)
|
19
|
+
end
|
20
|
+
|
21
|
+
def storage_to_output(data, kwargs)
|
22
|
+
if kwargs&.key?(:symbolize_names)
|
23
|
+
kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
|
24
|
+
end
|
25
|
+
msgpack_factory.load(data, kwargs)
|
26
|
+
end
|
27
|
+
|
28
|
+
def input_to_output(data, kwargs)
|
29
|
+
::JSON.parse(data, **(kwargs || {}))
|
30
|
+
end
|
31
|
+
|
32
|
+
def precompile(path)
|
33
|
+
Bootsnap::CompileCache::Native.precompile(
|
34
|
+
cache_dir,
|
35
|
+
path.to_s,
|
36
|
+
self,
|
37
|
+
)
|
38
|
+
end
|
39
|
+
|
40
|
+
def install!(cache_dir)
|
41
|
+
self.cache_dir = cache_dir
|
42
|
+
init!
|
43
|
+
if ::JSON.respond_to?(:load_file)
|
44
|
+
::JSON.singleton_class.prepend(Patch)
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
def init!
|
49
|
+
require("json")
|
50
|
+
require("msgpack")
|
51
|
+
|
52
|
+
self.msgpack_factory = MessagePack::Factory.new
|
53
|
+
self.supported_options = [:symbolize_names]
|
54
|
+
if supports_freeze?
|
55
|
+
self.supported_options = [:freeze]
|
56
|
+
end
|
57
|
+
supported_options.freeze
|
58
|
+
end
|
59
|
+
|
60
|
+
private
|
61
|
+
|
62
|
+
def supports_freeze?
|
63
|
+
::JSON.parse('["foo"]', freeze: true).first.frozen? &&
|
64
|
+
MessagePack.load(MessagePack.dump("foo"), freeze: true).frozen?
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
module Patch
|
69
|
+
def load_file(path, *args)
|
70
|
+
return super if args.size > 1
|
71
|
+
|
72
|
+
if (kwargs = args.first)
|
73
|
+
return super unless kwargs.is_a?(Hash)
|
74
|
+
return super unless (kwargs.keys - ::Bootsnap::CompileCache::JSON.supported_options).empty?
|
75
|
+
end
|
76
|
+
|
77
|
+
begin
|
78
|
+
::Bootsnap::CompileCache::Native.fetch(
|
79
|
+
Bootsnap::CompileCache::JSON.cache_dir,
|
80
|
+
File.realpath(path),
|
81
|
+
::Bootsnap::CompileCache::JSON,
|
82
|
+
kwargs,
|
83
|
+
)
|
84
|
+
rescue Errno::EACCES
|
85
|
+
::Bootsnap::CompileCache.permission_error(path)
|
86
|
+
end
|
87
|
+
end
|
88
|
+
|
89
|
+
ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
|
90
|
+
end
|
91
|
+
end
|
92
|
+
end
|
93
|
+
end
|