bootsnap 1.6.0 → 1.18.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,19 +1,33 @@
1
1
  # frozen_string_literal: true
2
- require("mkmf")
3
- $CFLAGS << ' -O3 '
4
- $CFLAGS << ' -std=c99'
5
2
 
6
- # ruby.h has some -Wpedantic fails in some cases
7
- # (e.g. https://github.com/Shopify/bootsnap/issues/15)
8
- unless ['0', '', nil].include?(ENV['BOOTSNAP_PEDANTIC'])
9
- $CFLAGS << ' -Wall'
10
- $CFLAGS << ' -Werror'
11
- $CFLAGS << ' -Wextra'
12
- $CFLAGS << ' -Wpedantic'
3
+ require "mkmf"
13
4
 
14
- $CFLAGS << ' -Wno-unused-parameter' # VALUE self has to be there but we don't care what it is.
15
- $CFLAGS << ' -Wno-keyword-macro' # hiding return
16
- $CFLAGS << ' -Wno-gcc-compat' # ruby.h 2.6.0 on macos 10.14, dunno
17
- end
5
+ if %w[ruby truffleruby].include?(RUBY_ENGINE)
6
+ have_func "fdatasync", "unistd.h"
7
+
8
+ unless RUBY_PLATFORM.match?(/mswin|mingw|cygwin/)
9
+ append_cppflags ["-D_GNU_SOURCE"] # Needed of O_NOATIME
10
+ end
11
+
12
+ append_cflags ["-O3", "-std=c99"]
18
13
 
19
- create_makefile("bootsnap/bootsnap")
14
+ # ruby.h has some -Wpedantic fails in some cases
15
+ # (e.g. https://github.com/Shopify/bootsnap/issues/15)
16
+ unless ["0", "", nil].include?(ENV["BOOTSNAP_PEDANTIC"])
17
+ append_cflags([
18
+ "-Wall",
19
+ "-Werror",
20
+ "-Wextra",
21
+ "-Wpedantic",
22
+
23
+ "-Wno-unused-parameter", # VALUE self has to be there but we don't care what it is.
24
+ "-Wno-keyword-macro", # hiding return
25
+ "-Wno-gcc-compat", # ruby.h 2.6.0 on macos 10.14, dunno
26
+ "-Wno-compound-token-split-by-macro",
27
+ ])
28
+ end
29
+
30
+ create_makefile("bootsnap/bootsnap")
31
+ else
32
+ File.write("Makefile", dummy_makefile($srcdir).join)
33
+ end
@@ -1,6 +1,7 @@
1
1
  # frozen_string_literal: true
2
+
2
3
  module Bootsnap
3
- extend(self)
4
+ extend self
4
5
 
5
6
  def bundler?
6
7
  return false unless defined?(::Bundler)
@@ -37,7 +37,11 @@ module Bootsnap
37
37
 
38
38
  def initialize(jobs)
39
39
  @jobs = jobs
40
- @pipe_out, @to_io = IO.pipe
40
+ @pipe_out, @to_io = IO.pipe(binmode: true)
41
+ # Set the writer encoding to binary since IO.pipe only sets it for the reader.
42
+ # https://github.com/rails/rails/issues/16514#issuecomment-52313290
43
+ @to_io.set_encoding(Encoding::BINARY)
44
+
41
45
  @pid = nil
42
46
  end
43
47
 
@@ -59,6 +63,7 @@ module Bootsnap
59
63
  loop do
60
64
  job, *args = Marshal.load(@pipe_out)
61
65
  return if job == :exit
66
+
62
67
  @jobs.fetch(job).call(*args)
63
68
  end
64
69
  rescue IOError
data/lib/bootsnap/cli.rb CHANGED
@@ -1,10 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
- require 'bootsnap'
4
- require 'bootsnap/cli/worker_pool'
5
- require 'optparse'
6
- require 'fileutils'
7
- require 'etc'
3
+ require "bootsnap"
4
+ require "bootsnap/cli/worker_pool"
5
+ require "optparse"
6
+ require "fileutils"
7
+ require "etc"
8
8
 
9
9
  module Bootsnap
10
10
  class CLI
@@ -21,51 +21,60 @@ module Bootsnap
21
21
 
22
22
  attr_reader :cache_dir, :argv
23
23
 
24
- attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :jobs
24
+ attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :json, :jobs
25
25
 
26
26
  def initialize(argv)
27
27
  @argv = argv
28
- self.cache_dir = ENV.fetch('BOOTSNAP_CACHE_DIR', 'tmp/cache')
28
+ self.cache_dir = ENV.fetch("BOOTSNAP_CACHE_DIR", "tmp/cache")
29
29
  self.compile_gemfile = false
30
30
  self.exclude = nil
31
31
  self.verbose = false
32
32
  self.jobs = Etc.nprocessors
33
33
  self.iseq = true
34
34
  self.yaml = true
35
+ self.json = true
35
36
  end
36
37
 
37
38
  def precompile_command(*sources)
38
- require 'bootsnap/compile_cache/iseq'
39
- require 'bootsnap/compile_cache/yaml'
39
+ require "bootsnap/compile_cache/iseq"
40
+ require "bootsnap/compile_cache/yaml"
41
+ require "bootsnap/compile_cache/json"
40
42
 
41
43
  fix_default_encoding do
42
- Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
44
+ Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
43
45
  Bootsnap::CompileCache::YAML.init!
44
- Bootsnap::CompileCache::YAML.cache_dir = self.cache_dir
46
+ Bootsnap::CompileCache::YAML.cache_dir = cache_dir
47
+ Bootsnap::CompileCache::JSON.init!
48
+ Bootsnap::CompileCache::JSON.cache_dir = cache_dir
45
49
 
46
50
  @work_pool = WorkerPool.create(size: jobs, jobs: {
47
51
  ruby: method(:precompile_ruby),
48
52
  yaml: method(:precompile_yaml),
53
+ json: method(:precompile_json),
49
54
  })
50
55
  @work_pool.spawn
51
56
 
52
57
  main_sources = sources.map { |d| File.expand_path(d) }
53
58
  precompile_ruby_files(main_sources)
54
59
  precompile_yaml_files(main_sources)
60
+ precompile_json_files(main_sources)
55
61
 
56
62
  if compile_gemfile
57
- # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
58
- gem_exclude = Regexp.union([exclude, '/spec/', '/test/'].compact)
59
- precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
60
-
61
- # Gems that include YAML files usually don't put them in `lib/`.
63
+ # Gems that include JSON or YAML files usually don't put them in `lib/`.
62
64
  # So we look at the gem root.
63
- gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gem\/[^/]+}
64
- gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
65
+ # Similarly, gems that include Rails engines generally file Ruby files in `app/`.
66
+ # However some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
67
+ gem_exclude = Regexp.union([exclude, "/spec/", "/test/", "/features/"].compact)
68
+
69
+ gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gems/[^/]+}
70
+ gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] || p }.uniq
71
+
72
+ precompile_ruby_files(gem_paths, exclude: gem_exclude)
65
73
  precompile_yaml_files(gem_paths, exclude: gem_exclude)
74
+ precompile_json_files(gem_paths, exclude: gem_exclude)
66
75
  end
67
76
 
68
- if exitstatus = @work_pool.shutdown
77
+ if (exitstatus = @work_pool.shutdown)
69
78
  exit(exitstatus)
70
79
  end
71
80
  end
@@ -82,7 +91,7 @@ module Bootsnap
82
91
  if dir_sort
83
92
  def list_files(path, pattern)
84
93
  if File.directory?(path)
85
- Dir[File.join(path, pattern), sort: false]
94
+ Dir[File.join(path, pattern), sort: false]
86
95
  elsif File.exist?(path)
87
96
  [path]
88
97
  else
@@ -92,7 +101,7 @@ module Bootsnap
92
101
  else
93
102
  def list_files(path, pattern)
94
103
  if File.directory?(path)
95
- Dir[File.join(path, pattern)]
104
+ Dir[File.join(path, pattern)]
96
105
  elsif File.exist?(path)
97
106
  [path]
98
107
  else
@@ -119,9 +128,9 @@ module Bootsnap
119
128
 
120
129
  load_paths.each do |path|
121
130
  if !exclude || !exclude.match?(path)
122
- list_files(path, '**/*.{yml,yaml}').each do |yaml_file|
131
+ list_files(path, "**/*.{yml,yaml}").each do |yaml_file|
123
132
  # We ignore hidden files to not match the various .ci.yml files
124
- if !yaml_file.include?('/.') && (!exclude || !exclude.match?(yaml_file))
133
+ if !File.basename(yaml_file).start_with?(".") && (!exclude || !exclude.match?(yaml_file))
125
134
  @work_pool.push(:yaml, yaml_file)
126
135
  end
127
136
  end
@@ -131,8 +140,31 @@ module Bootsnap
131
140
 
132
141
  def precompile_yaml(*yaml_files)
133
142
  Array(yaml_files).each do |yaml_file|
134
- if CompileCache::YAML.precompile(yaml_file, cache_dir: cache_dir)
135
- STDERR.puts(yaml_file) if verbose
143
+ if CompileCache::YAML.precompile(yaml_file) && verbose
144
+ $stderr.puts(yaml_file)
145
+ end
146
+ end
147
+ end
148
+
149
+ def precompile_json_files(load_paths, exclude: self.exclude)
150
+ return unless json
151
+
152
+ load_paths.each do |path|
153
+ if !exclude || !exclude.match?(path)
154
+ list_files(path, "**/*.json").each do |json_file|
155
+ # We ignore hidden files to not match the various .config.json files
156
+ if !File.basename(json_file).start_with?(".") && (!exclude || !exclude.match?(json_file))
157
+ @work_pool.push(:json, json_file)
158
+ end
159
+ end
160
+ end
161
+ end
162
+ end
163
+
164
+ def precompile_json(*json_files)
165
+ Array(json_files).each do |json_file|
166
+ if CompileCache::JSON.precompile(json_file) && verbose
167
+ $stderr.puts(json_file)
136
168
  end
137
169
  end
138
170
  end
@@ -142,7 +174,7 @@ module Bootsnap
142
174
 
143
175
  load_paths.each do |path|
144
176
  if !exclude || !exclude.match?(path)
145
- list_files(path, '**/*.rb').each do |ruby_file|
177
+ list_files(path, "**/{*.rb,*.rake,Rakefile}").each do |ruby_file|
146
178
  if !exclude || !exclude.match?(ruby_file)
147
179
  @work_pool.push(:ruby, ruby_file)
148
180
  end
@@ -153,8 +185,8 @@ module Bootsnap
153
185
 
154
186
  def precompile_ruby(*ruby_files)
155
187
  Array(ruby_files).each do |ruby_file|
156
- if CompileCache::ISeq.precompile(ruby_file, cache_dir: cache_dir)
157
- STDERR.puts(ruby_file) if verbose
188
+ if CompileCache::ISeq.precompile(ruby_file) && verbose
189
+ $stderr.puts(ruby_file)
158
190
  end
159
191
  end
160
192
  end
@@ -173,14 +205,14 @@ module Bootsnap
173
205
  end
174
206
 
175
207
  def invalid_usage!(message)
176
- STDERR.puts message
177
- STDERR.puts
178
- STDERR.puts parser
208
+ $stderr.puts message
209
+ $stderr.puts
210
+ $stderr.puts parser
179
211
  1
180
212
  end
181
213
 
182
214
  def cache_dir=(dir)
183
- @cache_dir = File.expand_path(File.join(dir, 'bootsnap/compile-cache'))
215
+ @cache_dir = File.expand_path(File.join(dir, "bootsnap/compile-cache"))
184
216
  end
185
217
 
186
218
  def exclude_pattern(pattern)
@@ -195,24 +227,24 @@ module Bootsnap
195
227
  opts.separator "GLOBAL OPTIONS"
196
228
  opts.separator ""
197
229
 
198
- help = <<~EOS
230
+ help = <<~HELP
199
231
  Path to the bootsnap cache directory. Defaults to tmp/cache
200
- EOS
201
- opts.on('--cache-dir DIR', help.strip) do |dir|
232
+ HELP
233
+ opts.on("--cache-dir DIR", help.strip) do |dir|
202
234
  self.cache_dir = dir
203
235
  end
204
236
 
205
- help = <<~EOS
237
+ help = <<~HELP
206
238
  Print precompiled paths.
207
- EOS
208
- opts.on('--verbose', '-v', help.strip) do
239
+ HELP
240
+ opts.on("--verbose", "-v", help.strip) do
209
241
  self.verbose = true
210
242
  end
211
243
 
212
- help = <<~EOS
244
+ help = <<~HELP
213
245
  Number of workers to use. Default to number of processors, set to 0 to disable multi-processing.
214
- EOS
215
- opts.on('--jobs JOBS', '-j', help.strip) do |jobs|
246
+ HELP
247
+ opts.on("--jobs JOBS", "-j", help.strip) do |jobs|
216
248
  self.jobs = Integer(jobs)
217
249
  end
218
250
 
@@ -221,25 +253,30 @@ module Bootsnap
221
253
  opts.separator ""
222
254
  opts.separator " precompile [DIRECTORIES...]: Precompile all .rb files in the passed directories"
223
255
 
224
- help = <<~EOS
256
+ help = <<~HELP
225
257
  Precompile the gems in Gemfile
226
- EOS
227
- opts.on('--gemfile', help) { self.compile_gemfile = true }
258
+ HELP
259
+ opts.on("--gemfile", help) { self.compile_gemfile = true }
228
260
 
229
- help = <<~EOS
261
+ help = <<~HELP
230
262
  Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api'
231
- EOS
232
- opts.on('--exclude PATTERN', help) { |pattern| exclude_pattern(pattern) }
263
+ HELP
264
+ opts.on("--exclude PATTERN", help) { |pattern| exclude_pattern(pattern) }
233
265
 
234
- help = <<~EOS
266
+ help = <<~HELP
235
267
  Disable ISeq (.rb) precompilation.
236
- EOS
237
- opts.on('--no-iseq', help) { self.iseq = false }
268
+ HELP
269
+ opts.on("--no-iseq", help) { self.iseq = false }
238
270
 
239
- help = <<~EOS
271
+ help = <<~HELP
240
272
  Disable YAML precompilation.
241
- EOS
242
- opts.on('--no-yaml', help) { self.yaml = false }
273
+ HELP
274
+ opts.on("--no-yaml", help) { self.yaml = false }
275
+
276
+ help = <<~HELP
277
+ Disable JSON precompilation.
278
+ HELP
279
+ opts.on("--no-json", help) { self.json = false }
243
280
  end
244
281
  end
245
282
  end
@@ -1,25 +1,59 @@
1
1
  # frozen_string_literal: true
2
- require('bootsnap/bootsnap')
3
- require('zlib')
2
+
3
+ require "bootsnap/bootsnap"
4
+ require "zlib"
4
5
 
5
6
  module Bootsnap
6
7
  module CompileCache
7
8
  module ISeq
8
9
  class << self
9
- attr_accessor(:cache_dir)
10
+ attr_reader(:cache_dir)
11
+
12
+ def cache_dir=(cache_dir)
13
+ @cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}iseq" : "#{cache_dir}-iseq"
14
+ end
15
+
16
+ def supported?
17
+ CompileCache.supported? && defined?(RubyVM)
18
+ end
19
+ end
20
+
21
+ has_ruby_bug_18250 = begin # https://bugs.ruby-lang.org/issues/18250
22
+ if defined? RubyVM::InstructionSequence
23
+ RubyVM::InstructionSequence.compile("def foo(*); ->{ super }; end; def foo(**); ->{ super }; end").to_binary
24
+ end
25
+ false
26
+ rescue TypeError
27
+ true
10
28
  end
11
29
 
12
- def self.input_to_storage(_, path)
13
- RubyVM::InstructionSequence.compile_file(path).to_binary
14
- rescue SyntaxError
15
- raise(Uncompilable, 'syntax error')
30
+ if has_ruby_bug_18250
31
+ def self.input_to_storage(_, path)
32
+ iseq = begin
33
+ RubyVM::InstructionSequence.compile_file(path)
34
+ rescue SyntaxError
35
+ return UNCOMPILABLE # syntax error
36
+ end
37
+
38
+ begin
39
+ iseq.to_binary
40
+ rescue TypeError
41
+ UNCOMPILABLE # ruby bug #18250
42
+ end
43
+ end
44
+ else
45
+ def self.input_to_storage(_, path)
46
+ RubyVM::InstructionSequence.compile_file(path).to_binary
47
+ rescue SyntaxError
48
+ UNCOMPILABLE # syntax error
49
+ end
16
50
  end
17
51
 
18
52
  def self.storage_to_output(binary, _args)
19
53
  RubyVM::InstructionSequence.load_from_binary(binary)
20
- rescue RuntimeError => e
21
- if e.message == 'broken binary format'
22
- STDERR.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
54
+ rescue RuntimeError => error
55
+ if error.message == "broken binary format"
56
+ $stderr.puts("[Bootsnap::CompileCache] warning: rejecting broken binary")
23
57
  nil
24
58
  else
25
59
  raise
@@ -35,7 +69,7 @@ module Bootsnap
35
69
  )
36
70
  end
37
71
 
38
- def self.precompile(path, cache_dir: ISeq.cache_dir)
72
+ def self.precompile(path)
39
73
  Bootsnap::CompileCache::Native.precompile(
40
74
  cache_dir,
41
75
  path.to_s,
@@ -53,10 +87,8 @@ module Bootsnap
53
87
  return nil if defined?(Coverage) && Bootsnap::CompileCache::Native.coverage_running?
54
88
 
55
89
  Bootsnap::CompileCache::ISeq.fetch(path.to_s)
56
- rescue Errno::EACCES
57
- Bootsnap::CompileCache.permission_error(path)
58
- rescue RuntimeError => e
59
- if e.message =~ /unmatched platform/
90
+ rescue RuntimeError => error
91
+ if error.message =~ /unmatched platform/
60
92
  puts("unmatched platform for file #{path}")
61
93
  end
62
94
  raise
@@ -73,11 +105,15 @@ module Bootsnap
73
105
  crc = Zlib.crc32(option.inspect)
74
106
  Bootsnap::CompileCache::Native.compile_option_crc32 = crc
75
107
  end
76
- compile_option_updated
108
+ compile_option_updated if supported?
77
109
 
78
110
  def self.install!(cache_dir)
79
111
  Bootsnap::CompileCache::ISeq.cache_dir = cache_dir
112
+
113
+ return unless supported?
114
+
80
115
  Bootsnap::CompileCache::ISeq.compile_option_updated
116
+
81
117
  class << RubyVM::InstructionSequence
82
118
  prepend(InstructionSequenceMixin)
83
119
  end
@@ -0,0 +1,89 @@
1
+ # frozen_string_literal: true
2
+
3
+ require "bootsnap/bootsnap"
4
+
5
+ module Bootsnap
6
+ module CompileCache
7
+ module JSON
8
+ class << self
9
+ attr_accessor(:msgpack_factory, :supported_options)
10
+ attr_reader(:cache_dir)
11
+
12
+ def cache_dir=(cache_dir)
13
+ @cache_dir = cache_dir.end_with?("/") ? "#{cache_dir}json" : "#{cache_dir}-json"
14
+ end
15
+
16
+ def input_to_storage(payload, _)
17
+ obj = ::JSON.parse(payload)
18
+ msgpack_factory.dump(obj)
19
+ end
20
+
21
+ def storage_to_output(data, kwargs)
22
+ if kwargs&.key?(:symbolize_names)
23
+ kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
24
+ end
25
+ msgpack_factory.load(data, kwargs)
26
+ end
27
+
28
+ def input_to_output(data, kwargs)
29
+ ::JSON.parse(data, **(kwargs || {}))
30
+ end
31
+
32
+ def precompile(path)
33
+ Bootsnap::CompileCache::Native.precompile(
34
+ cache_dir,
35
+ path.to_s,
36
+ self,
37
+ )
38
+ end
39
+
40
+ def install!(cache_dir)
41
+ self.cache_dir = cache_dir
42
+ init!
43
+ if ::JSON.respond_to?(:load_file)
44
+ ::JSON.singleton_class.prepend(Patch)
45
+ end
46
+ end
47
+
48
+ def init!
49
+ require "json"
50
+ require "msgpack"
51
+
52
+ self.msgpack_factory = MessagePack::Factory.new
53
+ self.supported_options = [:symbolize_names]
54
+ if supports_freeze?
55
+ self.supported_options = [:freeze]
56
+ end
57
+ supported_options.freeze
58
+ end
59
+
60
+ private
61
+
62
+ def supports_freeze?
63
+ ::JSON.parse('["foo"]', freeze: true).first.frozen? &&
64
+ MessagePack.load(MessagePack.dump("foo"), freeze: true).frozen?
65
+ end
66
+ end
67
+
68
+ module Patch
69
+ def load_file(path, *args)
70
+ return super if args.size > 1
71
+
72
+ if (kwargs = args.first)
73
+ return super unless kwargs.is_a?(Hash)
74
+ return super unless (kwargs.keys - ::Bootsnap::CompileCache::JSON.supported_options).empty?
75
+ end
76
+
77
+ ::Bootsnap::CompileCache::Native.fetch(
78
+ Bootsnap::CompileCache::JSON.cache_dir,
79
+ File.realpath(path),
80
+ ::Bootsnap::CompileCache::JSON,
81
+ kwargs,
82
+ )
83
+ end
84
+
85
+ ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
86
+ end
87
+ end
88
+ end
89
+ end