bootsnap 1.5.1 → 1.6.0

Sign up to get free protection for your applications and to get access to all the features.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 9a15b298603bbdda820fa4aa3d37e32c72f181aa49aac240a02f076de2dd17eb
4
- data.tar.gz: bbce00645395d42d30cb3c89b35dcb910003e3f9c3b65afd3a94b7e9019b868a
3
+ metadata.gz: '029d63ba428f470d2cd2ef194d857e20689e7c8e377e2eaaaab83570f366034a'
4
+ data.tar.gz: 36a55f9d12dddef6ea3c4739f586c9236d7f4bc677a8b6747dfd7465d46eeca2
5
5
  SHA512:
6
- metadata.gz: 982d29eb952ba2c053fd78d244493fc2c75d148f11bed24c67e657c0b59d1d6cb4cfc964583087dfababe0b3f977aa9d183c67c69949a31b1dca65c5d51886a4
7
- data.tar.gz: 9e862ebb9a2ddb6cebdfec4835e78c9fcf14dd1451af73a6f04bf38edb362111f27cb70a04e381cb9550cdbdf4cf83d838aaf514126a41e1c9f1c1a12ab96417
6
+ metadata.gz: 131ec17c4e4912f387c18778250e689467ebfcc80e91eb884237307af1a57a3c89d4fb20c772fbc0330123a796d631861a9cf145bd32c54183077bbc97d7fa6f
7
+ data.tar.gz: d0c92454c8a5d8b16a25908fd0ae134fc817c5977958bde8424baca2bb6c96e60eb648c9f770bf7c7f58a3dd98871d4e7b0e3a0950c269100fded45ca9fddfa3
@@ -1,3 +1,13 @@
1
+ # Unreleased
2
+
3
+ # 1.6.0
4
+
5
+ * Fix a Ruby 2.7/3.0 issue with `YAML.load_file` keyword arguments. (#342)
6
+ * `bootsnap precompile` CLI use multiple processes to complete faster. (#341)
7
+ * `bootsnap precompile` CLI also precompile YAML files. (#340)
8
+ * Changed the load path cache directory from `$BOOTSNAP_CACHE_DIR/bootsnap-load-path-cache` to `$BOOTSNAP_CACHE_DIR/bootsnap/load-path-cache` for ease of use. (#334)
9
+ * Changed the compile cache directory from `$BOOTSNAP_CACHE_DIR/bootsnap-compile-cache` to `$BOOTSNAP_CACHE_DIR/bootsnap/compile-cache` for ease of use. (#334)
10
+
1
11
  # 1.5.1
2
12
 
3
13
  * Workaround a Ruby bug in InstructionSequence.compile_file. (#332)
data/README.md CHANGED
@@ -29,7 +29,8 @@ If you are using Rails, add this to `config/boot.rb` immediately after `require
29
29
  require 'bootsnap/setup'
30
30
  ```
31
31
 
32
- Note that bootsnap writes to `tmp/cache`, and that directory *must* be writable. Rails will fail to
32
+ Note that bootsnap writes to `tmp/cache` (or the path specified by `ENV['BOOTSNAP_CACHE_DIR']`),
33
+ and that directory *must* be writable. Rails will fail to
33
34
  boot if it is not. If this is unacceptable (e.g. you are running in a read-only container and
34
35
  unwilling to mount in a writable tmpdir), you should remove this line or wrap it in a conditional.
35
36
 
@@ -70,7 +70,7 @@ struct bs_cache_key {
70
70
  STATIC_ASSERT(sizeof(struct bs_cache_key) == KEY_SIZE);
71
71
 
72
72
  /* Effectively a schema version. Bumping invalidates all previous caches */
73
- static const uint32_t current_version = 2;
73
+ static const uint32_t current_version = 3;
74
74
 
75
75
  /* hash of e.g. "x86_64-darwin17", invalidating when ruby is recompiled on a
76
76
  * new OS ABI, etc. */
@@ -92,13 +92,15 @@ static ID uncompilable;
92
92
  /* Functions exposed as module functions on Bootsnap::CompileCache::Native */
93
93
  static VALUE bs_compile_option_crc32_set(VALUE self, VALUE crc32_v);
94
94
  static VALUE bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler, VALUE args);
95
+ static VALUE bs_rb_precompile(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler);
95
96
 
96
97
  /* Helpers */
97
98
  static uint64_t fnv1a_64(const char *str);
98
- static void bs_cache_path(const char * cachedir, const char * path, const char * extra, char (* cache_path)[MAX_CACHEPATH_SIZE]);
99
+ static void bs_cache_path(const char * cachedir, const char * path, char (* cache_path)[MAX_CACHEPATH_SIZE]);
99
100
  static int bs_read_key(int fd, struct bs_cache_key * key);
100
101
  static int cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2);
101
102
  static VALUE bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args);
103
+ static VALUE bs_precompile(char * path, VALUE path_v, char * cache_path, VALUE handler);
102
104
  static int open_current_file(char * path, struct bs_cache_key * key, const char ** errno_provenance);
103
105
  static int fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE args, VALUE * output_data, int * exception_tag, const char ** errno_provenance);
104
106
  static uint32_t get_ruby_revision(void);
@@ -149,6 +151,7 @@ Init_bootsnap(void)
149
151
 
150
152
  rb_define_module_function(rb_mBootsnap_CompileCache_Native, "coverage_running?", bs_rb_coverage_running, 0);
151
153
  rb_define_module_function(rb_mBootsnap_CompileCache_Native, "fetch", bs_rb_fetch, 4);
154
+ rb_define_module_function(rb_mBootsnap_CompileCache_Native, "precompile", bs_rb_precompile, 3);
152
155
  rb_define_module_function(rb_mBootsnap_CompileCache_Native, "compile_option_crc32=", bs_compile_option_crc32_set, 1);
153
156
 
154
157
  current_umask = umask(0777);
@@ -264,13 +267,9 @@ get_ruby_platform(void)
264
267
  * The path will look something like: <cachedir>/12/34567890abcdef
265
268
  */
266
269
  static void
267
- bs_cache_path(const char * cachedir, const char * path, const char * extra, char (* cache_path)[MAX_CACHEPATH_SIZE])
270
+ bs_cache_path(const char * cachedir, const char * path, char (* cache_path)[MAX_CACHEPATH_SIZE])
268
271
  {
269
272
  uint64_t hash = fnv1a_64(path);
270
- if (extra) {
271
- hash ^= fnv1a_64(extra);
272
- }
273
-
274
273
  uint8_t first_byte = (hash >> (64 - 8));
275
274
  uint64_t remainder = hash & 0x00ffffffffffffff;
276
275
 
@@ -318,18 +317,39 @@ bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler, VALUE arg
318
317
  char * cachedir = RSTRING_PTR(cachedir_v);
319
318
  char * path = RSTRING_PTR(path_v);
320
319
  char cache_path[MAX_CACHEPATH_SIZE];
321
- char * extra = NULL;
322
- if (!NIL_P(args)) {
323
- VALUE args_serial = rb_marshal_dump(args, Qnil);
324
- extra = RSTRING_PTR(args_serial);
325
- }
326
320
 
327
321
  /* generate cache path to cache_path */
328
- bs_cache_path(cachedir, path, extra, &cache_path);
322
+ bs_cache_path(cachedir, path, &cache_path);
329
323
 
330
324
  return bs_fetch(path, path_v, cache_path, handler, args);
331
325
  }
332
326
 
327
+ /*
328
+ * Entrypoint for Bootsnap::CompileCache::Native.precompile.
329
+ * Similar to fetch, but it only generate the cache if missing
330
+ * and doesn't return the content.
331
+ */
332
+ static VALUE
333
+ bs_rb_precompile(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
334
+ {
335
+ FilePathValue(path_v);
336
+
337
+ Check_Type(cachedir_v, T_STRING);
338
+ Check_Type(path_v, T_STRING);
339
+
340
+ if (RSTRING_LEN(cachedir_v) > MAX_CACHEDIR_SIZE) {
341
+ rb_raise(rb_eArgError, "cachedir too long");
342
+ }
343
+
344
+ char * cachedir = RSTRING_PTR(cachedir_v);
345
+ char * path = RSTRING_PTR(path_v);
346
+ char cache_path[MAX_CACHEPATH_SIZE];
347
+
348
+ /* generate cache path to cache_path */
349
+ bs_cache_path(cachedir, path, &cache_path);
350
+
351
+ return bs_precompile(path, path_v, cache_path, handler);
352
+ }
333
353
  /*
334
354
  * Open the file we want to load/cache and generate a cache key for it if it
335
355
  * was loaded.
@@ -740,6 +760,79 @@ invalid_type_storage_data:
740
760
  #undef CLEANUP
741
761
  }
742
762
 
763
+ static VALUE
764
+ bs_precompile(char * path, VALUE path_v, char * cache_path, VALUE handler)
765
+ {
766
+ struct bs_cache_key cached_key, current_key;
767
+ char * contents = NULL;
768
+ int cache_fd = -1, current_fd = -1;
769
+ int res, valid_cache = 0, exception_tag = 0;
770
+ const char * errno_provenance = NULL;
771
+
772
+ VALUE input_data; /* data read from source file, e.g. YAML or ruby source */
773
+ VALUE storage_data; /* compiled data, e.g. msgpack / binary iseq */
774
+
775
+ /* Open the source file and generate a cache key for it */
776
+ current_fd = open_current_file(path, &current_key, &errno_provenance);
777
+ if (current_fd < 0) goto fail;
778
+
779
+ /* Open the cache key if it exists, and read its cache key in */
780
+ cache_fd = open_cache_file(cache_path, &cached_key, &errno_provenance);
781
+ if (cache_fd == CACHE_MISSING_OR_INVALID) {
782
+ /* This is ok: valid_cache remains false, we re-populate it. */
783
+ } else if (cache_fd < 0) {
784
+ goto fail;
785
+ } else {
786
+ /* True if the cache existed and no invalidating changes have occurred since
787
+ * it was generated. */
788
+ valid_cache = cache_key_equal(&current_key, &cached_key);
789
+ }
790
+
791
+ if (valid_cache) {
792
+ goto succeed;
793
+ }
794
+
795
+ close(cache_fd);
796
+ cache_fd = -1;
797
+ /* Cache is stale, invalid, or missing. Regenerate and write it out. */
798
+
799
+ /* Read the contents of the source file into a buffer */
800
+ if (bs_read_contents(current_fd, current_key.size, &contents, &errno_provenance) < 0) goto fail;
801
+ input_data = rb_str_new(contents, current_key.size);
802
+
803
+ /* Try to compile the input_data using input_to_storage(input_data) */
804
+ exception_tag = bs_input_to_storage(handler, Qnil, input_data, path_v, &storage_data);
805
+ if (exception_tag != 0) goto fail;
806
+
807
+ /* If input_to_storage raised Bootsnap::CompileCache::Uncompilable, don't try
808
+ * to cache anything; just return false */
809
+ if (storage_data == uncompilable) {
810
+ goto fail;
811
+ }
812
+ /* If storage_data isn't a string, we can't cache it */
813
+ if (!RB_TYPE_P(storage_data, T_STRING)) goto fail;
814
+
815
+ /* Write the cache key and storage_data to the cache directory */
816
+ res = atomic_write_cache_file(cache_path, &current_key, storage_data, &errno_provenance);
817
+ if (res < 0) goto fail;
818
+
819
+ goto succeed;
820
+
821
+ #define CLEANUP \
822
+ if (contents != NULL) xfree(contents); \
823
+ if (current_fd >= 0) close(current_fd); \
824
+ if (cache_fd >= 0) close(cache_fd);
825
+
826
+ succeed:
827
+ CLEANUP;
828
+ return Qtrue;
829
+ fail:
830
+ CLEANUP;
831
+ return Qfalse;
832
+ #undef CLEANUP
833
+ }
834
+
835
+
743
836
  /*****************************************************************************/
744
837
  /********************* Handler Wrappers **************************************/
745
838
  /*****************************************************************************
@@ -771,7 +864,6 @@ struct i2o_data {
771
864
 
772
865
  struct i2s_data {
773
866
  VALUE handler;
774
- VALUE args;
775
867
  VALUE input_data;
776
868
  VALUE pathval;
777
869
  };
@@ -789,7 +881,7 @@ bs_storage_to_output(VALUE handler, VALUE args, VALUE storage_data, VALUE * outp
789
881
  int state;
790
882
  struct s2o_data s2o_data = {
791
883
  .handler = handler,
792
- .args = args,
884
+ .args = args,
793
885
  .storage_data = storage_data,
794
886
  };
795
887
  *output_data = rb_protect(prot_storage_to_output, (VALUE)&s2o_data, &state);
@@ -818,7 +910,7 @@ static VALUE
818
910
  try_input_to_storage(VALUE arg)
819
911
  {
820
912
  struct i2s_data * data = (struct i2s_data *)arg;
821
- return rb_funcall(data->handler, rb_intern("input_to_storage"), 3, data->input_data, data->pathval, data->args);
913
+ return rb_funcall(data->handler, rb_intern("input_to_storage"), 2, data->input_data, data->pathval);
822
914
  }
823
915
 
824
916
  static VALUE
@@ -843,7 +935,6 @@ bs_input_to_storage(VALUE handler, VALUE args, VALUE input_data, VALUE pathval,
843
935
  int state;
844
936
  struct i2s_data i2s_data = {
845
937
  .handler = handler,
846
- .args = args,
847
938
  .input_data = input_data,
848
939
  .pathval = pathval,
849
940
  };
@@ -24,13 +24,13 @@ module Bootsnap
24
24
  setup_disable_trace if disable_trace
25
25
 
26
26
  Bootsnap::LoadPathCache.setup(
27
- cache_path: cache_dir + '/bootsnap-load-path-cache',
27
+ cache_path: cache_dir + '/bootsnap/load-path-cache',
28
28
  development_mode: development_mode,
29
29
  active_support: autoload_paths_cache
30
30
  ) if load_path_cache
31
31
 
32
32
  Bootsnap::CompileCache.setup(
33
- cache_dir: cache_dir + '/bootsnap-compile-cache',
33
+ cache_dir: cache_dir + '/bootsnap/compile-cache',
34
34
  iseq: compile_cache_iseq,
35
35
  yaml: compile_cache_yaml
36
36
  )
@@ -1,8 +1,10 @@
1
1
  # frozen_string_literal: true
2
2
 
3
3
  require 'bootsnap'
4
+ require 'bootsnap/cli/worker_pool'
4
5
  require 'optparse'
5
6
  require 'fileutils'
7
+ require 'etc'
6
8
 
7
9
  module Bootsnap
8
10
  class CLI
@@ -19,49 +21,68 @@ module Bootsnap
19
21
 
20
22
  attr_reader :cache_dir, :argv
21
23
 
22
- attr_accessor :compile_gemfile, :exclude
24
+ attr_accessor :compile_gemfile, :exclude, :verbose, :iseq, :yaml, :jobs
23
25
 
24
26
  def initialize(argv)
25
27
  @argv = argv
26
28
  self.cache_dir = ENV.fetch('BOOTSNAP_CACHE_DIR', 'tmp/cache')
27
29
  self.compile_gemfile = false
28
30
  self.exclude = nil
31
+ self.verbose = false
32
+ self.jobs = Etc.nprocessors
33
+ self.iseq = true
34
+ self.yaml = true
29
35
  end
30
36
 
31
37
  def precompile_command(*sources)
32
38
  require 'bootsnap/compile_cache/iseq'
39
+ require 'bootsnap/compile_cache/yaml'
33
40
 
34
41
  fix_default_encoding do
35
42
  Bootsnap::CompileCache::ISeq.cache_dir = self.cache_dir
43
+ Bootsnap::CompileCache::YAML.init!
44
+ Bootsnap::CompileCache::YAML.cache_dir = self.cache_dir
45
+
46
+ @work_pool = WorkerPool.create(size: jobs, jobs: {
47
+ ruby: method(:precompile_ruby),
48
+ yaml: method(:precompile_yaml),
49
+ })
50
+ @work_pool.spawn
51
+
52
+ main_sources = sources.map { |d| File.expand_path(d) }
53
+ precompile_ruby_files(main_sources)
54
+ precompile_yaml_files(main_sources)
36
55
 
37
56
  if compile_gemfile
38
- sources += $LOAD_PATH
57
+ # Some gems embed their tests, they're very unlikely to be loaded, so not worth precompiling.
58
+ gem_exclude = Regexp.union([exclude, '/spec/', '/test/'].compact)
59
+ precompile_ruby_files($LOAD_PATH.map { |d| File.expand_path(d) }, exclude: gem_exclude)
60
+
61
+ # Gems that include YAML files usually don't put them in `lib/`.
62
+ # So we look at the gem root.
63
+ gem_pattern = %r{^#{Regexp.escape(Bundler.bundle_path.to_s)}/?(?:bundler/)?gem\/[^/]+}
64
+ gem_paths = $LOAD_PATH.map { |p| p[gem_pattern] }.compact.uniq
65
+ precompile_yaml_files(gem_paths, exclude: gem_exclude)
39
66
  end
40
67
 
41
- sources.map { |d| File.expand_path(d) }.each do |path|
42
- if !exclude || !exclude.match?(path)
43
- list_ruby_files(path).each do |ruby_file|
44
- if !exclude || !exclude.match?(ruby_file)
45
- CompileCache::ISeq.fetch(ruby_file, cache_dir: cache_dir)
46
- end
47
- end
48
- end
68
+ if exitstatus = @work_pool.shutdown
69
+ exit(exitstatus)
49
70
  end
50
71
  end
51
72
  0
52
73
  end
53
74
 
54
75
  dir_sort = begin
55
- Dir['.', sort: false]
76
+ Dir[__FILE__, sort: false]
56
77
  true
57
78
  rescue ArgumentError, TypeError
58
79
  false
59
80
  end
60
81
 
61
82
  if dir_sort
62
- def list_ruby_files(path)
83
+ def list_files(path, pattern)
63
84
  if File.directory?(path)
64
- Dir[File.join(path, '**/*.rb'), sort: false]
85
+ Dir[File.join(path, pattern), sort: false]
65
86
  elsif File.exist?(path)
66
87
  [path]
67
88
  else
@@ -69,9 +90,9 @@ module Bootsnap
69
90
  end
70
91
  end
71
92
  else
72
- def list_ruby_files(path)
93
+ def list_files(path, pattern)
73
94
  if File.directory?(path)
74
- Dir[File.join(path, '**/*.rb')]
95
+ Dir[File.join(path, pattern)]
75
96
  elsif File.exist?(path)
76
97
  [path]
77
98
  else
@@ -93,6 +114,51 @@ module Bootsnap
93
114
 
94
115
  private
95
116
 
117
+ def precompile_yaml_files(load_paths, exclude: self.exclude)
118
+ return unless yaml
119
+
120
+ load_paths.each do |path|
121
+ if !exclude || !exclude.match?(path)
122
+ list_files(path, '**/*.{yml,yaml}').each do |yaml_file|
123
+ # We ignore hidden files to not match the various .ci.yml files
124
+ if !yaml_file.include?('/.') && (!exclude || !exclude.match?(yaml_file))
125
+ @work_pool.push(:yaml, yaml_file)
126
+ end
127
+ end
128
+ end
129
+ end
130
+ end
131
+
132
+ def precompile_yaml(*yaml_files)
133
+ Array(yaml_files).each do |yaml_file|
134
+ if CompileCache::YAML.precompile(yaml_file, cache_dir: cache_dir)
135
+ STDERR.puts(yaml_file) if verbose
136
+ end
137
+ end
138
+ end
139
+
140
+ def precompile_ruby_files(load_paths, exclude: self.exclude)
141
+ return unless iseq
142
+
143
+ load_paths.each do |path|
144
+ if !exclude || !exclude.match?(path)
145
+ list_files(path, '**/*.rb').each do |ruby_file|
146
+ if !exclude || !exclude.match?(ruby_file)
147
+ @work_pool.push(:ruby, ruby_file)
148
+ end
149
+ end
150
+ end
151
+ end
152
+ end
153
+
154
+ def precompile_ruby(*ruby_files)
155
+ Array(ruby_files).each do |ruby_file|
156
+ if CompileCache::ISeq.precompile(ruby_file, cache_dir: cache_dir)
157
+ STDERR.puts(ruby_file) if verbose
158
+ end
159
+ end
160
+ end
161
+
96
162
  def fix_default_encoding
97
163
  if Encoding.default_external == Encoding::US_ASCII
98
164
  Encoding.default_external = Encoding::UTF_8
@@ -114,7 +180,12 @@ module Bootsnap
114
180
  end
115
181
 
116
182
  def cache_dir=(dir)
117
- @cache_dir = File.expand_path(File.join(dir, 'bootsnap-compile-cache'))
183
+ @cache_dir = File.expand_path(File.join(dir, 'bootsnap/compile-cache'))
184
+ end
185
+
186
+ def exclude_pattern(pattern)
187
+ (@exclude_patterns ||= []) << Regexp.new(pattern)
188
+ self.exclude = Regexp.union(@exclude_patterns)
118
189
  end
119
190
 
120
191
  def parser
@@ -131,6 +202,20 @@ module Bootsnap
131
202
  self.cache_dir = dir
132
203
  end
133
204
 
205
+ help = <<~EOS
206
+ Print precompiled paths.
207
+ EOS
208
+ opts.on('--verbose', '-v', help.strip) do
209
+ self.verbose = true
210
+ end
211
+
212
+ help = <<~EOS
213
+ Number of workers to use. Default to number of processors, set to 0 to disable multi-processing.
214
+ EOS
215
+ opts.on('--jobs JOBS', '-j', help.strip) do |jobs|
216
+ self.jobs = Integer(jobs)
217
+ end
218
+
134
219
  opts.separator ""
135
220
  opts.separator "COMMANDS"
136
221
  opts.separator ""
@@ -144,7 +229,17 @@ module Bootsnap
144
229
  help = <<~EOS
145
230
  Path pattern to not precompile. e.g. --exclude 'aws-sdk|google-api'
146
231
  EOS
147
- opts.on('--exclude PATTERN', help) { |pattern| self.exclude = Regexp.new(pattern) }
232
+ opts.on('--exclude PATTERN', help) { |pattern| exclude_pattern(pattern) }
233
+
234
+ help = <<~EOS
235
+ Disable ISeq (.rb) precompilation.
236
+ EOS
237
+ opts.on('--no-iseq', help) { self.iseq = false }
238
+
239
+ help = <<~EOS
240
+ Disable YAML precompilation.
241
+ EOS
242
+ opts.on('--no-yaml', help) { self.yaml = false }
148
243
  end
149
244
  end
150
245
  end
@@ -0,0 +1,131 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Bootsnap
4
+ class CLI
5
+ class WorkerPool
6
+ class << self
7
+ def create(size:, jobs:)
8
+ if size > 0 && Process.respond_to?(:fork)
9
+ new(size: size, jobs: jobs)
10
+ else
11
+ Inline.new(jobs: jobs)
12
+ end
13
+ end
14
+ end
15
+
16
+ class Inline
17
+ def initialize(jobs: {})
18
+ @jobs = jobs
19
+ end
20
+
21
+ def push(job, *args)
22
+ @jobs.fetch(job).call(*args)
23
+ nil
24
+ end
25
+
26
+ def spawn
27
+ # noop
28
+ end
29
+
30
+ def shutdown
31
+ # noop
32
+ end
33
+ end
34
+
35
+ class Worker
36
+ attr_reader :to_io, :pid
37
+
38
+ def initialize(jobs)
39
+ @jobs = jobs
40
+ @pipe_out, @to_io = IO.pipe
41
+ @pid = nil
42
+ end
43
+
44
+ def write(message, block: true)
45
+ payload = Marshal.dump(message)
46
+ if block
47
+ to_io.write(payload)
48
+ true
49
+ else
50
+ to_io.write_nonblock(payload, exception: false) != :wait_writable
51
+ end
52
+ end
53
+
54
+ def close
55
+ to_io.close
56
+ end
57
+
58
+ def work_loop
59
+ loop do
60
+ job, *args = Marshal.load(@pipe_out)
61
+ return if job == :exit
62
+ @jobs.fetch(job).call(*args)
63
+ end
64
+ rescue IOError
65
+ nil
66
+ end
67
+
68
+ def spawn
69
+ @pid = Process.fork do
70
+ to_io.close
71
+ work_loop
72
+ exit!(0)
73
+ end
74
+ @pipe_out.close
75
+ true
76
+ end
77
+ end
78
+
79
+ def initialize(size:, jobs: {})
80
+ @size = size
81
+ @jobs = jobs
82
+ @queue = Queue.new
83
+ @pids = []
84
+ end
85
+
86
+ def spawn
87
+ @workers = @size.times.map { Worker.new(@jobs) }
88
+ @workers.each(&:spawn)
89
+ @dispatcher_thread = Thread.new { dispatch_loop }
90
+ @dispatcher_thread.abort_on_exception = true
91
+ true
92
+ end
93
+
94
+ def dispatch_loop
95
+ loop do
96
+ case job = @queue.pop
97
+ when nil
98
+ @workers.each do |worker|
99
+ worker.write([:exit])
100
+ worker.close
101
+ end
102
+ return true
103
+ else
104
+ unless @workers.sample.write(job, block: false)
105
+ free_worker.write(job)
106
+ end
107
+ end
108
+ end
109
+ end
110
+
111
+ def free_worker
112
+ IO.select(nil, @workers)[1].sample
113
+ end
114
+
115
+ def push(*args)
116
+ @queue.push(args)
117
+ nil
118
+ end
119
+
120
+ def shutdown
121
+ @queue.close
122
+ @dispatcher_thread.join
123
+ @workers.each do |worker|
124
+ _pid, status = Process.wait2(worker.pid)
125
+ return status.exitstatus unless status.success?
126
+ end
127
+ nil
128
+ end
129
+ end
130
+ end
131
+ end
@@ -9,7 +9,7 @@ module Bootsnap
9
9
  attr_accessor(:cache_dir)
10
10
  end
11
11
 
12
- def self.input_to_storage(_, path, _args)
12
+ def self.input_to_storage(_, path)
13
13
  RubyVM::InstructionSequence.compile_file(path).to_binary
14
14
  rescue SyntaxError
15
15
  raise(Uncompilable, 'syntax error')
@@ -35,6 +35,14 @@ module Bootsnap
35
35
  )
36
36
  end
37
37
 
38
+ def self.precompile(path, cache_dir: ISeq.cache_dir)
39
+ Bootsnap::CompileCache::Native.precompile(
40
+ cache_dir,
41
+ path.to_s,
42
+ Bootsnap::CompileCache::ISeq,
43
+ )
44
+ end
45
+
38
46
  def self.input_to_output(_data, _kwargs)
39
47
  nil # ruby handles this
40
48
  end
@@ -7,32 +7,34 @@ module Bootsnap
7
7
  class << self
8
8
  attr_accessor(:msgpack_factory, :cache_dir, :supported_options)
9
9
 
10
- def input_to_storage(contents, _, kwargs)
10
+ def input_to_storage(contents, _)
11
11
  raise(Uncompilable) if contents.index("!ruby/object")
12
- obj = ::YAML.load(contents, **(kwargs || {}))
12
+ obj = ::YAML.load(contents)
13
13
  msgpack_factory.dump(obj)
14
14
  rescue NoMethodError, RangeError
15
- # if the object included things that we can't serialize, fall back to
16
- # Marshal. It's a bit slower, but can encode anything yaml can.
17
- # NoMethodError is unexpected types; RangeError is Bignums
18
- Marshal.dump(obj)
15
+ # The object included things that we can't serialize
16
+ raise(Uncompilable)
19
17
  end
20
18
 
21
19
  def storage_to_output(data, kwargs)
22
- # This could have a meaning in messagepack, and we're being a little lazy
23
- # about it. -- but a leading 0x04 would indicate the contents of the YAML
24
- # is a positive integer, which is rare, to say the least.
25
- if data[0] == 0x04.chr && data[1] == 0x08.chr
26
- Marshal.load(data)
27
- else
28
- msgpack_factory.load(data, **(kwargs || {}))
20
+ if kwargs && kwargs.key?(:symbolize_names)
21
+ kwargs[:symbolize_keys] = kwargs.delete(:symbolize_names)
29
22
  end
23
+ msgpack_factory.load(data, kwargs)
30
24
  end
31
25
 
32
26
  def input_to_output(data, kwargs)
33
27
  ::YAML.load(data, **(kwargs || {}))
34
28
  end
35
29
 
30
+ def precompile(path, cache_dir: YAML.cache_dir)
31
+ Bootsnap::CompileCache::Native.precompile(
32
+ cache_dir,
33
+ path.to_s,
34
+ Bootsnap::CompileCache::YAML,
35
+ )
36
+ end
37
+
36
38
  def install!(cache_dir)
37
39
  self.cache_dir = cache_dir
38
40
  init!
@@ -42,12 +44,31 @@ module Bootsnap
42
44
  def init!
43
45
  require('yaml')
44
46
  require('msgpack')
47
+ require('date')
45
48
 
46
49
  # MessagePack serializes symbols as strings by default.
47
50
  # We want them to roundtrip cleanly, so we use a custom factory.
48
51
  # see: https://github.com/msgpack/msgpack-ruby/pull/122
49
52
  factory = MessagePack::Factory.new
50
53
  factory.register_type(0x00, Symbol)
54
+ factory.register_type(
55
+ MessagePack::Timestamp::TYPE, # or just -1
56
+ Time,
57
+ packer: MessagePack::Time::Packer,
58
+ unpacker: MessagePack::Time::Unpacker
59
+ )
60
+
61
+ marshal_fallback = {
62
+ packer: ->(value) { Marshal.dump(value) },
63
+ unpacker: ->(payload) { Marshal.load(payload) },
64
+ }
65
+ {
66
+ Date => 0x01,
67
+ Regexp => 0x02,
68
+ }.each do |type, code|
69
+ factory.register_type(code, type, marshal_fallback)
70
+ end
71
+
51
72
  self.msgpack_factory = factory
52
73
 
53
74
  self.supported_options = []
@@ -65,8 +86,6 @@ module Bootsnap
65
86
  end
66
87
 
67
88
  module Patch
68
- extend self
69
-
70
89
  def load_file(path, *args)
71
90
  return super if args.size > 1
72
91
  if kwargs = args.first
@@ -85,6 +104,8 @@ module Bootsnap
85
104
  ::Bootsnap::CompileCache.permission_error(path)
86
105
  end
87
106
  end
107
+
108
+ ruby2_keywords :load_file if respond_to?(:ruby2_keywords, true)
88
109
  end
89
110
  end
90
111
  end
@@ -1,4 +1,4 @@
1
1
  # frozen_string_literal: true
2
2
  module Bootsnap
3
- VERSION = "1.5.1"
3
+ VERSION = "1.6.0"
4
4
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: bootsnap
3
3
  version: !ruby/object:Gem::Version
4
- version: 1.5.1
4
+ version: 1.6.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Burke Libbey
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2020-11-10 00:00:00.000000000 Z
11
+ date: 2021-01-27 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: bundler
@@ -113,6 +113,7 @@ files:
113
113
  - lib/bootsnap.rb
114
114
  - lib/bootsnap/bundler.rb
115
115
  - lib/bootsnap/cli.rb
116
+ - lib/bootsnap/cli/worker_pool.rb
116
117
  - lib/bootsnap/compile_cache.rb
117
118
  - lib/bootsnap/compile_cache/iseq.rb
118
119
  - lib/bootsnap/compile_cache/yaml.rb