bootsnap 1.4.5 → 1.11.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +188 -0
- data/LICENSE.txt +1 -1
- data/README.md +48 -17
- data/exe/bootsnap +5 -0
- data/ext/bootsnap/bootsnap.c +297 -118
- data/ext/bootsnap/extconf.rb +22 -14
- data/lib/bootsnap/bundler.rb +2 -0
- data/lib/bootsnap/cli/worker_pool.rb +136 -0
- data/lib/bootsnap/cli.rb +281 -0
- data/lib/bootsnap/compile_cache/iseq.rb +63 -18
- data/lib/bootsnap/compile_cache/json.rb +88 -0
- data/lib/bootsnap/compile_cache/yaml.rb +331 -42
- data/lib/bootsnap/compile_cache.rb +22 -8
- data/lib/bootsnap/explicit_require.rb +5 -3
- data/lib/bootsnap/load_path_cache/cache.rb +75 -38
- data/lib/bootsnap/load_path_cache/change_observer.rb +6 -1
- data/lib/bootsnap/load_path_cache/core_ext/kernel_require.rb +37 -64
- data/lib/bootsnap/load_path_cache/core_ext/loaded_features.rb +2 -0
- data/lib/bootsnap/load_path_cache/loaded_features_index.rb +62 -28
- data/lib/bootsnap/load_path_cache/path.rb +31 -6
- data/lib/bootsnap/load_path_cache/path_scanner.rb +54 -29
- data/lib/bootsnap/load_path_cache/store.rb +60 -16
- data/lib/bootsnap/load_path_cache.rb +17 -38
- data/lib/bootsnap/setup.rb +3 -36
- data/lib/bootsnap/version.rb +3 -1
- data/lib/bootsnap.rb +137 -36
- metadata +15 -99
- data/.github/CODEOWNERS +0 -2
- data/.github/probots.yml +0 -2
- data/.gitignore +0 -17
- data/.rubocop.yml +0 -20
- data/.travis.yml +0 -21
- data/CODE_OF_CONDUCT.md +0 -74
- data/CONTRIBUTING.md +0 -21
- data/Gemfile +0 -8
- data/README.jp.md +0 -231
- data/Rakefile +0 -12
- data/bin/ci +0 -10
- data/bin/console +0 -14
- data/bin/setup +0 -8
- data/bin/test-minimal-support +0 -7
- data/bin/testunit +0 -8
- data/bootsnap.gemspec +0 -45
- data/dev.yml +0 -10
- data/lib/bootsnap/load_path_cache/core_ext/active_support.rb +0 -106
- data/lib/bootsnap/load_path_cache/realpath_cache.rb +0 -32
- data/shipit.rubygems.yml +0 -0
data/ext/bootsnap/bootsnap.c
CHANGED
@@ -14,6 +14,7 @@
|
|
14
14
|
#include "bootsnap.h"
|
15
15
|
#include "ruby.h"
|
16
16
|
#include <stdint.h>
|
17
|
+
#include <stdbool.h>
|
17
18
|
#include <sys/types.h>
|
18
19
|
#include <errno.h>
|
19
20
|
#include <fcntl.h>
|
@@ -21,6 +22,9 @@
|
|
21
22
|
#ifndef _WIN32
|
22
23
|
#include <sys/utsname.h>
|
23
24
|
#endif
|
25
|
+
#ifdef __GLIBC__
|
26
|
+
#include <gnu/libc-version.h>
|
27
|
+
#endif
|
24
28
|
|
25
29
|
/* 1000 is an arbitrary limit; FNV64 plus some slashes brings the cap down to
|
26
30
|
* 981 for the cache dir */
|
@@ -29,6 +33,12 @@
|
|
29
33
|
|
30
34
|
#define KEY_SIZE 64
|
31
35
|
|
36
|
+
#define MAX_CREATE_TEMPFILE_ATTEMPT 3
|
37
|
+
|
38
|
+
#ifndef RB_UNLIKELY
|
39
|
+
#define RB_UNLIKELY(x) (x)
|
40
|
+
#endif
|
41
|
+
|
32
42
|
/*
|
33
43
|
* An instance of this key is written as the first 64 bytes of each cache file.
|
34
44
|
* The mtime and size members track whether the file contents have changed, and
|
@@ -65,7 +75,7 @@ struct bs_cache_key {
|
|
65
75
|
STATIC_ASSERT(sizeof(struct bs_cache_key) == KEY_SIZE);
|
66
76
|
|
67
77
|
/* Effectively a schema version. Bumping invalidates all previous caches */
|
68
|
-
static const uint32_t current_version =
|
78
|
+
static const uint32_t current_version = 4;
|
69
79
|
|
70
80
|
/* hash of e.g. "x86_64-darwin17", invalidating when ruby is recompiled on a
|
71
81
|
* new OS ABI, etc. */
|
@@ -81,21 +91,26 @@ static mode_t current_umask;
|
|
81
91
|
static VALUE rb_mBootsnap;
|
82
92
|
static VALUE rb_mBootsnap_CompileCache;
|
83
93
|
static VALUE rb_mBootsnap_CompileCache_Native;
|
84
|
-
static VALUE
|
85
|
-
static ID
|
94
|
+
static VALUE rb_cBootsnap_CompileCache_UNCOMPILABLE;
|
95
|
+
static ID instrumentation_method;
|
96
|
+
static VALUE sym_miss;
|
97
|
+
static VALUE sym_stale;
|
98
|
+
static bool instrumentation_enabled = false;
|
86
99
|
|
87
100
|
/* Functions exposed as module functions on Bootsnap::CompileCache::Native */
|
101
|
+
static VALUE bs_instrumentation_enabled_set(VALUE self, VALUE enabled);
|
88
102
|
static VALUE bs_compile_option_crc32_set(VALUE self, VALUE crc32_v);
|
89
|
-
static VALUE bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler);
|
103
|
+
static VALUE bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler, VALUE args);
|
104
|
+
static VALUE bs_rb_precompile(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler);
|
90
105
|
|
91
106
|
/* Helpers */
|
92
|
-
static
|
93
|
-
static void bs_cache_path(const char * cachedir, const char * path, char ** cache_path);
|
107
|
+
static void bs_cache_path(const char * cachedir, const VALUE path, char (* cache_path)[MAX_CACHEPATH_SIZE]);
|
94
108
|
static int bs_read_key(int fd, struct bs_cache_key * key);
|
95
109
|
static int cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2);
|
96
|
-
static VALUE bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler);
|
97
|
-
static
|
98
|
-
static int
|
110
|
+
static VALUE bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args);
|
111
|
+
static VALUE bs_precompile(char * path, VALUE path_v, char * cache_path, VALUE handler);
|
112
|
+
static int open_current_file(char * path, struct bs_cache_key * key, const char ** errno_provenance);
|
113
|
+
static int fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE args, VALUE * output_data, int * exception_tag, const char ** errno_provenance);
|
99
114
|
static uint32_t get_ruby_revision(void);
|
100
115
|
static uint32_t get_ruby_platform(void);
|
101
116
|
|
@@ -103,12 +118,10 @@ static uint32_t get_ruby_platform(void);
|
|
103
118
|
* Helper functions to call ruby methods on handler object without crashing on
|
104
119
|
* exception.
|
105
120
|
*/
|
106
|
-
static int bs_storage_to_output(VALUE handler, VALUE storage_data, VALUE * output_data);
|
107
|
-
static VALUE prot_storage_to_output(VALUE arg);
|
121
|
+
static int bs_storage_to_output(VALUE handler, VALUE args, VALUE storage_data, VALUE * output_data);
|
108
122
|
static VALUE prot_input_to_output(VALUE arg);
|
109
|
-
static void bs_input_to_output(VALUE handler, VALUE input_data, VALUE * output_data, int * exception_tag);
|
110
|
-
static
|
111
|
-
static int bs_input_to_storage(VALUE handler, VALUE input_data, VALUE pathval, VALUE * storage_data);
|
123
|
+
static void bs_input_to_output(VALUE handler, VALUE args, VALUE input_data, VALUE * output_data, int * exception_tag);
|
124
|
+
static int bs_input_to_storage(VALUE handler, VALUE args, VALUE input_data, VALUE pathval, VALUE * storage_data);
|
112
125
|
struct s2o_data;
|
113
126
|
struct i2o_data;
|
114
127
|
struct i2s_data;
|
@@ -122,6 +135,12 @@ bs_rb_coverage_running(VALUE self)
|
|
122
135
|
return RTEST(cov) ? Qtrue : Qfalse;
|
123
136
|
}
|
124
137
|
|
138
|
+
static VALUE
|
139
|
+
bs_rb_get_path(VALUE self, VALUE fname)
|
140
|
+
{
|
141
|
+
return rb_get_path(fname);
|
142
|
+
}
|
143
|
+
|
125
144
|
/*
|
126
145
|
* Ruby C extensions are initialized by calling Init_<extname>.
|
127
146
|
*
|
@@ -133,23 +152,42 @@ void
|
|
133
152
|
Init_bootsnap(void)
|
134
153
|
{
|
135
154
|
rb_mBootsnap = rb_define_module("Bootsnap");
|
155
|
+
|
156
|
+
rb_define_singleton_method(rb_mBootsnap, "rb_get_path", bs_rb_get_path, 1);
|
157
|
+
|
136
158
|
rb_mBootsnap_CompileCache = rb_define_module_under(rb_mBootsnap, "CompileCache");
|
137
159
|
rb_mBootsnap_CompileCache_Native = rb_define_module_under(rb_mBootsnap_CompileCache, "Native");
|
138
|
-
|
160
|
+
rb_cBootsnap_CompileCache_UNCOMPILABLE = rb_const_get(rb_mBootsnap_CompileCache, rb_intern("UNCOMPILABLE"));
|
161
|
+
rb_global_variable(&rb_cBootsnap_CompileCache_UNCOMPILABLE);
|
139
162
|
|
140
163
|
current_ruby_revision = get_ruby_revision();
|
141
164
|
current_ruby_platform = get_ruby_platform();
|
142
165
|
|
143
|
-
|
166
|
+
instrumentation_method = rb_intern("_instrument");
|
167
|
+
|
168
|
+
sym_miss = ID2SYM(rb_intern("miss"));
|
169
|
+
rb_global_variable(&sym_miss);
|
144
170
|
|
171
|
+
sym_stale = ID2SYM(rb_intern("stale"));
|
172
|
+
rb_global_variable(&sym_stale);
|
173
|
+
|
174
|
+
rb_define_module_function(rb_mBootsnap, "instrumentation_enabled=", bs_instrumentation_enabled_set, 1);
|
145
175
|
rb_define_module_function(rb_mBootsnap_CompileCache_Native, "coverage_running?", bs_rb_coverage_running, 0);
|
146
|
-
rb_define_module_function(rb_mBootsnap_CompileCache_Native, "fetch", bs_rb_fetch,
|
176
|
+
rb_define_module_function(rb_mBootsnap_CompileCache_Native, "fetch", bs_rb_fetch, 4);
|
177
|
+
rb_define_module_function(rb_mBootsnap_CompileCache_Native, "precompile", bs_rb_precompile, 3);
|
147
178
|
rb_define_module_function(rb_mBootsnap_CompileCache_Native, "compile_option_crc32=", bs_compile_option_crc32_set, 1);
|
148
179
|
|
149
180
|
current_umask = umask(0777);
|
150
181
|
umask(current_umask);
|
151
182
|
}
|
152
183
|
|
184
|
+
static VALUE
|
185
|
+
bs_instrumentation_enabled_set(VALUE self, VALUE enabled)
|
186
|
+
{
|
187
|
+
instrumentation_enabled = RTEST(enabled);
|
188
|
+
return enabled;
|
189
|
+
}
|
190
|
+
|
153
191
|
/*
|
154
192
|
* Bootsnap's ruby code registers a hook that notifies us via this function
|
155
193
|
* when compile_option changes. These changes invalidate all existing caches.
|
@@ -178,7 +216,7 @@ bs_compile_option_crc32_set(VALUE self, VALUE crc32_v)
|
|
178
216
|
* - 32 bits doesn't feel collision-resistant enough; 64 is nice.
|
179
217
|
*/
|
180
218
|
static uint64_t
|
181
|
-
|
219
|
+
fnv1a_64_iter_cstr(uint64_t h, const char *str)
|
182
220
|
{
|
183
221
|
unsigned char *s = (unsigned char *)str;
|
184
222
|
|
@@ -191,7 +229,21 @@ fnv1a_64_iter(uint64_t h, const char *str)
|
|
191
229
|
}
|
192
230
|
|
193
231
|
static uint64_t
|
194
|
-
|
232
|
+
fnv1a_64_iter(uint64_t h, const VALUE str)
|
233
|
+
{
|
234
|
+
unsigned char *s = (unsigned char *)RSTRING_PTR(str);
|
235
|
+
unsigned char *str_end = (unsigned char *)RSTRING_PTR(str) + RSTRING_LEN(str);
|
236
|
+
|
237
|
+
while (s < str_end) {
|
238
|
+
h ^= (uint64_t)*s++;
|
239
|
+
h += (h << 1) + (h << 4) + (h << 5) + (h << 7) + (h << 8) + (h << 40);
|
240
|
+
}
|
241
|
+
|
242
|
+
return h;
|
243
|
+
}
|
244
|
+
|
245
|
+
static uint64_t
|
246
|
+
fnv1a_64(const VALUE str)
|
195
247
|
{
|
196
248
|
uint64_t h = (uint64_t)0xcbf29ce484222325ULL;
|
197
249
|
return fnv1a_64_iter(h, str);
|
@@ -212,7 +264,7 @@ get_ruby_revision(void)
|
|
212
264
|
} else {
|
213
265
|
uint64_t hash;
|
214
266
|
|
215
|
-
hash = fnv1a_64(
|
267
|
+
hash = fnv1a_64(ruby_revision);
|
216
268
|
return (uint32_t)(hash >> 32);
|
217
269
|
}
|
218
270
|
}
|
@@ -232,16 +284,19 @@ get_ruby_platform(void)
|
|
232
284
|
VALUE ruby_platform;
|
233
285
|
|
234
286
|
ruby_platform = rb_const_get(rb_cObject, rb_intern("RUBY_PLATFORM"));
|
235
|
-
hash = fnv1a_64(
|
287
|
+
hash = fnv1a_64(ruby_platform);
|
236
288
|
|
237
289
|
#ifdef _WIN32
|
238
290
|
return (uint32_t)(hash >> 32) ^ (uint32_t)GetVersion();
|
291
|
+
#elif defined(__GLIBC__)
|
292
|
+
hash = fnv1a_64_iter_cstr(hash, gnu_get_libc_version());
|
293
|
+
return (uint32_t)(hash >> 32);
|
239
294
|
#else
|
240
295
|
struct utsname utsname;
|
241
296
|
|
242
297
|
/* Not worth crashing if this fails; lose extra cache invalidation potential */
|
243
298
|
if (uname(&utsname) >= 0) {
|
244
|
-
hash =
|
299
|
+
hash = fnv1a_64_iter_cstr(hash, utsname.version);
|
245
300
|
}
|
246
301
|
|
247
302
|
return (uint32_t)(hash >> 32);
|
@@ -256,14 +311,13 @@ get_ruby_platform(void)
|
|
256
311
|
* The path will look something like: <cachedir>/12/34567890abcdef
|
257
312
|
*/
|
258
313
|
static void
|
259
|
-
bs_cache_path(const char * cachedir, const
|
314
|
+
bs_cache_path(const char * cachedir, const VALUE path, char (* cache_path)[MAX_CACHEPATH_SIZE])
|
260
315
|
{
|
261
316
|
uint64_t hash = fnv1a_64(path);
|
262
|
-
|
263
317
|
uint8_t first_byte = (hash >> (64 - 8));
|
264
318
|
uint64_t remainder = hash & 0x00ffffffffffffff;
|
265
319
|
|
266
|
-
sprintf(*cache_path, "%s/%
|
320
|
+
sprintf(*cache_path, "%s/%02"PRIx8"/%014"PRIx64, cachedir, first_byte, remainder);
|
267
321
|
}
|
268
322
|
|
269
323
|
/*
|
@@ -293,7 +347,7 @@ cache_key_equal(struct bs_cache_key * k1, struct bs_cache_key * k2)
|
|
293
347
|
* conversions on the ruby VALUE arguments before passing them along.
|
294
348
|
*/
|
295
349
|
static VALUE
|
296
|
-
bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
|
350
|
+
bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler, VALUE args)
|
297
351
|
{
|
298
352
|
FilePathValue(path_v);
|
299
353
|
|
@@ -308,27 +362,51 @@ bs_rb_fetch(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
|
|
308
362
|
char * path = RSTRING_PTR(path_v);
|
309
363
|
char cache_path[MAX_CACHEPATH_SIZE];
|
310
364
|
|
311
|
-
|
312
|
-
|
313
|
-
bs_cache_path(cachedir, path, &tmp);
|
314
|
-
}
|
365
|
+
/* generate cache path to cache_path */
|
366
|
+
bs_cache_path(cachedir, path_v, &cache_path);
|
315
367
|
|
316
|
-
return bs_fetch(path, path_v, cache_path, handler);
|
368
|
+
return bs_fetch(path, path_v, cache_path, handler, args);
|
317
369
|
}
|
318
370
|
|
371
|
+
/*
|
372
|
+
* Entrypoint for Bootsnap::CompileCache::Native.precompile.
|
373
|
+
* Similar to fetch, but it only generate the cache if missing
|
374
|
+
* and doesn't return the content.
|
375
|
+
*/
|
376
|
+
static VALUE
|
377
|
+
bs_rb_precompile(VALUE self, VALUE cachedir_v, VALUE path_v, VALUE handler)
|
378
|
+
{
|
379
|
+
FilePathValue(path_v);
|
380
|
+
|
381
|
+
Check_Type(cachedir_v, T_STRING);
|
382
|
+
Check_Type(path_v, T_STRING);
|
383
|
+
|
384
|
+
if (RSTRING_LEN(cachedir_v) > MAX_CACHEDIR_SIZE) {
|
385
|
+
rb_raise(rb_eArgError, "cachedir too long");
|
386
|
+
}
|
387
|
+
|
388
|
+
char * cachedir = RSTRING_PTR(cachedir_v);
|
389
|
+
char * path = RSTRING_PTR(path_v);
|
390
|
+
char cache_path[MAX_CACHEPATH_SIZE];
|
391
|
+
|
392
|
+
/* generate cache path to cache_path */
|
393
|
+
bs_cache_path(cachedir, path_v, &cache_path);
|
394
|
+
|
395
|
+
return bs_precompile(path, path_v, cache_path, handler);
|
396
|
+
}
|
319
397
|
/*
|
320
398
|
* Open the file we want to load/cache and generate a cache key for it if it
|
321
399
|
* was loaded.
|
322
400
|
*/
|
323
401
|
static int
|
324
|
-
open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenance)
|
402
|
+
open_current_file(char * path, struct bs_cache_key * key, const char ** errno_provenance)
|
325
403
|
{
|
326
404
|
struct stat statbuf;
|
327
405
|
int fd;
|
328
406
|
|
329
407
|
fd = open(path, O_RDONLY);
|
330
408
|
if (fd < 0) {
|
331
|
-
*errno_provenance =
|
409
|
+
*errno_provenance = "bs_fetch:open_current_file:open";
|
332
410
|
return fd;
|
333
411
|
}
|
334
412
|
#ifdef _WIN32
|
@@ -336,7 +414,7 @@ open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenan
|
|
336
414
|
#endif
|
337
415
|
|
338
416
|
if (fstat(fd, &statbuf) < 0) {
|
339
|
-
*errno_provenance =
|
417
|
+
*errno_provenance = "bs_fetch:open_current_file:fstat";
|
340
418
|
close(fd);
|
341
419
|
return -1;
|
342
420
|
}
|
@@ -352,7 +430,9 @@ open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenan
|
|
352
430
|
}
|
353
431
|
|
354
432
|
#define ERROR_WITH_ERRNO -1
|
355
|
-
#define
|
433
|
+
#define CACHE_MISS -2
|
434
|
+
#define CACHE_STALE -3
|
435
|
+
#define CACHE_UNCOMPILABLE -4
|
356
436
|
|
357
437
|
/*
|
358
438
|
* Read the cache key from the given fd, which must have position 0 (e.g.
|
@@ -360,15 +440,16 @@ open_current_file(char * path, struct bs_cache_key * key, char ** errno_provenan
|
|
360
440
|
*
|
361
441
|
* Possible return values:
|
362
442
|
* - 0 (OK, key was loaded)
|
363
|
-
* - CACHE_MISSING_OR_INVALID (-2)
|
364
443
|
* - ERROR_WITH_ERRNO (-1, errno is set)
|
444
|
+
* - CACHE_MISS (-2)
|
445
|
+
* - CACHE_STALE (-3)
|
365
446
|
*/
|
366
447
|
static int
|
367
448
|
bs_read_key(int fd, struct bs_cache_key * key)
|
368
449
|
{
|
369
450
|
ssize_t nread = read(fd, key, KEY_SIZE);
|
370
451
|
if (nread < 0) return ERROR_WITH_ERRNO;
|
371
|
-
if (nread < KEY_SIZE) return
|
452
|
+
if (nread < KEY_SIZE) return CACHE_STALE;
|
372
453
|
return 0;
|
373
454
|
}
|
374
455
|
|
@@ -378,19 +459,19 @@ bs_read_key(int fd, struct bs_cache_key * key)
|
|
378
459
|
*
|
379
460
|
* Possible return values:
|
380
461
|
* - 0 (OK, key was loaded)
|
381
|
-
* -
|
462
|
+
* - CACHE_MISS (-2)
|
463
|
+
* - CACHE_STALE (-3)
|
382
464
|
* - ERROR_WITH_ERRNO (-1, errno is set)
|
383
465
|
*/
|
384
466
|
static int
|
385
|
-
open_cache_file(const char * path, struct bs_cache_key * key, char ** errno_provenance)
|
467
|
+
open_cache_file(const char * path, struct bs_cache_key * key, const char ** errno_provenance)
|
386
468
|
{
|
387
469
|
int fd, res;
|
388
470
|
|
389
471
|
fd = open(path, O_RDONLY);
|
390
472
|
if (fd < 0) {
|
391
|
-
*errno_provenance =
|
392
|
-
|
393
|
-
return ERROR_WITH_ERRNO;
|
473
|
+
*errno_provenance = "bs_fetch:open_cache_file:open";
|
474
|
+
return CACHE_MISS;
|
394
475
|
}
|
395
476
|
#ifdef _WIN32
|
396
477
|
setmode(fd, O_BINARY);
|
@@ -398,7 +479,7 @@ open_cache_file(const char * path, struct bs_cache_key * key, char ** errno_prov
|
|
398
479
|
|
399
480
|
res = bs_read_key(fd, key);
|
400
481
|
if (res < 0) {
|
401
|
-
*errno_provenance =
|
482
|
+
*errno_provenance = "bs_fetch:open_cache_file:read";
|
402
483
|
close(fd);
|
403
484
|
return res;
|
404
485
|
}
|
@@ -422,7 +503,7 @@ open_cache_file(const char * path, struct bs_cache_key * key, char ** errno_prov
|
|
422
503
|
* or exception, will be the final data returnable to the user.
|
423
504
|
*/
|
424
505
|
static int
|
425
|
-
fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data, int * exception_tag, char ** errno_provenance)
|
506
|
+
fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE args, VALUE * output_data, int * exception_tag, const char ** errno_provenance)
|
426
507
|
{
|
427
508
|
char * data = NULL;
|
428
509
|
ssize_t nread;
|
@@ -431,26 +512,30 @@ fetch_cached_data(int fd, ssize_t data_size, VALUE handler, VALUE * output_data,
|
|
431
512
|
VALUE storage_data;
|
432
513
|
|
433
514
|
if (data_size > 100000000000) {
|
434
|
-
*errno_provenance =
|
515
|
+
*errno_provenance = "bs_fetch:fetch_cached_data:datasize";
|
435
516
|
errno = EINVAL; /* because wtf? */
|
436
|
-
ret =
|
517
|
+
ret = ERROR_WITH_ERRNO;
|
437
518
|
goto done;
|
438
519
|
}
|
439
520
|
data = ALLOC_N(char, data_size);
|
440
521
|
nread = read(fd, data, data_size);
|
441
522
|
if (nread < 0) {
|
442
|
-
*errno_provenance =
|
443
|
-
ret =
|
523
|
+
*errno_provenance = "bs_fetch:fetch_cached_data:read";
|
524
|
+
ret = ERROR_WITH_ERRNO;
|
444
525
|
goto done;
|
445
526
|
}
|
446
527
|
if (nread != data_size) {
|
447
|
-
ret =
|
528
|
+
ret = CACHE_STALE;
|
448
529
|
goto done;
|
449
530
|
}
|
450
531
|
|
451
|
-
storage_data =
|
532
|
+
storage_data = rb_str_new(data, data_size);
|
452
533
|
|
453
|
-
*exception_tag = bs_storage_to_output(handler, storage_data, output_data);
|
534
|
+
*exception_tag = bs_storage_to_output(handler, args, storage_data, output_data);
|
535
|
+
if (*output_data == rb_cBootsnap_CompileCache_UNCOMPILABLE) {
|
536
|
+
ret = CACHE_UNCOMPILABLE;
|
537
|
+
goto done;
|
538
|
+
}
|
454
539
|
ret = 0;
|
455
540
|
done:
|
456
541
|
if (data != NULL) xfree(data);
|
@@ -491,29 +576,36 @@ mkpath(char * file_path, mode_t mode)
|
|
491
576
|
* path.
|
492
577
|
*/
|
493
578
|
static int
|
494
|
-
atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char ** errno_provenance)
|
579
|
+
atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, const char ** errno_provenance)
|
495
580
|
{
|
496
581
|
char template[MAX_CACHEPATH_SIZE + 20];
|
497
582
|
char * tmp_path;
|
498
|
-
int fd, ret;
|
583
|
+
int fd, ret, attempt;
|
499
584
|
ssize_t nwrite;
|
500
585
|
|
501
|
-
|
502
|
-
|
586
|
+
for (attempt = 0; attempt < MAX_CREATE_TEMPFILE_ATTEMPT; ++attempt) {
|
587
|
+
tmp_path = strncpy(template, path, MAX_CACHEPATH_SIZE);
|
588
|
+
strcat(tmp_path, ".tmp.XXXXXX");
|
503
589
|
|
504
|
-
|
505
|
-
|
506
|
-
|
507
|
-
|
508
|
-
|
509
|
-
|
510
|
-
}
|
511
|
-
fd = open(tmp_path, O_WRONLY | O_CREAT, 0664);
|
512
|
-
if (fd < 0) {
|
513
|
-
*errno_provenance = (char *)"bs_fetch:atomic_write_cache_file:open";
|
590
|
+
// mkstemp modifies the template to be the actual created path
|
591
|
+
fd = mkstemp(tmp_path);
|
592
|
+
if (fd > 0) break;
|
593
|
+
|
594
|
+
if (attempt == 0 && mkpath(tmp_path, 0775) < 0) {
|
595
|
+
*errno_provenance = "bs_fetch:atomic_write_cache_file:mkpath";
|
514
596
|
return -1;
|
515
597
|
}
|
516
598
|
}
|
599
|
+
if (fd < 0) {
|
600
|
+
*errno_provenance = "bs_fetch:atomic_write_cache_file:mkstemp";
|
601
|
+
return -1;
|
602
|
+
}
|
603
|
+
|
604
|
+
if (chmod(tmp_path, 0644) < 0) {
|
605
|
+
*errno_provenance = "bs_fetch:atomic_write_cache_file:chmod";
|
606
|
+
return -1;
|
607
|
+
}
|
608
|
+
|
517
609
|
#ifdef _WIN32
|
518
610
|
setmode(fd, O_BINARY);
|
519
611
|
#endif
|
@@ -521,11 +613,11 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char
|
|
521
613
|
key->data_size = RSTRING_LEN(data);
|
522
614
|
nwrite = write(fd, key, KEY_SIZE);
|
523
615
|
if (nwrite < 0) {
|
524
|
-
*errno_provenance =
|
616
|
+
*errno_provenance = "bs_fetch:atomic_write_cache_file:write";
|
525
617
|
return -1;
|
526
618
|
}
|
527
619
|
if (nwrite != KEY_SIZE) {
|
528
|
-
*errno_provenance =
|
620
|
+
*errno_provenance = "bs_fetch:atomic_write_cache_file:keysize";
|
529
621
|
errno = EIO; /* Lies but whatever */
|
530
622
|
return -1;
|
531
623
|
}
|
@@ -533,7 +625,7 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char
|
|
533
625
|
nwrite = write(fd, RSTRING_PTR(data), RSTRING_LEN(data));
|
534
626
|
if (nwrite < 0) return -1;
|
535
627
|
if (nwrite != RSTRING_LEN(data)) {
|
536
|
-
*errno_provenance =
|
628
|
+
*errno_provenance = "bs_fetch:atomic_write_cache_file:writelength";
|
537
629
|
errno = EIO; /* Lies but whatever */
|
538
630
|
return -1;
|
539
631
|
}
|
@@ -541,12 +633,12 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char
|
|
541
633
|
close(fd);
|
542
634
|
ret = rename(tmp_path, path);
|
543
635
|
if (ret < 0) {
|
544
|
-
*errno_provenance =
|
636
|
+
*errno_provenance = "bs_fetch:atomic_write_cache_file:rename";
|
545
637
|
return -1;
|
546
638
|
}
|
547
639
|
ret = chmod(path, 0664 & ~current_umask);
|
548
640
|
if (ret < 0) {
|
549
|
-
*errno_provenance =
|
641
|
+
*errno_provenance = "bs_fetch:atomic_write_cache_file:chmod";
|
550
642
|
}
|
551
643
|
return ret;
|
552
644
|
}
|
@@ -555,13 +647,13 @@ atomic_write_cache_file(char * path, struct bs_cache_key * key, VALUE data, char
|
|
555
647
|
/* Read contents from an fd, whose contents are asserted to be +size+ bytes
|
556
648
|
* long, into a buffer */
|
557
649
|
static ssize_t
|
558
|
-
bs_read_contents(int fd, size_t size, char ** contents, char ** errno_provenance)
|
650
|
+
bs_read_contents(int fd, size_t size, char ** contents, const char ** errno_provenance)
|
559
651
|
{
|
560
652
|
ssize_t nread;
|
561
653
|
*contents = ALLOC_N(char, size);
|
562
654
|
nread = read(fd, *contents, size);
|
563
655
|
if (nread < 0) {
|
564
|
-
*errno_provenance =
|
656
|
+
*errno_provenance = "bs_fetch:bs_read_contents:read";
|
565
657
|
}
|
566
658
|
return nread;
|
567
659
|
}
|
@@ -611,13 +703,13 @@ bs_read_contents(int fd, size_t size, char ** contents, char ** errno_provenance
|
|
611
703
|
* - Return storage_to_output(storage_data)
|
612
704
|
*/
|
613
705
|
static VALUE
|
614
|
-
bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
|
706
|
+
bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler, VALUE args)
|
615
707
|
{
|
616
708
|
struct bs_cache_key cached_key, current_key;
|
617
709
|
char * contents = NULL;
|
618
710
|
int cache_fd = -1, current_fd = -1;
|
619
711
|
int res, valid_cache = 0, exception_tag = 0;
|
620
|
-
char * errno_provenance = NULL;
|
712
|
+
const char * errno_provenance = NULL;
|
621
713
|
|
622
714
|
VALUE input_data; /* data read from source file, e.g. YAML or ruby source */
|
623
715
|
VALUE storage_data; /* compiled data, e.g. msgpack / binary iseq */
|
@@ -631,26 +723,42 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
|
|
631
723
|
|
632
724
|
/* Open the cache key if it exists, and read its cache key in */
|
633
725
|
cache_fd = open_cache_file(cache_path, &cached_key, &errno_provenance);
|
634
|
-
if (cache_fd ==
|
726
|
+
if (cache_fd == CACHE_MISS || cache_fd == CACHE_STALE) {
|
635
727
|
/* This is ok: valid_cache remains false, we re-populate it. */
|
728
|
+
if (RB_UNLIKELY(instrumentation_enabled)) {
|
729
|
+
rb_funcall(rb_mBootsnap, instrumentation_method, 2, cache_fd == CACHE_MISS ? sym_miss : sym_stale, path_v);
|
730
|
+
}
|
636
731
|
} else if (cache_fd < 0) {
|
637
732
|
goto fail_errno;
|
638
733
|
} else {
|
639
734
|
/* True if the cache existed and no invalidating changes have occurred since
|
640
735
|
* it was generated. */
|
641
736
|
valid_cache = cache_key_equal(¤t_key, &cached_key);
|
737
|
+
if (RB_UNLIKELY(instrumentation_enabled)) {
|
738
|
+
if (!valid_cache) {
|
739
|
+
rb_funcall(rb_mBootsnap, instrumentation_method, 2, sym_stale, path_v);
|
740
|
+
}
|
741
|
+
}
|
642
742
|
}
|
643
743
|
|
644
744
|
if (valid_cache) {
|
645
745
|
/* Fetch the cache data and return it if we're able to load it successfully */
|
646
746
|
res = fetch_cached_data(
|
647
|
-
cache_fd, (ssize_t)cached_key.data_size, handler,
|
747
|
+
cache_fd, (ssize_t)cached_key.data_size, handler, args,
|
648
748
|
&output_data, &exception_tag, &errno_provenance
|
649
749
|
);
|
650
|
-
if (exception_tag != 0)
|
651
|
-
else if (res ==
|
652
|
-
|
653
|
-
|
750
|
+
if (exception_tag != 0) goto raise;
|
751
|
+
else if (res == CACHE_UNCOMPILABLE) {
|
752
|
+
/* If fetch_cached_data returned `Uncompilable` we fallback to `input_to_output`
|
753
|
+
This happens if we have say, an unsafe YAML cache, but try to load it in safe mode */
|
754
|
+
if (bs_read_contents(current_fd, current_key.size, &contents, &errno_provenance) < 0) goto fail_errno;
|
755
|
+
input_data = rb_str_new(contents, current_key.size);
|
756
|
+
bs_input_to_output(handler, args, input_data, &output_data, &exception_tag);
|
757
|
+
if (exception_tag != 0) goto raise;
|
758
|
+
goto succeed;
|
759
|
+
} else if (res == CACHE_MISS || res == CACHE_STALE) valid_cache = 0;
|
760
|
+
else if (res == ERROR_WITH_ERRNO) goto fail_errno;
|
761
|
+
else if (!NIL_P(output_data)) goto succeed; /* fast-path, goal */
|
654
762
|
}
|
655
763
|
close(cache_fd);
|
656
764
|
cache_fd = -1;
|
@@ -658,37 +766,47 @@ bs_fetch(char * path, VALUE path_v, char * cache_path, VALUE handler)
|
|
658
766
|
|
659
767
|
/* Read the contents of the source file into a buffer */
|
660
768
|
if (bs_read_contents(current_fd, current_key.size, &contents, &errno_provenance) < 0) goto fail_errno;
|
661
|
-
input_data =
|
769
|
+
input_data = rb_str_new(contents, current_key.size);
|
662
770
|
|
663
771
|
/* Try to compile the input_data using input_to_storage(input_data) */
|
664
|
-
exception_tag = bs_input_to_storage(handler, input_data, path_v, &storage_data);
|
772
|
+
exception_tag = bs_input_to_storage(handler, args, input_data, path_v, &storage_data);
|
665
773
|
if (exception_tag != 0) goto raise;
|
666
774
|
/* If input_to_storage raised Bootsnap::CompileCache::Uncompilable, don't try
|
667
775
|
* to cache anything; just return input_to_output(input_data) */
|
668
|
-
if (storage_data ==
|
669
|
-
bs_input_to_output(handler, input_data, &output_data, &exception_tag);
|
776
|
+
if (storage_data == rb_cBootsnap_CompileCache_UNCOMPILABLE) {
|
777
|
+
bs_input_to_output(handler, args, input_data, &output_data, &exception_tag);
|
670
778
|
if (exception_tag != 0) goto raise;
|
671
779
|
goto succeed;
|
672
780
|
}
|
673
781
|
/* If storage_data isn't a string, we can't cache it */
|
674
782
|
if (!RB_TYPE_P(storage_data, T_STRING)) goto invalid_type_storage_data;
|
675
783
|
|
676
|
-
/*
|
677
|
-
|
678
|
-
|
784
|
+
/* Attempt to write the cache key and storage_data to the cache directory.
|
785
|
+
* We do however ignore any failures to persist the cache, as it's better
|
786
|
+
* to move along, than to interrupt the process.
|
787
|
+
*/
|
788
|
+
atomic_write_cache_file(cache_path, ¤t_key, storage_data, &errno_provenance);
|
679
789
|
|
680
790
|
/* Having written the cache, now convert storage_data to output_data */
|
681
|
-
exception_tag = bs_storage_to_output(handler, storage_data, &output_data);
|
791
|
+
exception_tag = bs_storage_to_output(handler, args, storage_data, &output_data);
|
682
792
|
if (exception_tag != 0) goto raise;
|
683
793
|
|
684
|
-
|
685
|
-
|
686
|
-
|
794
|
+
if (output_data == rb_cBootsnap_CompileCache_UNCOMPILABLE) {
|
795
|
+
/* If storage_to_output returned `Uncompilable` we fallback to `input_to_output` */
|
796
|
+
bs_input_to_output(handler, args, input_data, &output_data, &exception_tag);
|
797
|
+
if (exception_tag != 0) goto raise;
|
798
|
+
} else if (NIL_P(output_data)) {
|
799
|
+
/* If output_data is nil, delete the cache entry and generate the output
|
800
|
+
* using input_to_output */
|
687
801
|
if (unlink(cache_path) < 0) {
|
688
|
-
|
689
|
-
|
802
|
+
/* If the cache was already deleted, it might be that another process did it before us.
|
803
|
+
* No point raising an error */
|
804
|
+
if (errno != ENOENT) {
|
805
|
+
errno_provenance = "bs_fetch:unlink";
|
806
|
+
goto fail_errno;
|
807
|
+
}
|
690
808
|
}
|
691
|
-
bs_input_to_output(handler, input_data, &output_data, &exception_tag);
|
809
|
+
bs_input_to_output(handler, args, input_data, &output_data, &exception_tag);
|
692
810
|
if (exception_tag != 0) goto raise;
|
693
811
|
}
|
694
812
|
|
@@ -719,6 +837,79 @@ invalid_type_storage_data:
|
|
719
837
|
#undef CLEANUP
|
720
838
|
}
|
721
839
|
|
840
|
+
static VALUE
|
841
|
+
bs_precompile(char * path, VALUE path_v, char * cache_path, VALUE handler)
|
842
|
+
{
|
843
|
+
struct bs_cache_key cached_key, current_key;
|
844
|
+
char * contents = NULL;
|
845
|
+
int cache_fd = -1, current_fd = -1;
|
846
|
+
int res, valid_cache = 0, exception_tag = 0;
|
847
|
+
const char * errno_provenance = NULL;
|
848
|
+
|
849
|
+
VALUE input_data; /* data read from source file, e.g. YAML or ruby source */
|
850
|
+
VALUE storage_data; /* compiled data, e.g. msgpack / binary iseq */
|
851
|
+
|
852
|
+
/* Open the source file and generate a cache key for it */
|
853
|
+
current_fd = open_current_file(path, ¤t_key, &errno_provenance);
|
854
|
+
if (current_fd < 0) goto fail;
|
855
|
+
|
856
|
+
/* Open the cache key if it exists, and read its cache key in */
|
857
|
+
cache_fd = open_cache_file(cache_path, &cached_key, &errno_provenance);
|
858
|
+
if (cache_fd == CACHE_MISS || cache_fd == CACHE_STALE) {
|
859
|
+
/* This is ok: valid_cache remains false, we re-populate it. */
|
860
|
+
} else if (cache_fd < 0) {
|
861
|
+
goto fail;
|
862
|
+
} else {
|
863
|
+
/* True if the cache existed and no invalidating changes have occurred since
|
864
|
+
* it was generated. */
|
865
|
+
valid_cache = cache_key_equal(¤t_key, &cached_key);
|
866
|
+
}
|
867
|
+
|
868
|
+
if (valid_cache) {
|
869
|
+
goto succeed;
|
870
|
+
}
|
871
|
+
|
872
|
+
close(cache_fd);
|
873
|
+
cache_fd = -1;
|
874
|
+
/* Cache is stale, invalid, or missing. Regenerate and write it out. */
|
875
|
+
|
876
|
+
/* Read the contents of the source file into a buffer */
|
877
|
+
if (bs_read_contents(current_fd, current_key.size, &contents, &errno_provenance) < 0) goto fail;
|
878
|
+
input_data = rb_str_new(contents, current_key.size);
|
879
|
+
|
880
|
+
/* Try to compile the input_data using input_to_storage(input_data) */
|
881
|
+
exception_tag = bs_input_to_storage(handler, Qnil, input_data, path_v, &storage_data);
|
882
|
+
if (exception_tag != 0) goto fail;
|
883
|
+
|
884
|
+
/* If input_to_storage raised Bootsnap::CompileCache::Uncompilable, don't try
|
885
|
+
* to cache anything; just return false */
|
886
|
+
if (storage_data == rb_cBootsnap_CompileCache_UNCOMPILABLE) {
|
887
|
+
goto fail;
|
888
|
+
}
|
889
|
+
/* If storage_data isn't a string, we can't cache it */
|
890
|
+
if (!RB_TYPE_P(storage_data, T_STRING)) goto fail;
|
891
|
+
|
892
|
+
/* Write the cache key and storage_data to the cache directory */
|
893
|
+
res = atomic_write_cache_file(cache_path, ¤t_key, storage_data, &errno_provenance);
|
894
|
+
if (res < 0) goto fail;
|
895
|
+
|
896
|
+
goto succeed;
|
897
|
+
|
898
|
+
#define CLEANUP \
|
899
|
+
if (contents != NULL) xfree(contents); \
|
900
|
+
if (current_fd >= 0) close(current_fd); \
|
901
|
+
if (cache_fd >= 0) close(cache_fd);
|
902
|
+
|
903
|
+
succeed:
|
904
|
+
CLEANUP;
|
905
|
+
return Qtrue;
|
906
|
+
fail:
|
907
|
+
CLEANUP;
|
908
|
+
return Qfalse;
|
909
|
+
#undef CLEANUP
|
910
|
+
}
|
911
|
+
|
912
|
+
|
722
913
|
/*****************************************************************************/
|
723
914
|
/********************* Handler Wrappers **************************************/
|
724
915
|
/*****************************************************************************
|
@@ -738,11 +929,13 @@ invalid_type_storage_data:
|
|
738
929
|
|
739
930
|
struct s2o_data {
|
740
931
|
VALUE handler;
|
932
|
+
VALUE args;
|
741
933
|
VALUE storage_data;
|
742
934
|
};
|
743
935
|
|
744
936
|
struct i2o_data {
|
745
937
|
VALUE handler;
|
938
|
+
VALUE args;
|
746
939
|
VALUE input_data;
|
747
940
|
};
|
748
941
|
|
@@ -753,29 +946,31 @@ struct i2s_data {
|
|
753
946
|
};
|
754
947
|
|
755
948
|
static VALUE
|
756
|
-
|
949
|
+
try_storage_to_output(VALUE arg)
|
757
950
|
{
|
758
951
|
struct s2o_data * data = (struct s2o_data *)arg;
|
759
|
-
return rb_funcall(data->handler, rb_intern("storage_to_output"),
|
952
|
+
return rb_funcall(data->handler, rb_intern("storage_to_output"), 2, data->storage_data, data->args);
|
760
953
|
}
|
761
954
|
|
762
955
|
static int
|
763
|
-
bs_storage_to_output(VALUE handler, VALUE storage_data, VALUE * output_data)
|
956
|
+
bs_storage_to_output(VALUE handler, VALUE args, VALUE storage_data, VALUE * output_data)
|
764
957
|
{
|
765
958
|
int state;
|
766
959
|
struct s2o_data s2o_data = {
|
767
960
|
.handler = handler,
|
961
|
+
.args = args,
|
768
962
|
.storage_data = storage_data,
|
769
963
|
};
|
770
|
-
*output_data = rb_protect(
|
964
|
+
*output_data = rb_protect(try_storage_to_output, (VALUE)&s2o_data, &state);
|
771
965
|
return state;
|
772
966
|
}
|
773
967
|
|
774
968
|
static void
|
775
|
-
bs_input_to_output(VALUE handler, VALUE input_data, VALUE * output_data, int * exception_tag)
|
969
|
+
bs_input_to_output(VALUE handler, VALUE args, VALUE input_data, VALUE * output_data, int * exception_tag)
|
776
970
|
{
|
777
971
|
struct i2o_data i2o_data = {
|
778
972
|
.handler = handler,
|
973
|
+
.args = args,
|
779
974
|
.input_data = input_data,
|
780
975
|
};
|
781
976
|
*output_data = rb_protect(prot_input_to_output, (VALUE)&i2o_data, exception_tag);
|
@@ -785,7 +980,7 @@ static VALUE
|
|
785
980
|
prot_input_to_output(VALUE arg)
|
786
981
|
{
|
787
982
|
struct i2o_data * data = (struct i2o_data *)arg;
|
788
|
-
return rb_funcall(data->handler, rb_intern("input_to_output"),
|
983
|
+
return rb_funcall(data->handler, rb_intern("input_to_output"), 2, data->input_data, data->args);
|
789
984
|
}
|
790
985
|
|
791
986
|
static VALUE
|
@@ -795,24 +990,8 @@ try_input_to_storage(VALUE arg)
|
|
795
990
|
return rb_funcall(data->handler, rb_intern("input_to_storage"), 2, data->input_data, data->pathval);
|
796
991
|
}
|
797
992
|
|
798
|
-
static VALUE
|
799
|
-
rescue_input_to_storage(VALUE arg)
|
800
|
-
{
|
801
|
-
return uncompilable;
|
802
|
-
}
|
803
|
-
|
804
|
-
static VALUE
|
805
|
-
prot_input_to_storage(VALUE arg)
|
806
|
-
{
|
807
|
-
struct i2s_data * data = (struct i2s_data *)arg;
|
808
|
-
return rb_rescue2(
|
809
|
-
try_input_to_storage, (VALUE)data,
|
810
|
-
rescue_input_to_storage, Qnil,
|
811
|
-
rb_eBootsnap_CompileCache_Uncompilable, 0);
|
812
|
-
}
|
813
|
-
|
814
993
|
static int
|
815
|
-
bs_input_to_storage(VALUE handler, VALUE input_data, VALUE pathval, VALUE * storage_data)
|
994
|
+
bs_input_to_storage(VALUE handler, VALUE args, VALUE input_data, VALUE pathval, VALUE * storage_data)
|
816
995
|
{
|
817
996
|
int state;
|
818
997
|
struct i2s_data i2s_data = {
|
@@ -820,6 +999,6 @@ bs_input_to_storage(VALUE handler, VALUE input_data, VALUE pathval, VALUE * stor
|
|
820
999
|
.input_data = input_data,
|
821
1000
|
.pathval = pathval,
|
822
1001
|
};
|
823
|
-
*storage_data = rb_protect(
|
1002
|
+
*storage_data = rb_protect(try_input_to_storage, (VALUE)&i2s_data, &state);
|
824
1003
|
return state;
|
825
1004
|
}
|