prism 0.28.0 → 0.30.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +41 -1
- data/CONTRIBUTING.md +0 -4
- data/README.md +1 -0
- data/config.yml +95 -26
- data/docs/fuzzing.md +1 -1
- data/docs/ripper_translation.md +22 -0
- data/ext/prism/api_node.c +70 -52
- data/ext/prism/extconf.rb +27 -23
- data/ext/prism/extension.c +107 -372
- data/ext/prism/extension.h +1 -1
- data/include/prism/ast.h +170 -102
- data/include/prism/diagnostic.h +18 -3
- data/include/prism/node.h +0 -21
- data/include/prism/parser.h +23 -25
- data/include/prism/regexp.h +17 -8
- data/include/prism/static_literals.h +3 -2
- data/include/prism/util/pm_char.h +1 -2
- data/include/prism/util/pm_constant_pool.h +0 -8
- data/include/prism/util/pm_integer.h +16 -9
- data/include/prism/util/pm_string.h +0 -8
- data/include/prism/version.h +2 -2
- data/include/prism.h +0 -11
- data/lib/prism/compiler.rb +3 -0
- data/lib/prism/desugar_compiler.rb +4 -4
- data/lib/prism/dispatcher.rb +14 -0
- data/lib/prism/dot_visitor.rb +54 -35
- data/lib/prism/dsl.rb +23 -18
- data/lib/prism/ffi.rb +25 -4
- data/lib/prism/inspect_visitor.rb +26 -24
- data/lib/prism/mutation_compiler.rb +6 -1
- data/lib/prism/node.rb +314 -389
- data/lib/prism/node_ext.rb +175 -17
- data/lib/prism/parse_result/comments.rb +1 -8
- data/lib/prism/parse_result/newlines.rb +102 -12
- data/lib/prism/parse_result.rb +17 -0
- data/lib/prism/reflection.rb +11 -9
- data/lib/prism/serialize.rb +91 -68
- data/lib/prism/translation/parser/compiler.rb +288 -138
- data/lib/prism/translation/parser.rb +7 -2
- data/lib/prism/translation/ripper.rb +24 -22
- data/lib/prism/translation/ruby_parser.rb +32 -14
- data/lib/prism/visitor.rb +3 -0
- data/lib/prism.rb +0 -4
- data/prism.gemspec +2 -4
- data/rbi/prism/node.rbi +114 -57
- data/rbi/prism/node_ext.rbi +5 -0
- data/rbi/prism/parse_result.rbi +1 -1
- data/rbi/prism/visitor.rbi +3 -0
- data/rbi/prism.rbi +6 -0
- data/sig/prism/dsl.rbs +13 -10
- data/sig/prism/lex_compat.rbs +10 -0
- data/sig/prism/mutation_compiler.rbs +1 -0
- data/sig/prism/node.rbs +72 -48
- data/sig/prism/node_ext.rbs +4 -0
- data/sig/prism/visitor.rbs +1 -0
- data/sig/prism.rbs +21 -0
- data/src/diagnostic.c +56 -27
- data/src/node.c +432 -1690
- data/src/prettyprint.c +97 -54
- data/src/prism.c +1286 -1196
- data/src/regexp.c +133 -68
- data/src/serialize.c +22 -17
- data/src/static_literals.c +63 -84
- data/src/token_type.c +4 -4
- data/src/util/pm_constant_pool.c +0 -8
- data/src/util/pm_integer.c +39 -11
- data/src/util/pm_string.c +0 -12
- data/src/util/pm_strpbrk.c +32 -6
- metadata +3 -5
- data/include/prism/util/pm_string_list.h +0 -44
- data/lib/prism/debug.rb +0 -249
- data/src/util/pm_string_list.c +0 -28
data/src/static_literals.c
CHANGED
@@ -58,6 +58,25 @@ murmur_hash(const uint8_t *key, size_t length) {
|
|
58
58
|
return hash;
|
59
59
|
}
|
60
60
|
|
61
|
+
/**
|
62
|
+
* Hash the value of an integer and return it.
|
63
|
+
*/
|
64
|
+
static uint32_t
|
65
|
+
integer_hash(const pm_integer_t *integer) {
|
66
|
+
uint32_t hash;
|
67
|
+
if (integer->values) {
|
68
|
+
hash = murmur_hash((const uint8_t *) integer->values, sizeof(uint32_t) * integer->length);
|
69
|
+
} else {
|
70
|
+
hash = murmur_hash((const uint8_t *) &integer->value, sizeof(uint32_t));
|
71
|
+
}
|
72
|
+
|
73
|
+
if (integer->negative) {
|
74
|
+
hash ^= murmur_scramble((uint32_t) 1);
|
75
|
+
}
|
76
|
+
|
77
|
+
return hash;
|
78
|
+
}
|
79
|
+
|
61
80
|
/**
|
62
81
|
* Return the hash of the given node. It is important that nodes that have
|
63
82
|
* equivalent static literal values have the same hash. This is because we use
|
@@ -68,19 +87,8 @@ node_hash(const pm_static_literals_metadata_t *metadata, const pm_node_t *node)
|
|
68
87
|
switch (PM_NODE_TYPE(node)) {
|
69
88
|
case PM_INTEGER_NODE: {
|
70
89
|
// Integers hash their value.
|
71
|
-
const
|
72
|
-
|
73
|
-
if (integer->values) {
|
74
|
-
hash = murmur_hash((const uint8_t *) integer->values, sizeof(uint32_t) * integer->length);
|
75
|
-
} else {
|
76
|
-
hash = murmur_hash((const uint8_t *) &integer->value, sizeof(uint32_t));
|
77
|
-
}
|
78
|
-
|
79
|
-
if (integer->negative) {
|
80
|
-
hash ^= murmur_scramble((uint32_t) 1);
|
81
|
-
}
|
82
|
-
|
83
|
-
return hash;
|
90
|
+
const pm_integer_node_t *cast = (const pm_integer_node_t *) node;
|
91
|
+
return integer_hash(&cast->value);
|
84
92
|
}
|
85
93
|
case PM_SOURCE_LINE_NODE: {
|
86
94
|
// Source lines hash their line number.
|
@@ -94,11 +102,9 @@ node_hash(const pm_static_literals_metadata_t *metadata, const pm_node_t *node)
|
|
94
102
|
return murmur_hash((const uint8_t *) value, sizeof(double));
|
95
103
|
}
|
96
104
|
case PM_RATIONAL_NODE: {
|
97
|
-
// Rationals hash their
|
98
|
-
|
99
|
-
|
100
|
-
const pm_node_t *numeric = ((const pm_rational_node_t *) node)->numeric;
|
101
|
-
return node_hash(metadata, numeric) ^ murmur_scramble((uint32_t) node->type);
|
105
|
+
// Rationals hash their numerator and denominator.
|
106
|
+
const pm_rational_node_t *cast = (const pm_rational_node_t *) node;
|
107
|
+
return integer_hash(&cast->numerator) ^ integer_hash(&cast->denominator) ^ murmur_scramble((uint32_t) cast->base.type);
|
102
108
|
}
|
103
109
|
case PM_IMAGINARY_NODE: {
|
104
110
|
// Imaginaries hash their numeric value. Because their numeric value
|
@@ -148,7 +154,7 @@ node_hash(const pm_static_literals_metadata_t *metadata, const pm_node_t *node)
|
|
148
154
|
* and must be able to compare all node types that will be stored in this hash.
|
149
155
|
*/
|
150
156
|
static pm_node_t *
|
151
|
-
pm_node_hash_insert(pm_node_hash_t *hash, const pm_static_literals_metadata_t *metadata, pm_node_t *node, int (*compare)(const pm_static_literals_metadata_t *metadata, const pm_node_t *left, const pm_node_t *right)) {
|
157
|
+
pm_node_hash_insert(pm_node_hash_t *hash, const pm_static_literals_metadata_t *metadata, pm_node_t *node, bool replace, int (*compare)(const pm_static_literals_metadata_t *metadata, const pm_node_t *left, const pm_node_t *right)) {
|
152
158
|
// If we are out of space, we need to resize the hash. This will cause all
|
153
159
|
// of the nodes to be rehashed and reinserted into the new hash.
|
154
160
|
if (hash->size * 2 >= hash->capacity) {
|
@@ -196,9 +202,14 @@ pm_node_hash_insert(pm_node_hash_t *hash, const pm_static_literals_metadata_t *m
|
|
196
202
|
// already in the hash. Otherwise, we can just increment the size and insert
|
197
203
|
// the new node.
|
198
204
|
pm_node_t *result = hash->nodes[index];
|
199
|
-
if (result == NULL) hash->size++;
|
200
205
|
|
201
|
-
|
206
|
+
if (result == NULL) {
|
207
|
+
hash->size++;
|
208
|
+
hash->nodes[index] = node;
|
209
|
+
} else if (replace) {
|
210
|
+
hash->nodes[index] = node;
|
211
|
+
}
|
212
|
+
|
202
213
|
return result;
|
203
214
|
}
|
204
215
|
|
@@ -275,8 +286,15 @@ pm_compare_number_nodes(const pm_static_literals_metadata_t *metadata, const pm_
|
|
275
286
|
switch (PM_NODE_TYPE(left)) {
|
276
287
|
case PM_IMAGINARY_NODE:
|
277
288
|
return pm_compare_number_nodes(metadata, ((const pm_imaginary_node_t *) left)->numeric, ((const pm_imaginary_node_t *) right)->numeric);
|
278
|
-
case PM_RATIONAL_NODE:
|
279
|
-
|
289
|
+
case PM_RATIONAL_NODE: {
|
290
|
+
const pm_rational_node_t *left_rational = (const pm_rational_node_t *) left;
|
291
|
+
const pm_rational_node_t *right_rational = (const pm_rational_node_t *) right;
|
292
|
+
|
293
|
+
int result = pm_integer_compare(&left_rational->denominator, &right_rational->denominator);
|
294
|
+
if (result != 0) return result;
|
295
|
+
|
296
|
+
return pm_integer_compare(&left_rational->numerator, &right_rational->numerator);
|
297
|
+
}
|
280
298
|
case PM_INTEGER_NODE:
|
281
299
|
return pm_compare_integer_nodes(metadata, left, right);
|
282
300
|
case PM_FLOAT_NODE:
|
@@ -335,7 +353,7 @@ pm_compare_regular_expression_nodes(PRISM_ATTRIBUTE_UNUSED const pm_static_liter
|
|
335
353
|
* Add a node to the set of static literals.
|
336
354
|
*/
|
337
355
|
pm_node_t *
|
338
|
-
pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node) {
|
356
|
+
pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line, pm_static_literals_t *literals, pm_node_t *node, bool replace) {
|
339
357
|
switch (PM_NODE_TYPE(node)) {
|
340
358
|
case PM_INTEGER_NODE:
|
341
359
|
case PM_SOURCE_LINE_NODE:
|
@@ -347,6 +365,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line
|
|
347
365
|
.encoding_name = NULL
|
348
366
|
},
|
349
367
|
node,
|
368
|
+
replace,
|
350
369
|
pm_compare_integer_nodes
|
351
370
|
);
|
352
371
|
case PM_FLOAT_NODE:
|
@@ -358,6 +377,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line
|
|
358
377
|
.encoding_name = NULL
|
359
378
|
},
|
360
379
|
node,
|
380
|
+
replace,
|
361
381
|
pm_compare_float_nodes
|
362
382
|
);
|
363
383
|
case PM_RATIONAL_NODE:
|
@@ -370,6 +390,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line
|
|
370
390
|
.encoding_name = NULL
|
371
391
|
},
|
372
392
|
node,
|
393
|
+
replace,
|
373
394
|
pm_compare_number_nodes
|
374
395
|
);
|
375
396
|
case PM_STRING_NODE:
|
@@ -382,6 +403,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line
|
|
382
403
|
.encoding_name = NULL
|
383
404
|
},
|
384
405
|
node,
|
406
|
+
replace,
|
385
407
|
pm_compare_string_nodes
|
386
408
|
);
|
387
409
|
case PM_REGULAR_EXPRESSION_NODE:
|
@@ -393,6 +415,7 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line
|
|
393
415
|
.encoding_name = NULL
|
394
416
|
},
|
395
417
|
node,
|
418
|
+
replace,
|
396
419
|
pm_compare_regular_expression_nodes
|
397
420
|
);
|
398
421
|
case PM_SYMBOL_NODE:
|
@@ -404,26 +427,27 @@ pm_static_literals_add(const pm_newline_list_t *newline_list, int32_t start_line
|
|
404
427
|
.encoding_name = NULL
|
405
428
|
},
|
406
429
|
node,
|
430
|
+
replace,
|
407
431
|
pm_compare_string_nodes
|
408
432
|
);
|
409
433
|
case PM_TRUE_NODE: {
|
410
434
|
pm_node_t *duplicated = literals->true_node;
|
411
|
-
literals->true_node = node;
|
435
|
+
if ((duplicated == NULL) || replace) literals->true_node = node;
|
412
436
|
return duplicated;
|
413
437
|
}
|
414
438
|
case PM_FALSE_NODE: {
|
415
439
|
pm_node_t *duplicated = literals->false_node;
|
416
|
-
literals->false_node = node;
|
440
|
+
if ((duplicated == NULL) || replace) literals->false_node = node;
|
417
441
|
return duplicated;
|
418
442
|
}
|
419
443
|
case PM_NIL_NODE: {
|
420
444
|
pm_node_t *duplicated = literals->nil_node;
|
421
|
-
literals->nil_node = node;
|
445
|
+
if ((duplicated == NULL) || replace) literals->nil_node = node;
|
422
446
|
return duplicated;
|
423
447
|
}
|
424
448
|
case PM_SOURCE_ENCODING_NODE: {
|
425
449
|
pm_node_t *duplicated = literals->source_encoding_node;
|
426
|
-
literals->source_encoding_node = node;
|
450
|
+
if ((duplicated == NULL) || replace) literals->source_encoding_node = node;
|
427
451
|
return duplicated;
|
428
452
|
}
|
429
453
|
default:
|
@@ -456,7 +480,7 @@ pm_static_literal_positive_p(const pm_node_t *node) {
|
|
456
480
|
case PM_INTEGER_NODE:
|
457
481
|
return !((const pm_integer_node_t *) node)->value.negative;
|
458
482
|
case PM_RATIONAL_NODE:
|
459
|
-
return
|
483
|
+
return !((const pm_rational_node_t *) node)->numerator.negative;
|
460
484
|
case PM_IMAGINARY_NODE:
|
461
485
|
return pm_static_literal_positive_p(((const pm_imaginary_node_t *) node)->numeric);
|
462
486
|
default:
|
@@ -465,43 +489,6 @@ pm_static_literal_positive_p(const pm_node_t *node) {
|
|
465
489
|
}
|
466
490
|
}
|
467
491
|
|
468
|
-
/**
|
469
|
-
* Inspect a rational node that wraps a float node. This is going to be a
|
470
|
-
* poor-man's version of the Ruby `Rational#to_s` method, because we're not
|
471
|
-
* going to try to reduce the rational by finding the GCD. We'll leave that for
|
472
|
-
* a future improvement.
|
473
|
-
*/
|
474
|
-
static void
|
475
|
-
pm_rational_inspect(pm_buffer_t *buffer, pm_rational_node_t *node) {
|
476
|
-
const uint8_t *start = node->base.location.start;
|
477
|
-
const uint8_t *end = node->base.location.end - 1; // r
|
478
|
-
|
479
|
-
while (start < end && *start == '0') start++; // 0.1 -> .1
|
480
|
-
while (end > start && end[-1] == '0') end--; // 1.0 -> 1.
|
481
|
-
size_t length = (size_t) (end - start);
|
482
|
-
|
483
|
-
const uint8_t *point = memchr(start, '.', length);
|
484
|
-
assert(point && "should have a decimal point");
|
485
|
-
|
486
|
-
uint8_t *digits = malloc(length - 1);
|
487
|
-
if (digits == NULL) return;
|
488
|
-
|
489
|
-
memcpy(digits, start, (unsigned long) (point - start));
|
490
|
-
memcpy(digits + (point - start), point + 1, (unsigned long) (end - point - 1));
|
491
|
-
|
492
|
-
pm_integer_t numerator = { 0 };
|
493
|
-
pm_integer_parse(&numerator, PM_INTEGER_BASE_DECIMAL, digits, digits + length - 1);
|
494
|
-
|
495
|
-
pm_buffer_append_byte(buffer, '(');
|
496
|
-
pm_integer_string(buffer, &numerator);
|
497
|
-
pm_buffer_append_string(buffer, "/1", 2);
|
498
|
-
for (size_t index = 0; index < (size_t) (end - point - 1); index++) pm_buffer_append_byte(buffer, '0');
|
499
|
-
pm_buffer_append_byte(buffer, ')');
|
500
|
-
|
501
|
-
pm_integer_free(&numerator);
|
502
|
-
free(digits);
|
503
|
-
}
|
504
|
-
|
505
492
|
/**
|
506
493
|
* Create a string-based representation of the given static literal.
|
507
494
|
*/
|
@@ -544,7 +531,9 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met
|
|
544
531
|
pm_buffer_append_string(buffer, "(0", 2);
|
545
532
|
if (pm_static_literal_positive_p(numeric)) pm_buffer_append_byte(buffer, '+');
|
546
533
|
pm_static_literal_inspect_node(buffer, metadata, numeric);
|
547
|
-
if (PM_NODE_TYPE_P(numeric, PM_RATIONAL_NODE))
|
534
|
+
if (PM_NODE_TYPE_P(numeric, PM_RATIONAL_NODE)) {
|
535
|
+
pm_buffer_append_byte(buffer, '*');
|
536
|
+
}
|
548
537
|
pm_buffer_append_string(buffer, "i)", 2);
|
549
538
|
break;
|
550
539
|
}
|
@@ -555,22 +544,12 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met
|
|
555
544
|
pm_buffer_append_string(buffer, "nil", 3);
|
556
545
|
break;
|
557
546
|
case PM_RATIONAL_NODE: {
|
558
|
-
const
|
559
|
-
|
560
|
-
|
561
|
-
|
562
|
-
|
563
|
-
|
564
|
-
pm_buffer_append_string(buffer, "/1)", 3);
|
565
|
-
break;
|
566
|
-
case PM_FLOAT_NODE:
|
567
|
-
pm_rational_inspect(buffer, (pm_rational_node_t *) node);
|
568
|
-
break;
|
569
|
-
default:
|
570
|
-
assert(false && "unreachable");
|
571
|
-
break;
|
572
|
-
}
|
573
|
-
|
547
|
+
const pm_rational_node_t *rational = (const pm_rational_node_t *) node;
|
548
|
+
pm_buffer_append_byte(buffer, '(');
|
549
|
+
pm_integer_string(buffer, &rational->numerator);
|
550
|
+
pm_buffer_append_byte(buffer, '/');
|
551
|
+
pm_integer_string(buffer, &rational->denominator);
|
552
|
+
pm_buffer_append_byte(buffer, ')');
|
574
553
|
break;
|
575
554
|
}
|
576
555
|
case PM_REGULAR_EXPRESSION_NODE: {
|
@@ -624,7 +603,7 @@ pm_static_literal_inspect_node(pm_buffer_t *buffer, const pm_static_literals_met
|
|
624
603
|
/**
|
625
604
|
* Create a string-based representation of the given static literal.
|
626
605
|
*/
|
627
|
-
|
606
|
+
void
|
628
607
|
pm_static_literal_inspect(pm_buffer_t *buffer, const pm_newline_list_t *newline_list, int32_t start_line, const char *encoding_name, const pm_node_t *node) {
|
629
608
|
pm_static_literal_inspect_node(
|
630
609
|
buffer,
|
data/src/token_type.c
CHANGED
@@ -362,7 +362,7 @@ const char *
|
|
362
362
|
pm_token_type_human(pm_token_type_t token_type) {
|
363
363
|
switch (token_type) {
|
364
364
|
case PM_TOKEN_EOF:
|
365
|
-
return "end
|
365
|
+
return "end-of-input";
|
366
366
|
case PM_TOKEN_MISSING:
|
367
367
|
return "missing token";
|
368
368
|
case PM_TOKEN_NOT_PROVIDED:
|
@@ -422,9 +422,9 @@ pm_token_type_human(pm_token_type_t token_type) {
|
|
422
422
|
case PM_TOKEN_DOT:
|
423
423
|
return "'.'";
|
424
424
|
case PM_TOKEN_DOT_DOT:
|
425
|
-
return "
|
425
|
+
return "..";
|
426
426
|
case PM_TOKEN_DOT_DOT_DOT:
|
427
|
-
return "
|
427
|
+
return "...";
|
428
428
|
case PM_TOKEN_EMBDOC_BEGIN:
|
429
429
|
return "'=begin'";
|
430
430
|
case PM_TOKEN_EMBDOC_END:
|
@@ -684,7 +684,7 @@ pm_token_type_human(pm_token_type_t token_type) {
|
|
684
684
|
case PM_TOKEN_USTAR:
|
685
685
|
return "*";
|
686
686
|
case PM_TOKEN_USTAR_STAR:
|
687
|
-
return "
|
687
|
+
return "**";
|
688
688
|
case PM_TOKEN_WORDS_SEP:
|
689
689
|
return "string separator";
|
690
690
|
case PM_TOKEN___END__:
|
data/src/util/pm_constant_pool.c
CHANGED
@@ -61,14 +61,6 @@ pm_constant_id_list_includes(pm_constant_id_list_t *list, pm_constant_id_t id) {
|
|
61
61
|
return false;
|
62
62
|
}
|
63
63
|
|
64
|
-
/**
|
65
|
-
* Get the memory size of a list of constant ids.
|
66
|
-
*/
|
67
|
-
size_t
|
68
|
-
pm_constant_id_list_memsize(pm_constant_id_list_t *list) {
|
69
|
-
return sizeof(pm_constant_id_list_t) + (list->capacity * sizeof(pm_constant_id_t));
|
70
|
-
}
|
71
|
-
|
72
64
|
/**
|
73
65
|
* Free the memory associated with a list of constant ids.
|
74
66
|
*/
|
data/src/util/pm_integer.c
CHANGED
@@ -48,7 +48,7 @@ big_add(pm_integer_t *destination, pm_integer_t *left, pm_integer_t *right, uint
|
|
48
48
|
|
49
49
|
/**
|
50
50
|
* Internal use for karatsuba_multiply. Calculates `a - b - c` with the given
|
51
|
-
* base. Assume a, b, c, a - b - c all to be
|
51
|
+
* base. Assume a, b, c, a - b - c all to be positive.
|
52
52
|
* Return pm_integer_t with values allocated. Not normalized.
|
53
53
|
*/
|
54
54
|
static void
|
@@ -471,15 +471,18 @@ pm_integer_parse_big(pm_integer_t *integer, uint32_t multiplier, const uint8_t *
|
|
471
471
|
* has already been validated, as internal validation checks are not performed
|
472
472
|
* here.
|
473
473
|
*/
|
474
|
-
|
474
|
+
void
|
475
475
|
pm_integer_parse(pm_integer_t *integer, pm_integer_base_t base, const uint8_t *start, const uint8_t *end) {
|
476
|
-
// Ignore unary +. Unary
|
476
|
+
// Ignore unary +. Unary - is parsed differently and will not end up here.
|
477
477
|
// Instead, it will modify the parsed integer later.
|
478
478
|
if (*start == '+') start++;
|
479
479
|
|
480
480
|
// Determine the multiplier from the base, and skip past any prefixes.
|
481
481
|
uint32_t multiplier = 10;
|
482
482
|
switch (base) {
|
483
|
+
case PM_INTEGER_BASE_DEFAULT:
|
484
|
+
while (*start == '0') start++; // 01 -> 1
|
485
|
+
break;
|
483
486
|
case PM_INTEGER_BASE_BINARY:
|
484
487
|
start += 2; // 0b
|
485
488
|
multiplier = 2;
|
@@ -533,14 +536,6 @@ pm_integer_parse(pm_integer_t *integer, pm_integer_base_t base, const uint8_t *s
|
|
533
536
|
integer->value = (uint32_t) value;
|
534
537
|
}
|
535
538
|
|
536
|
-
/**
|
537
|
-
* Return the memory size of the integer.
|
538
|
-
*/
|
539
|
-
size_t
|
540
|
-
pm_integer_memsize(const pm_integer_t *integer) {
|
541
|
-
return sizeof(pm_integer_t) + integer->length * sizeof(uint32_t);
|
542
|
-
}
|
543
|
-
|
544
539
|
/**
|
545
540
|
* Compare two integers. This function returns -1 if the left integer is less
|
546
541
|
* than the right integer, 0 if they are equal, and 1 if the left integer is
|
@@ -572,6 +567,39 @@ pm_integer_compare(const pm_integer_t *left, const pm_integer_t *right) {
|
|
572
567
|
return 0;
|
573
568
|
}
|
574
569
|
|
570
|
+
/**
|
571
|
+
* Reduce a ratio of integers to its simplest form.
|
572
|
+
*/
|
573
|
+
void pm_integers_reduce(pm_integer_t *numerator, pm_integer_t *denominator) {
|
574
|
+
// If either the numerator or denominator do not fit into a 32-bit integer,
|
575
|
+
// then this function is a no-op. In the future, we may consider reducing
|
576
|
+
// even the larger numbers, but for now we're going to keep it simple.
|
577
|
+
if (
|
578
|
+
// If the numerator doesn't fit into a 32-bit integer, return early.
|
579
|
+
numerator->length != 0 ||
|
580
|
+
// If the denominator doesn't fit into a 32-bit integer, return early.
|
581
|
+
denominator->length != 0 ||
|
582
|
+
// If the numerator is 0, then return early.
|
583
|
+
numerator->value == 0 ||
|
584
|
+
// If the denominator is 1, then return early.
|
585
|
+
denominator->value == 1
|
586
|
+
) return;
|
587
|
+
|
588
|
+
// Find the greatest common divisor of the numerator and denominator.
|
589
|
+
uint32_t divisor = numerator->value;
|
590
|
+
uint32_t remainder = denominator->value;
|
591
|
+
|
592
|
+
while (remainder != 0) {
|
593
|
+
uint32_t temporary = remainder;
|
594
|
+
remainder = divisor % remainder;
|
595
|
+
divisor = temporary;
|
596
|
+
}
|
597
|
+
|
598
|
+
// Divide the numerator and denominator by the greatest common divisor.
|
599
|
+
numerator->value /= divisor;
|
600
|
+
denominator->value /= divisor;
|
601
|
+
}
|
602
|
+
|
575
603
|
/**
|
576
604
|
* Convert an integer to a decimal string.
|
577
605
|
*/
|
data/src/util/pm_string.c
CHANGED
@@ -245,18 +245,6 @@ pm_string_file_init(pm_string_t *string, const char *filepath) {
|
|
245
245
|
#endif
|
246
246
|
}
|
247
247
|
|
248
|
-
/**
|
249
|
-
* Returns the memory size associated with the string.
|
250
|
-
*/
|
251
|
-
size_t
|
252
|
-
pm_string_memsize(const pm_string_t *string) {
|
253
|
-
size_t size = sizeof(pm_string_t);
|
254
|
-
if (string->type == PM_STRING_OWNED) {
|
255
|
-
size += string->length;
|
256
|
-
}
|
257
|
-
return size;
|
258
|
-
}
|
259
|
-
|
260
248
|
/**
|
261
249
|
* Ensure the string is owned. If it is not, then reinitialize it as owned and
|
262
250
|
* copy over the previous source.
|
data/src/util/pm_strpbrk.c
CHANGED
@@ -8,6 +8,27 @@ pm_strpbrk_invalid_multibyte_character(pm_parser_t *parser, const uint8_t *start
|
|
8
8
|
pm_diagnostic_list_append_format(&parser->error_list, start, end, PM_ERR_INVALID_MULTIBYTE_CHARACTER, *start);
|
9
9
|
}
|
10
10
|
|
11
|
+
/**
|
12
|
+
* Set the explicit encoding for the parser to the current encoding.
|
13
|
+
*/
|
14
|
+
static inline void
|
15
|
+
pm_strpbrk_explicit_encoding_set(pm_parser_t *parser, const uint8_t *source, size_t width) {
|
16
|
+
if (parser->explicit_encoding != NULL) {
|
17
|
+
if (parser->explicit_encoding == parser->encoding) {
|
18
|
+
// Okay, we already locked to this encoding.
|
19
|
+
} else if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
|
20
|
+
// Not okay, we already found a Unicode escape sequence and this
|
21
|
+
// conflicts.
|
22
|
+
pm_diagnostic_list_append_format(&parser->error_list, source, source + width, PM_ERR_MIXED_ENCODING, parser->encoding->name);
|
23
|
+
} else {
|
24
|
+
// Should not be anything else.
|
25
|
+
assert(false && "unreachable");
|
26
|
+
}
|
27
|
+
}
|
28
|
+
|
29
|
+
parser->explicit_encoding = parser->encoding;
|
30
|
+
}
|
31
|
+
|
11
32
|
/**
|
12
33
|
* This is the default path.
|
13
34
|
*/
|
@@ -52,7 +73,7 @@ pm_strpbrk_utf8(pm_parser_t *parser, const uint8_t *source, const uint8_t *chars
|
|
52
73
|
* This is the path when the encoding is ASCII-8BIT.
|
53
74
|
*/
|
54
75
|
static inline const uint8_t *
|
55
|
-
pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maximum) {
|
76
|
+
pm_strpbrk_ascii_8bit(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
|
56
77
|
size_t index = 0;
|
57
78
|
|
58
79
|
while (index < maximum) {
|
@@ -60,6 +81,7 @@ pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maxi
|
|
60
81
|
return source + index;
|
61
82
|
}
|
62
83
|
|
84
|
+
if (validate && source[index] >= 0x80) pm_strpbrk_explicit_encoding_set(parser, source, 1);
|
63
85
|
index++;
|
64
86
|
}
|
65
87
|
|
@@ -72,6 +94,7 @@ pm_strpbrk_ascii_8bit(const uint8_t *source, const uint8_t *charset, size_t maxi
|
|
72
94
|
static inline const uint8_t *
|
73
95
|
pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
|
74
96
|
size_t index = 0;
|
97
|
+
const pm_encoding_t *encoding = parser->encoding;
|
75
98
|
|
76
99
|
while (index < maximum) {
|
77
100
|
if (strchr((const char *) charset, source[index]) != NULL) {
|
@@ -81,7 +104,8 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
|
|
81
104
|
if (source[index] < 0x80) {
|
82
105
|
index++;
|
83
106
|
} else {
|
84
|
-
size_t width =
|
107
|
+
size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
|
108
|
+
if (validate) pm_strpbrk_explicit_encoding_set(parser, source, width);
|
85
109
|
|
86
110
|
if (width > 0) {
|
87
111
|
index += width;
|
@@ -96,7 +120,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
|
|
96
120
|
|
97
121
|
do {
|
98
122
|
index++;
|
99
|
-
} while (index < maximum &&
|
123
|
+
} while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
|
100
124
|
|
101
125
|
pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
|
102
126
|
}
|
@@ -113,6 +137,7 @@ pm_strpbrk_multi_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
|
|
113
137
|
static inline const uint8_t *
|
114
138
|
pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, size_t maximum, bool validate) {
|
115
139
|
size_t index = 0;
|
140
|
+
const pm_encoding_t *encoding = parser->encoding;
|
116
141
|
|
117
142
|
while (index < maximum) {
|
118
143
|
if (strchr((const char *) charset, source[index]) != NULL) {
|
@@ -122,7 +147,8 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
|
|
122
147
|
if (source[index] < 0x80 || !validate) {
|
123
148
|
index++;
|
124
149
|
} else {
|
125
|
-
size_t width =
|
150
|
+
size_t width = encoding->char_width(source + index, (ptrdiff_t) (maximum - index));
|
151
|
+
pm_strpbrk_explicit_encoding_set(parser, source, width);
|
126
152
|
|
127
153
|
if (width > 0) {
|
128
154
|
index += width;
|
@@ -135,7 +161,7 @@ pm_strpbrk_single_byte(pm_parser_t *parser, const uint8_t *source, const uint8_t
|
|
135
161
|
|
136
162
|
do {
|
137
163
|
index++;
|
138
|
-
} while (index < maximum &&
|
164
|
+
} while (index < maximum && encoding->char_width(source + index, (ptrdiff_t) (maximum - index)) == 0);
|
139
165
|
|
140
166
|
pm_strpbrk_invalid_multibyte_character(parser, source + start, source + index);
|
141
167
|
}
|
@@ -171,7 +197,7 @@ pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, p
|
|
171
197
|
} else if (!parser->encoding_changed) {
|
172
198
|
return pm_strpbrk_utf8(parser, source, charset, (size_t) length, validate);
|
173
199
|
} else if (parser->encoding == PM_ENCODING_ASCII_8BIT_ENTRY) {
|
174
|
-
return pm_strpbrk_ascii_8bit(source, charset, (size_t) length);
|
200
|
+
return pm_strpbrk_ascii_8bit(parser, source, charset, (size_t) length, validate);
|
175
201
|
} else if (parser->encoding->multibyte) {
|
176
202
|
return pm_strpbrk_multi_byte(parser, source, charset, (size_t) length, validate);
|
177
203
|
} else {
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: prism
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.30.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Shopify
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2024-
|
11
|
+
date: 2024-06-07 00:00:00.000000000 Z
|
12
12
|
dependencies: []
|
13
13
|
description:
|
14
14
|
email:
|
@@ -69,13 +69,11 @@ files:
|
|
69
69
|
- include/prism/util/pm_memchr.h
|
70
70
|
- include/prism/util/pm_newline_list.h
|
71
71
|
- include/prism/util/pm_string.h
|
72
|
-
- include/prism/util/pm_string_list.h
|
73
72
|
- include/prism/util/pm_strncasecmp.h
|
74
73
|
- include/prism/util/pm_strpbrk.h
|
75
74
|
- include/prism/version.h
|
76
75
|
- lib/prism.rb
|
77
76
|
- lib/prism/compiler.rb
|
78
|
-
- lib/prism/debug.rb
|
79
77
|
- lib/prism/desugar_compiler.rb
|
80
78
|
- lib/prism/dispatcher.rb
|
81
79
|
- lib/prism/dot_visitor.rb
|
@@ -126,6 +124,7 @@ files:
|
|
126
124
|
- sig/prism/dot_visitor.rbs
|
127
125
|
- sig/prism/dsl.rbs
|
128
126
|
- sig/prism/inspect_visitor.rbs
|
127
|
+
- sig/prism/lex_compat.rbs
|
129
128
|
- sig/prism/mutation_compiler.rbs
|
130
129
|
- sig/prism/node.rbs
|
131
130
|
- sig/prism/node_ext.rbs
|
@@ -154,7 +153,6 @@ files:
|
|
154
153
|
- src/util/pm_memchr.c
|
155
154
|
- src/util/pm_newline_list.c
|
156
155
|
- src/util/pm_string.c
|
157
|
-
- src/util/pm_string_list.c
|
158
156
|
- src/util/pm_strncasecmp.c
|
159
157
|
- src/util/pm_strpbrk.c
|
160
158
|
homepage: https://github.com/ruby/prism
|
@@ -1,44 +0,0 @@
|
|
1
|
-
/**
|
2
|
-
* @file pm_string_list.h
|
3
|
-
*
|
4
|
-
* A list of strings.
|
5
|
-
*/
|
6
|
-
#ifndef PRISM_STRING_LIST_H
|
7
|
-
#define PRISM_STRING_LIST_H
|
8
|
-
|
9
|
-
#include "prism/defines.h"
|
10
|
-
#include "prism/util/pm_string.h"
|
11
|
-
|
12
|
-
#include <stddef.h>
|
13
|
-
#include <stdlib.h>
|
14
|
-
|
15
|
-
/**
|
16
|
-
* A list of strings.
|
17
|
-
*/
|
18
|
-
typedef struct {
|
19
|
-
/** The length of the string list. */
|
20
|
-
size_t length;
|
21
|
-
|
22
|
-
/** The capacity of the string list that has been allocated. */
|
23
|
-
size_t capacity;
|
24
|
-
|
25
|
-
/** A pointer to the start of the string list. */
|
26
|
-
pm_string_t *strings;
|
27
|
-
} pm_string_list_t;
|
28
|
-
|
29
|
-
/**
|
30
|
-
* Append a pm_string_t to the given string list.
|
31
|
-
*
|
32
|
-
* @param string_list The string list to append to.
|
33
|
-
* @param string The string to append.
|
34
|
-
*/
|
35
|
-
void pm_string_list_append(pm_string_list_t *string_list, pm_string_t *string);
|
36
|
-
|
37
|
-
/**
|
38
|
-
* Free the memory associated with the string list.
|
39
|
-
*
|
40
|
-
* @param string_list The string list to free.
|
41
|
-
*/
|
42
|
-
PRISM_EXPORTED_FUNCTION void pm_string_list_free(pm_string_list_t *string_list);
|
43
|
-
|
44
|
-
#endif
|