prism 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -1
- data/Makefile +6 -0
- data/README.md +1 -1
- data/config.yml +50 -35
- data/docs/fuzzing.md +1 -1
- data/docs/serialization.md +28 -29
- data/ext/prism/api_node.c +802 -770
- data/ext/prism/api_pack.c +20 -9
- data/ext/prism/extension.c +464 -162
- data/ext/prism/extension.h +1 -1
- data/include/prism/ast.h +3173 -763
- data/include/prism/defines.h +32 -9
- data/include/prism/diagnostic.h +36 -3
- data/include/prism/enc/pm_encoding.h +118 -28
- data/include/prism/node.h +38 -13
- data/include/prism/options.h +204 -0
- data/include/prism/pack.h +44 -33
- data/include/prism/parser.h +445 -200
- data/include/prism/prettyprint.h +12 -1
- data/include/prism/regexp.h +16 -2
- data/include/prism/util/pm_buffer.h +94 -16
- data/include/prism/util/pm_char.h +162 -48
- data/include/prism/util/pm_constant_pool.h +126 -32
- data/include/prism/util/pm_list.h +68 -38
- data/include/prism/util/pm_memchr.h +18 -3
- data/include/prism/util/pm_newline_list.h +70 -27
- data/include/prism/util/pm_state_stack.h +25 -7
- data/include/prism/util/pm_string.h +115 -27
- data/include/prism/util/pm_string_list.h +25 -6
- data/include/prism/util/pm_strncasecmp.h +32 -0
- data/include/prism/util/pm_strpbrk.h +31 -17
- data/include/prism/version.h +27 -2
- data/include/prism.h +224 -31
- data/lib/prism/compiler.rb +6 -3
- data/lib/prism/debug.rb +23 -7
- data/lib/prism/dispatcher.rb +33 -18
- data/lib/prism/dsl.rb +10 -5
- data/lib/prism/ffi.rb +132 -80
- data/lib/prism/lex_compat.rb +25 -15
- data/lib/prism/mutation_compiler.rb +10 -5
- data/lib/prism/node.rb +370 -135
- data/lib/prism/node_ext.rb +1 -1
- data/lib/prism/node_inspector.rb +1 -1
- data/lib/prism/pack.rb +79 -40
- data/lib/prism/parse_result/comments.rb +7 -2
- data/lib/prism/parse_result/newlines.rb +4 -0
- data/lib/prism/parse_result.rb +150 -30
- data/lib/prism/pattern.rb +11 -0
- data/lib/prism/ripper_compat.rb +28 -10
- data/lib/prism/serialize.rb +86 -54
- data/lib/prism/visitor.rb +10 -3
- data/lib/prism.rb +20 -2
- data/prism.gemspec +4 -2
- data/rbi/prism.rbi +104 -60
- data/rbi/prism_static.rbi +16 -2
- data/sig/prism.rbs +72 -43
- data/sig/prism_static.rbs +14 -1
- data/src/diagnostic.c +56 -53
- data/src/enc/pm_big5.c +1 -0
- data/src/enc/pm_euc_jp.c +1 -0
- data/src/enc/pm_gbk.c +1 -0
- data/src/enc/pm_shift_jis.c +1 -0
- data/src/enc/pm_tables.c +316 -80
- data/src/enc/pm_unicode.c +53 -8
- data/src/enc/pm_windows_31j.c +1 -0
- data/src/node.c +334 -321
- data/src/options.c +170 -0
- data/src/prettyprint.c +74 -47
- data/src/prism.c +1642 -856
- data/src/regexp.c +151 -95
- data/src/serialize.c +44 -20
- data/src/token_type.c +3 -1
- data/src/util/pm_buffer.c +45 -15
- data/src/util/pm_char.c +103 -57
- data/src/util/pm_constant_pool.c +51 -21
- data/src/util/pm_list.c +12 -4
- data/src/util/pm_memchr.c +5 -3
- data/src/util/pm_newline_list.c +20 -12
- data/src/util/pm_state_stack.c +9 -3
- data/src/util/pm_string.c +95 -85
- data/src/util/pm_string_list.c +14 -15
- data/src/util/pm_strncasecmp.c +10 -3
- data/src/util/pm_strpbrk.c +25 -19
- metadata +5 -3
- data/docs/prism.png +0 -0
data/src/options.c
ADDED
@@ -0,0 +1,170 @@
|
|
1
|
+
#include "prism/options.h"
|
2
|
+
|
3
|
+
/**
|
4
|
+
* Set the filepath option on the given options struct.
|
5
|
+
*/
|
6
|
+
PRISM_EXPORTED_FUNCTION void
|
7
|
+
pm_options_filepath_set(pm_options_t *options, const char *filepath) {
|
8
|
+
pm_string_constant_init(&options->filepath, filepath, strlen(filepath));
|
9
|
+
}
|
10
|
+
|
11
|
+
/**
|
12
|
+
* Set the encoding option on the given options struct.
|
13
|
+
*/
|
14
|
+
PRISM_EXPORTED_FUNCTION void
|
15
|
+
pm_options_encoding_set(pm_options_t *options, const char *encoding) {
|
16
|
+
pm_string_constant_init(&options->encoding, encoding, strlen(encoding));
|
17
|
+
}
|
18
|
+
|
19
|
+
/**
|
20
|
+
* Set the line option on the given options struct.
|
21
|
+
*/
|
22
|
+
PRISM_EXPORTED_FUNCTION void
|
23
|
+
pm_options_line_set(pm_options_t *options, uint32_t line) {
|
24
|
+
options->line = line;
|
25
|
+
}
|
26
|
+
|
27
|
+
/**
|
28
|
+
* Set the frozen string literal option on the given options struct.
|
29
|
+
*/
|
30
|
+
PRISM_EXPORTED_FUNCTION void
|
31
|
+
pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal) {
|
32
|
+
options->frozen_string_literal = frozen_string_literal;
|
33
|
+
}
|
34
|
+
|
35
|
+
/**
|
36
|
+
* Set the suppress warnings option on the given options struct.
|
37
|
+
*/
|
38
|
+
PRISM_EXPORTED_FUNCTION void
|
39
|
+
pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings) {
|
40
|
+
options->suppress_warnings = suppress_warnings;
|
41
|
+
}
|
42
|
+
|
43
|
+
/**
|
44
|
+
* Allocate and zero out the scopes array on the given options struct.
|
45
|
+
*/
|
46
|
+
PRISM_EXPORTED_FUNCTION void
|
47
|
+
pm_options_scopes_init(pm_options_t *options, size_t scopes_count) {
|
48
|
+
options->scopes_count = scopes_count;
|
49
|
+
options->scopes = calloc(scopes_count, sizeof(pm_options_scope_t));
|
50
|
+
if (options->scopes == NULL) abort();
|
51
|
+
}
|
52
|
+
|
53
|
+
/**
|
54
|
+
* Return a pointer to the scope at the given index within the given options.
|
55
|
+
*/
|
56
|
+
PRISM_EXPORTED_FUNCTION const pm_options_scope_t *
|
57
|
+
pm_options_scope_get(const pm_options_t *options, size_t index) {
|
58
|
+
return &options->scopes[index];
|
59
|
+
}
|
60
|
+
|
61
|
+
/**
|
62
|
+
* Create a new options scope struct. This will hold a set of locals that are in
|
63
|
+
* scope surrounding the code that is being parsed.
|
64
|
+
*/
|
65
|
+
PRISM_EXPORTED_FUNCTION void
|
66
|
+
pm_options_scope_init(pm_options_scope_t *scope, size_t locals_count) {
|
67
|
+
scope->locals_count = locals_count;
|
68
|
+
scope->locals = calloc(locals_count, sizeof(pm_string_t));
|
69
|
+
if (scope->locals == NULL) abort();
|
70
|
+
}
|
71
|
+
|
72
|
+
/**
|
73
|
+
* Return a pointer to the local at the given index within the given scope.
|
74
|
+
*/
|
75
|
+
PRISM_EXPORTED_FUNCTION const pm_string_t *
|
76
|
+
pm_options_scope_local_get(const pm_options_scope_t *scope, size_t index) {
|
77
|
+
return &scope->locals[index];
|
78
|
+
}
|
79
|
+
|
80
|
+
/**
|
81
|
+
* Free the internal memory associated with the options.
|
82
|
+
*/
|
83
|
+
PRISM_EXPORTED_FUNCTION void
|
84
|
+
pm_options_free(pm_options_t *options) {
|
85
|
+
pm_string_free(&options->filepath);
|
86
|
+
pm_string_free(&options->encoding);
|
87
|
+
|
88
|
+
for (size_t scope_index = 0; scope_index < options->scopes_count; scope_index++) {
|
89
|
+
pm_options_scope_t *scope = &options->scopes[scope_index];
|
90
|
+
|
91
|
+
for (size_t local_index = 0; local_index < scope->locals_count; local_index++) {
|
92
|
+
pm_string_free(&scope->locals[local_index]);
|
93
|
+
}
|
94
|
+
|
95
|
+
free(scope->locals);
|
96
|
+
}
|
97
|
+
|
98
|
+
free(options->scopes);
|
99
|
+
}
|
100
|
+
|
101
|
+
/**
|
102
|
+
* Read a 32-bit unsigned integer from a pointer. This function is used to read
|
103
|
+
* the options that are passed into the parser from the Ruby implementation. It
|
104
|
+
* handles aligned and unaligned reads.
|
105
|
+
*/
|
106
|
+
static uint32_t
|
107
|
+
pm_options_read_u32(const char *data) {
|
108
|
+
if (((uintptr_t) data) % sizeof(uint32_t) == 0) {
|
109
|
+
return *((uint32_t *) data);
|
110
|
+
} else {
|
111
|
+
uint32_t value;
|
112
|
+
memcpy(&value, data, sizeof(uint32_t));
|
113
|
+
return value;
|
114
|
+
}
|
115
|
+
}
|
116
|
+
|
117
|
+
/**
|
118
|
+
* Deserialize an options struct from the given binary string. This is used to
|
119
|
+
* pass options to the parser from an FFI call so that consumers of the library
|
120
|
+
* from an FFI perspective don't have to worry about the structure of our
|
121
|
+
* options structs. Since the source of these calls will be from Ruby
|
122
|
+
* implementation internals we assume it is from a trusted source.
|
123
|
+
*/
|
124
|
+
void
|
125
|
+
pm_options_read(pm_options_t *options, const char *data) {
|
126
|
+
uint32_t filepath_length = pm_options_read_u32(data);
|
127
|
+
data += 4;
|
128
|
+
|
129
|
+
if (filepath_length > 0) {
|
130
|
+
pm_string_constant_init(&options->filepath, data, filepath_length);
|
131
|
+
data += filepath_length;
|
132
|
+
}
|
133
|
+
|
134
|
+
options->line = pm_options_read_u32(data);
|
135
|
+
data += 4;
|
136
|
+
|
137
|
+
uint32_t encoding_length = pm_options_read_u32(data);
|
138
|
+
data += 4;
|
139
|
+
|
140
|
+
if (encoding_length > 0) {
|
141
|
+
pm_string_constant_init(&options->encoding, data, encoding_length);
|
142
|
+
data += encoding_length;
|
143
|
+
}
|
144
|
+
|
145
|
+
options->frozen_string_literal = *data++;
|
146
|
+
options->suppress_warnings = *data++;
|
147
|
+
|
148
|
+
uint32_t scopes_count = pm_options_read_u32(data);
|
149
|
+
data += 4;
|
150
|
+
|
151
|
+
if (scopes_count > 0) {
|
152
|
+
pm_options_scopes_init(options, scopes_count);
|
153
|
+
|
154
|
+
for (size_t scope_index = 0; scope_index < scopes_count; scope_index++) {
|
155
|
+
uint32_t locals_count = pm_options_read_u32(data);
|
156
|
+
data += 4;
|
157
|
+
|
158
|
+
pm_options_scope_t *scope = &options->scopes[scope_index];
|
159
|
+
pm_options_scope_init(scope, locals_count);
|
160
|
+
|
161
|
+
for (size_t local_index = 0; local_index < locals_count; local_index++) {
|
162
|
+
uint32_t local_length = pm_options_read_u32(data);
|
163
|
+
data += 4;
|
164
|
+
|
165
|
+
pm_string_constant_init(&scope->locals[local_index], data, local_length);
|
166
|
+
data += local_length;
|
167
|
+
}
|
168
|
+
}
|
169
|
+
}
|
170
|
+
}
|
data/src/prettyprint.c
CHANGED
@@ -5123,52 +5123,6 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm
|
|
5123
5123
|
|
5124
5124
|
break;
|
5125
5125
|
}
|
5126
|
-
case PM_KEYWORD_PARAMETER_NODE: {
|
5127
|
-
pm_keyword_parameter_node_t *cast = (pm_keyword_parameter_node_t *) node;
|
5128
|
-
pm_buffer_append_string(output_buffer, "@ KeywordParameterNode (location: ", 34);
|
5129
|
-
prettyprint_location(output_buffer, parser, &node->location);
|
5130
|
-
pm_buffer_append_string(output_buffer, ")\n", 2);
|
5131
|
-
|
5132
|
-
// name
|
5133
|
-
{
|
5134
|
-
pm_buffer_concat(output_buffer, prefix_buffer);
|
5135
|
-
pm_buffer_append_string(output_buffer, "├── name:", 15);
|
5136
|
-
pm_buffer_append_byte(output_buffer, ' ');
|
5137
|
-
prettyprint_constant(output_buffer, parser, cast->name);
|
5138
|
-
pm_buffer_append_byte(output_buffer, '\n');
|
5139
|
-
}
|
5140
|
-
|
5141
|
-
// name_loc
|
5142
|
-
{
|
5143
|
-
pm_buffer_concat(output_buffer, prefix_buffer);
|
5144
|
-
pm_buffer_append_string(output_buffer, "├── name_loc:", 19);
|
5145
|
-
pm_location_t *location = &cast->name_loc;
|
5146
|
-
pm_buffer_append_byte(output_buffer, ' ');
|
5147
|
-
prettyprint_location(output_buffer, parser, location);
|
5148
|
-
pm_buffer_append_string(output_buffer, " = \"", 4);
|
5149
|
-
prettyprint_source(output_buffer, location->start, (size_t) (location->end - location->start));
|
5150
|
-
pm_buffer_append_string(output_buffer, "\"\n", 2);
|
5151
|
-
}
|
5152
|
-
|
5153
|
-
// value
|
5154
|
-
{
|
5155
|
-
pm_buffer_concat(output_buffer, prefix_buffer);
|
5156
|
-
pm_buffer_append_string(output_buffer, "└── value:", 16);
|
5157
|
-
if (cast->value == NULL) {
|
5158
|
-
pm_buffer_append_string(output_buffer, " ∅\n", 5);
|
5159
|
-
} else {
|
5160
|
-
pm_buffer_append_byte(output_buffer, '\n');
|
5161
|
-
|
5162
|
-
size_t prefix_length = prefix_buffer->length;
|
5163
|
-
pm_buffer_append_string(prefix_buffer, " ", 4);
|
5164
|
-
pm_buffer_concat(output_buffer, prefix_buffer);
|
5165
|
-
prettyprint_node(output_buffer, parser, (pm_node_t *) cast->value, prefix_buffer);
|
5166
|
-
prefix_buffer->length = prefix_length;
|
5167
|
-
}
|
5168
|
-
}
|
5169
|
-
|
5170
|
-
break;
|
5171
|
-
}
|
5172
5126
|
case PM_KEYWORD_REST_PARAMETER_NODE: {
|
5173
5127
|
pm_keyword_rest_parameter_node_t *cast = (pm_keyword_rest_parameter_node_t *) node;
|
5174
5128
|
pm_buffer_append_string(output_buffer, "@ KeywordRestParameterNode (location: ", 38);
|
@@ -6256,6 +6210,48 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm
|
|
6256
6210
|
|
6257
6211
|
break;
|
6258
6212
|
}
|
6213
|
+
case PM_OPTIONAL_KEYWORD_PARAMETER_NODE: {
|
6214
|
+
pm_optional_keyword_parameter_node_t *cast = (pm_optional_keyword_parameter_node_t *) node;
|
6215
|
+
pm_buffer_append_string(output_buffer, "@ OptionalKeywordParameterNode (location: ", 42);
|
6216
|
+
prettyprint_location(output_buffer, parser, &node->location);
|
6217
|
+
pm_buffer_append_string(output_buffer, ")\n", 2);
|
6218
|
+
|
6219
|
+
// name
|
6220
|
+
{
|
6221
|
+
pm_buffer_concat(output_buffer, prefix_buffer);
|
6222
|
+
pm_buffer_append_string(output_buffer, "├── name:", 15);
|
6223
|
+
pm_buffer_append_byte(output_buffer, ' ');
|
6224
|
+
prettyprint_constant(output_buffer, parser, cast->name);
|
6225
|
+
pm_buffer_append_byte(output_buffer, '\n');
|
6226
|
+
}
|
6227
|
+
|
6228
|
+
// name_loc
|
6229
|
+
{
|
6230
|
+
pm_buffer_concat(output_buffer, prefix_buffer);
|
6231
|
+
pm_buffer_append_string(output_buffer, "├── name_loc:", 19);
|
6232
|
+
pm_location_t *location = &cast->name_loc;
|
6233
|
+
pm_buffer_append_byte(output_buffer, ' ');
|
6234
|
+
prettyprint_location(output_buffer, parser, location);
|
6235
|
+
pm_buffer_append_string(output_buffer, " = \"", 4);
|
6236
|
+
prettyprint_source(output_buffer, location->start, (size_t) (location->end - location->start));
|
6237
|
+
pm_buffer_append_string(output_buffer, "\"\n", 2);
|
6238
|
+
}
|
6239
|
+
|
6240
|
+
// value
|
6241
|
+
{
|
6242
|
+
pm_buffer_concat(output_buffer, prefix_buffer);
|
6243
|
+
pm_buffer_append_string(output_buffer, "└── value:", 16);
|
6244
|
+
pm_buffer_append_byte(output_buffer, '\n');
|
6245
|
+
|
6246
|
+
size_t prefix_length = prefix_buffer->length;
|
6247
|
+
pm_buffer_append_string(prefix_buffer, " ", 4);
|
6248
|
+
pm_buffer_concat(output_buffer, prefix_buffer);
|
6249
|
+
prettyprint_node(output_buffer, parser, (pm_node_t *) cast->value, prefix_buffer);
|
6250
|
+
prefix_buffer->length = prefix_length;
|
6251
|
+
}
|
6252
|
+
|
6253
|
+
break;
|
6254
|
+
}
|
6259
6255
|
case PM_OPTIONAL_PARAMETER_NODE: {
|
6260
6256
|
pm_optional_parameter_node_t *cast = (pm_optional_parameter_node_t *) node;
|
6261
6257
|
pm_buffer_append_string(output_buffer, "@ OptionalParameterNode (location: ", 35);
|
@@ -7007,6 +7003,35 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm
|
|
7007
7003
|
|
7008
7004
|
break;
|
7009
7005
|
}
|
7006
|
+
case PM_REQUIRED_KEYWORD_PARAMETER_NODE: {
|
7007
|
+
pm_required_keyword_parameter_node_t *cast = (pm_required_keyword_parameter_node_t *) node;
|
7008
|
+
pm_buffer_append_string(output_buffer, "@ RequiredKeywordParameterNode (location: ", 42);
|
7009
|
+
prettyprint_location(output_buffer, parser, &node->location);
|
7010
|
+
pm_buffer_append_string(output_buffer, ")\n", 2);
|
7011
|
+
|
7012
|
+
// name
|
7013
|
+
{
|
7014
|
+
pm_buffer_concat(output_buffer, prefix_buffer);
|
7015
|
+
pm_buffer_append_string(output_buffer, "├── name:", 15);
|
7016
|
+
pm_buffer_append_byte(output_buffer, ' ');
|
7017
|
+
prettyprint_constant(output_buffer, parser, cast->name);
|
7018
|
+
pm_buffer_append_byte(output_buffer, '\n');
|
7019
|
+
}
|
7020
|
+
|
7021
|
+
// name_loc
|
7022
|
+
{
|
7023
|
+
pm_buffer_concat(output_buffer, prefix_buffer);
|
7024
|
+
pm_buffer_append_string(output_buffer, "└── name_loc:", 19);
|
7025
|
+
pm_location_t *location = &cast->name_loc;
|
7026
|
+
pm_buffer_append_byte(output_buffer, ' ');
|
7027
|
+
prettyprint_location(output_buffer, parser, location);
|
7028
|
+
pm_buffer_append_string(output_buffer, " = \"", 4);
|
7029
|
+
prettyprint_source(output_buffer, location->start, (size_t) (location->end - location->start));
|
7030
|
+
pm_buffer_append_string(output_buffer, "\"\n", 2);
|
7031
|
+
}
|
7032
|
+
|
7033
|
+
break;
|
7034
|
+
}
|
7010
7035
|
case PM_REQUIRED_PARAMETER_NODE: {
|
7011
7036
|
pm_required_parameter_node_t *cast = (pm_required_parameter_node_t *) node;
|
7012
7037
|
pm_buffer_append_string(output_buffer, "@ RequiredParameterNode (location: ", 35);
|
@@ -8211,7 +8236,9 @@ prettyprint_node(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm
|
|
8211
8236
|
}
|
8212
8237
|
}
|
8213
8238
|
|
8214
|
-
|
8239
|
+
/**
|
8240
|
+
* Pretty-prints the AST represented by the given node to the given buffer.
|
8241
|
+
*/
|
8215
8242
|
PRISM_EXPORTED_FUNCTION void
|
8216
8243
|
pm_prettyprint(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_node_t *node) {
|
8217
8244
|
pm_buffer_t prefix_buffer = { 0 };
|