yarp 0.10.0 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +50 -1
- data/CONTRIBUTING.md +7 -0
- data/config.yml +259 -49
- data/docs/configuration.md +0 -1
- data/docs/mapping.md +91 -91
- data/docs/serialization.md +23 -20
- data/ext/yarp/api_node.c +1268 -419
- data/ext/yarp/extension.c +9 -2
- data/ext/yarp/extension.h +2 -2
- data/include/yarp/ast.h +471 -318
- data/include/yarp/diagnostic.h +203 -1
- data/include/yarp/enc/yp_encoding.h +1 -1
- data/include/yarp/node.h +0 -4
- data/include/yarp/parser.h +44 -16
- data/include/yarp/util/yp_char.h +22 -6
- data/include/yarp/util/yp_constant_pool.h +11 -4
- data/include/yarp/version.h +2 -2
- data/lib/yarp/desugar_visitor.rb +19 -19
- data/lib/yarp/mutation_visitor.rb +50 -15
- data/lib/yarp/node.rb +6455 -443
- data/lib/yarp/parse_result/comments.rb +172 -0
- data/lib/yarp/parse_result/newlines.rb +60 -0
- data/lib/yarp/pattern.rb +239 -0
- data/lib/yarp/serialize.rb +173 -140
- data/lib/yarp.rb +151 -76
- data/src/diagnostic.c +259 -2
- data/src/enc/yp_unicode.c +5 -5
- data/src/node.c +984 -872
- data/src/prettyprint.c +461 -203
- data/src/serialize.c +380 -185
- data/src/unescape.c +20 -20
- data/src/util/yp_char.c +59 -16
- data/src/util/yp_constant_pool.c +97 -13
- data/src/util/yp_newline_list.c +5 -1
- data/src/util/yp_string_list.c +4 -1
- data/src/yarp.c +2313 -1675
- data/yarp.gemspec +4 -1
- metadata +5 -2
data/include/yarp/diagnostic.h
CHANGED
@@ -6,6 +6,7 @@
|
|
6
6
|
|
7
7
|
#include <stdbool.h>
|
8
8
|
#include <stdlib.h>
|
9
|
+
#include <assert.h>
|
9
10
|
|
10
11
|
// This struct represents a diagnostic found during parsing.
|
11
12
|
typedef struct {
|
@@ -15,8 +16,209 @@ typedef struct {
|
|
15
16
|
const char *message;
|
16
17
|
} yp_diagnostic_t;
|
17
18
|
|
19
|
+
typedef enum {
|
20
|
+
YP_ERR_ALIAS_ARGUMENT,
|
21
|
+
YP_ERR_AMPAMPEQ_MULTI_ASSIGN,
|
22
|
+
YP_ERR_ARGUMENT_AFTER_BLOCK,
|
23
|
+
YP_ERR_ARGUMENT_BARE_HASH,
|
24
|
+
YP_ERR_ARGUMENT_BLOCK_MULTI,
|
25
|
+
YP_ERR_ARGUMENT_FORMAL_CLASS,
|
26
|
+
YP_ERR_ARGUMENT_FORMAL_CONSTANT,
|
27
|
+
YP_ERR_ARGUMENT_FORMAL_GLOBAL,
|
28
|
+
YP_ERR_ARGUMENT_FORMAL_IVAR,
|
29
|
+
YP_ERR_ARGUMENT_NO_FORWARDING_AMP,
|
30
|
+
YP_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES,
|
31
|
+
YP_ERR_ARGUMENT_NO_FORWARDING_STAR,
|
32
|
+
YP_ERR_ARGUMENT_SPLAT_AFTER_ASSOC_SPLAT,
|
33
|
+
YP_ERR_ARGUMENT_SPLAT_AFTER_SPLAT,
|
34
|
+
YP_ERR_ARGUMENT_TERM_PAREN,
|
35
|
+
YP_ERR_ARGUMENT_UNEXPECTED_BLOCK,
|
36
|
+
YP_ERR_ARRAY_ELEMENT,
|
37
|
+
YP_ERR_ARRAY_EXPRESSION,
|
38
|
+
YP_ERR_ARRAY_EXPRESSION_AFTER_STAR,
|
39
|
+
YP_ERR_ARRAY_SEPARATOR,
|
40
|
+
YP_ERR_ARRAY_TERM,
|
41
|
+
YP_ERR_BEGIN_LONELY_ELSE,
|
42
|
+
YP_ERR_BEGIN_TERM,
|
43
|
+
YP_ERR_BEGIN_UPCASE_BRACE,
|
44
|
+
YP_ERR_BEGIN_UPCASE_TERM,
|
45
|
+
YP_ERR_BLOCK_PARAM_LOCAL_VARIABLE,
|
46
|
+
YP_ERR_BLOCK_PARAM_PIPE_TERM,
|
47
|
+
YP_ERR_BLOCK_TERM_BRACE,
|
48
|
+
YP_ERR_BLOCK_TERM_END,
|
49
|
+
YP_ERR_CANNOT_PARSE_EXPRESSION,
|
50
|
+
YP_ERR_CANNOT_PARSE_STRING_PART,
|
51
|
+
YP_ERR_CASE_EXPRESSION_AFTER_CASE,
|
52
|
+
YP_ERR_CASE_EXPRESSION_AFTER_WHEN,
|
53
|
+
YP_ERR_CASE_MISSING_CONDITIONS,
|
54
|
+
YP_ERR_CASE_TERM,
|
55
|
+
YP_ERR_CLASS_IN_METHOD,
|
56
|
+
YP_ERR_CLASS_NAME,
|
57
|
+
YP_ERR_CLASS_SUPERCLASS,
|
58
|
+
YP_ERR_CLASS_TERM,
|
59
|
+
YP_ERR_CONDITIONAL_ELSIF_PREDICATE,
|
60
|
+
YP_ERR_CONDITIONAL_IF_PREDICATE,
|
61
|
+
YP_ERR_CONDITIONAL_TERM,
|
62
|
+
YP_ERR_CONDITIONAL_TERM_ELSE,
|
63
|
+
YP_ERR_CONDITIONAL_UNLESS_PREDICATE,
|
64
|
+
YP_ERR_CONDITIONAL_UNTIL_PREDICATE,
|
65
|
+
YP_ERR_CONDITIONAL_WHILE_PREDICATE,
|
66
|
+
YP_ERR_CONSTANT_PATH_COLON_COLON_CONSTANT,
|
67
|
+
YP_ERR_DEF_ENDLESS,
|
68
|
+
YP_ERR_DEF_ENDLESS_SETTER,
|
69
|
+
YP_ERR_DEF_NAME,
|
70
|
+
YP_ERR_DEF_NAME_AFTER_RECEIVER,
|
71
|
+
YP_ERR_DEF_PARAMS_TERM,
|
72
|
+
YP_ERR_DEF_PARAMS_TERM_PAREN,
|
73
|
+
YP_ERR_DEF_RECEIVER,
|
74
|
+
YP_ERR_DEF_RECEIVER_TERM,
|
75
|
+
YP_ERR_DEF_TERM,
|
76
|
+
YP_ERR_DEFINED_EXPRESSION,
|
77
|
+
YP_ERR_EMBDOC_TERM,
|
78
|
+
YP_ERR_EMBEXPR_END,
|
79
|
+
YP_ERR_EMBVAR_INVALID,
|
80
|
+
YP_ERR_END_UPCASE_BRACE,
|
81
|
+
YP_ERR_END_UPCASE_TERM,
|
82
|
+
YP_ERR_ESCAPE_INVALID_CONTROL,
|
83
|
+
YP_ERR_ESCAPE_INVALID_CONTROL_REPEAT,
|
84
|
+
YP_ERR_ESCAPE_INVALID_HEXADECIMAL,
|
85
|
+
YP_ERR_ESCAPE_INVALID_META,
|
86
|
+
YP_ERR_ESCAPE_INVALID_META_REPEAT,
|
87
|
+
YP_ERR_ESCAPE_INVALID_UNICODE,
|
88
|
+
YP_ERR_ESCAPE_INVALID_UNICODE_CM_FLAGS,
|
89
|
+
YP_ERR_ESCAPE_INVALID_UNICODE_LITERAL,
|
90
|
+
YP_ERR_ESCAPE_INVALID_UNICODE_LONG,
|
91
|
+
YP_ERR_ESCAPE_INVALID_UNICODE_TERM,
|
92
|
+
YP_ERR_EXPECT_ARGUMENT,
|
93
|
+
YP_ERR_EXPECT_EOL_AFTER_STATEMENT,
|
94
|
+
YP_ERR_EXPECT_EXPRESSION_AFTER_AMPAMPEQ,
|
95
|
+
YP_ERR_EXPECT_EXPRESSION_AFTER_PIPEPIPEEQ,
|
96
|
+
YP_ERR_EXPECT_EXPRESSION_AFTER_COMMA,
|
97
|
+
YP_ERR_EXPECT_EXPRESSION_AFTER_EQUAL,
|
98
|
+
YP_ERR_EXPECT_EXPRESSION_AFTER_LESS_LESS,
|
99
|
+
YP_ERR_EXPECT_EXPRESSION_AFTER_LPAREN,
|
100
|
+
YP_ERR_EXPECT_EXPRESSION_AFTER_QUESTION,
|
101
|
+
YP_ERR_EXPECT_EXPRESSION_AFTER_OPERATOR,
|
102
|
+
YP_ERR_EXPECT_EXPRESSION_AFTER_SPLAT,
|
103
|
+
YP_ERR_EXPECT_EXPRESSION_AFTER_SPLAT_HASH,
|
104
|
+
YP_ERR_EXPECT_EXPRESSION_AFTER_STAR,
|
105
|
+
YP_ERR_EXPECT_IDENT_REQ_PARAMETER,
|
106
|
+
YP_ERR_EXPECT_LPAREN_REQ_PARAMETER,
|
107
|
+
YP_ERR_EXPECT_RBRACKET,
|
108
|
+
YP_ERR_EXPECT_RPAREN,
|
109
|
+
YP_ERR_EXPECT_RPAREN_AFTER_MULTI,
|
110
|
+
YP_ERR_EXPECT_RPAREN_REQ_PARAMETER,
|
111
|
+
YP_ERR_EXPECT_STRING_CONTENT,
|
112
|
+
YP_ERR_EXPECT_WHEN_DELIMITER,
|
113
|
+
YP_ERR_EXPRESSION_BARE_HASH,
|
114
|
+
YP_ERR_FOR_COLLECTION,
|
115
|
+
YP_ERR_FOR_IN,
|
116
|
+
YP_ERR_FOR_INDEX,
|
117
|
+
YP_ERR_FOR_TERM,
|
118
|
+
YP_ERR_HASH_EXPRESSION_AFTER_LABEL,
|
119
|
+
YP_ERR_HASH_KEY,
|
120
|
+
YP_ERR_HASH_ROCKET,
|
121
|
+
YP_ERR_HASH_TERM,
|
122
|
+
YP_ERR_HASH_VALUE,
|
123
|
+
YP_ERR_HEREDOC_TERM,
|
124
|
+
YP_ERR_INCOMPLETE_QUESTION_MARK,
|
125
|
+
YP_ERR_INCOMPLETE_VARIABLE_CLASS,
|
126
|
+
YP_ERR_INCOMPLETE_VARIABLE_INSTANCE,
|
127
|
+
YP_ERR_INVALID_ENCODING_MAGIC_COMMENT,
|
128
|
+
YP_ERR_INVALID_FLOAT_EXPONENT,
|
129
|
+
YP_ERR_INVALID_NUMBER_BINARY,
|
130
|
+
YP_ERR_INVALID_NUMBER_DECIMAL,
|
131
|
+
YP_ERR_INVALID_NUMBER_HEXADECIMAL,
|
132
|
+
YP_ERR_INVALID_NUMBER_OCTAL,
|
133
|
+
YP_ERR_INVALID_NUMBER_UNDERSCORE,
|
134
|
+
YP_ERR_INVALID_PERCENT,
|
135
|
+
YP_ERR_INVALID_TOKEN,
|
136
|
+
YP_ERR_INVALID_VARIABLE_GLOBAL,
|
137
|
+
YP_ERR_LAMBDA_OPEN,
|
138
|
+
YP_ERR_LAMBDA_TERM_BRACE,
|
139
|
+
YP_ERR_LAMBDA_TERM_END,
|
140
|
+
YP_ERR_LIST_I_LOWER_ELEMENT,
|
141
|
+
YP_ERR_LIST_I_LOWER_TERM,
|
142
|
+
YP_ERR_LIST_I_UPPER_ELEMENT,
|
143
|
+
YP_ERR_LIST_I_UPPER_TERM,
|
144
|
+
YP_ERR_LIST_W_LOWER_ELEMENT,
|
145
|
+
YP_ERR_LIST_W_LOWER_TERM,
|
146
|
+
YP_ERR_LIST_W_UPPER_ELEMENT,
|
147
|
+
YP_ERR_LIST_W_UPPER_TERM,
|
148
|
+
YP_ERR_MALLOC_FAILED,
|
149
|
+
YP_ERR_MODULE_IN_METHOD,
|
150
|
+
YP_ERR_MODULE_NAME,
|
151
|
+
YP_ERR_MODULE_TERM,
|
152
|
+
YP_ERR_MULTI_ASSIGN_MULTI_SPLATS,
|
153
|
+
YP_ERR_NOT_EXPRESSION,
|
154
|
+
YP_ERR_NUMBER_LITERAL_UNDERSCORE,
|
155
|
+
YP_ERR_NUMBERED_PARAMETER_NOT_ALLOWED,
|
156
|
+
YP_ERR_NUMBERED_PARAMETER_OUTER_SCOPE,
|
157
|
+
YP_ERR_OPERATOR_MULTI_ASSIGN,
|
158
|
+
YP_ERR_OPERATOR_WRITE_BLOCK,
|
159
|
+
YP_ERR_PARAMETER_ASSOC_SPLAT_MULTI,
|
160
|
+
YP_ERR_PARAMETER_BLOCK_MULTI,
|
161
|
+
YP_ERR_PARAMETER_NAME_REPEAT,
|
162
|
+
YP_ERR_PARAMETER_NO_DEFAULT,
|
163
|
+
YP_ERR_PARAMETER_NO_DEFAULT_KW,
|
164
|
+
YP_ERR_PARAMETER_NUMBERED_RESERVED,
|
165
|
+
YP_ERR_PARAMETER_ORDER,
|
166
|
+
YP_ERR_PARAMETER_SPLAT_MULTI,
|
167
|
+
YP_ERR_PARAMETER_STAR,
|
168
|
+
YP_ERR_PARAMETER_WILD_LOOSE_COMMA,
|
169
|
+
YP_ERR_PATTERN_EXPRESSION_AFTER_BRACKET,
|
170
|
+
YP_ERR_PATTERN_EXPRESSION_AFTER_HROCKET,
|
171
|
+
YP_ERR_PATTERN_EXPRESSION_AFTER_COMMA,
|
172
|
+
YP_ERR_PATTERN_EXPRESSION_AFTER_IN,
|
173
|
+
YP_ERR_PATTERN_EXPRESSION_AFTER_KEY,
|
174
|
+
YP_ERR_PATTERN_EXPRESSION_AFTER_PAREN,
|
175
|
+
YP_ERR_PATTERN_EXPRESSION_AFTER_PIN,
|
176
|
+
YP_ERR_PATTERN_EXPRESSION_AFTER_PIPE,
|
177
|
+
YP_ERR_PATTERN_EXPRESSION_AFTER_RANGE,
|
178
|
+
YP_ERR_PATTERN_HASH_KEY,
|
179
|
+
YP_ERR_PATTERN_HASH_KEY_LABEL,
|
180
|
+
YP_ERR_PATTERN_IDENT_AFTER_HROCKET,
|
181
|
+
YP_ERR_PATTERN_LABEL_AFTER_COMMA,
|
182
|
+
YP_ERR_PATTERN_REST,
|
183
|
+
YP_ERR_PATTERN_TERM_BRACE,
|
184
|
+
YP_ERR_PATTERN_TERM_BRACKET,
|
185
|
+
YP_ERR_PATTERN_TERM_PAREN,
|
186
|
+
YP_ERR_PIPEPIPEEQ_MULTI_ASSIGN,
|
187
|
+
YP_ERR_REGEXP_TERM,
|
188
|
+
YP_ERR_RESCUE_EXPRESSION,
|
189
|
+
YP_ERR_RESCUE_MODIFIER_VALUE,
|
190
|
+
YP_ERR_RESCUE_TERM,
|
191
|
+
YP_ERR_RESCUE_VARIABLE,
|
192
|
+
YP_ERR_RETURN_INVALID,
|
193
|
+
YP_ERR_STRING_CONCATENATION,
|
194
|
+
YP_ERR_STRING_INTERPOLATED_TERM,
|
195
|
+
YP_ERR_STRING_LITERAL_TERM,
|
196
|
+
YP_ERR_SYMBOL_INVALID,
|
197
|
+
YP_ERR_SYMBOL_TERM_DYNAMIC,
|
198
|
+
YP_ERR_SYMBOL_TERM_INTERPOLATED,
|
199
|
+
YP_ERR_TERNARY_COLON,
|
200
|
+
YP_ERR_TERNARY_EXPRESSION_FALSE,
|
201
|
+
YP_ERR_TERNARY_EXPRESSION_TRUE,
|
202
|
+
YP_ERR_UNARY_RECEIVER_BANG,
|
203
|
+
YP_ERR_UNARY_RECEIVER_MINUS,
|
204
|
+
YP_ERR_UNARY_RECEIVER_PLUS,
|
205
|
+
YP_ERR_UNARY_RECEIVER_TILDE,
|
206
|
+
YP_ERR_UNDEF_ARGUMENT,
|
207
|
+
YP_ERR_UNTIL_TERM,
|
208
|
+
YP_ERR_WHILE_TERM,
|
209
|
+
YP_ERR_WRITE_TARGET_READONLY,
|
210
|
+
YP_ERR_WRITE_TARGET_UNEXPECTED,
|
211
|
+
YP_ERR_XSTRING_TERM,
|
212
|
+
YP_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS,
|
213
|
+
YP_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS,
|
214
|
+
YP_WARN_AMBIGUOUS_PREFIX_STAR,
|
215
|
+
YP_WARN_AMBIGUOUS_SLASH,
|
216
|
+
/* This must be the last member. */
|
217
|
+
YP_DIAGNOSTIC_ID_LEN,
|
218
|
+
} yp_diagnostic_id_t;
|
219
|
+
|
18
220
|
// Append a diagnostic to the given list of diagnostics.
|
19
|
-
bool yp_diagnostic_list_append(yp_list_t *list, const uint8_t *start, const uint8_t *end,
|
221
|
+
bool yp_diagnostic_list_append(yp_list_t *list, const uint8_t *start, const uint8_t *end, yp_diagnostic_id_t diag_id);
|
20
222
|
|
21
223
|
// Deallocate the internal state of the given diagnostic list.
|
22
224
|
void yp_diagnostic_list_free(yp_list_t *list);
|
@@ -60,7 +60,7 @@ size_t yp_encoding_utf_8_alnum_char(const uint8_t *b, ptrdiff_t n);
|
|
60
60
|
|
61
61
|
// This lookup table is referenced in both the UTF-8 encoding file and the
|
62
62
|
// parser directly in order to speed up the default encoding processing.
|
63
|
-
extern uint8_t yp_encoding_unicode_table[256];
|
63
|
+
extern const uint8_t yp_encoding_unicode_table[256];
|
64
64
|
|
65
65
|
// These are the encodings that are supported by the parser. They are defined in
|
66
66
|
// their own files in the src/enc directory.
|
data/include/yarp/node.h
CHANGED
@@ -4,9 +4,6 @@
|
|
4
4
|
#include "yarp/defines.h"
|
5
5
|
#include "yarp/parser.h"
|
6
6
|
|
7
|
-
// Append a token to the given list.
|
8
|
-
void yp_location_list_append(yp_location_list_t *list, const yp_token_t *token);
|
9
|
-
|
10
7
|
// Append a new node onto the end of the node list.
|
11
8
|
void yp_node_list_append(yp_node_list_t *list, yp_node_t *node);
|
12
9
|
|
@@ -31,7 +28,6 @@ YP_EXPORTED_FUNCTION void yp_node_memsize(yp_node_t *node, yp_memsize_t *memsize
|
|
31
28
|
YP_EXPORTED_FUNCTION const char * yp_node_type_to_str(yp_node_type_t node_type);
|
32
29
|
|
33
30
|
#define YP_EMPTY_NODE_LIST ((yp_node_list_t) { .nodes = NULL, .size = 0, .capacity = 0 })
|
34
|
-
#define YP_EMPTY_LOCATION_LIST ((yp_location_list_t) { .locations = NULL, .size = 0, .capacity = 0 })
|
35
31
|
|
36
32
|
#endif // YARP_NODE_H
|
37
33
|
|
data/include/yarp/parser.h
CHANGED
@@ -277,12 +277,22 @@ typedef struct yp_scope {
|
|
277
277
|
// The IDs of the locals in the given scope.
|
278
278
|
yp_constant_id_list_t locals;
|
279
279
|
|
280
|
+
// A pointer to the previous scope in the linked list.
|
281
|
+
struct yp_scope *previous;
|
282
|
+
|
280
283
|
// A boolean indicating whether or not this scope can see into its parent.
|
281
284
|
// If closed is true, then the scope cannot see into its parent.
|
282
285
|
bool closed;
|
283
286
|
|
284
|
-
// A
|
285
|
-
|
287
|
+
// A boolean indicating whether or not this scope has explicit parameters.
|
288
|
+
// This is necessary to determine whether or not numbered parameters are
|
289
|
+
// allowed.
|
290
|
+
bool explicit_params;
|
291
|
+
|
292
|
+
// A boolean indicating whether or not this scope has numbered parameters.
|
293
|
+
// This is necessary to determine if child blocks are allowed to use
|
294
|
+
// numbered parameters.
|
295
|
+
bool numbered_params;
|
286
296
|
} yp_scope_t;
|
287
297
|
|
288
298
|
// This struct represents the overall parser. It contains a reference to the
|
@@ -291,7 +301,6 @@ typedef struct yp_scope {
|
|
291
301
|
// it's considering.
|
292
302
|
struct yp_parser {
|
293
303
|
yp_lex_state_t lex_state; // the current state of the lexer
|
294
|
-
bool command_start; // whether or not we're at the beginning of a command
|
295
304
|
int enclosure_nesting; // tracks the current nesting of (), [], and {}
|
296
305
|
|
297
306
|
// Used to temporarily track the nesting of enclosures to determine if a {
|
@@ -338,17 +347,11 @@ struct yp_parser {
|
|
338
347
|
yp_scope_t *current_scope; // the current local scope
|
339
348
|
|
340
349
|
yp_context_node_t *current_context; // the current parsing context
|
341
|
-
bool recovering; // whether or not we're currently recovering from a syntax error
|
342
350
|
|
343
351
|
// The encoding functions for the current file is attached to the parser as
|
344
352
|
// it's parsing so that it can change with a magic comment.
|
345
353
|
yp_encoding_t encoding;
|
346
354
|
|
347
|
-
// Whether or not the encoding has been changed by a magic comment. We use
|
348
|
-
// this to provide a fast path for the lexer instead of going through the
|
349
|
-
// function pointer.
|
350
|
-
bool encoding_changed;
|
351
|
-
|
352
355
|
// When the encoding that is being used to parse the source is changed by
|
353
356
|
// YARP, we provide the ability here to call out to a user-defined function.
|
354
357
|
yp_encoding_changed_callback_t encoding_changed_callback;
|
@@ -367,13 +370,6 @@ struct yp_parser {
|
|
367
370
|
// be called whenever a new token is lexed by the parser.
|
368
371
|
yp_lex_callback_t *lex_callback;
|
369
372
|
|
370
|
-
// This flag indicates that we are currently parsing a pattern matching
|
371
|
-
// expression and impacts that calculation of newlines.
|
372
|
-
bool pattern_matching_newlines;
|
373
|
-
|
374
|
-
// This flag indicates that we are currently parsing a keyword argument.
|
375
|
-
bool in_keyword_arg;
|
376
|
-
|
377
373
|
// This is the path of the file being parsed
|
378
374
|
// We use the filepath when constructing SourceFileNodes
|
379
375
|
yp_string_t filepath_string;
|
@@ -384,6 +380,38 @@ struct yp_parser {
|
|
384
380
|
|
385
381
|
// This is the list of newline offsets in the source file.
|
386
382
|
yp_newline_list_t newline_list;
|
383
|
+
|
384
|
+
// We want to add a flag to integer nodes that indicates their base. We only
|
385
|
+
// want to parse these once, but we don't have space on the token itself to
|
386
|
+
// communicate this information. So we store it here and pass it through
|
387
|
+
// when we find tokens that we need it for.
|
388
|
+
yp_node_flags_t integer_base;
|
389
|
+
|
390
|
+
// Whether or not we're at the beginning of a command
|
391
|
+
bool command_start;
|
392
|
+
|
393
|
+
// Whether or not we're currently recovering from a syntax error
|
394
|
+
bool recovering;
|
395
|
+
|
396
|
+
// Whether or not the encoding has been changed by a magic comment. We use
|
397
|
+
// this to provide a fast path for the lexer instead of going through the
|
398
|
+
// function pointer.
|
399
|
+
bool encoding_changed;
|
400
|
+
|
401
|
+
// This flag indicates that we are currently parsing a pattern matching
|
402
|
+
// expression and impacts that calculation of newlines.
|
403
|
+
bool pattern_matching_newlines;
|
404
|
+
|
405
|
+
// This flag indicates that we are currently parsing a keyword argument.
|
406
|
+
bool in_keyword_arg;
|
407
|
+
|
408
|
+
// Whether or not the parser has seen a token that has semantic meaning
|
409
|
+
// (i.e., a token that is not a comment or whitespace).
|
410
|
+
bool semantic_token_seen;
|
411
|
+
|
412
|
+
// Whether or not we have found a frozen_string_literal magic comment with
|
413
|
+
// a true value.
|
414
|
+
bool frozen_string_literal;
|
387
415
|
};
|
388
416
|
|
389
417
|
#endif // YARP_PARSER_H
|
data/include/yarp/util/yp_char.h
CHANGED
@@ -15,7 +15,7 @@ size_t yp_strspn_whitespace(const uint8_t *string, ptrdiff_t length);
|
|
15
15
|
// whitespace while also tracking the location of each newline. Disallows
|
16
16
|
// searching past the given maximum number of characters.
|
17
17
|
size_t
|
18
|
-
yp_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, yp_newline_list_t *newline_list
|
18
|
+
yp_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, yp_newline_list_t *newline_list);
|
19
19
|
|
20
20
|
// Returns the number of characters at the start of the string that are inline
|
21
21
|
// whitespace. Disallows searching past the given maximum number of characters.
|
@@ -31,19 +31,31 @@ size_t yp_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length);
|
|
31
31
|
size_t yp_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length);
|
32
32
|
|
33
33
|
// Returns the number of characters at the start of the string that are octal
|
34
|
-
// digits or underscores.
|
34
|
+
// digits or underscores. Disallows searching past the given maximum number of
|
35
35
|
// characters.
|
36
|
-
|
36
|
+
//
|
37
|
+
// If multiple underscores are found in a row or if an underscore is
|
38
|
+
// found at the end of the number, then the invalid pointer is set to the index
|
39
|
+
// of the first invalid underscore.
|
40
|
+
size_t yp_strspn_octal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
|
37
41
|
|
38
42
|
// Returns the number of characters at the start of the string that are decimal
|
39
43
|
// digits or underscores. Disallows searching past the given maximum number of
|
40
44
|
// characters.
|
41
|
-
|
45
|
+
//
|
46
|
+
// If multiple underscores are found in a row or if an underscore is
|
47
|
+
// found at the end of the number, then the invalid pointer is set to the index
|
48
|
+
// of the first invalid underscore.
|
49
|
+
size_t yp_strspn_decimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
|
42
50
|
|
43
51
|
// Returns the number of characters at the start of the string that are
|
44
52
|
// hexadecimal digits or underscores. Disallows searching past the given maximum
|
45
53
|
// number of characters.
|
46
|
-
|
54
|
+
//
|
55
|
+
// If multiple underscores are found in a row or if an underscore is
|
56
|
+
// found at the end of the number, then the invalid pointer is set to the index
|
57
|
+
// of the first invalid underscore.
|
58
|
+
size_t yp_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
|
47
59
|
|
48
60
|
// Returns the number of characters at the start of the string that are regexp
|
49
61
|
// options. Disallows searching past the given maximum number of characters.
|
@@ -52,7 +64,11 @@ size_t yp_strspn_regexp_option(const uint8_t *string, ptrdiff_t length);
|
|
52
64
|
// Returns the number of characters at the start of the string that are binary
|
53
65
|
// digits or underscores. Disallows searching past the given maximum number of
|
54
66
|
// characters.
|
55
|
-
|
67
|
+
//
|
68
|
+
// If multiple underscores are found in a row or if an underscore is
|
69
|
+
// found at the end of the number, then the invalid pointer is set to the index
|
70
|
+
// of the first invalid underscore.
|
71
|
+
size_t yp_strspn_binary_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
|
56
72
|
|
57
73
|
// Returns true if the given character is a whitespace character.
|
58
74
|
bool yp_char_is_whitespace(const uint8_t b);
|
@@ -8,6 +8,7 @@
|
|
8
8
|
|
9
9
|
#include "yarp/defines.h"
|
10
10
|
|
11
|
+
#include <assert.h>
|
11
12
|
#include <stdbool.h>
|
12
13
|
#include <stdint.h>
|
13
14
|
#include <stdlib.h>
|
@@ -39,7 +40,8 @@ size_t yp_constant_id_list_memsize(yp_constant_id_list_t *list);
|
|
39
40
|
void yp_constant_id_list_free(yp_constant_id_list_t *list);
|
40
41
|
|
41
42
|
typedef struct {
|
42
|
-
|
43
|
+
unsigned int id: 31;
|
44
|
+
bool owned: 1;
|
43
45
|
const uint8_t *start;
|
44
46
|
size_t length;
|
45
47
|
size_t hash;
|
@@ -57,9 +59,14 @@ typedef struct {
|
|
57
59
|
// Initialize a new constant pool with a given capacity.
|
58
60
|
bool yp_constant_pool_init(yp_constant_pool_t *pool, size_t capacity);
|
59
61
|
|
60
|
-
// Insert a constant into a constant pool
|
61
|
-
// if any potential calls to resize fail.
|
62
|
-
yp_constant_id_t
|
62
|
+
// Insert a constant into a constant pool that is a slice of a source string.
|
63
|
+
// Returns the id of the constant, or 0 if any potential calls to resize fail.
|
64
|
+
yp_constant_id_t yp_constant_pool_insert_shared(yp_constant_pool_t *pool, const uint8_t *start, size_t length);
|
65
|
+
|
66
|
+
// Insert a constant into a constant pool from memory that is now owned by the
|
67
|
+
// constant pool. Returns the id of the constant, or 0 if any potential calls to
|
68
|
+
// resize fail.
|
69
|
+
yp_constant_id_t yp_constant_pool_insert_owned(yp_constant_pool_t *pool, const uint8_t *start, size_t length);
|
63
70
|
|
64
71
|
// Free the memory associated with a constant pool.
|
65
72
|
void yp_constant_pool_free(yp_constant_pool_t *pool);
|
data/include/yarp/version.h
CHANGED
data/lib/yarp/desugar_visitor.rb
CHANGED
@@ -8,7 +8,7 @@ module YARP
|
|
8
8
|
#
|
9
9
|
# @@foo && @@foo = bar
|
10
10
|
def visit_class_variable_and_write_node(node)
|
11
|
-
desugar_and_write_node(node, ClassVariableReadNode, ClassVariableWriteNode,
|
11
|
+
desugar_and_write_node(node, ClassVariableReadNode, ClassVariableWriteNode, node.name)
|
12
12
|
end
|
13
13
|
|
14
14
|
# @@foo ||= bar
|
@@ -17,7 +17,7 @@ module YARP
|
|
17
17
|
#
|
18
18
|
# defined?(@@foo) ? @@foo : @@foo = bar
|
19
19
|
def visit_class_variable_or_write_node(node)
|
20
|
-
desugar_or_write_defined_node(node, ClassVariableReadNode, ClassVariableWriteNode,
|
20
|
+
desugar_or_write_defined_node(node, ClassVariableReadNode, ClassVariableWriteNode, node.name)
|
21
21
|
end
|
22
22
|
|
23
23
|
# @@foo += bar
|
@@ -26,7 +26,7 @@ module YARP
|
|
26
26
|
#
|
27
27
|
# @@foo = @@foo + bar
|
28
28
|
def visit_class_variable_operator_write_node(node)
|
29
|
-
desugar_operator_write_node(node, ClassVariableReadNode, ClassVariableWriteNode,
|
29
|
+
desugar_operator_write_node(node, ClassVariableReadNode, ClassVariableWriteNode, node.name)
|
30
30
|
end
|
31
31
|
|
32
32
|
# Foo &&= bar
|
@@ -35,7 +35,7 @@ module YARP
|
|
35
35
|
#
|
36
36
|
# Foo && Foo = bar
|
37
37
|
def visit_constant_and_write_node(node)
|
38
|
-
desugar_and_write_node(node, ConstantReadNode, ConstantWriteNode)
|
38
|
+
desugar_and_write_node(node, ConstantReadNode, ConstantWriteNode, node.name)
|
39
39
|
end
|
40
40
|
|
41
41
|
# Foo ||= bar
|
@@ -44,7 +44,7 @@ module YARP
|
|
44
44
|
#
|
45
45
|
# defined?(Foo) ? Foo : Foo = bar
|
46
46
|
def visit_constant_or_write_node(node)
|
47
|
-
desugar_or_write_defined_node(node, ConstantReadNode, ConstantWriteNode)
|
47
|
+
desugar_or_write_defined_node(node, ConstantReadNode, ConstantWriteNode, node.name)
|
48
48
|
end
|
49
49
|
|
50
50
|
# Foo += bar
|
@@ -53,7 +53,7 @@ module YARP
|
|
53
53
|
#
|
54
54
|
# Foo = Foo + bar
|
55
55
|
def visit_constant_operator_write_node(node)
|
56
|
-
desugar_operator_write_node(node, ConstantReadNode, ConstantWriteNode)
|
56
|
+
desugar_operator_write_node(node, ConstantReadNode, ConstantWriteNode, node.name)
|
57
57
|
end
|
58
58
|
|
59
59
|
# $foo &&= bar
|
@@ -62,7 +62,7 @@ module YARP
|
|
62
62
|
#
|
63
63
|
# $foo && $foo = bar
|
64
64
|
def visit_global_variable_and_write_node(node)
|
65
|
-
desugar_and_write_node(node, GlobalVariableReadNode, GlobalVariableWriteNode)
|
65
|
+
desugar_and_write_node(node, GlobalVariableReadNode, GlobalVariableWriteNode, node.name)
|
66
66
|
end
|
67
67
|
|
68
68
|
# $foo ||= bar
|
@@ -71,7 +71,7 @@ module YARP
|
|
71
71
|
#
|
72
72
|
# defined?($foo) ? $foo : $foo = bar
|
73
73
|
def visit_global_variable_or_write_node(node)
|
74
|
-
desugar_or_write_defined_node(node, GlobalVariableReadNode, GlobalVariableWriteNode)
|
74
|
+
desugar_or_write_defined_node(node, GlobalVariableReadNode, GlobalVariableWriteNode, node.name)
|
75
75
|
end
|
76
76
|
|
77
77
|
# $foo += bar
|
@@ -80,7 +80,7 @@ module YARP
|
|
80
80
|
#
|
81
81
|
# $foo = $foo + bar
|
82
82
|
def visit_global_variable_operator_write_node(node)
|
83
|
-
desugar_operator_write_node(node, GlobalVariableReadNode, GlobalVariableWriteNode)
|
83
|
+
desugar_operator_write_node(node, GlobalVariableReadNode, GlobalVariableWriteNode, node.name)
|
84
84
|
end
|
85
85
|
|
86
86
|
# @foo &&= bar
|
@@ -89,7 +89,7 @@ module YARP
|
|
89
89
|
#
|
90
90
|
# @foo && @foo = bar
|
91
91
|
def visit_instance_variable_and_write_node(node)
|
92
|
-
desugar_and_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode,
|
92
|
+
desugar_and_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode, node.name)
|
93
93
|
end
|
94
94
|
|
95
95
|
# @foo ||= bar
|
@@ -98,7 +98,7 @@ module YARP
|
|
98
98
|
#
|
99
99
|
# @foo || @foo = bar
|
100
100
|
def visit_instance_variable_or_write_node(node)
|
101
|
-
desugar_or_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode,
|
101
|
+
desugar_or_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode, node.name)
|
102
102
|
end
|
103
103
|
|
104
104
|
# @foo += bar
|
@@ -107,7 +107,7 @@ module YARP
|
|
107
107
|
#
|
108
108
|
# @foo = @foo + bar
|
109
109
|
def visit_instance_variable_operator_write_node(node)
|
110
|
-
desugar_operator_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode,
|
110
|
+
desugar_operator_write_node(node, InstanceVariableReadNode, InstanceVariableWriteNode, node.name)
|
111
111
|
end
|
112
112
|
|
113
113
|
# foo &&= bar
|
@@ -116,7 +116,7 @@ module YARP
|
|
116
116
|
#
|
117
117
|
# foo && foo = bar
|
118
118
|
def visit_local_variable_and_write_node(node)
|
119
|
-
desugar_and_write_node(node, LocalVariableReadNode, LocalVariableWriteNode,
|
119
|
+
desugar_and_write_node(node, LocalVariableReadNode, LocalVariableWriteNode, node.name, node.depth)
|
120
120
|
end
|
121
121
|
|
122
122
|
# foo ||= bar
|
@@ -125,7 +125,7 @@ module YARP
|
|
125
125
|
#
|
126
126
|
# foo || foo = bar
|
127
127
|
def visit_local_variable_or_write_node(node)
|
128
|
-
desugar_or_write_node(node, LocalVariableReadNode, LocalVariableWriteNode,
|
128
|
+
desugar_or_write_node(node, LocalVariableReadNode, LocalVariableWriteNode, node.name, node.depth)
|
129
129
|
end
|
130
130
|
|
131
131
|
# foo += bar
|
@@ -134,13 +134,13 @@ module YARP
|
|
134
134
|
#
|
135
135
|
# foo = foo + bar
|
136
136
|
def visit_local_variable_operator_write_node(node)
|
137
|
-
desugar_operator_write_node(node, LocalVariableReadNode, LocalVariableWriteNode,
|
137
|
+
desugar_operator_write_node(node, LocalVariableReadNode, LocalVariableWriteNode, node.name, node.depth)
|
138
138
|
end
|
139
139
|
|
140
140
|
private
|
141
141
|
|
142
142
|
# Desugar `x &&= y` to `x && x = y`
|
143
|
-
def desugar_and_write_node(node, read_class, write_class, arguments
|
143
|
+
def desugar_and_write_node(node, read_class, write_class, *arguments)
|
144
144
|
AndNode.new(
|
145
145
|
read_class.new(*arguments, node.name_loc),
|
146
146
|
write_class.new(*arguments, node.name_loc, node.value, node.operator_loc, node.location),
|
@@ -150,7 +150,7 @@ module YARP
|
|
150
150
|
end
|
151
151
|
|
152
152
|
# Desugar `x += y` to `x = x + y`
|
153
|
-
def desugar_operator_write_node(node, read_class, write_class, arguments
|
153
|
+
def desugar_operator_write_node(node, read_class, write_class, *arguments)
|
154
154
|
write_class.new(
|
155
155
|
*arguments,
|
156
156
|
node.name_loc,
|
@@ -172,7 +172,7 @@ module YARP
|
|
172
172
|
end
|
173
173
|
|
174
174
|
# Desugar `x ||= y` to `x || x = y`
|
175
|
-
def desugar_or_write_node(node, read_class, write_class, arguments
|
175
|
+
def desugar_or_write_node(node, read_class, write_class, *arguments)
|
176
176
|
OrNode.new(
|
177
177
|
read_class.new(*arguments, node.name_loc),
|
178
178
|
write_class.new(*arguments, node.name_loc, node.value, node.operator_loc, node.location),
|
@@ -182,7 +182,7 @@ module YARP
|
|
182
182
|
end
|
183
183
|
|
184
184
|
# Desugar `x ||= y` to `defined?(x) ? x : x = y`
|
185
|
-
def desugar_or_write_defined_node(node, read_class, write_class, arguments
|
185
|
+
def desugar_or_write_defined_node(node, read_class, write_class, *arguments)
|
186
186
|
IfNode.new(
|
187
187
|
node.operator_loc,
|
188
188
|
DefinedNode.new(nil, read_class.new(*arguments, node.name_loc), nil, node.operator_loc, node.name_loc),
|