prism 0.19.0 → 0.24.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +102 -1
- data/Makefile +5 -0
- data/README.md +9 -6
- data/config.yml +236 -38
- data/docs/build_system.md +19 -2
- data/docs/cruby_compilation.md +27 -0
- data/docs/parser_translation.md +34 -0
- data/docs/parsing_rules.md +19 -0
- data/docs/releasing.md +84 -16
- data/docs/ruby_api.md +1 -1
- data/docs/ruby_parser_translation.md +19 -0
- data/docs/serialization.md +19 -5
- data/ext/prism/api_node.c +1989 -1525
- data/ext/prism/extension.c +130 -30
- data/ext/prism/extension.h +2 -2
- data/include/prism/ast.h +1700 -505
- data/include/prism/defines.h +8 -0
- data/include/prism/diagnostic.h +49 -7
- data/include/prism/encoding.h +17 -0
- data/include/prism/options.h +40 -14
- data/include/prism/parser.h +34 -18
- data/include/prism/util/pm_buffer.h +9 -0
- data/include/prism/util/pm_constant_pool.h +18 -0
- data/include/prism/util/pm_newline_list.h +4 -14
- data/include/prism/util/pm_strpbrk.h +4 -1
- data/include/prism/version.h +2 -2
- data/include/prism.h +19 -2
- data/lib/prism/debug.rb +11 -5
- data/lib/prism/desugar_compiler.rb +225 -80
- data/lib/prism/dot_visitor.rb +36 -14
- data/lib/prism/dsl.rb +302 -299
- data/lib/prism/ffi.rb +107 -76
- data/lib/prism/lex_compat.rb +17 -1
- data/lib/prism/node.rb +4580 -2607
- data/lib/prism/node_ext.rb +27 -4
- data/lib/prism/parse_result.rb +75 -29
- data/lib/prism/serialize.rb +633 -305
- data/lib/prism/translation/parser/compiler.rb +1838 -0
- data/lib/prism/translation/parser/lexer.rb +335 -0
- data/lib/prism/translation/parser/rubocop.rb +45 -0
- data/lib/prism/translation/parser.rb +190 -0
- data/lib/prism/translation/parser33.rb +12 -0
- data/lib/prism/translation/parser34.rb +12 -0
- data/lib/prism/translation/ripper.rb +696 -0
- data/lib/prism/translation/ruby_parser.rb +1521 -0
- data/lib/prism/translation.rb +11 -0
- data/lib/prism.rb +1 -1
- data/prism.gemspec +18 -7
- data/rbi/prism.rbi +150 -88
- data/rbi/prism_static.rbi +15 -3
- data/sig/prism.rbs +996 -961
- data/sig/prism_static.rbs +123 -46
- data/src/diagnostic.c +264 -219
- data/src/encoding.c +21 -26
- data/src/node.c +2 -6
- data/src/options.c +29 -5
- data/src/prettyprint.c +176 -44
- data/src/prism.c +1499 -564
- data/src/serialize.c +35 -21
- data/src/token_type.c +353 -4
- data/src/util/pm_buffer.c +11 -0
- data/src/util/pm_constant_pool.c +37 -11
- data/src/util/pm_newline_list.c +6 -15
- data/src/util/pm_string.c +0 -7
- data/src/util/pm_strpbrk.c +122 -14
- metadata +16 -5
- data/docs/building.md +0 -29
- data/lib/prism/ripper_compat.rb +0 -207
data/include/prism/defines.h
CHANGED
@@ -16,6 +16,14 @@
|
|
16
16
|
#include <stdio.h>
|
17
17
|
#include <string.h>
|
18
18
|
|
19
|
+
/**
|
20
|
+
* We want to be able to use the PRI* macros for printing out integers, but on
|
21
|
+
* some platforms they aren't included unless this is already defined.
|
22
|
+
*/
|
23
|
+
#define __STDC_FORMAT_MACROS
|
24
|
+
|
25
|
+
#include <inttypes.h>
|
26
|
+
|
19
27
|
/**
|
20
28
|
* By default, we compile with -fvisibility=hidden. When this is enabled, we
|
21
29
|
* need to mark certain functions as being publically-visible. This macro does
|
data/include/prism/diagnostic.h
CHANGED
@@ -14,6 +14,28 @@
|
|
14
14
|
#include <stdlib.h>
|
15
15
|
#include <assert.h>
|
16
16
|
|
17
|
+
/**
|
18
|
+
* The levels of errors generated during parsing.
|
19
|
+
*/
|
20
|
+
typedef enum {
|
21
|
+
/** For errors that cannot be recovered from. */
|
22
|
+
PM_ERROR_LEVEL_FATAL = 0,
|
23
|
+
|
24
|
+
/** For errors that should raise an argument error. */
|
25
|
+
PM_ERROR_LEVEL_ARGUMENT = 1
|
26
|
+
} pm_error_level_t;
|
27
|
+
|
28
|
+
/**
|
29
|
+
* The levels of warnings generated during parsing.
|
30
|
+
*/
|
31
|
+
typedef enum {
|
32
|
+
/** For warnings which should be emitted if $VERBOSE != nil. */
|
33
|
+
PM_WARNING_LEVEL_DEFAULT = 0,
|
34
|
+
|
35
|
+
/** For warnings which should be emitted if $VERBOSE == true. */
|
36
|
+
PM_WARNING_LEVEL_VERBOSE = 1
|
37
|
+
} pm_warning_level_t;
|
38
|
+
|
17
39
|
/**
|
18
40
|
* This struct represents a diagnostic generated during parsing.
|
19
41
|
*
|
@@ -35,6 +57,12 @@ typedef struct {
|
|
35
57
|
* diagnostic is freed.
|
36
58
|
*/
|
37
59
|
bool owned;
|
60
|
+
|
61
|
+
/**
|
62
|
+
* The level of the diagnostic, see `pm_error_level_t` and
|
63
|
+
* `pm_warning_level_t` for possible values.
|
64
|
+
*/
|
65
|
+
uint8_t level;
|
38
66
|
} pm_diagnostic_t;
|
39
67
|
|
40
68
|
/**
|
@@ -42,17 +70,24 @@ typedef struct {
|
|
42
70
|
* of errors between the parser and the user.
|
43
71
|
*/
|
44
72
|
typedef enum {
|
73
|
+
// This is a special error that we can potentially replace by others. For
|
74
|
+
// an example of how this is used, see parse_expression_prefix.
|
75
|
+
PM_ERR_CANNOT_PARSE_EXPRESSION,
|
76
|
+
|
77
|
+
// These are the error codes.
|
45
78
|
PM_ERR_ALIAS_ARGUMENT,
|
46
79
|
PM_ERR_AMPAMPEQ_MULTI_ASSIGN,
|
47
80
|
PM_ERR_ARGUMENT_AFTER_BLOCK,
|
48
81
|
PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES,
|
49
82
|
PM_ERR_ARGUMENT_BARE_HASH,
|
83
|
+
PM_ERR_ARGUMENT_BLOCK_FORWARDING,
|
50
84
|
PM_ERR_ARGUMENT_BLOCK_MULTI,
|
51
85
|
PM_ERR_ARGUMENT_FORMAL_CLASS,
|
52
86
|
PM_ERR_ARGUMENT_FORMAL_CONSTANT,
|
53
87
|
PM_ERR_ARGUMENT_FORMAL_GLOBAL,
|
54
88
|
PM_ERR_ARGUMENT_FORMAL_IVAR,
|
55
89
|
PM_ERR_ARGUMENT_FORWARDING_UNBOUND,
|
90
|
+
PM_ERR_ARGUMENT_IN,
|
56
91
|
PM_ERR_ARGUMENT_NO_FORWARDING_AMP,
|
57
92
|
PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES,
|
58
93
|
PM_ERR_ARGUMENT_NO_FORWARDING_STAR,
|
@@ -74,7 +109,6 @@ typedef enum {
|
|
74
109
|
PM_ERR_BLOCK_PARAM_PIPE_TERM,
|
75
110
|
PM_ERR_BLOCK_TERM_BRACE,
|
76
111
|
PM_ERR_BLOCK_TERM_END,
|
77
|
-
PM_ERR_CANNOT_PARSE_EXPRESSION,
|
78
112
|
PM_ERR_CANNOT_PARSE_STRING_PART,
|
79
113
|
PM_ERR_CASE_EXPRESSION_AFTER_CASE,
|
80
114
|
PM_ERR_CASE_EXPRESSION_AFTER_WHEN,
|
@@ -162,9 +196,12 @@ typedef enum {
|
|
162
196
|
PM_ERR_INVALID_NUMBER_HEXADECIMAL,
|
163
197
|
PM_ERR_INVALID_NUMBER_OCTAL,
|
164
198
|
PM_ERR_INVALID_NUMBER_UNDERSCORE,
|
199
|
+
PM_ERR_INVALID_CHARACTER,
|
200
|
+
PM_ERR_INVALID_MULTIBYTE_CHARACTER,
|
201
|
+
PM_ERR_INVALID_PRINTABLE_CHARACTER,
|
165
202
|
PM_ERR_INVALID_PERCENT,
|
166
|
-
PM_ERR_INVALID_TOKEN,
|
167
203
|
PM_ERR_INVALID_VARIABLE_GLOBAL,
|
204
|
+
PM_ERR_IT_NOT_ALLOWED,
|
168
205
|
PM_ERR_LAMBDA_OPEN,
|
169
206
|
PM_ERR_LAMBDA_TERM_BRACE,
|
170
207
|
PM_ERR_LAMBDA_TERM_END,
|
@@ -182,7 +219,9 @@ typedef enum {
|
|
182
219
|
PM_ERR_MODULE_NAME,
|
183
220
|
PM_ERR_MODULE_TERM,
|
184
221
|
PM_ERR_MULTI_ASSIGN_MULTI_SPLATS,
|
222
|
+
PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST,
|
185
223
|
PM_ERR_NOT_EXPRESSION,
|
224
|
+
PM_ERR_NO_LOCAL_VARIABLE,
|
186
225
|
PM_ERR_NUMBER_LITERAL_UNDERSCORE,
|
187
226
|
PM_ERR_NUMBERED_PARAMETER_NOT_ALLOWED,
|
188
227
|
PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE,
|
@@ -227,12 +266,14 @@ typedef enum {
|
|
227
266
|
PM_ERR_RESCUE_TERM,
|
228
267
|
PM_ERR_RESCUE_VARIABLE,
|
229
268
|
PM_ERR_RETURN_INVALID,
|
269
|
+
PM_ERR_SINGLETON_FOR_LITERALS,
|
230
270
|
PM_ERR_STATEMENT_ALIAS,
|
231
271
|
PM_ERR_STATEMENT_POSTEXE_END,
|
232
272
|
PM_ERR_STATEMENT_PREEXE_BEGIN,
|
233
273
|
PM_ERR_STATEMENT_UNDEF,
|
234
274
|
PM_ERR_STRING_CONCATENATION,
|
235
275
|
PM_ERR_STRING_INTERPOLATED_TERM,
|
276
|
+
PM_ERR_STRING_LITERAL_EOF,
|
236
277
|
PM_ERR_STRING_LITERAL_TERM,
|
237
278
|
PM_ERR_SYMBOL_INVALID,
|
238
279
|
PM_ERR_SYMBOL_TERM_DYNAMIC,
|
@@ -240,10 +281,9 @@ typedef enum {
|
|
240
281
|
PM_ERR_TERNARY_COLON,
|
241
282
|
PM_ERR_TERNARY_EXPRESSION_FALSE,
|
242
283
|
PM_ERR_TERNARY_EXPRESSION_TRUE,
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
PM_ERR_UNARY_RECEIVER_TILDE,
|
284
|
+
PM_ERR_UNARY_RECEIVER,
|
285
|
+
PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT,
|
286
|
+
PM_ERR_UNEXPECTED_TOKEN_IGNORE,
|
247
287
|
PM_ERR_UNDEF_ARGUMENT,
|
248
288
|
PM_ERR_UNTIL_TERM,
|
249
289
|
PM_ERR_VOID_EXPRESSION,
|
@@ -252,13 +292,15 @@ typedef enum {
|
|
252
292
|
PM_ERR_WRITE_TARGET_READONLY,
|
253
293
|
PM_ERR_WRITE_TARGET_UNEXPECTED,
|
254
294
|
PM_ERR_XSTRING_TERM,
|
295
|
+
|
296
|
+
// These are the warning codes.
|
255
297
|
PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS,
|
256
298
|
PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS,
|
257
299
|
PM_WARN_AMBIGUOUS_PREFIX_STAR,
|
258
300
|
PM_WARN_AMBIGUOUS_SLASH,
|
259
301
|
PM_WARN_END_IN_METHOD,
|
260
302
|
|
261
|
-
|
303
|
+
// This is the number of diagnostic codes.
|
262
304
|
PM_DIAGNOSTIC_ID_LEN,
|
263
305
|
} pm_diagnostic_id_t;
|
264
306
|
|
data/include/prism/encoding.h
CHANGED
@@ -79,6 +79,16 @@ typedef struct {
|
|
79
79
|
*/
|
80
80
|
#define PRISM_ENCODING_UPPERCASE_BIT 1 << 2
|
81
81
|
|
82
|
+
/**
|
83
|
+
* Return the size of the next character in the UTF-8 encoding.
|
84
|
+
*
|
85
|
+
* @param b The bytes to read.
|
86
|
+
* @param n The number of bytes that can be read.
|
87
|
+
* @returns The number of bytes that the next character takes if it is valid in
|
88
|
+
* the encoding, or 0 if it is not.
|
89
|
+
*/
|
90
|
+
size_t pm_encoding_utf_8_char_width(const uint8_t *b, ptrdiff_t n);
|
91
|
+
|
82
92
|
/**
|
83
93
|
* Return the size of the next character in the UTF-8 encoding if it is an
|
84
94
|
* alphabetical character.
|
@@ -235,6 +245,13 @@ extern const pm_encoding_t pm_encodings[PM_ENCODING_MAXIMUM];
|
|
235
245
|
*/
|
236
246
|
#define PM_ENCODING_US_ASCII_ENTRY (&pm_encodings[PM_ENCODING_US_ASCII])
|
237
247
|
|
248
|
+
/**
|
249
|
+
* This is the ASCII-8BIT encoding. We need a reference to it so that pm_strpbrk
|
250
|
+
* can compare against it because invalid multibyte characters are not a thing
|
251
|
+
* in this encoding.
|
252
|
+
*/
|
253
|
+
#define PM_ENCODING_ASCII_8BIT_ENTRY (&pm_encodings[PM_ENCODING_ASCII_8BIT])
|
254
|
+
|
238
255
|
/**
|
239
256
|
* Parse the given name of an encoding and return a pointer to the corresponding
|
240
257
|
* encoding struct if one can be found, otherwise return NULL.
|
data/include/prism/options.h
CHANGED
@@ -24,6 +24,19 @@ typedef struct pm_options_scope {
|
|
24
24
|
pm_string_t *locals;
|
25
25
|
} pm_options_scope_t;
|
26
26
|
|
27
|
+
/**
|
28
|
+
* The version of Ruby syntax that we should be parsing with. This is used to
|
29
|
+
* allow consumers to specify which behavior they want in case they need to
|
30
|
+
* parse in the same way as a specific version of CRuby would have.
|
31
|
+
*/
|
32
|
+
typedef enum {
|
33
|
+
/** The current version of prism. */
|
34
|
+
PM_OPTIONS_VERSION_LATEST = 0,
|
35
|
+
|
36
|
+
/** The vendored version of prism in CRuby 3.3.0. */
|
37
|
+
PM_OPTIONS_VERSION_CRUBY_3_3_0 = 1
|
38
|
+
} pm_options_version_t;
|
39
|
+
|
27
40
|
/**
|
28
41
|
* The options that can be passed to the parser.
|
29
42
|
*/
|
@@ -33,7 +46,7 @@ typedef struct {
|
|
33
46
|
|
34
47
|
/**
|
35
48
|
* The line within the file that the parse starts on. This value is
|
36
|
-
*
|
49
|
+
* 1-indexed.
|
37
50
|
*/
|
38
51
|
int32_t line;
|
39
52
|
|
@@ -51,19 +64,20 @@ typedef struct {
|
|
51
64
|
/**
|
52
65
|
* The scopes surrounding the code that is being parsed. For most parses
|
53
66
|
* this will be NULL, but for evals it will be the locals that are in scope
|
54
|
-
* surrounding the eval.
|
67
|
+
* surrounding the eval. Scopes are ordered from the outermost scope to the
|
68
|
+
* innermost one.
|
55
69
|
*/
|
56
70
|
pm_options_scope_t *scopes;
|
57
71
|
|
58
|
-
/** Whether or not the frozen string literal option has been set. */
|
59
|
-
bool frozen_string_literal;
|
60
|
-
|
61
72
|
/**
|
62
|
-
*
|
63
|
-
*
|
64
|
-
*
|
73
|
+
* The version of prism that we should be parsing with. This is used to
|
74
|
+
* allow consumers to specify which behavior they want in case they need to
|
75
|
+
* parse exactly as a specific version of CRuby.
|
65
76
|
*/
|
66
|
-
|
77
|
+
pm_options_version_t version;
|
78
|
+
|
79
|
+
/** Whether or not the frozen string literal option has been set. */
|
80
|
+
bool frozen_string_literal;
|
67
81
|
} pm_options_t;
|
68
82
|
|
69
83
|
/**
|
@@ -99,12 +113,16 @@ PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, cons
|
|
99
113
|
PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal);
|
100
114
|
|
101
115
|
/**
|
102
|
-
* Set the
|
103
|
-
*
|
104
|
-
*
|
105
|
-
*
|
116
|
+
* Set the version option on the given options struct by parsing the given
|
117
|
+
* string. If the string contains an invalid option, this returns false.
|
118
|
+
* Otherwise, it returns true.
|
119
|
+
*
|
120
|
+
* @param options The options struct to set the version on.
|
121
|
+
* @param version The version to set.
|
122
|
+
* @param length The length of the version string.
|
123
|
+
* @return Whether or not the version was parsed successfully.
|
106
124
|
*/
|
107
|
-
PRISM_EXPORTED_FUNCTION
|
125
|
+
PRISM_EXPORTED_FUNCTION bool pm_options_version_set(pm_options_t *options, const char *version, size_t length);
|
108
126
|
|
109
127
|
/**
|
110
128
|
* Allocate and zero out the scopes array on the given options struct.
|
@@ -167,9 +185,17 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
|
|
167
185
|
* | ... | the encoding bytes |
|
168
186
|
* | `1` | frozen string literal |
|
169
187
|
* | `1` | suppress warnings |
|
188
|
+
* | `1` | the version |
|
170
189
|
* | `4` | the number of scopes |
|
171
190
|
* | ... | the scopes |
|
172
191
|
*
|
192
|
+
* The version field is an enum, so it should be one of the following values:
|
193
|
+
*
|
194
|
+
* | value | version |
|
195
|
+
* | ----- | ------------------------- |
|
196
|
+
* | `0` | use the latest version of prism |
|
197
|
+
* | `1` | use the version of prism that is vendored in CRuby 3.3.0 |
|
198
|
+
*
|
173
199
|
* Each scope is layed out as follows:
|
174
200
|
*
|
175
201
|
* | # bytes | field |
|
data/include/prism/parser.h
CHANGED
@@ -9,6 +9,7 @@
|
|
9
9
|
#include "prism/ast.h"
|
10
10
|
#include "prism/defines.h"
|
11
11
|
#include "prism/encoding.h"
|
12
|
+
#include "prism/options.h"
|
12
13
|
#include "prism/util/pm_constant_pool.h"
|
13
14
|
#include "prism/util/pm_list.h"
|
14
15
|
#include "prism/util/pm_newline_list.h"
|
@@ -17,12 +18,6 @@
|
|
17
18
|
|
18
19
|
#include <stdbool.h>
|
19
20
|
|
20
|
-
// TODO: remove this by renaming the original flag
|
21
|
-
/**
|
22
|
-
* Temporary alias for the PM_NODE_FLAG_STATIC_KEYS flag.
|
23
|
-
*/
|
24
|
-
#define PM_KEYWORD_HASH_NODE_FLAGS_SYMBOL_KEYS PM_KEYWORD_HASH_NODE_FLAGS_STATIC_KEYS
|
25
|
-
|
26
21
|
/**
|
27
22
|
* This enum provides various bits that represent different kinds of states that
|
28
23
|
* the lexer can track. This is used to determine which kind of token to return
|
@@ -264,6 +259,9 @@ typedef struct pm_parser pm_parser_t;
|
|
264
259
|
* token that is understood by a parent context but not by the current context.
|
265
260
|
*/
|
266
261
|
typedef enum {
|
262
|
+
/** a null context, used for returning a value from a function */
|
263
|
+
PM_CONTEXT_NONE = 0,
|
264
|
+
|
267
265
|
/** a begin statement */
|
268
266
|
PM_CONTEXT_BEGIN,
|
269
267
|
|
@@ -471,6 +469,19 @@ typedef struct pm_scope {
|
|
471
469
|
*/
|
472
470
|
bool explicit_params;
|
473
471
|
|
472
|
+
/**
|
473
|
+
* Booleans indicating whether the parameters for this scope have declared
|
474
|
+
* forwarding parameters.
|
475
|
+
*
|
476
|
+
* For example, some combinations of:
|
477
|
+
* def foo(*); end
|
478
|
+
* def foo(**); end
|
479
|
+
* def foo(&); end
|
480
|
+
* def foo(...); end
|
481
|
+
*/
|
482
|
+
|
483
|
+
uint8_t forwarding_params;
|
484
|
+
|
474
485
|
/**
|
475
486
|
* An integer indicating the number of numbered parameters on this scope.
|
476
487
|
* This is necessary to determine if child blocks are allowed to use
|
@@ -480,6 +491,11 @@ typedef struct pm_scope {
|
|
480
491
|
uint8_t numbered_parameters;
|
481
492
|
} pm_scope_t;
|
482
493
|
|
494
|
+
static const uint8_t PM_FORWARDING_POSITIONALS = 0x1;
|
495
|
+
static const uint8_t PM_FORWARDING_KEYWORDS = 0x2;
|
496
|
+
static const uint8_t PM_FORWARDING_BLOCK = 0x4;
|
497
|
+
static const uint8_t PM_FORWARDING_ALL = 0x8;
|
498
|
+
|
483
499
|
/**
|
484
500
|
* This struct represents the overall parser. It contains a reference to the
|
485
501
|
* source file, as well as pointers that indicate where in the source it's
|
@@ -562,7 +578,11 @@ struct pm_parser {
|
|
562
578
|
/** The list of magic comments that have been found while parsing. */
|
563
579
|
pm_list_t magic_comment_list;
|
564
580
|
|
565
|
-
/**
|
581
|
+
/**
|
582
|
+
* An optional location that represents the location of the __END__ marker
|
583
|
+
* and the rest of the content of the file. This content is loaded into the
|
584
|
+
* DATA constant when the file being parsed is the main file being executed.
|
585
|
+
*/
|
566
586
|
pm_location_t data_loc;
|
567
587
|
|
568
588
|
/** The list of warnings that have been found while parsing. */
|
@@ -606,7 +626,7 @@ struct pm_parser {
|
|
606
626
|
* This is the path of the file being parsed. We use the filepath when
|
607
627
|
* constructing SourceFileNodes.
|
608
628
|
*/
|
609
|
-
pm_string_t
|
629
|
+
pm_string_t filepath;
|
610
630
|
|
611
631
|
/**
|
612
632
|
* This constant pool keeps all of the constants defined throughout the file
|
@@ -668,6 +688,12 @@ struct pm_parser {
|
|
668
688
|
*/
|
669
689
|
const pm_encoding_t *explicit_encoding;
|
670
690
|
|
691
|
+
/** The current parameter name id on parsing its default value. */
|
692
|
+
pm_constant_id_t current_param_name;
|
693
|
+
|
694
|
+
/** The version of prism that we should use to parse. */
|
695
|
+
pm_options_version_t version;
|
696
|
+
|
671
697
|
/** Whether or not we're at the beginning of a command. */
|
672
698
|
bool command_start;
|
673
699
|
|
@@ -690,9 +716,6 @@ struct pm_parser {
|
|
690
716
|
/** This flag indicates that we are currently parsing a keyword argument. */
|
691
717
|
bool in_keyword_arg;
|
692
718
|
|
693
|
-
/** The current parameter name id on parsing its default value. */
|
694
|
-
pm_constant_id_t current_param_name;
|
695
|
-
|
696
719
|
/**
|
697
720
|
* Whether or not the parser has seen a token that has semantic meaning
|
698
721
|
* (i.e., a token that is not a comment or whitespace).
|
@@ -704,13 +727,6 @@ struct pm_parser {
|
|
704
727
|
* a true value.
|
705
728
|
*/
|
706
729
|
bool frozen_string_literal;
|
707
|
-
|
708
|
-
/**
|
709
|
-
* Whether or not we should emit warnings. This will be set to false if the
|
710
|
-
* consumer of the library specified it, usually because they are parsing
|
711
|
-
* when $VERBOSE is nil.
|
712
|
-
*/
|
713
|
-
bool suppress_warnings;
|
714
730
|
};
|
715
731
|
|
716
732
|
#endif
|
@@ -128,6 +128,15 @@ void pm_buffer_append_varuint(pm_buffer_t *buffer, uint32_t value);
|
|
128
128
|
*/
|
129
129
|
void pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value);
|
130
130
|
|
131
|
+
/**
|
132
|
+
* Prepend the given string to the buffer.
|
133
|
+
*
|
134
|
+
* @param buffer The buffer to prepend to.
|
135
|
+
* @param value The string to prepend.
|
136
|
+
* @param length The length of the string to prepend.
|
137
|
+
*/
|
138
|
+
void pm_buffer_prepend_string(pm_buffer_t *buffer, const char *value, size_t length);
|
139
|
+
|
131
140
|
/**
|
132
141
|
* Concatenate one buffer onto another.
|
133
142
|
*
|
@@ -18,6 +18,13 @@
|
|
18
18
|
#include <stdlib.h>
|
19
19
|
#include <string.h>
|
20
20
|
|
21
|
+
/**
|
22
|
+
* When we allocate constants into the pool, we reserve 0 to mean that the slot
|
23
|
+
* is not yet filled. This constant is reused in other places to indicate the
|
24
|
+
* lack of a constant id.
|
25
|
+
*/
|
26
|
+
#define PM_CONSTANT_ID_UNSET 0
|
27
|
+
|
21
28
|
/**
|
22
29
|
* A constant id is a unique identifier for a constant in the constant pool.
|
23
30
|
*/
|
@@ -147,6 +154,17 @@ bool pm_constant_pool_init(pm_constant_pool_t *pool, uint32_t capacity);
|
|
147
154
|
*/
|
148
155
|
pm_constant_t * pm_constant_pool_id_to_constant(const pm_constant_pool_t *pool, pm_constant_id_t constant_id);
|
149
156
|
|
157
|
+
/**
|
158
|
+
* Find a constant in a constant pool. Returns the id of the constant, or 0 if
|
159
|
+
* the constant is not found.
|
160
|
+
*
|
161
|
+
* @param pool The pool to find the constant in.
|
162
|
+
* @param start A pointer to the start of the constant.
|
163
|
+
* @param length The length of the constant.
|
164
|
+
* @return The id of the constant.
|
165
|
+
*/
|
166
|
+
pm_constant_id_t pm_constant_pool_find(const pm_constant_pool_t *pool, const uint8_t *start, size_t length);
|
167
|
+
|
150
168
|
/**
|
151
169
|
* Insert a constant into a constant pool that is a slice of a source string.
|
152
170
|
* Returns the id of the constant, or 0 if any potential calls to resize fail.
|
@@ -44,10 +44,10 @@ typedef struct {
|
|
44
44
|
*/
|
45
45
|
typedef struct {
|
46
46
|
/** The line number. */
|
47
|
-
|
47
|
+
int32_t line;
|
48
48
|
|
49
49
|
/** The column number. */
|
50
|
-
|
50
|
+
uint32_t column;
|
51
51
|
} pm_line_column_t;
|
52
52
|
|
53
53
|
/**
|
@@ -72,17 +72,6 @@ bool pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t
|
|
72
72
|
*/
|
73
73
|
bool pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor);
|
74
74
|
|
75
|
-
/**
|
76
|
-
* Conditionally append a new offset to the newline list, if the value passed in
|
77
|
-
* is a newline.
|
78
|
-
*
|
79
|
-
* @param list The list to append to.
|
80
|
-
* @param cursor A pointer to the offset to append.
|
81
|
-
* @return True if the reallocation of the offsets succeeds (if one was
|
82
|
-
* necessary), otherwise false.
|
83
|
-
*/
|
84
|
-
bool pm_newline_list_check_append(pm_newline_list_t *list, const uint8_t *cursor);
|
85
|
-
|
86
75
|
/**
|
87
76
|
* Returns the line and column of the given offset. If the offset is not in the
|
88
77
|
* list, the line and column of the closest offset less than the given offset
|
@@ -90,9 +79,10 @@ bool pm_newline_list_check_append(pm_newline_list_t *list, const uint8_t *cursor
|
|
90
79
|
*
|
91
80
|
* @param list The list to search.
|
92
81
|
* @param cursor A pointer to the offset to search for.
|
82
|
+
* @param start_line The line to start counting from.
|
93
83
|
* @return The line and column of the given offset.
|
94
84
|
*/
|
95
|
-
pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor);
|
85
|
+
pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line);
|
96
86
|
|
97
87
|
/**
|
98
88
|
* Free the internal memory allocated for the newline list.
|
@@ -7,6 +7,7 @@
|
|
7
7
|
#define PRISM_STRPBRK_H
|
8
8
|
|
9
9
|
#include "prism/defines.h"
|
10
|
+
#include "prism/diagnostic.h"
|
10
11
|
#include "prism/parser.h"
|
11
12
|
|
12
13
|
#include <stddef.h>
|
@@ -35,9 +36,11 @@
|
|
35
36
|
* @param source The source to search.
|
36
37
|
* @param charset The charset to search for.
|
37
38
|
* @param length The maximum number of bytes to search.
|
39
|
+
* @param validate Whether to validate that the source string is valid in the
|
40
|
+
* current encoding of the parser.
|
38
41
|
* @return A pointer to the first character in the source string that is in the
|
39
42
|
* charset, or NULL if no such character exists.
|
40
43
|
*/
|
41
|
-
const uint8_t * pm_strpbrk(
|
44
|
+
const uint8_t * pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length, bool validate);
|
42
45
|
|
43
46
|
#endif
|
data/include/prism/version.h
CHANGED
@@ -14,7 +14,7 @@
|
|
14
14
|
/**
|
15
15
|
* The minor version of the Prism library as an int.
|
16
16
|
*/
|
17
|
-
#define PRISM_VERSION_MINOR
|
17
|
+
#define PRISM_VERSION_MINOR 24
|
18
18
|
|
19
19
|
/**
|
20
20
|
* The patch version of the Prism library as an int.
|
@@ -24,6 +24,6 @@
|
|
24
24
|
/**
|
25
25
|
* The version of the Prism library as a constant string.
|
26
26
|
*/
|
27
|
-
#define PRISM_VERSION "0.
|
27
|
+
#define PRISM_VERSION "0.24.0"
|
28
28
|
|
29
29
|
#endif
|
data/include/prism.h
CHANGED
@@ -168,7 +168,24 @@ PRISM_EXPORTED_FUNCTION bool pm_parse_success_p(const uint8_t *source, size_t si
|
|
168
168
|
* @param token_type The token type to convert to a string.
|
169
169
|
* @return A string representation of the given token type.
|
170
170
|
*/
|
171
|
-
PRISM_EXPORTED_FUNCTION const char *
|
171
|
+
PRISM_EXPORTED_FUNCTION const char * pm_token_type_name(pm_token_type_t token_type);
|
172
|
+
|
173
|
+
/**
|
174
|
+
* Returns the human name of the given token type.
|
175
|
+
*
|
176
|
+
* @param token_type The token type to convert to a human name.
|
177
|
+
* @return The human name of the given token type.
|
178
|
+
*/
|
179
|
+
const char * pm_token_type_human(pm_token_type_t token_type);
|
180
|
+
|
181
|
+
/**
|
182
|
+
* Format the errors on the parser into the given buffer.
|
183
|
+
*
|
184
|
+
* @param parser The parser to format the errors for.
|
185
|
+
* @param buffer The buffer to format the errors into.
|
186
|
+
* @param colorize Whether or not to colorize the errors with ANSI escape sequences.
|
187
|
+
*/
|
188
|
+
PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool colorize);
|
172
189
|
|
173
190
|
/**
|
174
191
|
* @mainpage
|
@@ -260,7 +277,7 @@ PRISM_EXPORTED_FUNCTION const char * pm_token_type_to_str(pm_token_type_t token_
|
|
260
277
|
* pm_buffer_t buffer = { 0 };
|
261
278
|
*
|
262
279
|
* pm_prettyprint(&buffer, &parser, root);
|
263
|
-
* printf("
|
280
|
+
* printf("%*.s\n", (int) buffer.length, buffer.value);
|
264
281
|
*
|
265
282
|
* pm_buffer_free(&buffer);
|
266
283
|
* pm_node_destroy(&parser, root);
|
data/lib/prism/debug.rb
CHANGED
@@ -138,12 +138,14 @@ module Prism
|
|
138
138
|
*params.keywords.grep(OptionalKeywordParameterNode).map(&:name),
|
139
139
|
]
|
140
140
|
|
141
|
+
sorted << AnonymousLocal if params.keywords.any?
|
142
|
+
|
141
143
|
if params.keyword_rest.is_a?(ForwardingParameterNode)
|
142
|
-
sorted.push(:*, :&, :"...")
|
144
|
+
sorted.push(:*, :**, :&, :"...")
|
145
|
+
elsif params.keyword_rest.is_a?(KeywordRestParameterNode)
|
146
|
+
sorted << (params.keyword_rest.name || :**)
|
143
147
|
end
|
144
148
|
|
145
|
-
sorted << AnonymousLocal if params.keywords.any?
|
146
|
-
|
147
149
|
# Recurse down the parameter tree to find any destructured
|
148
150
|
# parameters and add them after the other parameters.
|
149
151
|
param_stack = params.requireds.concat(params.posts).grep(MultiTargetNode).reverse
|
@@ -151,15 +153,19 @@ module Prism
|
|
151
153
|
case param
|
152
154
|
when MultiTargetNode
|
153
155
|
param_stack.concat(param.rights.reverse)
|
154
|
-
param_stack << param.rest
|
156
|
+
param_stack << param.rest if param.rest&.expression && !sorted.include?(param.rest.expression.name)
|
155
157
|
param_stack.concat(param.lefts.reverse)
|
156
158
|
when RequiredParameterNode
|
157
159
|
sorted << param.name
|
158
160
|
when SplatNode
|
159
|
-
sorted << param.expression.name
|
161
|
+
sorted << param.expression.name
|
160
162
|
end
|
161
163
|
end
|
162
164
|
|
165
|
+
if params.block
|
166
|
+
sorted << (params.block.name || :&)
|
167
|
+
end
|
168
|
+
|
163
169
|
names = sorted.concat(names - sorted)
|
164
170
|
end
|
165
171
|
|