prism 0.19.0 → 0.24.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +102 -1
- data/Makefile +5 -0
- data/README.md +9 -6
- data/config.yml +236 -38
- data/docs/build_system.md +19 -2
- data/docs/cruby_compilation.md +27 -0
- data/docs/parser_translation.md +34 -0
- data/docs/parsing_rules.md +19 -0
- data/docs/releasing.md +84 -16
- data/docs/ruby_api.md +1 -1
- data/docs/ruby_parser_translation.md +19 -0
- data/docs/serialization.md +19 -5
- data/ext/prism/api_node.c +1989 -1525
- data/ext/prism/extension.c +130 -30
- data/ext/prism/extension.h +2 -2
- data/include/prism/ast.h +1700 -505
- data/include/prism/defines.h +8 -0
- data/include/prism/diagnostic.h +49 -7
- data/include/prism/encoding.h +17 -0
- data/include/prism/options.h +40 -14
- data/include/prism/parser.h +34 -18
- data/include/prism/util/pm_buffer.h +9 -0
- data/include/prism/util/pm_constant_pool.h +18 -0
- data/include/prism/util/pm_newline_list.h +4 -14
- data/include/prism/util/pm_strpbrk.h +4 -1
- data/include/prism/version.h +2 -2
- data/include/prism.h +19 -2
- data/lib/prism/debug.rb +11 -5
- data/lib/prism/desugar_compiler.rb +225 -80
- data/lib/prism/dot_visitor.rb +36 -14
- data/lib/prism/dsl.rb +302 -299
- data/lib/prism/ffi.rb +107 -76
- data/lib/prism/lex_compat.rb +17 -1
- data/lib/prism/node.rb +4580 -2607
- data/lib/prism/node_ext.rb +27 -4
- data/lib/prism/parse_result.rb +75 -29
- data/lib/prism/serialize.rb +633 -305
- data/lib/prism/translation/parser/compiler.rb +1838 -0
- data/lib/prism/translation/parser/lexer.rb +335 -0
- data/lib/prism/translation/parser/rubocop.rb +45 -0
- data/lib/prism/translation/parser.rb +190 -0
- data/lib/prism/translation/parser33.rb +12 -0
- data/lib/prism/translation/parser34.rb +12 -0
- data/lib/prism/translation/ripper.rb +696 -0
- data/lib/prism/translation/ruby_parser.rb +1521 -0
- data/lib/prism/translation.rb +11 -0
- data/lib/prism.rb +1 -1
- data/prism.gemspec +18 -7
- data/rbi/prism.rbi +150 -88
- data/rbi/prism_static.rbi +15 -3
- data/sig/prism.rbs +996 -961
- data/sig/prism_static.rbs +123 -46
- data/src/diagnostic.c +264 -219
- data/src/encoding.c +21 -26
- data/src/node.c +2 -6
- data/src/options.c +29 -5
- data/src/prettyprint.c +176 -44
- data/src/prism.c +1499 -564
- data/src/serialize.c +35 -21
- data/src/token_type.c +353 -4
- data/src/util/pm_buffer.c +11 -0
- data/src/util/pm_constant_pool.c +37 -11
- data/src/util/pm_newline_list.c +6 -15
- data/src/util/pm_string.c +0 -7
- data/src/util/pm_strpbrk.c +122 -14
- metadata +16 -5
- data/docs/building.md +0 -29
- data/lib/prism/ripper_compat.rb +0 -207
data/include/prism/defines.h
CHANGED
@@ -16,6 +16,14 @@
|
|
16
16
|
#include <stdio.h>
|
17
17
|
#include <string.h>
|
18
18
|
|
19
|
+
/**
|
20
|
+
* We want to be able to use the PRI* macros for printing out integers, but on
|
21
|
+
* some platforms they aren't included unless this is already defined.
|
22
|
+
*/
|
23
|
+
#define __STDC_FORMAT_MACROS
|
24
|
+
|
25
|
+
#include <inttypes.h>
|
26
|
+
|
19
27
|
/**
|
20
28
|
* By default, we compile with -fvisibility=hidden. When this is enabled, we
|
21
29
|
* need to mark certain functions as being publically-visible. This macro does
|
data/include/prism/diagnostic.h
CHANGED
@@ -14,6 +14,28 @@
|
|
14
14
|
#include <stdlib.h>
|
15
15
|
#include <assert.h>
|
16
16
|
|
17
|
+
/**
|
18
|
+
* The levels of errors generated during parsing.
|
19
|
+
*/
|
20
|
+
typedef enum {
|
21
|
+
/** For errors that cannot be recovered from. */
|
22
|
+
PM_ERROR_LEVEL_FATAL = 0,
|
23
|
+
|
24
|
+
/** For errors that should raise an argument error. */
|
25
|
+
PM_ERROR_LEVEL_ARGUMENT = 1
|
26
|
+
} pm_error_level_t;
|
27
|
+
|
28
|
+
/**
|
29
|
+
* The levels of warnings generated during parsing.
|
30
|
+
*/
|
31
|
+
typedef enum {
|
32
|
+
/** For warnings which should be emitted if $VERBOSE != nil. */
|
33
|
+
PM_WARNING_LEVEL_DEFAULT = 0,
|
34
|
+
|
35
|
+
/** For warnings which should be emitted if $VERBOSE == true. */
|
36
|
+
PM_WARNING_LEVEL_VERBOSE = 1
|
37
|
+
} pm_warning_level_t;
|
38
|
+
|
17
39
|
/**
|
18
40
|
* This struct represents a diagnostic generated during parsing.
|
19
41
|
*
|
@@ -35,6 +57,12 @@ typedef struct {
|
|
35
57
|
* diagnostic is freed.
|
36
58
|
*/
|
37
59
|
bool owned;
|
60
|
+
|
61
|
+
/**
|
62
|
+
* The level of the diagnostic, see `pm_error_level_t` and
|
63
|
+
* `pm_warning_level_t` for possible values.
|
64
|
+
*/
|
65
|
+
uint8_t level;
|
38
66
|
} pm_diagnostic_t;
|
39
67
|
|
40
68
|
/**
|
@@ -42,17 +70,24 @@ typedef struct {
|
|
42
70
|
* of errors between the parser and the user.
|
43
71
|
*/
|
44
72
|
typedef enum {
|
73
|
+
// This is a special error that we can potentially replace by others. For
|
74
|
+
// an example of how this is used, see parse_expression_prefix.
|
75
|
+
PM_ERR_CANNOT_PARSE_EXPRESSION,
|
76
|
+
|
77
|
+
// These are the error codes.
|
45
78
|
PM_ERR_ALIAS_ARGUMENT,
|
46
79
|
PM_ERR_AMPAMPEQ_MULTI_ASSIGN,
|
47
80
|
PM_ERR_ARGUMENT_AFTER_BLOCK,
|
48
81
|
PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES,
|
49
82
|
PM_ERR_ARGUMENT_BARE_HASH,
|
83
|
+
PM_ERR_ARGUMENT_BLOCK_FORWARDING,
|
50
84
|
PM_ERR_ARGUMENT_BLOCK_MULTI,
|
51
85
|
PM_ERR_ARGUMENT_FORMAL_CLASS,
|
52
86
|
PM_ERR_ARGUMENT_FORMAL_CONSTANT,
|
53
87
|
PM_ERR_ARGUMENT_FORMAL_GLOBAL,
|
54
88
|
PM_ERR_ARGUMENT_FORMAL_IVAR,
|
55
89
|
PM_ERR_ARGUMENT_FORWARDING_UNBOUND,
|
90
|
+
PM_ERR_ARGUMENT_IN,
|
56
91
|
PM_ERR_ARGUMENT_NO_FORWARDING_AMP,
|
57
92
|
PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES,
|
58
93
|
PM_ERR_ARGUMENT_NO_FORWARDING_STAR,
|
@@ -74,7 +109,6 @@ typedef enum {
|
|
74
109
|
PM_ERR_BLOCK_PARAM_PIPE_TERM,
|
75
110
|
PM_ERR_BLOCK_TERM_BRACE,
|
76
111
|
PM_ERR_BLOCK_TERM_END,
|
77
|
-
PM_ERR_CANNOT_PARSE_EXPRESSION,
|
78
112
|
PM_ERR_CANNOT_PARSE_STRING_PART,
|
79
113
|
PM_ERR_CASE_EXPRESSION_AFTER_CASE,
|
80
114
|
PM_ERR_CASE_EXPRESSION_AFTER_WHEN,
|
@@ -162,9 +196,12 @@ typedef enum {
|
|
162
196
|
PM_ERR_INVALID_NUMBER_HEXADECIMAL,
|
163
197
|
PM_ERR_INVALID_NUMBER_OCTAL,
|
164
198
|
PM_ERR_INVALID_NUMBER_UNDERSCORE,
|
199
|
+
PM_ERR_INVALID_CHARACTER,
|
200
|
+
PM_ERR_INVALID_MULTIBYTE_CHARACTER,
|
201
|
+
PM_ERR_INVALID_PRINTABLE_CHARACTER,
|
165
202
|
PM_ERR_INVALID_PERCENT,
|
166
|
-
PM_ERR_INVALID_TOKEN,
|
167
203
|
PM_ERR_INVALID_VARIABLE_GLOBAL,
|
204
|
+
PM_ERR_IT_NOT_ALLOWED,
|
168
205
|
PM_ERR_LAMBDA_OPEN,
|
169
206
|
PM_ERR_LAMBDA_TERM_BRACE,
|
170
207
|
PM_ERR_LAMBDA_TERM_END,
|
@@ -182,7 +219,9 @@ typedef enum {
|
|
182
219
|
PM_ERR_MODULE_NAME,
|
183
220
|
PM_ERR_MODULE_TERM,
|
184
221
|
PM_ERR_MULTI_ASSIGN_MULTI_SPLATS,
|
222
|
+
PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST,
|
185
223
|
PM_ERR_NOT_EXPRESSION,
|
224
|
+
PM_ERR_NO_LOCAL_VARIABLE,
|
186
225
|
PM_ERR_NUMBER_LITERAL_UNDERSCORE,
|
187
226
|
PM_ERR_NUMBERED_PARAMETER_NOT_ALLOWED,
|
188
227
|
PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE,
|
@@ -227,12 +266,14 @@ typedef enum {
|
|
227
266
|
PM_ERR_RESCUE_TERM,
|
228
267
|
PM_ERR_RESCUE_VARIABLE,
|
229
268
|
PM_ERR_RETURN_INVALID,
|
269
|
+
PM_ERR_SINGLETON_FOR_LITERALS,
|
230
270
|
PM_ERR_STATEMENT_ALIAS,
|
231
271
|
PM_ERR_STATEMENT_POSTEXE_END,
|
232
272
|
PM_ERR_STATEMENT_PREEXE_BEGIN,
|
233
273
|
PM_ERR_STATEMENT_UNDEF,
|
234
274
|
PM_ERR_STRING_CONCATENATION,
|
235
275
|
PM_ERR_STRING_INTERPOLATED_TERM,
|
276
|
+
PM_ERR_STRING_LITERAL_EOF,
|
236
277
|
PM_ERR_STRING_LITERAL_TERM,
|
237
278
|
PM_ERR_SYMBOL_INVALID,
|
238
279
|
PM_ERR_SYMBOL_TERM_DYNAMIC,
|
@@ -240,10 +281,9 @@ typedef enum {
|
|
240
281
|
PM_ERR_TERNARY_COLON,
|
241
282
|
PM_ERR_TERNARY_EXPRESSION_FALSE,
|
242
283
|
PM_ERR_TERNARY_EXPRESSION_TRUE,
|
243
|
-
|
244
|
-
|
245
|
-
|
246
|
-
PM_ERR_UNARY_RECEIVER_TILDE,
|
284
|
+
PM_ERR_UNARY_RECEIVER,
|
285
|
+
PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT,
|
286
|
+
PM_ERR_UNEXPECTED_TOKEN_IGNORE,
|
247
287
|
PM_ERR_UNDEF_ARGUMENT,
|
248
288
|
PM_ERR_UNTIL_TERM,
|
249
289
|
PM_ERR_VOID_EXPRESSION,
|
@@ -252,13 +292,15 @@ typedef enum {
|
|
252
292
|
PM_ERR_WRITE_TARGET_READONLY,
|
253
293
|
PM_ERR_WRITE_TARGET_UNEXPECTED,
|
254
294
|
PM_ERR_XSTRING_TERM,
|
295
|
+
|
296
|
+
// These are the warning codes.
|
255
297
|
PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS,
|
256
298
|
PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS,
|
257
299
|
PM_WARN_AMBIGUOUS_PREFIX_STAR,
|
258
300
|
PM_WARN_AMBIGUOUS_SLASH,
|
259
301
|
PM_WARN_END_IN_METHOD,
|
260
302
|
|
261
|
-
|
303
|
+
// This is the number of diagnostic codes.
|
262
304
|
PM_DIAGNOSTIC_ID_LEN,
|
263
305
|
} pm_diagnostic_id_t;
|
264
306
|
|
data/include/prism/encoding.h
CHANGED
@@ -79,6 +79,16 @@ typedef struct {
|
|
79
79
|
*/
|
80
80
|
#define PRISM_ENCODING_UPPERCASE_BIT 1 << 2
|
81
81
|
|
82
|
+
/**
|
83
|
+
* Return the size of the next character in the UTF-8 encoding.
|
84
|
+
*
|
85
|
+
* @param b The bytes to read.
|
86
|
+
* @param n The number of bytes that can be read.
|
87
|
+
* @returns The number of bytes that the next character takes if it is valid in
|
88
|
+
* the encoding, or 0 if it is not.
|
89
|
+
*/
|
90
|
+
size_t pm_encoding_utf_8_char_width(const uint8_t *b, ptrdiff_t n);
|
91
|
+
|
82
92
|
/**
|
83
93
|
* Return the size of the next character in the UTF-8 encoding if it is an
|
84
94
|
* alphabetical character.
|
@@ -235,6 +245,13 @@ extern const pm_encoding_t pm_encodings[PM_ENCODING_MAXIMUM];
|
|
235
245
|
*/
|
236
246
|
#define PM_ENCODING_US_ASCII_ENTRY (&pm_encodings[PM_ENCODING_US_ASCII])
|
237
247
|
|
248
|
+
/**
|
249
|
+
* This is the ASCII-8BIT encoding. We need a reference to it so that pm_strpbrk
|
250
|
+
* can compare against it because invalid multibyte characters are not a thing
|
251
|
+
* in this encoding.
|
252
|
+
*/
|
253
|
+
#define PM_ENCODING_ASCII_8BIT_ENTRY (&pm_encodings[PM_ENCODING_ASCII_8BIT])
|
254
|
+
|
238
255
|
/**
|
239
256
|
* Parse the given name of an encoding and return a pointer to the corresponding
|
240
257
|
* encoding struct if one can be found, otherwise return NULL.
|
data/include/prism/options.h
CHANGED
@@ -24,6 +24,19 @@ typedef struct pm_options_scope {
|
|
24
24
|
pm_string_t *locals;
|
25
25
|
} pm_options_scope_t;
|
26
26
|
|
27
|
+
/**
|
28
|
+
* The version of Ruby syntax that we should be parsing with. This is used to
|
29
|
+
* allow consumers to specify which behavior they want in case they need to
|
30
|
+
* parse in the same way as a specific version of CRuby would have.
|
31
|
+
*/
|
32
|
+
typedef enum {
|
33
|
+
/** The current version of prism. */
|
34
|
+
PM_OPTIONS_VERSION_LATEST = 0,
|
35
|
+
|
36
|
+
/** The vendored version of prism in CRuby 3.3.0. */
|
37
|
+
PM_OPTIONS_VERSION_CRUBY_3_3_0 = 1
|
38
|
+
} pm_options_version_t;
|
39
|
+
|
27
40
|
/**
|
28
41
|
* The options that can be passed to the parser.
|
29
42
|
*/
|
@@ -33,7 +46,7 @@ typedef struct {
|
|
33
46
|
|
34
47
|
/**
|
35
48
|
* The line within the file that the parse starts on. This value is
|
36
|
-
*
|
49
|
+
* 1-indexed.
|
37
50
|
*/
|
38
51
|
int32_t line;
|
39
52
|
|
@@ -51,19 +64,20 @@ typedef struct {
|
|
51
64
|
/**
|
52
65
|
* The scopes surrounding the code that is being parsed. For most parses
|
53
66
|
* this will be NULL, but for evals it will be the locals that are in scope
|
54
|
-
* surrounding the eval.
|
67
|
+
* surrounding the eval. Scopes are ordered from the outermost scope to the
|
68
|
+
* innermost one.
|
55
69
|
*/
|
56
70
|
pm_options_scope_t *scopes;
|
57
71
|
|
58
|
-
/** Whether or not the frozen string literal option has been set. */
|
59
|
-
bool frozen_string_literal;
|
60
|
-
|
61
72
|
/**
|
62
|
-
*
|
63
|
-
*
|
64
|
-
*
|
73
|
+
* The version of prism that we should be parsing with. This is used to
|
74
|
+
* allow consumers to specify which behavior they want in case they need to
|
75
|
+
* parse exactly as a specific version of CRuby.
|
65
76
|
*/
|
66
|
-
|
77
|
+
pm_options_version_t version;
|
78
|
+
|
79
|
+
/** Whether or not the frozen string literal option has been set. */
|
80
|
+
bool frozen_string_literal;
|
67
81
|
} pm_options_t;
|
68
82
|
|
69
83
|
/**
|
@@ -99,12 +113,16 @@ PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, cons
|
|
99
113
|
PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal);
|
100
114
|
|
101
115
|
/**
|
102
|
-
* Set the
|
103
|
-
*
|
104
|
-
*
|
105
|
-
*
|
116
|
+
* Set the version option on the given options struct by parsing the given
|
117
|
+
* string. If the string contains an invalid option, this returns false.
|
118
|
+
* Otherwise, it returns true.
|
119
|
+
*
|
120
|
+
* @param options The options struct to set the version on.
|
121
|
+
* @param version The version to set.
|
122
|
+
* @param length The length of the version string.
|
123
|
+
* @return Whether or not the version was parsed successfully.
|
106
124
|
*/
|
107
|
-
PRISM_EXPORTED_FUNCTION
|
125
|
+
PRISM_EXPORTED_FUNCTION bool pm_options_version_set(pm_options_t *options, const char *version, size_t length);
|
108
126
|
|
109
127
|
/**
|
110
128
|
* Allocate and zero out the scopes array on the given options struct.
|
@@ -167,9 +185,17 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
|
|
167
185
|
* | ... | the encoding bytes |
|
168
186
|
* | `1` | frozen string literal |
|
169
187
|
* | `1` | suppress warnings |
|
188
|
+
* | `1` | the version |
|
170
189
|
* | `4` | the number of scopes |
|
171
190
|
* | ... | the scopes |
|
172
191
|
*
|
192
|
+
* The version field is an enum, so it should be one of the following values:
|
193
|
+
*
|
194
|
+
* | value | version |
|
195
|
+
* | ----- | ------------------------- |
|
196
|
+
* | `0` | use the latest version of prism |
|
197
|
+
* | `1` | use the version of prism that is vendored in CRuby 3.3.0 |
|
198
|
+
*
|
173
199
|
* Each scope is layed out as follows:
|
174
200
|
*
|
175
201
|
* | # bytes | field |
|
data/include/prism/parser.h
CHANGED
@@ -9,6 +9,7 @@
|
|
9
9
|
#include "prism/ast.h"
|
10
10
|
#include "prism/defines.h"
|
11
11
|
#include "prism/encoding.h"
|
12
|
+
#include "prism/options.h"
|
12
13
|
#include "prism/util/pm_constant_pool.h"
|
13
14
|
#include "prism/util/pm_list.h"
|
14
15
|
#include "prism/util/pm_newline_list.h"
|
@@ -17,12 +18,6 @@
|
|
17
18
|
|
18
19
|
#include <stdbool.h>
|
19
20
|
|
20
|
-
// TODO: remove this by renaming the original flag
|
21
|
-
/**
|
22
|
-
* Temporary alias for the PM_NODE_FLAG_STATIC_KEYS flag.
|
23
|
-
*/
|
24
|
-
#define PM_KEYWORD_HASH_NODE_FLAGS_SYMBOL_KEYS PM_KEYWORD_HASH_NODE_FLAGS_STATIC_KEYS
|
25
|
-
|
26
21
|
/**
|
27
22
|
* This enum provides various bits that represent different kinds of states that
|
28
23
|
* the lexer can track. This is used to determine which kind of token to return
|
@@ -264,6 +259,9 @@ typedef struct pm_parser pm_parser_t;
|
|
264
259
|
* token that is understood by a parent context but not by the current context.
|
265
260
|
*/
|
266
261
|
typedef enum {
|
262
|
+
/** a null context, used for returning a value from a function */
|
263
|
+
PM_CONTEXT_NONE = 0,
|
264
|
+
|
267
265
|
/** a begin statement */
|
268
266
|
PM_CONTEXT_BEGIN,
|
269
267
|
|
@@ -471,6 +469,19 @@ typedef struct pm_scope {
|
|
471
469
|
*/
|
472
470
|
bool explicit_params;
|
473
471
|
|
472
|
+
/**
|
473
|
+
* Booleans indicating whether the parameters for this scope have declared
|
474
|
+
* forwarding parameters.
|
475
|
+
*
|
476
|
+
* For example, some combinations of:
|
477
|
+
* def foo(*); end
|
478
|
+
* def foo(**); end
|
479
|
+
* def foo(&); end
|
480
|
+
* def foo(...); end
|
481
|
+
*/
|
482
|
+
|
483
|
+
uint8_t forwarding_params;
|
484
|
+
|
474
485
|
/**
|
475
486
|
* An integer indicating the number of numbered parameters on this scope.
|
476
487
|
* This is necessary to determine if child blocks are allowed to use
|
@@ -480,6 +491,11 @@ typedef struct pm_scope {
|
|
480
491
|
uint8_t numbered_parameters;
|
481
492
|
} pm_scope_t;
|
482
493
|
|
494
|
+
static const uint8_t PM_FORWARDING_POSITIONALS = 0x1;
|
495
|
+
static const uint8_t PM_FORWARDING_KEYWORDS = 0x2;
|
496
|
+
static const uint8_t PM_FORWARDING_BLOCK = 0x4;
|
497
|
+
static const uint8_t PM_FORWARDING_ALL = 0x8;
|
498
|
+
|
483
499
|
/**
|
484
500
|
* This struct represents the overall parser. It contains a reference to the
|
485
501
|
* source file, as well as pointers that indicate where in the source it's
|
@@ -562,7 +578,11 @@ struct pm_parser {
|
|
562
578
|
/** The list of magic comments that have been found while parsing. */
|
563
579
|
pm_list_t magic_comment_list;
|
564
580
|
|
565
|
-
/**
|
581
|
+
/**
|
582
|
+
* An optional location that represents the location of the __END__ marker
|
583
|
+
* and the rest of the content of the file. This content is loaded into the
|
584
|
+
* DATA constant when the file being parsed is the main file being executed.
|
585
|
+
*/
|
566
586
|
pm_location_t data_loc;
|
567
587
|
|
568
588
|
/** The list of warnings that have been found while parsing. */
|
@@ -606,7 +626,7 @@ struct pm_parser {
|
|
606
626
|
* This is the path of the file being parsed. We use the filepath when
|
607
627
|
* constructing SourceFileNodes.
|
608
628
|
*/
|
609
|
-
pm_string_t
|
629
|
+
pm_string_t filepath;
|
610
630
|
|
611
631
|
/**
|
612
632
|
* This constant pool keeps all of the constants defined throughout the file
|
@@ -668,6 +688,12 @@ struct pm_parser {
|
|
668
688
|
*/
|
669
689
|
const pm_encoding_t *explicit_encoding;
|
670
690
|
|
691
|
+
/** The current parameter name id on parsing its default value. */
|
692
|
+
pm_constant_id_t current_param_name;
|
693
|
+
|
694
|
+
/** The version of prism that we should use to parse. */
|
695
|
+
pm_options_version_t version;
|
696
|
+
|
671
697
|
/** Whether or not we're at the beginning of a command. */
|
672
698
|
bool command_start;
|
673
699
|
|
@@ -690,9 +716,6 @@ struct pm_parser {
|
|
690
716
|
/** This flag indicates that we are currently parsing a keyword argument. */
|
691
717
|
bool in_keyword_arg;
|
692
718
|
|
693
|
-
/** The current parameter name id on parsing its default value. */
|
694
|
-
pm_constant_id_t current_param_name;
|
695
|
-
|
696
719
|
/**
|
697
720
|
* Whether or not the parser has seen a token that has semantic meaning
|
698
721
|
* (i.e., a token that is not a comment or whitespace).
|
@@ -704,13 +727,6 @@ struct pm_parser {
|
|
704
727
|
* a true value.
|
705
728
|
*/
|
706
729
|
bool frozen_string_literal;
|
707
|
-
|
708
|
-
/**
|
709
|
-
* Whether or not we should emit warnings. This will be set to false if the
|
710
|
-
* consumer of the library specified it, usually because they are parsing
|
711
|
-
* when $VERBOSE is nil.
|
712
|
-
*/
|
713
|
-
bool suppress_warnings;
|
714
730
|
};
|
715
731
|
|
716
732
|
#endif
|
@@ -128,6 +128,15 @@ void pm_buffer_append_varuint(pm_buffer_t *buffer, uint32_t value);
|
|
128
128
|
*/
|
129
129
|
void pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value);
|
130
130
|
|
131
|
+
/**
|
132
|
+
* Prepend the given string to the buffer.
|
133
|
+
*
|
134
|
+
* @param buffer The buffer to prepend to.
|
135
|
+
* @param value The string to prepend.
|
136
|
+
* @param length The length of the string to prepend.
|
137
|
+
*/
|
138
|
+
void pm_buffer_prepend_string(pm_buffer_t *buffer, const char *value, size_t length);
|
139
|
+
|
131
140
|
/**
|
132
141
|
* Concatenate one buffer onto another.
|
133
142
|
*
|
@@ -18,6 +18,13 @@
|
|
18
18
|
#include <stdlib.h>
|
19
19
|
#include <string.h>
|
20
20
|
|
21
|
+
/**
|
22
|
+
* When we allocate constants into the pool, we reserve 0 to mean that the slot
|
23
|
+
* is not yet filled. This constant is reused in other places to indicate the
|
24
|
+
* lack of a constant id.
|
25
|
+
*/
|
26
|
+
#define PM_CONSTANT_ID_UNSET 0
|
27
|
+
|
21
28
|
/**
|
22
29
|
* A constant id is a unique identifier for a constant in the constant pool.
|
23
30
|
*/
|
@@ -147,6 +154,17 @@ bool pm_constant_pool_init(pm_constant_pool_t *pool, uint32_t capacity);
|
|
147
154
|
*/
|
148
155
|
pm_constant_t * pm_constant_pool_id_to_constant(const pm_constant_pool_t *pool, pm_constant_id_t constant_id);
|
149
156
|
|
157
|
+
/**
|
158
|
+
* Find a constant in a constant pool. Returns the id of the constant, or 0 if
|
159
|
+
* the constant is not found.
|
160
|
+
*
|
161
|
+
* @param pool The pool to find the constant in.
|
162
|
+
* @param start A pointer to the start of the constant.
|
163
|
+
* @param length The length of the constant.
|
164
|
+
* @return The id of the constant.
|
165
|
+
*/
|
166
|
+
pm_constant_id_t pm_constant_pool_find(const pm_constant_pool_t *pool, const uint8_t *start, size_t length);
|
167
|
+
|
150
168
|
/**
|
151
169
|
* Insert a constant into a constant pool that is a slice of a source string.
|
152
170
|
* Returns the id of the constant, or 0 if any potential calls to resize fail.
|
@@ -44,10 +44,10 @@ typedef struct {
|
|
44
44
|
*/
|
45
45
|
typedef struct {
|
46
46
|
/** The line number. */
|
47
|
-
|
47
|
+
int32_t line;
|
48
48
|
|
49
49
|
/** The column number. */
|
50
|
-
|
50
|
+
uint32_t column;
|
51
51
|
} pm_line_column_t;
|
52
52
|
|
53
53
|
/**
|
@@ -72,17 +72,6 @@ bool pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t
|
|
72
72
|
*/
|
73
73
|
bool pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor);
|
74
74
|
|
75
|
-
/**
|
76
|
-
* Conditionally append a new offset to the newline list, if the value passed in
|
77
|
-
* is a newline.
|
78
|
-
*
|
79
|
-
* @param list The list to append to.
|
80
|
-
* @param cursor A pointer to the offset to append.
|
81
|
-
* @return True if the reallocation of the offsets succeeds (if one was
|
82
|
-
* necessary), otherwise false.
|
83
|
-
*/
|
84
|
-
bool pm_newline_list_check_append(pm_newline_list_t *list, const uint8_t *cursor);
|
85
|
-
|
86
75
|
/**
|
87
76
|
* Returns the line and column of the given offset. If the offset is not in the
|
88
77
|
* list, the line and column of the closest offset less than the given offset
|
@@ -90,9 +79,10 @@ bool pm_newline_list_check_append(pm_newline_list_t *list, const uint8_t *cursor
|
|
90
79
|
*
|
91
80
|
* @param list The list to search.
|
92
81
|
* @param cursor A pointer to the offset to search for.
|
82
|
+
* @param start_line The line to start counting from.
|
93
83
|
* @return The line and column of the given offset.
|
94
84
|
*/
|
95
|
-
pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor);
|
85
|
+
pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line);
|
96
86
|
|
97
87
|
/**
|
98
88
|
* Free the internal memory allocated for the newline list.
|
@@ -7,6 +7,7 @@
|
|
7
7
|
#define PRISM_STRPBRK_H
|
8
8
|
|
9
9
|
#include "prism/defines.h"
|
10
|
+
#include "prism/diagnostic.h"
|
10
11
|
#include "prism/parser.h"
|
11
12
|
|
12
13
|
#include <stddef.h>
|
@@ -35,9 +36,11 @@
|
|
35
36
|
* @param source The source to search.
|
36
37
|
* @param charset The charset to search for.
|
37
38
|
* @param length The maximum number of bytes to search.
|
39
|
+
* @param validate Whether to validate that the source string is valid in the
|
40
|
+
* current encoding of the parser.
|
38
41
|
* @return A pointer to the first character in the source string that is in the
|
39
42
|
* charset, or NULL if no such character exists.
|
40
43
|
*/
|
41
|
-
const uint8_t * pm_strpbrk(
|
44
|
+
const uint8_t * pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length, bool validate);
|
42
45
|
|
43
46
|
#endif
|
data/include/prism/version.h
CHANGED
@@ -14,7 +14,7 @@
|
|
14
14
|
/**
|
15
15
|
* The minor version of the Prism library as an int.
|
16
16
|
*/
|
17
|
-
#define PRISM_VERSION_MINOR
|
17
|
+
#define PRISM_VERSION_MINOR 24
|
18
18
|
|
19
19
|
/**
|
20
20
|
* The patch version of the Prism library as an int.
|
@@ -24,6 +24,6 @@
|
|
24
24
|
/**
|
25
25
|
* The version of the Prism library as a constant string.
|
26
26
|
*/
|
27
|
-
#define PRISM_VERSION "0.
|
27
|
+
#define PRISM_VERSION "0.24.0"
|
28
28
|
|
29
29
|
#endif
|
data/include/prism.h
CHANGED
@@ -168,7 +168,24 @@ PRISM_EXPORTED_FUNCTION bool pm_parse_success_p(const uint8_t *source, size_t si
|
|
168
168
|
* @param token_type The token type to convert to a string.
|
169
169
|
* @return A string representation of the given token type.
|
170
170
|
*/
|
171
|
-
PRISM_EXPORTED_FUNCTION const char *
|
171
|
+
PRISM_EXPORTED_FUNCTION const char * pm_token_type_name(pm_token_type_t token_type);
|
172
|
+
|
173
|
+
/**
|
174
|
+
* Returns the human name of the given token type.
|
175
|
+
*
|
176
|
+
* @param token_type The token type to convert to a human name.
|
177
|
+
* @return The human name of the given token type.
|
178
|
+
*/
|
179
|
+
const char * pm_token_type_human(pm_token_type_t token_type);
|
180
|
+
|
181
|
+
/**
|
182
|
+
* Format the errors on the parser into the given buffer.
|
183
|
+
*
|
184
|
+
* @param parser The parser to format the errors for.
|
185
|
+
* @param buffer The buffer to format the errors into.
|
186
|
+
* @param colorize Whether or not to colorize the errors with ANSI escape sequences.
|
187
|
+
*/
|
188
|
+
PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool colorize);
|
172
189
|
|
173
190
|
/**
|
174
191
|
* @mainpage
|
@@ -260,7 +277,7 @@ PRISM_EXPORTED_FUNCTION const char * pm_token_type_to_str(pm_token_type_t token_
|
|
260
277
|
* pm_buffer_t buffer = { 0 };
|
261
278
|
*
|
262
279
|
* pm_prettyprint(&buffer, &parser, root);
|
263
|
-
* printf("
|
280
|
+
* printf("%*.s\n", (int) buffer.length, buffer.value);
|
264
281
|
*
|
265
282
|
* pm_buffer_free(&buffer);
|
266
283
|
* pm_node_destroy(&parser, root);
|
data/lib/prism/debug.rb
CHANGED
@@ -138,12 +138,14 @@ module Prism
|
|
138
138
|
*params.keywords.grep(OptionalKeywordParameterNode).map(&:name),
|
139
139
|
]
|
140
140
|
|
141
|
+
sorted << AnonymousLocal if params.keywords.any?
|
142
|
+
|
141
143
|
if params.keyword_rest.is_a?(ForwardingParameterNode)
|
142
|
-
sorted.push(:*, :&, :"...")
|
144
|
+
sorted.push(:*, :**, :&, :"...")
|
145
|
+
elsif params.keyword_rest.is_a?(KeywordRestParameterNode)
|
146
|
+
sorted << (params.keyword_rest.name || :**)
|
143
147
|
end
|
144
148
|
|
145
|
-
sorted << AnonymousLocal if params.keywords.any?
|
146
|
-
|
147
149
|
# Recurse down the parameter tree to find any destructured
|
148
150
|
# parameters and add them after the other parameters.
|
149
151
|
param_stack = params.requireds.concat(params.posts).grep(MultiTargetNode).reverse
|
@@ -151,15 +153,19 @@ module Prism
|
|
151
153
|
case param
|
152
154
|
when MultiTargetNode
|
153
155
|
param_stack.concat(param.rights.reverse)
|
154
|
-
param_stack << param.rest
|
156
|
+
param_stack << param.rest if param.rest&.expression && !sorted.include?(param.rest.expression.name)
|
155
157
|
param_stack.concat(param.lefts.reverse)
|
156
158
|
when RequiredParameterNode
|
157
159
|
sorted << param.name
|
158
160
|
when SplatNode
|
159
|
-
sorted << param.expression.name
|
161
|
+
sorted << param.expression.name
|
160
162
|
end
|
161
163
|
end
|
162
164
|
|
165
|
+
if params.block
|
166
|
+
sorted << (params.block.name || :&)
|
167
|
+
end
|
168
|
+
|
163
169
|
names = sorted.concat(names - sorted)
|
164
170
|
end
|
165
171
|
|