prism 0.19.0 → 0.24.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (69) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +102 -1
  3. data/Makefile +5 -0
  4. data/README.md +9 -6
  5. data/config.yml +236 -38
  6. data/docs/build_system.md +19 -2
  7. data/docs/cruby_compilation.md +27 -0
  8. data/docs/parser_translation.md +34 -0
  9. data/docs/parsing_rules.md +19 -0
  10. data/docs/releasing.md +84 -16
  11. data/docs/ruby_api.md +1 -1
  12. data/docs/ruby_parser_translation.md +19 -0
  13. data/docs/serialization.md +19 -5
  14. data/ext/prism/api_node.c +1989 -1525
  15. data/ext/prism/extension.c +130 -30
  16. data/ext/prism/extension.h +2 -2
  17. data/include/prism/ast.h +1700 -505
  18. data/include/prism/defines.h +8 -0
  19. data/include/prism/diagnostic.h +49 -7
  20. data/include/prism/encoding.h +17 -0
  21. data/include/prism/options.h +40 -14
  22. data/include/prism/parser.h +34 -18
  23. data/include/prism/util/pm_buffer.h +9 -0
  24. data/include/prism/util/pm_constant_pool.h +18 -0
  25. data/include/prism/util/pm_newline_list.h +4 -14
  26. data/include/prism/util/pm_strpbrk.h +4 -1
  27. data/include/prism/version.h +2 -2
  28. data/include/prism.h +19 -2
  29. data/lib/prism/debug.rb +11 -5
  30. data/lib/prism/desugar_compiler.rb +225 -80
  31. data/lib/prism/dot_visitor.rb +36 -14
  32. data/lib/prism/dsl.rb +302 -299
  33. data/lib/prism/ffi.rb +107 -76
  34. data/lib/prism/lex_compat.rb +17 -1
  35. data/lib/prism/node.rb +4580 -2607
  36. data/lib/prism/node_ext.rb +27 -4
  37. data/lib/prism/parse_result.rb +75 -29
  38. data/lib/prism/serialize.rb +633 -305
  39. data/lib/prism/translation/parser/compiler.rb +1838 -0
  40. data/lib/prism/translation/parser/lexer.rb +335 -0
  41. data/lib/prism/translation/parser/rubocop.rb +45 -0
  42. data/lib/prism/translation/parser.rb +190 -0
  43. data/lib/prism/translation/parser33.rb +12 -0
  44. data/lib/prism/translation/parser34.rb +12 -0
  45. data/lib/prism/translation/ripper.rb +696 -0
  46. data/lib/prism/translation/ruby_parser.rb +1521 -0
  47. data/lib/prism/translation.rb +11 -0
  48. data/lib/prism.rb +1 -1
  49. data/prism.gemspec +18 -7
  50. data/rbi/prism.rbi +150 -88
  51. data/rbi/prism_static.rbi +15 -3
  52. data/sig/prism.rbs +996 -961
  53. data/sig/prism_static.rbs +123 -46
  54. data/src/diagnostic.c +264 -219
  55. data/src/encoding.c +21 -26
  56. data/src/node.c +2 -6
  57. data/src/options.c +29 -5
  58. data/src/prettyprint.c +176 -44
  59. data/src/prism.c +1499 -564
  60. data/src/serialize.c +35 -21
  61. data/src/token_type.c +353 -4
  62. data/src/util/pm_buffer.c +11 -0
  63. data/src/util/pm_constant_pool.c +37 -11
  64. data/src/util/pm_newline_list.c +6 -15
  65. data/src/util/pm_string.c +0 -7
  66. data/src/util/pm_strpbrk.c +122 -14
  67. metadata +16 -5
  68. data/docs/building.md +0 -29
  69. data/lib/prism/ripper_compat.rb +0 -207
@@ -16,6 +16,14 @@
16
16
  #include <stdio.h>
17
17
  #include <string.h>
18
18
 
19
+ /**
20
+ * We want to be able to use the PRI* macros for printing out integers, but on
21
+ * some platforms they aren't included unless this is already defined.
22
+ */
23
+ #define __STDC_FORMAT_MACROS
24
+
25
+ #include <inttypes.h>
26
+
19
27
  /**
20
28
  * By default, we compile with -fvisibility=hidden. When this is enabled, we
21
29
  * need to mark certain functions as being publically-visible. This macro does
@@ -14,6 +14,28 @@
14
14
  #include <stdlib.h>
15
15
  #include <assert.h>
16
16
 
17
+ /**
18
+ * The levels of errors generated during parsing.
19
+ */
20
+ typedef enum {
21
+ /** For errors that cannot be recovered from. */
22
+ PM_ERROR_LEVEL_FATAL = 0,
23
+
24
+ /** For errors that should raise an argument error. */
25
+ PM_ERROR_LEVEL_ARGUMENT = 1
26
+ } pm_error_level_t;
27
+
28
+ /**
29
+ * The levels of warnings generated during parsing.
30
+ */
31
+ typedef enum {
32
+ /** For warnings which should be emitted if $VERBOSE != nil. */
33
+ PM_WARNING_LEVEL_DEFAULT = 0,
34
+
35
+ /** For warnings which should be emitted if $VERBOSE == true. */
36
+ PM_WARNING_LEVEL_VERBOSE = 1
37
+ } pm_warning_level_t;
38
+
17
39
  /**
18
40
  * This struct represents a diagnostic generated during parsing.
19
41
  *
@@ -35,6 +57,12 @@ typedef struct {
35
57
  * diagnostic is freed.
36
58
  */
37
59
  bool owned;
60
+
61
+ /**
62
+ * The level of the diagnostic, see `pm_error_level_t` and
63
+ * `pm_warning_level_t` for possible values.
64
+ */
65
+ uint8_t level;
38
66
  } pm_diagnostic_t;
39
67
 
40
68
  /**
@@ -42,17 +70,24 @@ typedef struct {
42
70
  * of errors between the parser and the user.
43
71
  */
44
72
  typedef enum {
73
+ // This is a special error that we can potentially replace by others. For
74
+ // an example of how this is used, see parse_expression_prefix.
75
+ PM_ERR_CANNOT_PARSE_EXPRESSION,
76
+
77
+ // These are the error codes.
45
78
  PM_ERR_ALIAS_ARGUMENT,
46
79
  PM_ERR_AMPAMPEQ_MULTI_ASSIGN,
47
80
  PM_ERR_ARGUMENT_AFTER_BLOCK,
48
81
  PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES,
49
82
  PM_ERR_ARGUMENT_BARE_HASH,
83
+ PM_ERR_ARGUMENT_BLOCK_FORWARDING,
50
84
  PM_ERR_ARGUMENT_BLOCK_MULTI,
51
85
  PM_ERR_ARGUMENT_FORMAL_CLASS,
52
86
  PM_ERR_ARGUMENT_FORMAL_CONSTANT,
53
87
  PM_ERR_ARGUMENT_FORMAL_GLOBAL,
54
88
  PM_ERR_ARGUMENT_FORMAL_IVAR,
55
89
  PM_ERR_ARGUMENT_FORWARDING_UNBOUND,
90
+ PM_ERR_ARGUMENT_IN,
56
91
  PM_ERR_ARGUMENT_NO_FORWARDING_AMP,
57
92
  PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES,
58
93
  PM_ERR_ARGUMENT_NO_FORWARDING_STAR,
@@ -74,7 +109,6 @@ typedef enum {
74
109
  PM_ERR_BLOCK_PARAM_PIPE_TERM,
75
110
  PM_ERR_BLOCK_TERM_BRACE,
76
111
  PM_ERR_BLOCK_TERM_END,
77
- PM_ERR_CANNOT_PARSE_EXPRESSION,
78
112
  PM_ERR_CANNOT_PARSE_STRING_PART,
79
113
  PM_ERR_CASE_EXPRESSION_AFTER_CASE,
80
114
  PM_ERR_CASE_EXPRESSION_AFTER_WHEN,
@@ -162,9 +196,12 @@ typedef enum {
162
196
  PM_ERR_INVALID_NUMBER_HEXADECIMAL,
163
197
  PM_ERR_INVALID_NUMBER_OCTAL,
164
198
  PM_ERR_INVALID_NUMBER_UNDERSCORE,
199
+ PM_ERR_INVALID_CHARACTER,
200
+ PM_ERR_INVALID_MULTIBYTE_CHARACTER,
201
+ PM_ERR_INVALID_PRINTABLE_CHARACTER,
165
202
  PM_ERR_INVALID_PERCENT,
166
- PM_ERR_INVALID_TOKEN,
167
203
  PM_ERR_INVALID_VARIABLE_GLOBAL,
204
+ PM_ERR_IT_NOT_ALLOWED,
168
205
  PM_ERR_LAMBDA_OPEN,
169
206
  PM_ERR_LAMBDA_TERM_BRACE,
170
207
  PM_ERR_LAMBDA_TERM_END,
@@ -182,7 +219,9 @@ typedef enum {
182
219
  PM_ERR_MODULE_NAME,
183
220
  PM_ERR_MODULE_TERM,
184
221
  PM_ERR_MULTI_ASSIGN_MULTI_SPLATS,
222
+ PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST,
185
223
  PM_ERR_NOT_EXPRESSION,
224
+ PM_ERR_NO_LOCAL_VARIABLE,
186
225
  PM_ERR_NUMBER_LITERAL_UNDERSCORE,
187
226
  PM_ERR_NUMBERED_PARAMETER_NOT_ALLOWED,
188
227
  PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE,
@@ -227,12 +266,14 @@ typedef enum {
227
266
  PM_ERR_RESCUE_TERM,
228
267
  PM_ERR_RESCUE_VARIABLE,
229
268
  PM_ERR_RETURN_INVALID,
269
+ PM_ERR_SINGLETON_FOR_LITERALS,
230
270
  PM_ERR_STATEMENT_ALIAS,
231
271
  PM_ERR_STATEMENT_POSTEXE_END,
232
272
  PM_ERR_STATEMENT_PREEXE_BEGIN,
233
273
  PM_ERR_STATEMENT_UNDEF,
234
274
  PM_ERR_STRING_CONCATENATION,
235
275
  PM_ERR_STRING_INTERPOLATED_TERM,
276
+ PM_ERR_STRING_LITERAL_EOF,
236
277
  PM_ERR_STRING_LITERAL_TERM,
237
278
  PM_ERR_SYMBOL_INVALID,
238
279
  PM_ERR_SYMBOL_TERM_DYNAMIC,
@@ -240,10 +281,9 @@ typedef enum {
240
281
  PM_ERR_TERNARY_COLON,
241
282
  PM_ERR_TERNARY_EXPRESSION_FALSE,
242
283
  PM_ERR_TERNARY_EXPRESSION_TRUE,
243
- PM_ERR_UNARY_RECEIVER_BANG,
244
- PM_ERR_UNARY_RECEIVER_MINUS,
245
- PM_ERR_UNARY_RECEIVER_PLUS,
246
- PM_ERR_UNARY_RECEIVER_TILDE,
284
+ PM_ERR_UNARY_RECEIVER,
285
+ PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT,
286
+ PM_ERR_UNEXPECTED_TOKEN_IGNORE,
247
287
  PM_ERR_UNDEF_ARGUMENT,
248
288
  PM_ERR_UNTIL_TERM,
249
289
  PM_ERR_VOID_EXPRESSION,
@@ -252,13 +292,15 @@ typedef enum {
252
292
  PM_ERR_WRITE_TARGET_READONLY,
253
293
  PM_ERR_WRITE_TARGET_UNEXPECTED,
254
294
  PM_ERR_XSTRING_TERM,
295
+
296
+ // These are the warning codes.
255
297
  PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS,
256
298
  PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS,
257
299
  PM_WARN_AMBIGUOUS_PREFIX_STAR,
258
300
  PM_WARN_AMBIGUOUS_SLASH,
259
301
  PM_WARN_END_IN_METHOD,
260
302
 
261
- /* This must be the last member. */
303
+ // This is the number of diagnostic codes.
262
304
  PM_DIAGNOSTIC_ID_LEN,
263
305
  } pm_diagnostic_id_t;
264
306
 
@@ -79,6 +79,16 @@ typedef struct {
79
79
  */
80
80
  #define PRISM_ENCODING_UPPERCASE_BIT 1 << 2
81
81
 
82
+ /**
83
+ * Return the size of the next character in the UTF-8 encoding.
84
+ *
85
+ * @param b The bytes to read.
86
+ * @param n The number of bytes that can be read.
87
+ * @returns The number of bytes that the next character takes if it is valid in
88
+ * the encoding, or 0 if it is not.
89
+ */
90
+ size_t pm_encoding_utf_8_char_width(const uint8_t *b, ptrdiff_t n);
91
+
82
92
  /**
83
93
  * Return the size of the next character in the UTF-8 encoding if it is an
84
94
  * alphabetical character.
@@ -235,6 +245,13 @@ extern const pm_encoding_t pm_encodings[PM_ENCODING_MAXIMUM];
235
245
  */
236
246
  #define PM_ENCODING_US_ASCII_ENTRY (&pm_encodings[PM_ENCODING_US_ASCII])
237
247
 
248
+ /**
249
+ * This is the ASCII-8BIT encoding. We need a reference to it so that pm_strpbrk
250
+ * can compare against it because invalid multibyte characters are not a thing
251
+ * in this encoding.
252
+ */
253
+ #define PM_ENCODING_ASCII_8BIT_ENTRY (&pm_encodings[PM_ENCODING_ASCII_8BIT])
254
+
238
255
  /**
239
256
  * Parse the given name of an encoding and return a pointer to the corresponding
240
257
  * encoding struct if one can be found, otherwise return NULL.
@@ -24,6 +24,19 @@ typedef struct pm_options_scope {
24
24
  pm_string_t *locals;
25
25
  } pm_options_scope_t;
26
26
 
27
+ /**
28
+ * The version of Ruby syntax that we should be parsing with. This is used to
29
+ * allow consumers to specify which behavior they want in case they need to
30
+ * parse in the same way as a specific version of CRuby would have.
31
+ */
32
+ typedef enum {
33
+ /** The current version of prism. */
34
+ PM_OPTIONS_VERSION_LATEST = 0,
35
+
36
+ /** The vendored version of prism in CRuby 3.3.0. */
37
+ PM_OPTIONS_VERSION_CRUBY_3_3_0 = 1
38
+ } pm_options_version_t;
39
+
27
40
  /**
28
41
  * The options that can be passed to the parser.
29
42
  */
@@ -33,7 +46,7 @@ typedef struct {
33
46
 
34
47
  /**
35
48
  * The line within the file that the parse starts on. This value is
36
- * 0-indexed.
49
+ * 1-indexed.
37
50
  */
38
51
  int32_t line;
39
52
 
@@ -51,19 +64,20 @@ typedef struct {
51
64
  /**
52
65
  * The scopes surrounding the code that is being parsed. For most parses
53
66
  * this will be NULL, but for evals it will be the locals that are in scope
54
- * surrounding the eval.
67
+ * surrounding the eval. Scopes are ordered from the outermost scope to the
68
+ * innermost one.
55
69
  */
56
70
  pm_options_scope_t *scopes;
57
71
 
58
- /** Whether or not the frozen string literal option has been set. */
59
- bool frozen_string_literal;
60
-
61
72
  /**
62
- * Whether or not we should suppress warnings. This is purposefully negated
63
- * so that the default is to not suppress warnings, which allows us to still
64
- * create an options struct with zeroed memory.
73
+ * The version of prism that we should be parsing with. This is used to
74
+ * allow consumers to specify which behavior they want in case they need to
75
+ * parse exactly as a specific version of CRuby.
65
76
  */
66
- bool suppress_warnings;
77
+ pm_options_version_t version;
78
+
79
+ /** Whether or not the frozen string literal option has been set. */
80
+ bool frozen_string_literal;
67
81
  } pm_options_t;
68
82
 
69
83
  /**
@@ -99,12 +113,16 @@ PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, cons
99
113
  PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal);
100
114
 
101
115
  /**
102
- * Set the suppress warnings option on the given options struct.
103
- *
104
- * @param options The options struct to set the suppress warnings value on.
105
- * @param suppress_warnings The suppress warnings value to set.
116
+ * Set the version option on the given options struct by parsing the given
117
+ * string. If the string contains an invalid option, this returns false.
118
+ * Otherwise, it returns true.
119
+ *
120
+ * @param options The options struct to set the version on.
121
+ * @param version The version to set.
122
+ * @param length The length of the version string.
123
+ * @return Whether or not the version was parsed successfully.
106
124
  */
107
- PRISM_EXPORTED_FUNCTION void pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings);
125
+ PRISM_EXPORTED_FUNCTION bool pm_options_version_set(pm_options_t *options, const char *version, size_t length);
108
126
 
109
127
  /**
110
128
  * Allocate and zero out the scopes array on the given options struct.
@@ -167,9 +185,17 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
167
185
  * | ... | the encoding bytes |
168
186
  * | `1` | frozen string literal |
169
187
  * | `1` | suppress warnings |
188
+ * | `1` | the version |
170
189
  * | `4` | the number of scopes |
171
190
  * | ... | the scopes |
172
191
  *
192
+ * The version field is an enum, so it should be one of the following values:
193
+ *
194
+ * | value | version |
195
+ * | ----- | ------------------------- |
196
+ * | `0` | use the latest version of prism |
197
+ * | `1` | use the version of prism that is vendored in CRuby 3.3.0 |
198
+ *
173
199
  * Each scope is layed out as follows:
174
200
  *
175
201
  * | # bytes | field |
@@ -9,6 +9,7 @@
9
9
  #include "prism/ast.h"
10
10
  #include "prism/defines.h"
11
11
  #include "prism/encoding.h"
12
+ #include "prism/options.h"
12
13
  #include "prism/util/pm_constant_pool.h"
13
14
  #include "prism/util/pm_list.h"
14
15
  #include "prism/util/pm_newline_list.h"
@@ -17,12 +18,6 @@
17
18
 
18
19
  #include <stdbool.h>
19
20
 
20
- // TODO: remove this by renaming the original flag
21
- /**
22
- * Temporary alias for the PM_NODE_FLAG_STATIC_KEYS flag.
23
- */
24
- #define PM_KEYWORD_HASH_NODE_FLAGS_SYMBOL_KEYS PM_KEYWORD_HASH_NODE_FLAGS_STATIC_KEYS
25
-
26
21
  /**
27
22
  * This enum provides various bits that represent different kinds of states that
28
23
  * the lexer can track. This is used to determine which kind of token to return
@@ -264,6 +259,9 @@ typedef struct pm_parser pm_parser_t;
264
259
  * token that is understood by a parent context but not by the current context.
265
260
  */
266
261
  typedef enum {
262
+ /** a null context, used for returning a value from a function */
263
+ PM_CONTEXT_NONE = 0,
264
+
267
265
  /** a begin statement */
268
266
  PM_CONTEXT_BEGIN,
269
267
 
@@ -471,6 +469,19 @@ typedef struct pm_scope {
471
469
  */
472
470
  bool explicit_params;
473
471
 
472
+ /**
473
+ * Booleans indicating whether the parameters for this scope have declared
474
+ * forwarding parameters.
475
+ *
476
+ * For example, some combinations of:
477
+ * def foo(*); end
478
+ * def foo(**); end
479
+ * def foo(&); end
480
+ * def foo(...); end
481
+ */
482
+
483
+ uint8_t forwarding_params;
484
+
474
485
  /**
475
486
  * An integer indicating the number of numbered parameters on this scope.
476
487
  * This is necessary to determine if child blocks are allowed to use
@@ -480,6 +491,11 @@ typedef struct pm_scope {
480
491
  uint8_t numbered_parameters;
481
492
  } pm_scope_t;
482
493
 
494
+ static const uint8_t PM_FORWARDING_POSITIONALS = 0x1;
495
+ static const uint8_t PM_FORWARDING_KEYWORDS = 0x2;
496
+ static const uint8_t PM_FORWARDING_BLOCK = 0x4;
497
+ static const uint8_t PM_FORWARDING_ALL = 0x8;
498
+
483
499
  /**
484
500
  * This struct represents the overall parser. It contains a reference to the
485
501
  * source file, as well as pointers that indicate where in the source it's
@@ -562,7 +578,11 @@ struct pm_parser {
562
578
  /** The list of magic comments that have been found while parsing. */
563
579
  pm_list_t magic_comment_list;
564
580
 
565
- /** The optional location of the __END__ keyword and its contents. */
581
+ /**
582
+ * An optional location that represents the location of the __END__ marker
583
+ * and the rest of the content of the file. This content is loaded into the
584
+ * DATA constant when the file being parsed is the main file being executed.
585
+ */
566
586
  pm_location_t data_loc;
567
587
 
568
588
  /** The list of warnings that have been found while parsing. */
@@ -606,7 +626,7 @@ struct pm_parser {
606
626
  * This is the path of the file being parsed. We use the filepath when
607
627
  * constructing SourceFileNodes.
608
628
  */
609
- pm_string_t filepath_string;
629
+ pm_string_t filepath;
610
630
 
611
631
  /**
612
632
  * This constant pool keeps all of the constants defined throughout the file
@@ -668,6 +688,12 @@ struct pm_parser {
668
688
  */
669
689
  const pm_encoding_t *explicit_encoding;
670
690
 
691
+ /** The current parameter name id on parsing its default value. */
692
+ pm_constant_id_t current_param_name;
693
+
694
+ /** The version of prism that we should use to parse. */
695
+ pm_options_version_t version;
696
+
671
697
  /** Whether or not we're at the beginning of a command. */
672
698
  bool command_start;
673
699
 
@@ -690,9 +716,6 @@ struct pm_parser {
690
716
  /** This flag indicates that we are currently parsing a keyword argument. */
691
717
  bool in_keyword_arg;
692
718
 
693
- /** The current parameter name id on parsing its default value. */
694
- pm_constant_id_t current_param_name;
695
-
696
719
  /**
697
720
  * Whether or not the parser has seen a token that has semantic meaning
698
721
  * (i.e., a token that is not a comment or whitespace).
@@ -704,13 +727,6 @@ struct pm_parser {
704
727
  * a true value.
705
728
  */
706
729
  bool frozen_string_literal;
707
-
708
- /**
709
- * Whether or not we should emit warnings. This will be set to false if the
710
- * consumer of the library specified it, usually because they are parsing
711
- * when $VERBOSE is nil.
712
- */
713
- bool suppress_warnings;
714
730
  };
715
731
 
716
732
  #endif
@@ -128,6 +128,15 @@ void pm_buffer_append_varuint(pm_buffer_t *buffer, uint32_t value);
128
128
  */
129
129
  void pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value);
130
130
 
131
+ /**
132
+ * Prepend the given string to the buffer.
133
+ *
134
+ * @param buffer The buffer to prepend to.
135
+ * @param value The string to prepend.
136
+ * @param length The length of the string to prepend.
137
+ */
138
+ void pm_buffer_prepend_string(pm_buffer_t *buffer, const char *value, size_t length);
139
+
131
140
  /**
132
141
  * Concatenate one buffer onto another.
133
142
  *
@@ -18,6 +18,13 @@
18
18
  #include <stdlib.h>
19
19
  #include <string.h>
20
20
 
21
+ /**
22
+ * When we allocate constants into the pool, we reserve 0 to mean that the slot
23
+ * is not yet filled. This constant is reused in other places to indicate the
24
+ * lack of a constant id.
25
+ */
26
+ #define PM_CONSTANT_ID_UNSET 0
27
+
21
28
  /**
22
29
  * A constant id is a unique identifier for a constant in the constant pool.
23
30
  */
@@ -147,6 +154,17 @@ bool pm_constant_pool_init(pm_constant_pool_t *pool, uint32_t capacity);
147
154
  */
148
155
  pm_constant_t * pm_constant_pool_id_to_constant(const pm_constant_pool_t *pool, pm_constant_id_t constant_id);
149
156
 
157
+ /**
158
+ * Find a constant in a constant pool. Returns the id of the constant, or 0 if
159
+ * the constant is not found.
160
+ *
161
+ * @param pool The pool to find the constant in.
162
+ * @param start A pointer to the start of the constant.
163
+ * @param length The length of the constant.
164
+ * @return The id of the constant.
165
+ */
166
+ pm_constant_id_t pm_constant_pool_find(const pm_constant_pool_t *pool, const uint8_t *start, size_t length);
167
+
150
168
  /**
151
169
  * Insert a constant into a constant pool that is a slice of a source string.
152
170
  * Returns the id of the constant, or 0 if any potential calls to resize fail.
@@ -44,10 +44,10 @@ typedef struct {
44
44
  */
45
45
  typedef struct {
46
46
  /** The line number. */
47
- size_t line;
47
+ int32_t line;
48
48
 
49
49
  /** The column number. */
50
- size_t column;
50
+ uint32_t column;
51
51
  } pm_line_column_t;
52
52
 
53
53
  /**
@@ -72,17 +72,6 @@ bool pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t
72
72
  */
73
73
  bool pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor);
74
74
 
75
- /**
76
- * Conditionally append a new offset to the newline list, if the value passed in
77
- * is a newline.
78
- *
79
- * @param list The list to append to.
80
- * @param cursor A pointer to the offset to append.
81
- * @return True if the reallocation of the offsets succeeds (if one was
82
- * necessary), otherwise false.
83
- */
84
- bool pm_newline_list_check_append(pm_newline_list_t *list, const uint8_t *cursor);
85
-
86
75
  /**
87
76
  * Returns the line and column of the given offset. If the offset is not in the
88
77
  * list, the line and column of the closest offset less than the given offset
@@ -90,9 +79,10 @@ bool pm_newline_list_check_append(pm_newline_list_t *list, const uint8_t *cursor
90
79
  *
91
80
  * @param list The list to search.
92
81
  * @param cursor A pointer to the offset to search for.
82
+ * @param start_line The line to start counting from.
93
83
  * @return The line and column of the given offset.
94
84
  */
95
- pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor);
85
+ pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line);
96
86
 
97
87
  /**
98
88
  * Free the internal memory allocated for the newline list.
@@ -7,6 +7,7 @@
7
7
  #define PRISM_STRPBRK_H
8
8
 
9
9
  #include "prism/defines.h"
10
+ #include "prism/diagnostic.h"
10
11
  #include "prism/parser.h"
11
12
 
12
13
  #include <stddef.h>
@@ -35,9 +36,11 @@
35
36
  * @param source The source to search.
36
37
  * @param charset The charset to search for.
37
38
  * @param length The maximum number of bytes to search.
39
+ * @param validate Whether to validate that the source string is valid in the
40
+ * current encoding of the parser.
38
41
  * @return A pointer to the first character in the source string that is in the
39
42
  * charset, or NULL if no such character exists.
40
43
  */
41
- const uint8_t * pm_strpbrk(const pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length);
44
+ const uint8_t * pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length, bool validate);
42
45
 
43
46
  #endif
@@ -14,7 +14,7 @@
14
14
  /**
15
15
  * The minor version of the Prism library as an int.
16
16
  */
17
- #define PRISM_VERSION_MINOR 19
17
+ #define PRISM_VERSION_MINOR 24
18
18
 
19
19
  /**
20
20
  * The patch version of the Prism library as an int.
@@ -24,6 +24,6 @@
24
24
  /**
25
25
  * The version of the Prism library as a constant string.
26
26
  */
27
- #define PRISM_VERSION "0.19.0"
27
+ #define PRISM_VERSION "0.24.0"
28
28
 
29
29
  #endif
data/include/prism.h CHANGED
@@ -168,7 +168,24 @@ PRISM_EXPORTED_FUNCTION bool pm_parse_success_p(const uint8_t *source, size_t si
168
168
  * @param token_type The token type to convert to a string.
169
169
  * @return A string representation of the given token type.
170
170
  */
171
- PRISM_EXPORTED_FUNCTION const char * pm_token_type_to_str(pm_token_type_t token_type);
171
+ PRISM_EXPORTED_FUNCTION const char * pm_token_type_name(pm_token_type_t token_type);
172
+
173
+ /**
174
+ * Returns the human name of the given token type.
175
+ *
176
+ * @param token_type The token type to convert to a human name.
177
+ * @return The human name of the given token type.
178
+ */
179
+ const char * pm_token_type_human(pm_token_type_t token_type);
180
+
181
+ /**
182
+ * Format the errors on the parser into the given buffer.
183
+ *
184
+ * @param parser The parser to format the errors for.
185
+ * @param buffer The buffer to format the errors into.
186
+ * @param colorize Whether or not to colorize the errors with ANSI escape sequences.
187
+ */
188
+ PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool colorize);
172
189
 
173
190
  /**
174
191
  * @mainpage
@@ -260,7 +277,7 @@ PRISM_EXPORTED_FUNCTION const char * pm_token_type_to_str(pm_token_type_t token_
260
277
  * pm_buffer_t buffer = { 0 };
261
278
  *
262
279
  * pm_prettyprint(&buffer, &parser, root);
263
- * printf("*.s%\n", (int) buffer.length, buffer.value);
280
+ * printf("%*.s\n", (int) buffer.length, buffer.value);
264
281
  *
265
282
  * pm_buffer_free(&buffer);
266
283
  * pm_node_destroy(&parser, root);
data/lib/prism/debug.rb CHANGED
@@ -138,12 +138,14 @@ module Prism
138
138
  *params.keywords.grep(OptionalKeywordParameterNode).map(&:name),
139
139
  ]
140
140
 
141
+ sorted << AnonymousLocal if params.keywords.any?
142
+
141
143
  if params.keyword_rest.is_a?(ForwardingParameterNode)
142
- sorted.push(:*, :&, :"...")
144
+ sorted.push(:*, :**, :&, :"...")
145
+ elsif params.keyword_rest.is_a?(KeywordRestParameterNode)
146
+ sorted << (params.keyword_rest.name || :**)
143
147
  end
144
148
 
145
- sorted << AnonymousLocal if params.keywords.any?
146
-
147
149
  # Recurse down the parameter tree to find any destructured
148
150
  # parameters and add them after the other parameters.
149
151
  param_stack = params.requireds.concat(params.posts).grep(MultiTargetNode).reverse
@@ -151,15 +153,19 @@ module Prism
151
153
  case param
152
154
  when MultiTargetNode
153
155
  param_stack.concat(param.rights.reverse)
154
- param_stack << param.rest
156
+ param_stack << param.rest if param.rest&.expression && !sorted.include?(param.rest.expression.name)
155
157
  param_stack.concat(param.lefts.reverse)
156
158
  when RequiredParameterNode
157
159
  sorted << param.name
158
160
  when SplatNode
159
- sorted << param.expression.name if param.expression
161
+ sorted << param.expression.name
160
162
  end
161
163
  end
162
164
 
165
+ if params.block
166
+ sorted << (params.block.name || :&)
167
+ end
168
+
163
169
  names = sorted.concat(names - sorted)
164
170
  end
165
171