prism 0.19.0 → 0.24.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +102 -1
  3. data/Makefile +5 -0
  4. data/README.md +9 -6
  5. data/config.yml +236 -38
  6. data/docs/build_system.md +19 -2
  7. data/docs/cruby_compilation.md +27 -0
  8. data/docs/parser_translation.md +34 -0
  9. data/docs/parsing_rules.md +19 -0
  10. data/docs/releasing.md +84 -16
  11. data/docs/ruby_api.md +1 -1
  12. data/docs/ruby_parser_translation.md +19 -0
  13. data/docs/serialization.md +19 -5
  14. data/ext/prism/api_node.c +1989 -1525
  15. data/ext/prism/extension.c +130 -30
  16. data/ext/prism/extension.h +2 -2
  17. data/include/prism/ast.h +1700 -505
  18. data/include/prism/defines.h +8 -0
  19. data/include/prism/diagnostic.h +49 -7
  20. data/include/prism/encoding.h +17 -0
  21. data/include/prism/options.h +40 -14
  22. data/include/prism/parser.h +34 -18
  23. data/include/prism/util/pm_buffer.h +9 -0
  24. data/include/prism/util/pm_constant_pool.h +18 -0
  25. data/include/prism/util/pm_newline_list.h +4 -14
  26. data/include/prism/util/pm_strpbrk.h +4 -1
  27. data/include/prism/version.h +2 -2
  28. data/include/prism.h +19 -2
  29. data/lib/prism/debug.rb +11 -5
  30. data/lib/prism/desugar_compiler.rb +225 -80
  31. data/lib/prism/dot_visitor.rb +36 -14
  32. data/lib/prism/dsl.rb +302 -299
  33. data/lib/prism/ffi.rb +107 -76
  34. data/lib/prism/lex_compat.rb +17 -1
  35. data/lib/prism/node.rb +4580 -2607
  36. data/lib/prism/node_ext.rb +27 -4
  37. data/lib/prism/parse_result.rb +75 -29
  38. data/lib/prism/serialize.rb +633 -305
  39. data/lib/prism/translation/parser/compiler.rb +1838 -0
  40. data/lib/prism/translation/parser/lexer.rb +335 -0
  41. data/lib/prism/translation/parser/rubocop.rb +45 -0
  42. data/lib/prism/translation/parser.rb +190 -0
  43. data/lib/prism/translation/parser33.rb +12 -0
  44. data/lib/prism/translation/parser34.rb +12 -0
  45. data/lib/prism/translation/ripper.rb +696 -0
  46. data/lib/prism/translation/ruby_parser.rb +1521 -0
  47. data/lib/prism/translation.rb +11 -0
  48. data/lib/prism.rb +1 -1
  49. data/prism.gemspec +18 -7
  50. data/rbi/prism.rbi +150 -88
  51. data/rbi/prism_static.rbi +15 -3
  52. data/sig/prism.rbs +996 -961
  53. data/sig/prism_static.rbs +123 -46
  54. data/src/diagnostic.c +264 -219
  55. data/src/encoding.c +21 -26
  56. data/src/node.c +2 -6
  57. data/src/options.c +29 -5
  58. data/src/prettyprint.c +176 -44
  59. data/src/prism.c +1499 -564
  60. data/src/serialize.c +35 -21
  61. data/src/token_type.c +353 -4
  62. data/src/util/pm_buffer.c +11 -0
  63. data/src/util/pm_constant_pool.c +37 -11
  64. data/src/util/pm_newline_list.c +6 -15
  65. data/src/util/pm_string.c +0 -7
  66. data/src/util/pm_strpbrk.c +122 -14
  67. metadata +16 -5
  68. data/docs/building.md +0 -29
  69. data/lib/prism/ripper_compat.rb +0 -207
@@ -16,6 +16,14 @@
16
16
  #include <stdio.h>
17
17
  #include <string.h>
18
18
 
19
+ /**
20
+ * We want to be able to use the PRI* macros for printing out integers, but on
21
+ * some platforms they aren't included unless this is already defined.
22
+ */
23
+ #define __STDC_FORMAT_MACROS
24
+
25
+ #include <inttypes.h>
26
+
19
27
  /**
20
28
  * By default, we compile with -fvisibility=hidden. When this is enabled, we
21
29
  * need to mark certain functions as being publically-visible. This macro does
@@ -14,6 +14,28 @@
14
14
  #include <stdlib.h>
15
15
  #include <assert.h>
16
16
 
17
+ /**
18
+ * The levels of errors generated during parsing.
19
+ */
20
+ typedef enum {
21
+ /** For errors that cannot be recovered from. */
22
+ PM_ERROR_LEVEL_FATAL = 0,
23
+
24
+ /** For errors that should raise an argument error. */
25
+ PM_ERROR_LEVEL_ARGUMENT = 1
26
+ } pm_error_level_t;
27
+
28
+ /**
29
+ * The levels of warnings generated during parsing.
30
+ */
31
+ typedef enum {
32
+ /** For warnings which should be emitted if $VERBOSE != nil. */
33
+ PM_WARNING_LEVEL_DEFAULT = 0,
34
+
35
+ /** For warnings which should be emitted if $VERBOSE == true. */
36
+ PM_WARNING_LEVEL_VERBOSE = 1
37
+ } pm_warning_level_t;
38
+
17
39
  /**
18
40
  * This struct represents a diagnostic generated during parsing.
19
41
  *
@@ -35,6 +57,12 @@ typedef struct {
35
57
  * diagnostic is freed.
36
58
  */
37
59
  bool owned;
60
+
61
+ /**
62
+ * The level of the diagnostic, see `pm_error_level_t` and
63
+ * `pm_warning_level_t` for possible values.
64
+ */
65
+ uint8_t level;
38
66
  } pm_diagnostic_t;
39
67
 
40
68
  /**
@@ -42,17 +70,24 @@ typedef struct {
42
70
  * of errors between the parser and the user.
43
71
  */
44
72
  typedef enum {
73
+ // This is a special error that we can potentially replace by others. For
74
+ // an example of how this is used, see parse_expression_prefix.
75
+ PM_ERR_CANNOT_PARSE_EXPRESSION,
76
+
77
+ // These are the error codes.
45
78
  PM_ERR_ALIAS_ARGUMENT,
46
79
  PM_ERR_AMPAMPEQ_MULTI_ASSIGN,
47
80
  PM_ERR_ARGUMENT_AFTER_BLOCK,
48
81
  PM_ERR_ARGUMENT_AFTER_FORWARDING_ELLIPSES,
49
82
  PM_ERR_ARGUMENT_BARE_HASH,
83
+ PM_ERR_ARGUMENT_BLOCK_FORWARDING,
50
84
  PM_ERR_ARGUMENT_BLOCK_MULTI,
51
85
  PM_ERR_ARGUMENT_FORMAL_CLASS,
52
86
  PM_ERR_ARGUMENT_FORMAL_CONSTANT,
53
87
  PM_ERR_ARGUMENT_FORMAL_GLOBAL,
54
88
  PM_ERR_ARGUMENT_FORMAL_IVAR,
55
89
  PM_ERR_ARGUMENT_FORWARDING_UNBOUND,
90
+ PM_ERR_ARGUMENT_IN,
56
91
  PM_ERR_ARGUMENT_NO_FORWARDING_AMP,
57
92
  PM_ERR_ARGUMENT_NO_FORWARDING_ELLIPSES,
58
93
  PM_ERR_ARGUMENT_NO_FORWARDING_STAR,
@@ -74,7 +109,6 @@ typedef enum {
74
109
  PM_ERR_BLOCK_PARAM_PIPE_TERM,
75
110
  PM_ERR_BLOCK_TERM_BRACE,
76
111
  PM_ERR_BLOCK_TERM_END,
77
- PM_ERR_CANNOT_PARSE_EXPRESSION,
78
112
  PM_ERR_CANNOT_PARSE_STRING_PART,
79
113
  PM_ERR_CASE_EXPRESSION_AFTER_CASE,
80
114
  PM_ERR_CASE_EXPRESSION_AFTER_WHEN,
@@ -162,9 +196,12 @@ typedef enum {
162
196
  PM_ERR_INVALID_NUMBER_HEXADECIMAL,
163
197
  PM_ERR_INVALID_NUMBER_OCTAL,
164
198
  PM_ERR_INVALID_NUMBER_UNDERSCORE,
199
+ PM_ERR_INVALID_CHARACTER,
200
+ PM_ERR_INVALID_MULTIBYTE_CHARACTER,
201
+ PM_ERR_INVALID_PRINTABLE_CHARACTER,
165
202
  PM_ERR_INVALID_PERCENT,
166
- PM_ERR_INVALID_TOKEN,
167
203
  PM_ERR_INVALID_VARIABLE_GLOBAL,
204
+ PM_ERR_IT_NOT_ALLOWED,
168
205
  PM_ERR_LAMBDA_OPEN,
169
206
  PM_ERR_LAMBDA_TERM_BRACE,
170
207
  PM_ERR_LAMBDA_TERM_END,
@@ -182,7 +219,9 @@ typedef enum {
182
219
  PM_ERR_MODULE_NAME,
183
220
  PM_ERR_MODULE_TERM,
184
221
  PM_ERR_MULTI_ASSIGN_MULTI_SPLATS,
222
+ PM_ERR_MULTI_ASSIGN_UNEXPECTED_REST,
185
223
  PM_ERR_NOT_EXPRESSION,
224
+ PM_ERR_NO_LOCAL_VARIABLE,
186
225
  PM_ERR_NUMBER_LITERAL_UNDERSCORE,
187
226
  PM_ERR_NUMBERED_PARAMETER_NOT_ALLOWED,
188
227
  PM_ERR_NUMBERED_PARAMETER_OUTER_SCOPE,
@@ -227,12 +266,14 @@ typedef enum {
227
266
  PM_ERR_RESCUE_TERM,
228
267
  PM_ERR_RESCUE_VARIABLE,
229
268
  PM_ERR_RETURN_INVALID,
269
+ PM_ERR_SINGLETON_FOR_LITERALS,
230
270
  PM_ERR_STATEMENT_ALIAS,
231
271
  PM_ERR_STATEMENT_POSTEXE_END,
232
272
  PM_ERR_STATEMENT_PREEXE_BEGIN,
233
273
  PM_ERR_STATEMENT_UNDEF,
234
274
  PM_ERR_STRING_CONCATENATION,
235
275
  PM_ERR_STRING_INTERPOLATED_TERM,
276
+ PM_ERR_STRING_LITERAL_EOF,
236
277
  PM_ERR_STRING_LITERAL_TERM,
237
278
  PM_ERR_SYMBOL_INVALID,
238
279
  PM_ERR_SYMBOL_TERM_DYNAMIC,
@@ -240,10 +281,9 @@ typedef enum {
240
281
  PM_ERR_TERNARY_COLON,
241
282
  PM_ERR_TERNARY_EXPRESSION_FALSE,
242
283
  PM_ERR_TERNARY_EXPRESSION_TRUE,
243
- PM_ERR_UNARY_RECEIVER_BANG,
244
- PM_ERR_UNARY_RECEIVER_MINUS,
245
- PM_ERR_UNARY_RECEIVER_PLUS,
246
- PM_ERR_UNARY_RECEIVER_TILDE,
284
+ PM_ERR_UNARY_RECEIVER,
285
+ PM_ERR_UNEXPECTED_TOKEN_CLOSE_CONTEXT,
286
+ PM_ERR_UNEXPECTED_TOKEN_IGNORE,
247
287
  PM_ERR_UNDEF_ARGUMENT,
248
288
  PM_ERR_UNTIL_TERM,
249
289
  PM_ERR_VOID_EXPRESSION,
@@ -252,13 +292,15 @@ typedef enum {
252
292
  PM_ERR_WRITE_TARGET_READONLY,
253
293
  PM_ERR_WRITE_TARGET_UNEXPECTED,
254
294
  PM_ERR_XSTRING_TERM,
295
+
296
+ // These are the warning codes.
255
297
  PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_MINUS,
256
298
  PM_WARN_AMBIGUOUS_FIRST_ARGUMENT_PLUS,
257
299
  PM_WARN_AMBIGUOUS_PREFIX_STAR,
258
300
  PM_WARN_AMBIGUOUS_SLASH,
259
301
  PM_WARN_END_IN_METHOD,
260
302
 
261
- /* This must be the last member. */
303
+ // This is the number of diagnostic codes.
262
304
  PM_DIAGNOSTIC_ID_LEN,
263
305
  } pm_diagnostic_id_t;
264
306
 
@@ -79,6 +79,16 @@ typedef struct {
79
79
  */
80
80
  #define PRISM_ENCODING_UPPERCASE_BIT 1 << 2
81
81
 
82
+ /**
83
+ * Return the size of the next character in the UTF-8 encoding.
84
+ *
85
+ * @param b The bytes to read.
86
+ * @param n The number of bytes that can be read.
87
+ * @returns The number of bytes that the next character takes if it is valid in
88
+ * the encoding, or 0 if it is not.
89
+ */
90
+ size_t pm_encoding_utf_8_char_width(const uint8_t *b, ptrdiff_t n);
91
+
82
92
  /**
83
93
  * Return the size of the next character in the UTF-8 encoding if it is an
84
94
  * alphabetical character.
@@ -235,6 +245,13 @@ extern const pm_encoding_t pm_encodings[PM_ENCODING_MAXIMUM];
235
245
  */
236
246
  #define PM_ENCODING_US_ASCII_ENTRY (&pm_encodings[PM_ENCODING_US_ASCII])
237
247
 
248
+ /**
249
+ * This is the ASCII-8BIT encoding. We need a reference to it so that pm_strpbrk
250
+ * can compare against it because invalid multibyte characters are not a thing
251
+ * in this encoding.
252
+ */
253
+ #define PM_ENCODING_ASCII_8BIT_ENTRY (&pm_encodings[PM_ENCODING_ASCII_8BIT])
254
+
238
255
  /**
239
256
  * Parse the given name of an encoding and return a pointer to the corresponding
240
257
  * encoding struct if one can be found, otherwise return NULL.
@@ -24,6 +24,19 @@ typedef struct pm_options_scope {
24
24
  pm_string_t *locals;
25
25
  } pm_options_scope_t;
26
26
 
27
+ /**
28
+ * The version of Ruby syntax that we should be parsing with. This is used to
29
+ * allow consumers to specify which behavior they want in case they need to
30
+ * parse in the same way as a specific version of CRuby would have.
31
+ */
32
+ typedef enum {
33
+ /** The current version of prism. */
34
+ PM_OPTIONS_VERSION_LATEST = 0,
35
+
36
+ /** The vendored version of prism in CRuby 3.3.0. */
37
+ PM_OPTIONS_VERSION_CRUBY_3_3_0 = 1
38
+ } pm_options_version_t;
39
+
27
40
  /**
28
41
  * The options that can be passed to the parser.
29
42
  */
@@ -33,7 +46,7 @@ typedef struct {
33
46
 
34
47
  /**
35
48
  * The line within the file that the parse starts on. This value is
36
- * 0-indexed.
49
+ * 1-indexed.
37
50
  */
38
51
  int32_t line;
39
52
 
@@ -51,19 +64,20 @@ typedef struct {
51
64
  /**
52
65
  * The scopes surrounding the code that is being parsed. For most parses
53
66
  * this will be NULL, but for evals it will be the locals that are in scope
54
- * surrounding the eval.
67
+ * surrounding the eval. Scopes are ordered from the outermost scope to the
68
+ * innermost one.
55
69
  */
56
70
  pm_options_scope_t *scopes;
57
71
 
58
- /** Whether or not the frozen string literal option has been set. */
59
- bool frozen_string_literal;
60
-
61
72
  /**
62
- * Whether or not we should suppress warnings. This is purposefully negated
63
- * so that the default is to not suppress warnings, which allows us to still
64
- * create an options struct with zeroed memory.
73
+ * The version of prism that we should be parsing with. This is used to
74
+ * allow consumers to specify which behavior they want in case they need to
75
+ * parse exactly as a specific version of CRuby.
65
76
  */
66
- bool suppress_warnings;
77
+ pm_options_version_t version;
78
+
79
+ /** Whether or not the frozen string literal option has been set. */
80
+ bool frozen_string_literal;
67
81
  } pm_options_t;
68
82
 
69
83
  /**
@@ -99,12 +113,16 @@ PRISM_EXPORTED_FUNCTION void pm_options_encoding_set(pm_options_t *options, cons
99
113
  PRISM_EXPORTED_FUNCTION void pm_options_frozen_string_literal_set(pm_options_t *options, bool frozen_string_literal);
100
114
 
101
115
  /**
102
- * Set the suppress warnings option on the given options struct.
103
- *
104
- * @param options The options struct to set the suppress warnings value on.
105
- * @param suppress_warnings The suppress warnings value to set.
116
+ * Set the version option on the given options struct by parsing the given
117
+ * string. If the string contains an invalid option, this returns false.
118
+ * Otherwise, it returns true.
119
+ *
120
+ * @param options The options struct to set the version on.
121
+ * @param version The version to set.
122
+ * @param length The length of the version string.
123
+ * @return Whether or not the version was parsed successfully.
106
124
  */
107
- PRISM_EXPORTED_FUNCTION void pm_options_suppress_warnings_set(pm_options_t *options, bool suppress_warnings);
125
+ PRISM_EXPORTED_FUNCTION bool pm_options_version_set(pm_options_t *options, const char *version, size_t length);
108
126
 
109
127
  /**
110
128
  * Allocate and zero out the scopes array on the given options struct.
@@ -167,9 +185,17 @@ PRISM_EXPORTED_FUNCTION void pm_options_free(pm_options_t *options);
167
185
  * | ... | the encoding bytes |
168
186
  * | `1` | frozen string literal |
169
187
  * | `1` | suppress warnings |
188
+ * | `1` | the version |
170
189
  * | `4` | the number of scopes |
171
190
  * | ... | the scopes |
172
191
  *
192
+ * The version field is an enum, so it should be one of the following values:
193
+ *
194
+ * | value | version |
195
+ * | ----- | ------------------------- |
196
+ * | `0` | use the latest version of prism |
197
+ * | `1` | use the version of prism that is vendored in CRuby 3.3.0 |
198
+ *
173
199
  * Each scope is layed out as follows:
174
200
  *
175
201
  * | # bytes | field |
@@ -9,6 +9,7 @@
9
9
  #include "prism/ast.h"
10
10
  #include "prism/defines.h"
11
11
  #include "prism/encoding.h"
12
+ #include "prism/options.h"
12
13
  #include "prism/util/pm_constant_pool.h"
13
14
  #include "prism/util/pm_list.h"
14
15
  #include "prism/util/pm_newline_list.h"
@@ -17,12 +18,6 @@
17
18
 
18
19
  #include <stdbool.h>
19
20
 
20
- // TODO: remove this by renaming the original flag
21
- /**
22
- * Temporary alias for the PM_NODE_FLAG_STATIC_KEYS flag.
23
- */
24
- #define PM_KEYWORD_HASH_NODE_FLAGS_SYMBOL_KEYS PM_KEYWORD_HASH_NODE_FLAGS_STATIC_KEYS
25
-
26
21
  /**
27
22
  * This enum provides various bits that represent different kinds of states that
28
23
  * the lexer can track. This is used to determine which kind of token to return
@@ -264,6 +259,9 @@ typedef struct pm_parser pm_parser_t;
264
259
  * token that is understood by a parent context but not by the current context.
265
260
  */
266
261
  typedef enum {
262
+ /** a null context, used for returning a value from a function */
263
+ PM_CONTEXT_NONE = 0,
264
+
267
265
  /** a begin statement */
268
266
  PM_CONTEXT_BEGIN,
269
267
 
@@ -471,6 +469,19 @@ typedef struct pm_scope {
471
469
  */
472
470
  bool explicit_params;
473
471
 
472
+ /**
473
+ * Booleans indicating whether the parameters for this scope have declared
474
+ * forwarding parameters.
475
+ *
476
+ * For example, some combinations of:
477
+ * def foo(*); end
478
+ * def foo(**); end
479
+ * def foo(&); end
480
+ * def foo(...); end
481
+ */
482
+
483
+ uint8_t forwarding_params;
484
+
474
485
  /**
475
486
  * An integer indicating the number of numbered parameters on this scope.
476
487
  * This is necessary to determine if child blocks are allowed to use
@@ -480,6 +491,11 @@ typedef struct pm_scope {
480
491
  uint8_t numbered_parameters;
481
492
  } pm_scope_t;
482
493
 
494
+ static const uint8_t PM_FORWARDING_POSITIONALS = 0x1;
495
+ static const uint8_t PM_FORWARDING_KEYWORDS = 0x2;
496
+ static const uint8_t PM_FORWARDING_BLOCK = 0x4;
497
+ static const uint8_t PM_FORWARDING_ALL = 0x8;
498
+
483
499
  /**
484
500
  * This struct represents the overall parser. It contains a reference to the
485
501
  * source file, as well as pointers that indicate where in the source it's
@@ -562,7 +578,11 @@ struct pm_parser {
562
578
  /** The list of magic comments that have been found while parsing. */
563
579
  pm_list_t magic_comment_list;
564
580
 
565
- /** The optional location of the __END__ keyword and its contents. */
581
+ /**
582
+ * An optional location that represents the location of the __END__ marker
583
+ * and the rest of the content of the file. This content is loaded into the
584
+ * DATA constant when the file being parsed is the main file being executed.
585
+ */
566
586
  pm_location_t data_loc;
567
587
 
568
588
  /** The list of warnings that have been found while parsing. */
@@ -606,7 +626,7 @@ struct pm_parser {
606
626
  * This is the path of the file being parsed. We use the filepath when
607
627
  * constructing SourceFileNodes.
608
628
  */
609
- pm_string_t filepath_string;
629
+ pm_string_t filepath;
610
630
 
611
631
  /**
612
632
  * This constant pool keeps all of the constants defined throughout the file
@@ -668,6 +688,12 @@ struct pm_parser {
668
688
  */
669
689
  const pm_encoding_t *explicit_encoding;
670
690
 
691
+ /** The current parameter name id on parsing its default value. */
692
+ pm_constant_id_t current_param_name;
693
+
694
+ /** The version of prism that we should use to parse. */
695
+ pm_options_version_t version;
696
+
671
697
  /** Whether or not we're at the beginning of a command. */
672
698
  bool command_start;
673
699
 
@@ -690,9 +716,6 @@ struct pm_parser {
690
716
  /** This flag indicates that we are currently parsing a keyword argument. */
691
717
  bool in_keyword_arg;
692
718
 
693
- /** The current parameter name id on parsing its default value. */
694
- pm_constant_id_t current_param_name;
695
-
696
719
  /**
697
720
  * Whether or not the parser has seen a token that has semantic meaning
698
721
  * (i.e., a token that is not a comment or whitespace).
@@ -704,13 +727,6 @@ struct pm_parser {
704
727
  * a true value.
705
728
  */
706
729
  bool frozen_string_literal;
707
-
708
- /**
709
- * Whether or not we should emit warnings. This will be set to false if the
710
- * consumer of the library specified it, usually because they are parsing
711
- * when $VERBOSE is nil.
712
- */
713
- bool suppress_warnings;
714
730
  };
715
731
 
716
732
  #endif
@@ -128,6 +128,15 @@ void pm_buffer_append_varuint(pm_buffer_t *buffer, uint32_t value);
128
128
  */
129
129
  void pm_buffer_append_varsint(pm_buffer_t *buffer, int32_t value);
130
130
 
131
+ /**
132
+ * Prepend the given string to the buffer.
133
+ *
134
+ * @param buffer The buffer to prepend to.
135
+ * @param value The string to prepend.
136
+ * @param length The length of the string to prepend.
137
+ */
138
+ void pm_buffer_prepend_string(pm_buffer_t *buffer, const char *value, size_t length);
139
+
131
140
  /**
132
141
  * Concatenate one buffer onto another.
133
142
  *
@@ -18,6 +18,13 @@
18
18
  #include <stdlib.h>
19
19
  #include <string.h>
20
20
 
21
+ /**
22
+ * When we allocate constants into the pool, we reserve 0 to mean that the slot
23
+ * is not yet filled. This constant is reused in other places to indicate the
24
+ * lack of a constant id.
25
+ */
26
+ #define PM_CONSTANT_ID_UNSET 0
27
+
21
28
  /**
22
29
  * A constant id is a unique identifier for a constant in the constant pool.
23
30
  */
@@ -147,6 +154,17 @@ bool pm_constant_pool_init(pm_constant_pool_t *pool, uint32_t capacity);
147
154
  */
148
155
  pm_constant_t * pm_constant_pool_id_to_constant(const pm_constant_pool_t *pool, pm_constant_id_t constant_id);
149
156
 
157
+ /**
158
+ * Find a constant in a constant pool. Returns the id of the constant, or 0 if
159
+ * the constant is not found.
160
+ *
161
+ * @param pool The pool to find the constant in.
162
+ * @param start A pointer to the start of the constant.
163
+ * @param length The length of the constant.
164
+ * @return The id of the constant.
165
+ */
166
+ pm_constant_id_t pm_constant_pool_find(const pm_constant_pool_t *pool, const uint8_t *start, size_t length);
167
+
150
168
  /**
151
169
  * Insert a constant into a constant pool that is a slice of a source string.
152
170
  * Returns the id of the constant, or 0 if any potential calls to resize fail.
@@ -44,10 +44,10 @@ typedef struct {
44
44
  */
45
45
  typedef struct {
46
46
  /** The line number. */
47
- size_t line;
47
+ int32_t line;
48
48
 
49
49
  /** The column number. */
50
- size_t column;
50
+ uint32_t column;
51
51
  } pm_line_column_t;
52
52
 
53
53
  /**
@@ -72,17 +72,6 @@ bool pm_newline_list_init(pm_newline_list_t *list, const uint8_t *start, size_t
72
72
  */
73
73
  bool pm_newline_list_append(pm_newline_list_t *list, const uint8_t *cursor);
74
74
 
75
- /**
76
- * Conditionally append a new offset to the newline list, if the value passed in
77
- * is a newline.
78
- *
79
- * @param list The list to append to.
80
- * @param cursor A pointer to the offset to append.
81
- * @return True if the reallocation of the offsets succeeds (if one was
82
- * necessary), otherwise false.
83
- */
84
- bool pm_newline_list_check_append(pm_newline_list_t *list, const uint8_t *cursor);
85
-
86
75
  /**
87
76
  * Returns the line and column of the given offset. If the offset is not in the
88
77
  * list, the line and column of the closest offset less than the given offset
@@ -90,9 +79,10 @@ bool pm_newline_list_check_append(pm_newline_list_t *list, const uint8_t *cursor
90
79
  *
91
80
  * @param list The list to search.
92
81
  * @param cursor A pointer to the offset to search for.
82
+ * @param start_line The line to start counting from.
93
83
  * @return The line and column of the given offset.
94
84
  */
95
- pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor);
85
+ pm_line_column_t pm_newline_list_line_column(const pm_newline_list_t *list, const uint8_t *cursor, int32_t start_line);
96
86
 
97
87
  /**
98
88
  * Free the internal memory allocated for the newline list.
@@ -7,6 +7,7 @@
7
7
  #define PRISM_STRPBRK_H
8
8
 
9
9
  #include "prism/defines.h"
10
+ #include "prism/diagnostic.h"
10
11
  #include "prism/parser.h"
11
12
 
12
13
  #include <stddef.h>
@@ -35,9 +36,11 @@
35
36
  * @param source The source to search.
36
37
  * @param charset The charset to search for.
37
38
  * @param length The maximum number of bytes to search.
39
+ * @param validate Whether to validate that the source string is valid in the
40
+ * current encoding of the parser.
38
41
  * @return A pointer to the first character in the source string that is in the
39
42
  * charset, or NULL if no such character exists.
40
43
  */
41
- const uint8_t * pm_strpbrk(const pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length);
44
+ const uint8_t * pm_strpbrk(pm_parser_t *parser, const uint8_t *source, const uint8_t *charset, ptrdiff_t length, bool validate);
42
45
 
43
46
  #endif
@@ -14,7 +14,7 @@
14
14
  /**
15
15
  * The minor version of the Prism library as an int.
16
16
  */
17
- #define PRISM_VERSION_MINOR 19
17
+ #define PRISM_VERSION_MINOR 24
18
18
 
19
19
  /**
20
20
  * The patch version of the Prism library as an int.
@@ -24,6 +24,6 @@
24
24
  /**
25
25
  * The version of the Prism library as a constant string.
26
26
  */
27
- #define PRISM_VERSION "0.19.0"
27
+ #define PRISM_VERSION "0.24.0"
28
28
 
29
29
  #endif
data/include/prism.h CHANGED
@@ -168,7 +168,24 @@ PRISM_EXPORTED_FUNCTION bool pm_parse_success_p(const uint8_t *source, size_t si
168
168
  * @param token_type The token type to convert to a string.
169
169
  * @return A string representation of the given token type.
170
170
  */
171
- PRISM_EXPORTED_FUNCTION const char * pm_token_type_to_str(pm_token_type_t token_type);
171
+ PRISM_EXPORTED_FUNCTION const char * pm_token_type_name(pm_token_type_t token_type);
172
+
173
+ /**
174
+ * Returns the human name of the given token type.
175
+ *
176
+ * @param token_type The token type to convert to a human name.
177
+ * @return The human name of the given token type.
178
+ */
179
+ const char * pm_token_type_human(pm_token_type_t token_type);
180
+
181
+ /**
182
+ * Format the errors on the parser into the given buffer.
183
+ *
184
+ * @param parser The parser to format the errors for.
185
+ * @param buffer The buffer to format the errors into.
186
+ * @param colorize Whether or not to colorize the errors with ANSI escape sequences.
187
+ */
188
+ PRISM_EXPORTED_FUNCTION void pm_parser_errors_format(const pm_parser_t *parser, pm_buffer_t *buffer, bool colorize);
172
189
 
173
190
  /**
174
191
  * @mainpage
@@ -260,7 +277,7 @@ PRISM_EXPORTED_FUNCTION const char * pm_token_type_to_str(pm_token_type_t token_
260
277
  * pm_buffer_t buffer = { 0 };
261
278
  *
262
279
  * pm_prettyprint(&buffer, &parser, root);
263
- * printf("*.s%\n", (int) buffer.length, buffer.value);
280
+ * printf("%*.s\n", (int) buffer.length, buffer.value);
264
281
  *
265
282
  * pm_buffer_free(&buffer);
266
283
  * pm_node_destroy(&parser, root);
data/lib/prism/debug.rb CHANGED
@@ -138,12 +138,14 @@ module Prism
138
138
  *params.keywords.grep(OptionalKeywordParameterNode).map(&:name),
139
139
  ]
140
140
 
141
+ sorted << AnonymousLocal if params.keywords.any?
142
+
141
143
  if params.keyword_rest.is_a?(ForwardingParameterNode)
142
- sorted.push(:*, :&, :"...")
144
+ sorted.push(:*, :**, :&, :"...")
145
+ elsif params.keyword_rest.is_a?(KeywordRestParameterNode)
146
+ sorted << (params.keyword_rest.name || :**)
143
147
  end
144
148
 
145
- sorted << AnonymousLocal if params.keywords.any?
146
-
147
149
  # Recurse down the parameter tree to find any destructured
148
150
  # parameters and add them after the other parameters.
149
151
  param_stack = params.requireds.concat(params.posts).grep(MultiTargetNode).reverse
@@ -151,15 +153,19 @@ module Prism
151
153
  case param
152
154
  when MultiTargetNode
153
155
  param_stack.concat(param.rights.reverse)
154
- param_stack << param.rest
156
+ param_stack << param.rest if param.rest&.expression && !sorted.include?(param.rest.expression.name)
155
157
  param_stack.concat(param.lefts.reverse)
156
158
  when RequiredParameterNode
157
159
  sorted << param.name
158
160
  when SplatNode
159
- sorted << param.expression.name if param.expression
161
+ sorted << param.expression.name
160
162
  end
161
163
  end
162
164
 
165
+ if params.block
166
+ sorted << (params.block.name || :&)
167
+ end
168
+
163
169
  names = sorted.concat(names - sorted)
164
170
  end
165
171