prism 0.16.0 → 0.17.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +16 -1
  3. data/Makefile +6 -0
  4. data/README.md +1 -1
  5. data/config.yml +50 -35
  6. data/docs/fuzzing.md +1 -1
  7. data/docs/serialization.md +28 -29
  8. data/ext/prism/api_node.c +802 -770
  9. data/ext/prism/api_pack.c +20 -9
  10. data/ext/prism/extension.c +464 -162
  11. data/ext/prism/extension.h +1 -1
  12. data/include/prism/ast.h +3173 -763
  13. data/include/prism/defines.h +32 -9
  14. data/include/prism/diagnostic.h +36 -3
  15. data/include/prism/enc/pm_encoding.h +118 -28
  16. data/include/prism/node.h +38 -13
  17. data/include/prism/options.h +204 -0
  18. data/include/prism/pack.h +44 -33
  19. data/include/prism/parser.h +445 -200
  20. data/include/prism/prettyprint.h +12 -1
  21. data/include/prism/regexp.h +16 -2
  22. data/include/prism/util/pm_buffer.h +94 -16
  23. data/include/prism/util/pm_char.h +162 -48
  24. data/include/prism/util/pm_constant_pool.h +126 -32
  25. data/include/prism/util/pm_list.h +68 -38
  26. data/include/prism/util/pm_memchr.h +18 -3
  27. data/include/prism/util/pm_newline_list.h +70 -27
  28. data/include/prism/util/pm_state_stack.h +25 -7
  29. data/include/prism/util/pm_string.h +115 -27
  30. data/include/prism/util/pm_string_list.h +25 -6
  31. data/include/prism/util/pm_strncasecmp.h +32 -0
  32. data/include/prism/util/pm_strpbrk.h +31 -17
  33. data/include/prism/version.h +27 -2
  34. data/include/prism.h +224 -31
  35. data/lib/prism/compiler.rb +6 -3
  36. data/lib/prism/debug.rb +23 -7
  37. data/lib/prism/dispatcher.rb +33 -18
  38. data/lib/prism/dsl.rb +10 -5
  39. data/lib/prism/ffi.rb +132 -80
  40. data/lib/prism/lex_compat.rb +25 -15
  41. data/lib/prism/mutation_compiler.rb +10 -5
  42. data/lib/prism/node.rb +370 -135
  43. data/lib/prism/node_ext.rb +1 -1
  44. data/lib/prism/node_inspector.rb +1 -1
  45. data/lib/prism/pack.rb +79 -40
  46. data/lib/prism/parse_result/comments.rb +7 -2
  47. data/lib/prism/parse_result/newlines.rb +4 -0
  48. data/lib/prism/parse_result.rb +150 -30
  49. data/lib/prism/pattern.rb +11 -0
  50. data/lib/prism/ripper_compat.rb +28 -10
  51. data/lib/prism/serialize.rb +86 -54
  52. data/lib/prism/visitor.rb +10 -3
  53. data/lib/prism.rb +20 -2
  54. data/prism.gemspec +4 -2
  55. data/rbi/prism.rbi +104 -60
  56. data/rbi/prism_static.rbi +16 -2
  57. data/sig/prism.rbs +72 -43
  58. data/sig/prism_static.rbs +14 -1
  59. data/src/diagnostic.c +56 -53
  60. data/src/enc/pm_big5.c +1 -0
  61. data/src/enc/pm_euc_jp.c +1 -0
  62. data/src/enc/pm_gbk.c +1 -0
  63. data/src/enc/pm_shift_jis.c +1 -0
  64. data/src/enc/pm_tables.c +316 -80
  65. data/src/enc/pm_unicode.c +53 -8
  66. data/src/enc/pm_windows_31j.c +1 -0
  67. data/src/node.c +334 -321
  68. data/src/options.c +170 -0
  69. data/src/prettyprint.c +74 -47
  70. data/src/prism.c +1642 -856
  71. data/src/regexp.c +151 -95
  72. data/src/serialize.c +44 -20
  73. data/src/token_type.c +3 -1
  74. data/src/util/pm_buffer.c +45 -15
  75. data/src/util/pm_char.c +103 -57
  76. data/src/util/pm_constant_pool.c +51 -21
  77. data/src/util/pm_list.c +12 -4
  78. data/src/util/pm_memchr.c +5 -3
  79. data/src/util/pm_newline_list.c +20 -12
  80. data/src/util/pm_state_stack.c +9 -3
  81. data/src/util/pm_string.c +95 -85
  82. data/src/util/pm_string_list.c +14 -15
  83. data/src/util/pm_strncasecmp.c +10 -3
  84. data/src/util/pm_strpbrk.c +25 -19
  85. metadata +5 -3
  86. data/docs/prism.png +0 -0
@@ -1,3 +1,8 @@
1
+ /**
2
+ * @file prettyprint.h
3
+ *
4
+ * An AST node pretty-printer.
5
+ */
1
6
  #ifndef PRISM_PRETTYPRINT_H
2
7
  #define PRISM_PRETTYPRINT_H
3
8
 
@@ -9,7 +14,13 @@
9
14
  #include "prism/parser.h"
10
15
  #include "prism/util/pm_buffer.h"
11
16
 
12
- // Pretty-prints the AST represented by the given node to the given buffer.
17
+ /**
18
+ * Pretty-prints the AST represented by the given node to the given buffer.
19
+ *
20
+ * @param output_buffer The buffer to write the pretty-printed AST to.
21
+ * @param parser The parser that parsed the AST.
22
+ * @param node The root node of the AST to pretty-print.
23
+ */
13
24
  PRISM_EXPORTED_FUNCTION void pm_prettyprint(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_node_t *node);
14
25
 
15
26
  #endif
@@ -1,3 +1,8 @@
1
+ /**
2
+ * @file regexp.h
3
+ *
4
+ * A regular expression parser.
5
+ */
1
6
  #ifndef PRISM_REGEXP_H
2
7
  #define PRISM_REGEXP_H
3
8
 
@@ -12,8 +17,17 @@
12
17
  #include <stddef.h>
13
18
  #include <string.h>
14
19
 
15
- // Parse a regular expression and extract the names of all of the named capture
16
- // groups.
20
+ /**
21
+ * Parse a regular expression and extract the names of all of the named capture
22
+ * groups.
23
+ *
24
+ * @param source The source code to parse.
25
+ * @param size The size of the source code.
26
+ * @param named_captures The list to add the names of the named capture groups.
27
+ * @param encoding_changed Whether or not the encoding changed from the default.
28
+ * @param encoding The encoding of the source code.
29
+ * @return Whether or not the parsing was successful.
30
+ */
17
31
  PRISM_EXPORTED_FUNCTION bool pm_regexp_named_capture_group_names(const uint8_t *source, size_t size, pm_string_list_t *named_captures, bool encoding_changed, pm_encoding_t *encoding);
18
32
 
19
33
  #endif
@@ -1,3 +1,8 @@
1
+ /**
2
+ * @file pm_buffer.h
3
+ *
4
+ * A wrapper around a contiguous block of allocated memory.
5
+ */
1
6
  #ifndef PRISM_BUFFER_H
2
7
  #define PRISM_BUFFER_H
3
8
 
@@ -9,52 +14,125 @@
9
14
  #include <stdlib.h>
10
15
  #include <string.h>
11
16
 
12
- // A pm_buffer_t is a simple memory buffer that stores data in a contiguous
13
- // block of memory. It is used to store the serialized representation of a
14
- // prism tree.
17
+ /**
18
+ * A pm_buffer_t is a simple memory buffer that stores data in a contiguous
19
+ * block of memory.
20
+ */
15
21
  typedef struct {
22
+ /** The length of the buffer in bytes. */
16
23
  size_t length;
24
+
25
+ /** The capacity of the buffer in bytes that has been allocated. */
17
26
  size_t capacity;
27
+
28
+ /** A pointer to the start of the buffer. */
18
29
  char *value;
19
30
  } pm_buffer_t;
20
31
 
21
- // Return the size of the pm_buffer_t struct.
32
+ /**
33
+ * Return the size of the pm_buffer_t struct.
34
+ *
35
+ * @returns The size of the pm_buffer_t struct.
36
+ */
22
37
  PRISM_EXPORTED_FUNCTION size_t pm_buffer_sizeof(void);
23
38
 
24
- // Initialize a pm_buffer_t with the given capacity.
39
+ /**
40
+ * Initialize a pm_buffer_t with the given capacity.
41
+ *
42
+ * @param buffer The buffer to initialize.
43
+ * @param capacity The capacity of the buffer.
44
+ * @returns True if the buffer was initialized successfully, false otherwise.
45
+ */
25
46
  bool pm_buffer_init_capacity(pm_buffer_t *buffer, size_t capacity);
26
47
 
27
- // Initialize a pm_buffer_t with its default values.
48
+ /**
49
+ * Initialize a pm_buffer_t with its default values.
50
+ *
51
+ * @param buffer The buffer to initialize.
52
+ * @returns True if the buffer was initialized successfully, false otherwise.
53
+ */
28
54
  PRISM_EXPORTED_FUNCTION bool pm_buffer_init(pm_buffer_t *buffer);
29
55
 
30
- // Return the value of the buffer.
56
+ /**
57
+ * Return the value of the buffer.
58
+ *
59
+ * @param buffer The buffer to get the value of.
60
+ * @returns The value of the buffer.
61
+ */
31
62
  PRISM_EXPORTED_FUNCTION char * pm_buffer_value(pm_buffer_t *buffer);
32
63
 
33
- // Return the length of the buffer.
64
+ /**
65
+ * Return the length of the buffer.
66
+ *
67
+ * @param buffer The buffer to get the length of.
68
+ * @returns The length of the buffer.
69
+ */
34
70
  PRISM_EXPORTED_FUNCTION size_t pm_buffer_length(pm_buffer_t *buffer);
35
71
 
36
- // Append the given amount of space as zeroes to the buffer.
72
+ /**
73
+ * Append the given amount of space as zeroes to the buffer.
74
+ *
75
+ * @param buffer The buffer to append to.
76
+ * @param length The amount of space to append and zero.
77
+ */
37
78
  void pm_buffer_append_zeroes(pm_buffer_t *buffer, size_t length);
38
79
 
39
- // Append a formatted string to the buffer.
80
+ /**
81
+ * Append a formatted string to the buffer.
82
+ *
83
+ * @param buffer The buffer to append to.
84
+ * @param format The format string to append.
85
+ * @param ... The arguments to the format string.
86
+ */
40
87
  void pm_buffer_append_format(pm_buffer_t *buffer, const char *format, ...) PRISM_ATTRIBUTE_FORMAT(2, 3);
41
88
 
42
- // Append a string to the buffer.
89
+ /**
90
+ * Append a string to the buffer.
91
+ *
92
+ * @param buffer The buffer to append to.
93
+ * @param value The string to append.
94
+ * @param length The length of the string to append.
95
+ */
43
96
  void pm_buffer_append_string(pm_buffer_t *buffer, const char *value, size_t length);
44
97
 
45
- // Append a list of bytes to the buffer.
98
+ /**
99
+ * Append a list of bytes to the buffer.
100
+ *
101
+ * @param buffer The buffer to append to.
102
+ * @param value The bytes to append.
103
+ * @param length The length of the bytes to append.
104
+ */
46
105
  void pm_buffer_append_bytes(pm_buffer_t *buffer, const uint8_t *value, size_t length);
47
106
 
48
- // Append a single byte to the buffer.
107
+ /**
108
+ * Append a single byte to the buffer.
109
+ *
110
+ * @param buffer The buffer to append to.
111
+ * @param value The byte to append.
112
+ */
49
113
  void pm_buffer_append_byte(pm_buffer_t *buffer, uint8_t value);
50
114
 
51
- // Append a 32-bit unsigned integer to the buffer.
115
+ /**
116
+ * Append a 32-bit unsigned integer to the buffer as a variable-length integer.
117
+ *
118
+ * @param buffer The buffer to append to.
119
+ * @param value The integer to append.
120
+ */
52
121
  void pm_buffer_append_varint(pm_buffer_t *buffer, uint32_t value);
53
122
 
54
- // Append one buffer onto another.
123
+ /**
124
+ * Concatenate one buffer onto another.
125
+ *
126
+ * @param destination The buffer to concatenate onto.
127
+ * @param source The buffer to concatenate.
128
+ */
55
129
  void pm_buffer_concat(pm_buffer_t *destination, const pm_buffer_t *source);
56
130
 
57
- // Free the memory associated with the buffer.
131
+ /**
132
+ * Free the memory associated with the buffer.
133
+ *
134
+ * @param buffer The buffer to free.
135
+ */
58
136
  PRISM_EXPORTED_FUNCTION void pm_buffer_free(pm_buffer_t *buffer);
59
137
 
60
138
  #endif
@@ -1,3 +1,8 @@
1
+ /**
2
+ * @file pm_char.h
3
+ *
4
+ * Functions for working with characters and strings.
5
+ */
1
6
  #ifndef PRISM_CHAR_H
2
7
  #define PRISM_CHAR_H
3
8
 
@@ -7,85 +12,194 @@
7
12
  #include <stdbool.h>
8
13
  #include <stddef.h>
9
14
 
10
- // Returns the number of characters at the start of the string that are
11
- // whitespace. Disallows searching past the given maximum number of characters.
15
+ /**
16
+ * Returns the number of characters at the start of the string that are
17
+ * whitespace. Disallows searching past the given maximum number of characters.
18
+ *
19
+ * @param string The string to search.
20
+ * @param length The maximum number of characters to search.
21
+ * @return The number of characters at the start of the string that are
22
+ * whitespace.
23
+ */
12
24
  size_t pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length);
13
25
 
14
- // Returns the number of characters at the start of the string that are
15
- // whitespace while also tracking the location of each newline. Disallows
16
- // searching past the given maximum number of characters.
26
+ /**
27
+ * Returns the number of characters at the start of the string that are
28
+ * whitespace while also tracking the location of each newline. Disallows
29
+ * searching past the given maximum number of characters.
30
+ *
31
+ * @param string The string to search.
32
+ * @param length The maximum number of characters to search.
33
+ * @param newline_list The list of newlines to populate.
34
+ * @return The number of characters at the start of the string that are
35
+ * whitespace.
36
+ */
17
37
  size_t
18
38
  pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list);
19
39
 
20
- // Returns the number of characters at the start of the string that are inline
21
- // whitespace. Disallows searching past the given maximum number of characters.
40
+ /**
41
+ * Returns the number of characters at the start of the string that are inline
42
+ * whitespace. Disallows searching past the given maximum number of characters.
43
+ *
44
+ * @param string The string to search.
45
+ * @param length The maximum number of characters to search.
46
+ * @return The number of characters at the start of the string that are inline
47
+ * whitespace.
48
+ */
22
49
  size_t pm_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length);
23
50
 
24
- // Returns the number of characters at the start of the string that are decimal
25
- // digits. Disallows searching past the given maximum number of characters.
51
+ /**
52
+ * Returns the number of characters at the start of the string that are decimal
53
+ * digits. Disallows searching past the given maximum number of characters.
54
+ *
55
+ * @param string The string to search.
56
+ * @param length The maximum number of characters to search.
57
+ * @return The number of characters at the start of the string that are decimal
58
+ * digits.
59
+ */
26
60
  size_t pm_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length);
27
61
 
28
- // Returns the number of characters at the start of the string that are
29
- // hexadecimal digits. Disallows searching past the given maximum number of
30
- // characters.
62
+ /**
63
+ * Returns the number of characters at the start of the string that are
64
+ * hexadecimal digits. Disallows searching past the given maximum number of
65
+ * characters.
66
+ *
67
+ * @param string The string to search.
68
+ * @param length The maximum number of characters to search.
69
+ * @return The number of characters at the start of the string that are
70
+ * hexadecimal digits.
71
+ */
31
72
  size_t pm_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length);
32
73
 
33
- // Returns the number of characters at the start of the string that are octal
34
- // digits or underscores. Disallows searching past the given maximum number of
35
- // characters.
36
- //
37
- // If multiple underscores are found in a row or if an underscore is
38
- // found at the end of the number, then the invalid pointer is set to the index
39
- // of the first invalid underscore.
74
+ /**
75
+ * Returns the number of characters at the start of the string that are octal
76
+ * digits or underscores. Disallows searching past the given maximum number of
77
+ * characters.
78
+ *
79
+ * If multiple underscores are found in a row or if an underscore is
80
+ * found at the end of the number, then the invalid pointer is set to the index
81
+ * of the first invalid underscore.
82
+ *
83
+ * @param string The string to search.
84
+ * @param length The maximum number of characters to search.
85
+ * @param invalid The pointer to set to the index of the first invalid
86
+ * underscore.
87
+ * @return The number of characters at the start of the string that are octal
88
+ * digits or underscores.
89
+ */
40
90
  size_t pm_strspn_octal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
41
91
 
42
- // Returns the number of characters at the start of the string that are decimal
43
- // digits or underscores. Disallows searching past the given maximum number of
44
- // characters.
45
- //
46
- // If multiple underscores are found in a row or if an underscore is
47
- // found at the end of the number, then the invalid pointer is set to the index
48
- // of the first invalid underscore.
92
+ /**
93
+ * Returns the number of characters at the start of the string that are decimal
94
+ * digits or underscores. Disallows searching past the given maximum number of
95
+ * characters.
96
+ *
97
+ * If multiple underscores are found in a row or if an underscore is
98
+ * found at the end of the number, then the invalid pointer is set to the index
99
+ * of the first invalid underscore.
100
+ *
101
+ * @param string The string to search.
102
+ * @param length The maximum number of characters to search.
103
+ * @param invalid The pointer to set to the index of the first invalid
104
+ * underscore.
105
+ * @return The number of characters at the start of the string that are decimal
106
+ * digits or underscores.
107
+ */
49
108
  size_t pm_strspn_decimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
50
109
 
51
- // Returns the number of characters at the start of the string that are
52
- // hexadecimal digits or underscores. Disallows searching past the given maximum
53
- // number of characters.
54
- //
55
- // If multiple underscores are found in a row or if an underscore is
56
- // found at the end of the number, then the invalid pointer is set to the index
57
- // of the first invalid underscore.
110
+ /**
111
+ * Returns the number of characters at the start of the string that are
112
+ * hexadecimal digits or underscores. Disallows searching past the given maximum
113
+ * number of characters.
114
+ *
115
+ * If multiple underscores are found in a row or if an underscore is
116
+ * found at the end of the number, then the invalid pointer is set to the index
117
+ * of the first invalid underscore.
118
+ *
119
+ * @param string The string to search.
120
+ * @param length The maximum number of characters to search.
121
+ * @param invalid The pointer to set to the index of the first invalid
122
+ * underscore.
123
+ * @return The number of characters at the start of the string that are
124
+ * hexadecimal digits or underscores.
125
+ */
58
126
  size_t pm_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
59
127
 
60
- // Returns the number of characters at the start of the string that are regexp
61
- // options. Disallows searching past the given maximum number of characters.
128
+ /**
129
+ * Returns the number of characters at the start of the string that are regexp
130
+ * options. Disallows searching past the given maximum number of characters.
131
+ *
132
+ * @param string The string to search.
133
+ * @param length The maximum number of characters to search.
134
+ * @return The number of characters at the start of the string that are regexp
135
+ * options.
136
+ */
62
137
  size_t pm_strspn_regexp_option(const uint8_t *string, ptrdiff_t length);
63
138
 
64
- // Returns the number of characters at the start of the string that are binary
65
- // digits or underscores. Disallows searching past the given maximum number of
66
- // characters.
67
- //
68
- // If multiple underscores are found in a row or if an underscore is
69
- // found at the end of the number, then the invalid pointer is set to the index
70
- // of the first invalid underscore.
139
+ /**
140
+ * Returns the number of characters at the start of the string that are binary
141
+ * digits or underscores. Disallows searching past the given maximum number of
142
+ * characters.
143
+ *
144
+ * If multiple underscores are found in a row or if an underscore is
145
+ * found at the end of the number, then the invalid pointer is set to the index
146
+ * of the first invalid underscore.
147
+ *
148
+ * @param string The string to search.
149
+ * @param length The maximum number of characters to search.
150
+ * @param invalid The pointer to set to the index of the first invalid
151
+ * underscore.
152
+ * @return The number of characters at the start of the string that are binary
153
+ * digits or underscores.
154
+ */
71
155
  size_t pm_strspn_binary_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
72
156
 
73
- // Returns true if the given character is a whitespace character.
157
+ /**
158
+ * Returns true if the given character is a whitespace character.
159
+ *
160
+ * @param b The character to check.
161
+ * @return True if the given character is a whitespace character.
162
+ */
74
163
  bool pm_char_is_whitespace(const uint8_t b);
75
164
 
76
- // Returns true if the given character is an inline whitespace character.
165
+ /**
166
+ * Returns true if the given character is an inline whitespace character.
167
+ *
168
+ * @param b The character to check.
169
+ * @return True if the given character is an inline whitespace character.
170
+ */
77
171
  bool pm_char_is_inline_whitespace(const uint8_t b);
78
172
 
79
- // Returns true if the given character is a binary digit.
173
+ /**
174
+ * Returns true if the given character is a binary digit.
175
+ *
176
+ * @param b The character to check.
177
+ * @return True if the given character is a binary digit.
178
+ */
80
179
  bool pm_char_is_binary_digit(const uint8_t b);
81
180
 
82
- // Returns true if the given character is an octal digit.
181
+ /**
182
+ * Returns true if the given character is an octal digit.
183
+ *
184
+ * @param b The character to check.
185
+ * @return True if the given character is an octal digit.
186
+ */
83
187
  bool pm_char_is_octal_digit(const uint8_t b);
84
188
 
85
- // Returns true if the given character is a decimal digit.
189
+ /**
190
+ * Returns true if the given character is a decimal digit.
191
+ *
192
+ * @param b The character to check.
193
+ * @return True if the given character is a decimal digit.
194
+ */
86
195
  bool pm_char_is_decimal_digit(const uint8_t b);
87
196
 
88
- // Returns true if the given character is a hexadecimal digit.
197
+ /**
198
+ * Returns true if the given character is a hexadecimal digit.
199
+ *
200
+ * @param b The character to check.
201
+ * @return True if the given character is a hexadecimal digit.
202
+ */
89
203
  bool pm_char_is_hexadecimal_digit(const uint8_t b);
90
204
 
91
205
  #endif