prism 0.16.0 → 0.17.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (86) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +16 -1
  3. data/Makefile +6 -0
  4. data/README.md +1 -1
  5. data/config.yml +50 -35
  6. data/docs/fuzzing.md +1 -1
  7. data/docs/serialization.md +28 -29
  8. data/ext/prism/api_node.c +802 -770
  9. data/ext/prism/api_pack.c +20 -9
  10. data/ext/prism/extension.c +464 -162
  11. data/ext/prism/extension.h +1 -1
  12. data/include/prism/ast.h +3173 -763
  13. data/include/prism/defines.h +32 -9
  14. data/include/prism/diagnostic.h +36 -3
  15. data/include/prism/enc/pm_encoding.h +118 -28
  16. data/include/prism/node.h +38 -13
  17. data/include/prism/options.h +204 -0
  18. data/include/prism/pack.h +44 -33
  19. data/include/prism/parser.h +445 -200
  20. data/include/prism/prettyprint.h +12 -1
  21. data/include/prism/regexp.h +16 -2
  22. data/include/prism/util/pm_buffer.h +94 -16
  23. data/include/prism/util/pm_char.h +162 -48
  24. data/include/prism/util/pm_constant_pool.h +126 -32
  25. data/include/prism/util/pm_list.h +68 -38
  26. data/include/prism/util/pm_memchr.h +18 -3
  27. data/include/prism/util/pm_newline_list.h +70 -27
  28. data/include/prism/util/pm_state_stack.h +25 -7
  29. data/include/prism/util/pm_string.h +115 -27
  30. data/include/prism/util/pm_string_list.h +25 -6
  31. data/include/prism/util/pm_strncasecmp.h +32 -0
  32. data/include/prism/util/pm_strpbrk.h +31 -17
  33. data/include/prism/version.h +27 -2
  34. data/include/prism.h +224 -31
  35. data/lib/prism/compiler.rb +6 -3
  36. data/lib/prism/debug.rb +23 -7
  37. data/lib/prism/dispatcher.rb +33 -18
  38. data/lib/prism/dsl.rb +10 -5
  39. data/lib/prism/ffi.rb +132 -80
  40. data/lib/prism/lex_compat.rb +25 -15
  41. data/lib/prism/mutation_compiler.rb +10 -5
  42. data/lib/prism/node.rb +370 -135
  43. data/lib/prism/node_ext.rb +1 -1
  44. data/lib/prism/node_inspector.rb +1 -1
  45. data/lib/prism/pack.rb +79 -40
  46. data/lib/prism/parse_result/comments.rb +7 -2
  47. data/lib/prism/parse_result/newlines.rb +4 -0
  48. data/lib/prism/parse_result.rb +150 -30
  49. data/lib/prism/pattern.rb +11 -0
  50. data/lib/prism/ripper_compat.rb +28 -10
  51. data/lib/prism/serialize.rb +86 -54
  52. data/lib/prism/visitor.rb +10 -3
  53. data/lib/prism.rb +20 -2
  54. data/prism.gemspec +4 -2
  55. data/rbi/prism.rbi +104 -60
  56. data/rbi/prism_static.rbi +16 -2
  57. data/sig/prism.rbs +72 -43
  58. data/sig/prism_static.rbs +14 -1
  59. data/src/diagnostic.c +56 -53
  60. data/src/enc/pm_big5.c +1 -0
  61. data/src/enc/pm_euc_jp.c +1 -0
  62. data/src/enc/pm_gbk.c +1 -0
  63. data/src/enc/pm_shift_jis.c +1 -0
  64. data/src/enc/pm_tables.c +316 -80
  65. data/src/enc/pm_unicode.c +53 -8
  66. data/src/enc/pm_windows_31j.c +1 -0
  67. data/src/node.c +334 -321
  68. data/src/options.c +170 -0
  69. data/src/prettyprint.c +74 -47
  70. data/src/prism.c +1642 -856
  71. data/src/regexp.c +151 -95
  72. data/src/serialize.c +44 -20
  73. data/src/token_type.c +3 -1
  74. data/src/util/pm_buffer.c +45 -15
  75. data/src/util/pm_char.c +103 -57
  76. data/src/util/pm_constant_pool.c +51 -21
  77. data/src/util/pm_list.c +12 -4
  78. data/src/util/pm_memchr.c +5 -3
  79. data/src/util/pm_newline_list.c +20 -12
  80. data/src/util/pm_state_stack.c +9 -3
  81. data/src/util/pm_string.c +95 -85
  82. data/src/util/pm_string_list.c +14 -15
  83. data/src/util/pm_strncasecmp.c +10 -3
  84. data/src/util/pm_strpbrk.c +25 -19
  85. metadata +5 -3
  86. data/docs/prism.png +0 -0
@@ -1,3 +1,8 @@
1
+ /**
2
+ * @file prettyprint.h
3
+ *
4
+ * An AST node pretty-printer.
5
+ */
1
6
  #ifndef PRISM_PRETTYPRINT_H
2
7
  #define PRISM_PRETTYPRINT_H
3
8
 
@@ -9,7 +14,13 @@
9
14
  #include "prism/parser.h"
10
15
  #include "prism/util/pm_buffer.h"
11
16
 
12
- // Pretty-prints the AST represented by the given node to the given buffer.
17
+ /**
18
+ * Pretty-prints the AST represented by the given node to the given buffer.
19
+ *
20
+ * @param output_buffer The buffer to write the pretty-printed AST to.
21
+ * @param parser The parser that parsed the AST.
22
+ * @param node The root node of the AST to pretty-print.
23
+ */
13
24
  PRISM_EXPORTED_FUNCTION void pm_prettyprint(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_node_t *node);
14
25
 
15
26
  #endif
@@ -1,3 +1,8 @@
1
+ /**
2
+ * @file regexp.h
3
+ *
4
+ * A regular expression parser.
5
+ */
1
6
  #ifndef PRISM_REGEXP_H
2
7
  #define PRISM_REGEXP_H
3
8
 
@@ -12,8 +17,17 @@
12
17
  #include <stddef.h>
13
18
  #include <string.h>
14
19
 
15
- // Parse a regular expression and extract the names of all of the named capture
16
- // groups.
20
+ /**
21
+ * Parse a regular expression and extract the names of all of the named capture
22
+ * groups.
23
+ *
24
+ * @param source The source code to parse.
25
+ * @param size The size of the source code.
26
+ * @param named_captures The list to add the names of the named capture groups.
27
+ * @param encoding_changed Whether or not the encoding changed from the default.
28
+ * @param encoding The encoding of the source code.
29
+ * @return Whether or not the parsing was successful.
30
+ */
17
31
  PRISM_EXPORTED_FUNCTION bool pm_regexp_named_capture_group_names(const uint8_t *source, size_t size, pm_string_list_t *named_captures, bool encoding_changed, pm_encoding_t *encoding);
18
32
 
19
33
  #endif
@@ -1,3 +1,8 @@
1
+ /**
2
+ * @file pm_buffer.h
3
+ *
4
+ * A wrapper around a contiguous block of allocated memory.
5
+ */
1
6
  #ifndef PRISM_BUFFER_H
2
7
  #define PRISM_BUFFER_H
3
8
 
@@ -9,52 +14,125 @@
9
14
  #include <stdlib.h>
10
15
  #include <string.h>
11
16
 
12
- // A pm_buffer_t is a simple memory buffer that stores data in a contiguous
13
- // block of memory. It is used to store the serialized representation of a
14
- // prism tree.
17
+ /**
18
+ * A pm_buffer_t is a simple memory buffer that stores data in a contiguous
19
+ * block of memory.
20
+ */
15
21
  typedef struct {
22
+ /** The length of the buffer in bytes. */
16
23
  size_t length;
24
+
25
+ /** The capacity of the buffer in bytes that has been allocated. */
17
26
  size_t capacity;
27
+
28
+ /** A pointer to the start of the buffer. */
18
29
  char *value;
19
30
  } pm_buffer_t;
20
31
 
21
- // Return the size of the pm_buffer_t struct.
32
+ /**
33
+ * Return the size of the pm_buffer_t struct.
34
+ *
35
+ * @returns The size of the pm_buffer_t struct.
36
+ */
22
37
  PRISM_EXPORTED_FUNCTION size_t pm_buffer_sizeof(void);
23
38
 
24
- // Initialize a pm_buffer_t with the given capacity.
39
+ /**
40
+ * Initialize a pm_buffer_t with the given capacity.
41
+ *
42
+ * @param buffer The buffer to initialize.
43
+ * @param capacity The capacity of the buffer.
44
+ * @returns True if the buffer was initialized successfully, false otherwise.
45
+ */
25
46
  bool pm_buffer_init_capacity(pm_buffer_t *buffer, size_t capacity);
26
47
 
27
- // Initialize a pm_buffer_t with its default values.
48
+ /**
49
+ * Initialize a pm_buffer_t with its default values.
50
+ *
51
+ * @param buffer The buffer to initialize.
52
+ * @returns True if the buffer was initialized successfully, false otherwise.
53
+ */
28
54
  PRISM_EXPORTED_FUNCTION bool pm_buffer_init(pm_buffer_t *buffer);
29
55
 
30
- // Return the value of the buffer.
56
+ /**
57
+ * Return the value of the buffer.
58
+ *
59
+ * @param buffer The buffer to get the value of.
60
+ * @returns The value of the buffer.
61
+ */
31
62
  PRISM_EXPORTED_FUNCTION char * pm_buffer_value(pm_buffer_t *buffer);
32
63
 
33
- // Return the length of the buffer.
64
+ /**
65
+ * Return the length of the buffer.
66
+ *
67
+ * @param buffer The buffer to get the length of.
68
+ * @returns The length of the buffer.
69
+ */
34
70
  PRISM_EXPORTED_FUNCTION size_t pm_buffer_length(pm_buffer_t *buffer);
35
71
 
36
- // Append the given amount of space as zeroes to the buffer.
72
+ /**
73
+ * Append the given amount of space as zeroes to the buffer.
74
+ *
75
+ * @param buffer The buffer to append to.
76
+ * @param length The amount of space to append and zero.
77
+ */
37
78
  void pm_buffer_append_zeroes(pm_buffer_t *buffer, size_t length);
38
79
 
39
- // Append a formatted string to the buffer.
80
+ /**
81
+ * Append a formatted string to the buffer.
82
+ *
83
+ * @param buffer The buffer to append to.
84
+ * @param format The format string to append.
85
+ * @param ... The arguments to the format string.
86
+ */
40
87
  void pm_buffer_append_format(pm_buffer_t *buffer, const char *format, ...) PRISM_ATTRIBUTE_FORMAT(2, 3);
41
88
 
42
- // Append a string to the buffer.
89
+ /**
90
+ * Append a string to the buffer.
91
+ *
92
+ * @param buffer The buffer to append to.
93
+ * @param value The string to append.
94
+ * @param length The length of the string to append.
95
+ */
43
96
  void pm_buffer_append_string(pm_buffer_t *buffer, const char *value, size_t length);
44
97
 
45
- // Append a list of bytes to the buffer.
98
+ /**
99
+ * Append a list of bytes to the buffer.
100
+ *
101
+ * @param buffer The buffer to append to.
102
+ * @param value The bytes to append.
103
+ * @param length The length of the bytes to append.
104
+ */
46
105
  void pm_buffer_append_bytes(pm_buffer_t *buffer, const uint8_t *value, size_t length);
47
106
 
48
- // Append a single byte to the buffer.
107
+ /**
108
+ * Append a single byte to the buffer.
109
+ *
110
+ * @param buffer The buffer to append to.
111
+ * @param value The byte to append.
112
+ */
49
113
  void pm_buffer_append_byte(pm_buffer_t *buffer, uint8_t value);
50
114
 
51
- // Append a 32-bit unsigned integer to the buffer.
115
+ /**
116
+ * Append a 32-bit unsigned integer to the buffer as a variable-length integer.
117
+ *
118
+ * @param buffer The buffer to append to.
119
+ * @param value The integer to append.
120
+ */
52
121
  void pm_buffer_append_varint(pm_buffer_t *buffer, uint32_t value);
53
122
 
54
- // Append one buffer onto another.
123
+ /**
124
+ * Concatenate one buffer onto another.
125
+ *
126
+ * @param destination The buffer to concatenate onto.
127
+ * @param source The buffer to concatenate.
128
+ */
55
129
  void pm_buffer_concat(pm_buffer_t *destination, const pm_buffer_t *source);
56
130
 
57
- // Free the memory associated with the buffer.
131
+ /**
132
+ * Free the memory associated with the buffer.
133
+ *
134
+ * @param buffer The buffer to free.
135
+ */
58
136
  PRISM_EXPORTED_FUNCTION void pm_buffer_free(pm_buffer_t *buffer);
59
137
 
60
138
  #endif
@@ -1,3 +1,8 @@
1
+ /**
2
+ * @file pm_char.h
3
+ *
4
+ * Functions for working with characters and strings.
5
+ */
1
6
  #ifndef PRISM_CHAR_H
2
7
  #define PRISM_CHAR_H
3
8
 
@@ -7,85 +12,194 @@
7
12
  #include <stdbool.h>
8
13
  #include <stddef.h>
9
14
 
10
- // Returns the number of characters at the start of the string that are
11
- // whitespace. Disallows searching past the given maximum number of characters.
15
+ /**
16
+ * Returns the number of characters at the start of the string that are
17
+ * whitespace. Disallows searching past the given maximum number of characters.
18
+ *
19
+ * @param string The string to search.
20
+ * @param length The maximum number of characters to search.
21
+ * @return The number of characters at the start of the string that are
22
+ * whitespace.
23
+ */
12
24
  size_t pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length);
13
25
 
14
- // Returns the number of characters at the start of the string that are
15
- // whitespace while also tracking the location of each newline. Disallows
16
- // searching past the given maximum number of characters.
26
+ /**
27
+ * Returns the number of characters at the start of the string that are
28
+ * whitespace while also tracking the location of each newline. Disallows
29
+ * searching past the given maximum number of characters.
30
+ *
31
+ * @param string The string to search.
32
+ * @param length The maximum number of characters to search.
33
+ * @param newline_list The list of newlines to populate.
34
+ * @return The number of characters at the start of the string that are
35
+ * whitespace.
36
+ */
17
37
  size_t
18
38
  pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list);
19
39
 
20
- // Returns the number of characters at the start of the string that are inline
21
- // whitespace. Disallows searching past the given maximum number of characters.
40
+ /**
41
+ * Returns the number of characters at the start of the string that are inline
42
+ * whitespace. Disallows searching past the given maximum number of characters.
43
+ *
44
+ * @param string The string to search.
45
+ * @param length The maximum number of characters to search.
46
+ * @return The number of characters at the start of the string that are inline
47
+ * whitespace.
48
+ */
22
49
  size_t pm_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length);
23
50
 
24
- // Returns the number of characters at the start of the string that are decimal
25
- // digits. Disallows searching past the given maximum number of characters.
51
+ /**
52
+ * Returns the number of characters at the start of the string that are decimal
53
+ * digits. Disallows searching past the given maximum number of characters.
54
+ *
55
+ * @param string The string to search.
56
+ * @param length The maximum number of characters to search.
57
+ * @return The number of characters at the start of the string that are decimal
58
+ * digits.
59
+ */
26
60
  size_t pm_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length);
27
61
 
28
- // Returns the number of characters at the start of the string that are
29
- // hexadecimal digits. Disallows searching past the given maximum number of
30
- // characters.
62
+ /**
63
+ * Returns the number of characters at the start of the string that are
64
+ * hexadecimal digits. Disallows searching past the given maximum number of
65
+ * characters.
66
+ *
67
+ * @param string The string to search.
68
+ * @param length The maximum number of characters to search.
69
+ * @return The number of characters at the start of the string that are
70
+ * hexadecimal digits.
71
+ */
31
72
  size_t pm_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length);
32
73
 
33
- // Returns the number of characters at the start of the string that are octal
34
- // digits or underscores. Disallows searching past the given maximum number of
35
- // characters.
36
- //
37
- // If multiple underscores are found in a row or if an underscore is
38
- // found at the end of the number, then the invalid pointer is set to the index
39
- // of the first invalid underscore.
74
+ /**
75
+ * Returns the number of characters at the start of the string that are octal
76
+ * digits or underscores. Disallows searching past the given maximum number of
77
+ * characters.
78
+ *
79
+ * If multiple underscores are found in a row or if an underscore is
80
+ * found at the end of the number, then the invalid pointer is set to the index
81
+ * of the first invalid underscore.
82
+ *
83
+ * @param string The string to search.
84
+ * @param length The maximum number of characters to search.
85
+ * @param invalid The pointer to set to the index of the first invalid
86
+ * underscore.
87
+ * @return The number of characters at the start of the string that are octal
88
+ * digits or underscores.
89
+ */
40
90
  size_t pm_strspn_octal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
41
91
 
42
- // Returns the number of characters at the start of the string that are decimal
43
- // digits or underscores. Disallows searching past the given maximum number of
44
- // characters.
45
- //
46
- // If multiple underscores are found in a row or if an underscore is
47
- // found at the end of the number, then the invalid pointer is set to the index
48
- // of the first invalid underscore.
92
+ /**
93
+ * Returns the number of characters at the start of the string that are decimal
94
+ * digits or underscores. Disallows searching past the given maximum number of
95
+ * characters.
96
+ *
97
+ * If multiple underscores are found in a row or if an underscore is
98
+ * found at the end of the number, then the invalid pointer is set to the index
99
+ * of the first invalid underscore.
100
+ *
101
+ * @param string The string to search.
102
+ * @param length The maximum number of characters to search.
103
+ * @param invalid The pointer to set to the index of the first invalid
104
+ * underscore.
105
+ * @return The number of characters at the start of the string that are decimal
106
+ * digits or underscores.
107
+ */
49
108
  size_t pm_strspn_decimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
50
109
 
51
- // Returns the number of characters at the start of the string that are
52
- // hexadecimal digits or underscores. Disallows searching past the given maximum
53
- // number of characters.
54
- //
55
- // If multiple underscores are found in a row or if an underscore is
56
- // found at the end of the number, then the invalid pointer is set to the index
57
- // of the first invalid underscore.
110
+ /**
111
+ * Returns the number of characters at the start of the string that are
112
+ * hexadecimal digits or underscores. Disallows searching past the given maximum
113
+ * number of characters.
114
+ *
115
+ * If multiple underscores are found in a row or if an underscore is
116
+ * found at the end of the number, then the invalid pointer is set to the index
117
+ * of the first invalid underscore.
118
+ *
119
+ * @param string The string to search.
120
+ * @param length The maximum number of characters to search.
121
+ * @param invalid The pointer to set to the index of the first invalid
122
+ * underscore.
123
+ * @return The number of characters at the start of the string that are
124
+ * hexadecimal digits or underscores.
125
+ */
58
126
  size_t pm_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
59
127
 
60
- // Returns the number of characters at the start of the string that are regexp
61
- // options. Disallows searching past the given maximum number of characters.
128
+ /**
129
+ * Returns the number of characters at the start of the string that are regexp
130
+ * options. Disallows searching past the given maximum number of characters.
131
+ *
132
+ * @param string The string to search.
133
+ * @param length The maximum number of characters to search.
134
+ * @return The number of characters at the start of the string that are regexp
135
+ * options.
136
+ */
62
137
  size_t pm_strspn_regexp_option(const uint8_t *string, ptrdiff_t length);
63
138
 
64
- // Returns the number of characters at the start of the string that are binary
65
- // digits or underscores. Disallows searching past the given maximum number of
66
- // characters.
67
- //
68
- // If multiple underscores are found in a row or if an underscore is
69
- // found at the end of the number, then the invalid pointer is set to the index
70
- // of the first invalid underscore.
139
+ /**
140
+ * Returns the number of characters at the start of the string that are binary
141
+ * digits or underscores. Disallows searching past the given maximum number of
142
+ * characters.
143
+ *
144
+ * If multiple underscores are found in a row or if an underscore is
145
+ * found at the end of the number, then the invalid pointer is set to the index
146
+ * of the first invalid underscore.
147
+ *
148
+ * @param string The string to search.
149
+ * @param length The maximum number of characters to search.
150
+ * @param invalid The pointer to set to the index of the first invalid
151
+ * underscore.
152
+ * @return The number of characters at the start of the string that are binary
153
+ * digits or underscores.
154
+ */
71
155
  size_t pm_strspn_binary_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
72
156
 
73
- // Returns true if the given character is a whitespace character.
157
+ /**
158
+ * Returns true if the given character is a whitespace character.
159
+ *
160
+ * @param b The character to check.
161
+ * @return True if the given character is a whitespace character.
162
+ */
74
163
  bool pm_char_is_whitespace(const uint8_t b);
75
164
 
76
- // Returns true if the given character is an inline whitespace character.
165
+ /**
166
+ * Returns true if the given character is an inline whitespace character.
167
+ *
168
+ * @param b The character to check.
169
+ * @return True if the given character is an inline whitespace character.
170
+ */
77
171
  bool pm_char_is_inline_whitespace(const uint8_t b);
78
172
 
79
- // Returns true if the given character is a binary digit.
173
+ /**
174
+ * Returns true if the given character is a binary digit.
175
+ *
176
+ * @param b The character to check.
177
+ * @return True if the given character is a binary digit.
178
+ */
80
179
  bool pm_char_is_binary_digit(const uint8_t b);
81
180
 
82
- // Returns true if the given character is an octal digit.
181
+ /**
182
+ * Returns true if the given character is an octal digit.
183
+ *
184
+ * @param b The character to check.
185
+ * @return True if the given character is an octal digit.
186
+ */
83
187
  bool pm_char_is_octal_digit(const uint8_t b);
84
188
 
85
- // Returns true if the given character is a decimal digit.
189
+ /**
190
+ * Returns true if the given character is a decimal digit.
191
+ *
192
+ * @param b The character to check.
193
+ * @return True if the given character is a decimal digit.
194
+ */
86
195
  bool pm_char_is_decimal_digit(const uint8_t b);
87
196
 
88
- // Returns true if the given character is a hexadecimal digit.
197
+ /**
198
+ * Returns true if the given character is a hexadecimal digit.
199
+ *
200
+ * @param b The character to check.
201
+ * @return True if the given character is a hexadecimal digit.
202
+ */
89
203
  bool pm_char_is_hexadecimal_digit(const uint8_t b);
90
204
 
91
205
  #endif