prism 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -1
- data/Makefile +6 -0
- data/README.md +1 -1
- data/config.yml +50 -35
- data/docs/fuzzing.md +1 -1
- data/docs/serialization.md +28 -29
- data/ext/prism/api_node.c +802 -770
- data/ext/prism/api_pack.c +20 -9
- data/ext/prism/extension.c +464 -162
- data/ext/prism/extension.h +1 -1
- data/include/prism/ast.h +3173 -763
- data/include/prism/defines.h +32 -9
- data/include/prism/diagnostic.h +36 -3
- data/include/prism/enc/pm_encoding.h +118 -28
- data/include/prism/node.h +38 -13
- data/include/prism/options.h +204 -0
- data/include/prism/pack.h +44 -33
- data/include/prism/parser.h +445 -200
- data/include/prism/prettyprint.h +12 -1
- data/include/prism/regexp.h +16 -2
- data/include/prism/util/pm_buffer.h +94 -16
- data/include/prism/util/pm_char.h +162 -48
- data/include/prism/util/pm_constant_pool.h +126 -32
- data/include/prism/util/pm_list.h +68 -38
- data/include/prism/util/pm_memchr.h +18 -3
- data/include/prism/util/pm_newline_list.h +70 -27
- data/include/prism/util/pm_state_stack.h +25 -7
- data/include/prism/util/pm_string.h +115 -27
- data/include/prism/util/pm_string_list.h +25 -6
- data/include/prism/util/pm_strncasecmp.h +32 -0
- data/include/prism/util/pm_strpbrk.h +31 -17
- data/include/prism/version.h +27 -2
- data/include/prism.h +224 -31
- data/lib/prism/compiler.rb +6 -3
- data/lib/prism/debug.rb +23 -7
- data/lib/prism/dispatcher.rb +33 -18
- data/lib/prism/dsl.rb +10 -5
- data/lib/prism/ffi.rb +132 -80
- data/lib/prism/lex_compat.rb +25 -15
- data/lib/prism/mutation_compiler.rb +10 -5
- data/lib/prism/node.rb +370 -135
- data/lib/prism/node_ext.rb +1 -1
- data/lib/prism/node_inspector.rb +1 -1
- data/lib/prism/pack.rb +79 -40
- data/lib/prism/parse_result/comments.rb +7 -2
- data/lib/prism/parse_result/newlines.rb +4 -0
- data/lib/prism/parse_result.rb +150 -30
- data/lib/prism/pattern.rb +11 -0
- data/lib/prism/ripper_compat.rb +28 -10
- data/lib/prism/serialize.rb +86 -54
- data/lib/prism/visitor.rb +10 -3
- data/lib/prism.rb +20 -2
- data/prism.gemspec +4 -2
- data/rbi/prism.rbi +104 -60
- data/rbi/prism_static.rbi +16 -2
- data/sig/prism.rbs +72 -43
- data/sig/prism_static.rbs +14 -1
- data/src/diagnostic.c +56 -53
- data/src/enc/pm_big5.c +1 -0
- data/src/enc/pm_euc_jp.c +1 -0
- data/src/enc/pm_gbk.c +1 -0
- data/src/enc/pm_shift_jis.c +1 -0
- data/src/enc/pm_tables.c +316 -80
- data/src/enc/pm_unicode.c +53 -8
- data/src/enc/pm_windows_31j.c +1 -0
- data/src/node.c +334 -321
- data/src/options.c +170 -0
- data/src/prettyprint.c +74 -47
- data/src/prism.c +1642 -856
- data/src/regexp.c +151 -95
- data/src/serialize.c +44 -20
- data/src/token_type.c +3 -1
- data/src/util/pm_buffer.c +45 -15
- data/src/util/pm_char.c +103 -57
- data/src/util/pm_constant_pool.c +51 -21
- data/src/util/pm_list.c +12 -4
- data/src/util/pm_memchr.c +5 -3
- data/src/util/pm_newline_list.c +20 -12
- data/src/util/pm_state_stack.c +9 -3
- data/src/util/pm_string.c +95 -85
- data/src/util/pm_string_list.c +14 -15
- data/src/util/pm_strncasecmp.c +10 -3
- data/src/util/pm_strpbrk.c +25 -19
- metadata +5 -3
- data/docs/prism.png +0 -0
data/include/prism/prettyprint.h
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
/**
|
2
|
+
* @file prettyprint.h
|
3
|
+
*
|
4
|
+
* An AST node pretty-printer.
|
5
|
+
*/
|
1
6
|
#ifndef PRISM_PRETTYPRINT_H
|
2
7
|
#define PRISM_PRETTYPRINT_H
|
3
8
|
|
@@ -9,7 +14,13 @@
|
|
9
14
|
#include "prism/parser.h"
|
10
15
|
#include "prism/util/pm_buffer.h"
|
11
16
|
|
12
|
-
|
17
|
+
/**
|
18
|
+
* Pretty-prints the AST represented by the given node to the given buffer.
|
19
|
+
*
|
20
|
+
* @param output_buffer The buffer to write the pretty-printed AST to.
|
21
|
+
* @param parser The parser that parsed the AST.
|
22
|
+
* @param node The root node of the AST to pretty-print.
|
23
|
+
*/
|
13
24
|
PRISM_EXPORTED_FUNCTION void pm_prettyprint(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_node_t *node);
|
14
25
|
|
15
26
|
#endif
|
data/include/prism/regexp.h
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
/**
|
2
|
+
* @file regexp.h
|
3
|
+
*
|
4
|
+
* A regular expression parser.
|
5
|
+
*/
|
1
6
|
#ifndef PRISM_REGEXP_H
|
2
7
|
#define PRISM_REGEXP_H
|
3
8
|
|
@@ -12,8 +17,17 @@
|
|
12
17
|
#include <stddef.h>
|
13
18
|
#include <string.h>
|
14
19
|
|
15
|
-
|
16
|
-
|
20
|
+
/**
|
21
|
+
* Parse a regular expression and extract the names of all of the named capture
|
22
|
+
* groups.
|
23
|
+
*
|
24
|
+
* @param source The source code to parse.
|
25
|
+
* @param size The size of the source code.
|
26
|
+
* @param named_captures The list to add the names of the named capture groups.
|
27
|
+
* @param encoding_changed Whether or not the encoding changed from the default.
|
28
|
+
* @param encoding The encoding of the source code.
|
29
|
+
* @return Whether or not the parsing was successful.
|
30
|
+
*/
|
17
31
|
PRISM_EXPORTED_FUNCTION bool pm_regexp_named_capture_group_names(const uint8_t *source, size_t size, pm_string_list_t *named_captures, bool encoding_changed, pm_encoding_t *encoding);
|
18
32
|
|
19
33
|
#endif
|
@@ -1,3 +1,8 @@
|
|
1
|
+
/**
|
2
|
+
* @file pm_buffer.h
|
3
|
+
*
|
4
|
+
* A wrapper around a contiguous block of allocated memory.
|
5
|
+
*/
|
1
6
|
#ifndef PRISM_BUFFER_H
|
2
7
|
#define PRISM_BUFFER_H
|
3
8
|
|
@@ -9,52 +14,125 @@
|
|
9
14
|
#include <stdlib.h>
|
10
15
|
#include <string.h>
|
11
16
|
|
12
|
-
|
13
|
-
|
14
|
-
|
17
|
+
/**
|
18
|
+
* A pm_buffer_t is a simple memory buffer that stores data in a contiguous
|
19
|
+
* block of memory.
|
20
|
+
*/
|
15
21
|
typedef struct {
|
22
|
+
/** The length of the buffer in bytes. */
|
16
23
|
size_t length;
|
24
|
+
|
25
|
+
/** The capacity of the buffer in bytes that has been allocated. */
|
17
26
|
size_t capacity;
|
27
|
+
|
28
|
+
/** A pointer to the start of the buffer. */
|
18
29
|
char *value;
|
19
30
|
} pm_buffer_t;
|
20
31
|
|
21
|
-
|
32
|
+
/**
|
33
|
+
* Return the size of the pm_buffer_t struct.
|
34
|
+
*
|
35
|
+
* @returns The size of the pm_buffer_t struct.
|
36
|
+
*/
|
22
37
|
PRISM_EXPORTED_FUNCTION size_t pm_buffer_sizeof(void);
|
23
38
|
|
24
|
-
|
39
|
+
/**
|
40
|
+
* Initialize a pm_buffer_t with the given capacity.
|
41
|
+
*
|
42
|
+
* @param buffer The buffer to initialize.
|
43
|
+
* @param capacity The capacity of the buffer.
|
44
|
+
* @returns True if the buffer was initialized successfully, false otherwise.
|
45
|
+
*/
|
25
46
|
bool pm_buffer_init_capacity(pm_buffer_t *buffer, size_t capacity);
|
26
47
|
|
27
|
-
|
48
|
+
/**
|
49
|
+
* Initialize a pm_buffer_t with its default values.
|
50
|
+
*
|
51
|
+
* @param buffer The buffer to initialize.
|
52
|
+
* @returns True if the buffer was initialized successfully, false otherwise.
|
53
|
+
*/
|
28
54
|
PRISM_EXPORTED_FUNCTION bool pm_buffer_init(pm_buffer_t *buffer);
|
29
55
|
|
30
|
-
|
56
|
+
/**
|
57
|
+
* Return the value of the buffer.
|
58
|
+
*
|
59
|
+
* @param buffer The buffer to get the value of.
|
60
|
+
* @returns The value of the buffer.
|
61
|
+
*/
|
31
62
|
PRISM_EXPORTED_FUNCTION char * pm_buffer_value(pm_buffer_t *buffer);
|
32
63
|
|
33
|
-
|
64
|
+
/**
|
65
|
+
* Return the length of the buffer.
|
66
|
+
*
|
67
|
+
* @param buffer The buffer to get the length of.
|
68
|
+
* @returns The length of the buffer.
|
69
|
+
*/
|
34
70
|
PRISM_EXPORTED_FUNCTION size_t pm_buffer_length(pm_buffer_t *buffer);
|
35
71
|
|
36
|
-
|
72
|
+
/**
|
73
|
+
* Append the given amount of space as zeroes to the buffer.
|
74
|
+
*
|
75
|
+
* @param buffer The buffer to append to.
|
76
|
+
* @param length The amount of space to append and zero.
|
77
|
+
*/
|
37
78
|
void pm_buffer_append_zeroes(pm_buffer_t *buffer, size_t length);
|
38
79
|
|
39
|
-
|
80
|
+
/**
|
81
|
+
* Append a formatted string to the buffer.
|
82
|
+
*
|
83
|
+
* @param buffer The buffer to append to.
|
84
|
+
* @param format The format string to append.
|
85
|
+
* @param ... The arguments to the format string.
|
86
|
+
*/
|
40
87
|
void pm_buffer_append_format(pm_buffer_t *buffer, const char *format, ...) PRISM_ATTRIBUTE_FORMAT(2, 3);
|
41
88
|
|
42
|
-
|
89
|
+
/**
|
90
|
+
* Append a string to the buffer.
|
91
|
+
*
|
92
|
+
* @param buffer The buffer to append to.
|
93
|
+
* @param value The string to append.
|
94
|
+
* @param length The length of the string to append.
|
95
|
+
*/
|
43
96
|
void pm_buffer_append_string(pm_buffer_t *buffer, const char *value, size_t length);
|
44
97
|
|
45
|
-
|
98
|
+
/**
|
99
|
+
* Append a list of bytes to the buffer.
|
100
|
+
*
|
101
|
+
* @param buffer The buffer to append to.
|
102
|
+
* @param value The bytes to append.
|
103
|
+
* @param length The length of the bytes to append.
|
104
|
+
*/
|
46
105
|
void pm_buffer_append_bytes(pm_buffer_t *buffer, const uint8_t *value, size_t length);
|
47
106
|
|
48
|
-
|
107
|
+
/**
|
108
|
+
* Append a single byte to the buffer.
|
109
|
+
*
|
110
|
+
* @param buffer The buffer to append to.
|
111
|
+
* @param value The byte to append.
|
112
|
+
*/
|
49
113
|
void pm_buffer_append_byte(pm_buffer_t *buffer, uint8_t value);
|
50
114
|
|
51
|
-
|
115
|
+
/**
|
116
|
+
* Append a 32-bit unsigned integer to the buffer as a variable-length integer.
|
117
|
+
*
|
118
|
+
* @param buffer The buffer to append to.
|
119
|
+
* @param value The integer to append.
|
120
|
+
*/
|
52
121
|
void pm_buffer_append_varint(pm_buffer_t *buffer, uint32_t value);
|
53
122
|
|
54
|
-
|
123
|
+
/**
|
124
|
+
* Concatenate one buffer onto another.
|
125
|
+
*
|
126
|
+
* @param destination The buffer to concatenate onto.
|
127
|
+
* @param source The buffer to concatenate.
|
128
|
+
*/
|
55
129
|
void pm_buffer_concat(pm_buffer_t *destination, const pm_buffer_t *source);
|
56
130
|
|
57
|
-
|
131
|
+
/**
|
132
|
+
* Free the memory associated with the buffer.
|
133
|
+
*
|
134
|
+
* @param buffer The buffer to free.
|
135
|
+
*/
|
58
136
|
PRISM_EXPORTED_FUNCTION void pm_buffer_free(pm_buffer_t *buffer);
|
59
137
|
|
60
138
|
#endif
|
@@ -1,3 +1,8 @@
|
|
1
|
+
/**
|
2
|
+
* @file pm_char.h
|
3
|
+
*
|
4
|
+
* Functions for working with characters and strings.
|
5
|
+
*/
|
1
6
|
#ifndef PRISM_CHAR_H
|
2
7
|
#define PRISM_CHAR_H
|
3
8
|
|
@@ -7,85 +12,194 @@
|
|
7
12
|
#include <stdbool.h>
|
8
13
|
#include <stddef.h>
|
9
14
|
|
10
|
-
|
11
|
-
|
15
|
+
/**
|
16
|
+
* Returns the number of characters at the start of the string that are
|
17
|
+
* whitespace. Disallows searching past the given maximum number of characters.
|
18
|
+
*
|
19
|
+
* @param string The string to search.
|
20
|
+
* @param length The maximum number of characters to search.
|
21
|
+
* @return The number of characters at the start of the string that are
|
22
|
+
* whitespace.
|
23
|
+
*/
|
12
24
|
size_t pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length);
|
13
25
|
|
14
|
-
|
15
|
-
|
16
|
-
|
26
|
+
/**
|
27
|
+
* Returns the number of characters at the start of the string that are
|
28
|
+
* whitespace while also tracking the location of each newline. Disallows
|
29
|
+
* searching past the given maximum number of characters.
|
30
|
+
*
|
31
|
+
* @param string The string to search.
|
32
|
+
* @param length The maximum number of characters to search.
|
33
|
+
* @param newline_list The list of newlines to populate.
|
34
|
+
* @return The number of characters at the start of the string that are
|
35
|
+
* whitespace.
|
36
|
+
*/
|
17
37
|
size_t
|
18
38
|
pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list);
|
19
39
|
|
20
|
-
|
21
|
-
|
40
|
+
/**
|
41
|
+
* Returns the number of characters at the start of the string that are inline
|
42
|
+
* whitespace. Disallows searching past the given maximum number of characters.
|
43
|
+
*
|
44
|
+
* @param string The string to search.
|
45
|
+
* @param length The maximum number of characters to search.
|
46
|
+
* @return The number of characters at the start of the string that are inline
|
47
|
+
* whitespace.
|
48
|
+
*/
|
22
49
|
size_t pm_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length);
|
23
50
|
|
24
|
-
|
25
|
-
|
51
|
+
/**
|
52
|
+
* Returns the number of characters at the start of the string that are decimal
|
53
|
+
* digits. Disallows searching past the given maximum number of characters.
|
54
|
+
*
|
55
|
+
* @param string The string to search.
|
56
|
+
* @param length The maximum number of characters to search.
|
57
|
+
* @return The number of characters at the start of the string that are decimal
|
58
|
+
* digits.
|
59
|
+
*/
|
26
60
|
size_t pm_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length);
|
27
61
|
|
28
|
-
|
29
|
-
|
30
|
-
|
62
|
+
/**
|
63
|
+
* Returns the number of characters at the start of the string that are
|
64
|
+
* hexadecimal digits. Disallows searching past the given maximum number of
|
65
|
+
* characters.
|
66
|
+
*
|
67
|
+
* @param string The string to search.
|
68
|
+
* @param length The maximum number of characters to search.
|
69
|
+
* @return The number of characters at the start of the string that are
|
70
|
+
* hexadecimal digits.
|
71
|
+
*/
|
31
72
|
size_t pm_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length);
|
32
73
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
74
|
+
/**
|
75
|
+
* Returns the number of characters at the start of the string that are octal
|
76
|
+
* digits or underscores. Disallows searching past the given maximum number of
|
77
|
+
* characters.
|
78
|
+
*
|
79
|
+
* If multiple underscores are found in a row or if an underscore is
|
80
|
+
* found at the end of the number, then the invalid pointer is set to the index
|
81
|
+
* of the first invalid underscore.
|
82
|
+
*
|
83
|
+
* @param string The string to search.
|
84
|
+
* @param length The maximum number of characters to search.
|
85
|
+
* @param invalid The pointer to set to the index of the first invalid
|
86
|
+
* underscore.
|
87
|
+
* @return The number of characters at the start of the string that are octal
|
88
|
+
* digits or underscores.
|
89
|
+
*/
|
40
90
|
size_t pm_strspn_octal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
|
41
91
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
92
|
+
/**
|
93
|
+
* Returns the number of characters at the start of the string that are decimal
|
94
|
+
* digits or underscores. Disallows searching past the given maximum number of
|
95
|
+
* characters.
|
96
|
+
*
|
97
|
+
* If multiple underscores are found in a row or if an underscore is
|
98
|
+
* found at the end of the number, then the invalid pointer is set to the index
|
99
|
+
* of the first invalid underscore.
|
100
|
+
*
|
101
|
+
* @param string The string to search.
|
102
|
+
* @param length The maximum number of characters to search.
|
103
|
+
* @param invalid The pointer to set to the index of the first invalid
|
104
|
+
* underscore.
|
105
|
+
* @return The number of characters at the start of the string that are decimal
|
106
|
+
* digits or underscores.
|
107
|
+
*/
|
49
108
|
size_t pm_strspn_decimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
|
50
109
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
110
|
+
/**
|
111
|
+
* Returns the number of characters at the start of the string that are
|
112
|
+
* hexadecimal digits or underscores. Disallows searching past the given maximum
|
113
|
+
* number of characters.
|
114
|
+
*
|
115
|
+
* If multiple underscores are found in a row or if an underscore is
|
116
|
+
* found at the end of the number, then the invalid pointer is set to the index
|
117
|
+
* of the first invalid underscore.
|
118
|
+
*
|
119
|
+
* @param string The string to search.
|
120
|
+
* @param length The maximum number of characters to search.
|
121
|
+
* @param invalid The pointer to set to the index of the first invalid
|
122
|
+
* underscore.
|
123
|
+
* @return The number of characters at the start of the string that are
|
124
|
+
* hexadecimal digits or underscores.
|
125
|
+
*/
|
58
126
|
size_t pm_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
|
59
127
|
|
60
|
-
|
61
|
-
|
128
|
+
/**
|
129
|
+
* Returns the number of characters at the start of the string that are regexp
|
130
|
+
* options. Disallows searching past the given maximum number of characters.
|
131
|
+
*
|
132
|
+
* @param string The string to search.
|
133
|
+
* @param length The maximum number of characters to search.
|
134
|
+
* @return The number of characters at the start of the string that are regexp
|
135
|
+
* options.
|
136
|
+
*/
|
62
137
|
size_t pm_strspn_regexp_option(const uint8_t *string, ptrdiff_t length);
|
63
138
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
139
|
+
/**
|
140
|
+
* Returns the number of characters at the start of the string that are binary
|
141
|
+
* digits or underscores. Disallows searching past the given maximum number of
|
142
|
+
* characters.
|
143
|
+
*
|
144
|
+
* If multiple underscores are found in a row or if an underscore is
|
145
|
+
* found at the end of the number, then the invalid pointer is set to the index
|
146
|
+
* of the first invalid underscore.
|
147
|
+
*
|
148
|
+
* @param string The string to search.
|
149
|
+
* @param length The maximum number of characters to search.
|
150
|
+
* @param invalid The pointer to set to the index of the first invalid
|
151
|
+
* underscore.
|
152
|
+
* @return The number of characters at the start of the string that are binary
|
153
|
+
* digits or underscores.
|
154
|
+
*/
|
71
155
|
size_t pm_strspn_binary_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
|
72
156
|
|
73
|
-
|
157
|
+
/**
|
158
|
+
* Returns true if the given character is a whitespace character.
|
159
|
+
*
|
160
|
+
* @param b The character to check.
|
161
|
+
* @return True if the given character is a whitespace character.
|
162
|
+
*/
|
74
163
|
bool pm_char_is_whitespace(const uint8_t b);
|
75
164
|
|
76
|
-
|
165
|
+
/**
|
166
|
+
* Returns true if the given character is an inline whitespace character.
|
167
|
+
*
|
168
|
+
* @param b The character to check.
|
169
|
+
* @return True if the given character is an inline whitespace character.
|
170
|
+
*/
|
77
171
|
bool pm_char_is_inline_whitespace(const uint8_t b);
|
78
172
|
|
79
|
-
|
173
|
+
/**
|
174
|
+
* Returns true if the given character is a binary digit.
|
175
|
+
*
|
176
|
+
* @param b The character to check.
|
177
|
+
* @return True if the given character is a binary digit.
|
178
|
+
*/
|
80
179
|
bool pm_char_is_binary_digit(const uint8_t b);
|
81
180
|
|
82
|
-
|
181
|
+
/**
|
182
|
+
* Returns true if the given character is an octal digit.
|
183
|
+
*
|
184
|
+
* @param b The character to check.
|
185
|
+
* @return True if the given character is an octal digit.
|
186
|
+
*/
|
83
187
|
bool pm_char_is_octal_digit(const uint8_t b);
|
84
188
|
|
85
|
-
|
189
|
+
/**
|
190
|
+
* Returns true if the given character is a decimal digit.
|
191
|
+
*
|
192
|
+
* @param b The character to check.
|
193
|
+
* @return True if the given character is a decimal digit.
|
194
|
+
*/
|
86
195
|
bool pm_char_is_decimal_digit(const uint8_t b);
|
87
196
|
|
88
|
-
|
197
|
+
/**
|
198
|
+
* Returns true if the given character is a hexadecimal digit.
|
199
|
+
*
|
200
|
+
* @param b The character to check.
|
201
|
+
* @return True if the given character is a hexadecimal digit.
|
202
|
+
*/
|
89
203
|
bool pm_char_is_hexadecimal_digit(const uint8_t b);
|
90
204
|
|
91
205
|
#endif
|