prism 0.16.0 → 0.17.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +16 -1
- data/Makefile +6 -0
- data/README.md +1 -1
- data/config.yml +50 -35
- data/docs/fuzzing.md +1 -1
- data/docs/serialization.md +28 -29
- data/ext/prism/api_node.c +802 -770
- data/ext/prism/api_pack.c +20 -9
- data/ext/prism/extension.c +464 -162
- data/ext/prism/extension.h +1 -1
- data/include/prism/ast.h +3173 -763
- data/include/prism/defines.h +32 -9
- data/include/prism/diagnostic.h +36 -3
- data/include/prism/enc/pm_encoding.h +118 -28
- data/include/prism/node.h +38 -13
- data/include/prism/options.h +204 -0
- data/include/prism/pack.h +44 -33
- data/include/prism/parser.h +445 -200
- data/include/prism/prettyprint.h +12 -1
- data/include/prism/regexp.h +16 -2
- data/include/prism/util/pm_buffer.h +94 -16
- data/include/prism/util/pm_char.h +162 -48
- data/include/prism/util/pm_constant_pool.h +126 -32
- data/include/prism/util/pm_list.h +68 -38
- data/include/prism/util/pm_memchr.h +18 -3
- data/include/prism/util/pm_newline_list.h +70 -27
- data/include/prism/util/pm_state_stack.h +25 -7
- data/include/prism/util/pm_string.h +115 -27
- data/include/prism/util/pm_string_list.h +25 -6
- data/include/prism/util/pm_strncasecmp.h +32 -0
- data/include/prism/util/pm_strpbrk.h +31 -17
- data/include/prism/version.h +27 -2
- data/include/prism.h +224 -31
- data/lib/prism/compiler.rb +6 -3
- data/lib/prism/debug.rb +23 -7
- data/lib/prism/dispatcher.rb +33 -18
- data/lib/prism/dsl.rb +10 -5
- data/lib/prism/ffi.rb +132 -80
- data/lib/prism/lex_compat.rb +25 -15
- data/lib/prism/mutation_compiler.rb +10 -5
- data/lib/prism/node.rb +370 -135
- data/lib/prism/node_ext.rb +1 -1
- data/lib/prism/node_inspector.rb +1 -1
- data/lib/prism/pack.rb +79 -40
- data/lib/prism/parse_result/comments.rb +7 -2
- data/lib/prism/parse_result/newlines.rb +4 -0
- data/lib/prism/parse_result.rb +150 -30
- data/lib/prism/pattern.rb +11 -0
- data/lib/prism/ripper_compat.rb +28 -10
- data/lib/prism/serialize.rb +86 -54
- data/lib/prism/visitor.rb +10 -3
- data/lib/prism.rb +20 -2
- data/prism.gemspec +4 -2
- data/rbi/prism.rbi +104 -60
- data/rbi/prism_static.rbi +16 -2
- data/sig/prism.rbs +72 -43
- data/sig/prism_static.rbs +14 -1
- data/src/diagnostic.c +56 -53
- data/src/enc/pm_big5.c +1 -0
- data/src/enc/pm_euc_jp.c +1 -0
- data/src/enc/pm_gbk.c +1 -0
- data/src/enc/pm_shift_jis.c +1 -0
- data/src/enc/pm_tables.c +316 -80
- data/src/enc/pm_unicode.c +53 -8
- data/src/enc/pm_windows_31j.c +1 -0
- data/src/node.c +334 -321
- data/src/options.c +170 -0
- data/src/prettyprint.c +74 -47
- data/src/prism.c +1642 -856
- data/src/regexp.c +151 -95
- data/src/serialize.c +44 -20
- data/src/token_type.c +3 -1
- data/src/util/pm_buffer.c +45 -15
- data/src/util/pm_char.c +103 -57
- data/src/util/pm_constant_pool.c +51 -21
- data/src/util/pm_list.c +12 -4
- data/src/util/pm_memchr.c +5 -3
- data/src/util/pm_newline_list.c +20 -12
- data/src/util/pm_state_stack.c +9 -3
- data/src/util/pm_string.c +95 -85
- data/src/util/pm_string_list.c +14 -15
- data/src/util/pm_strncasecmp.c +10 -3
- data/src/util/pm_strpbrk.c +25 -19
- metadata +5 -3
- data/docs/prism.png +0 -0
data/include/prism/prettyprint.h
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
/**
|
2
|
+
* @file prettyprint.h
|
3
|
+
*
|
4
|
+
* An AST node pretty-printer.
|
5
|
+
*/
|
1
6
|
#ifndef PRISM_PRETTYPRINT_H
|
2
7
|
#define PRISM_PRETTYPRINT_H
|
3
8
|
|
@@ -9,7 +14,13 @@
|
|
9
14
|
#include "prism/parser.h"
|
10
15
|
#include "prism/util/pm_buffer.h"
|
11
16
|
|
12
|
-
|
17
|
+
/**
|
18
|
+
* Pretty-prints the AST represented by the given node to the given buffer.
|
19
|
+
*
|
20
|
+
* @param output_buffer The buffer to write the pretty-printed AST to.
|
21
|
+
* @param parser The parser that parsed the AST.
|
22
|
+
* @param node The root node of the AST to pretty-print.
|
23
|
+
*/
|
13
24
|
PRISM_EXPORTED_FUNCTION void pm_prettyprint(pm_buffer_t *output_buffer, const pm_parser_t *parser, const pm_node_t *node);
|
14
25
|
|
15
26
|
#endif
|
data/include/prism/regexp.h
CHANGED
@@ -1,3 +1,8 @@
|
|
1
|
+
/**
|
2
|
+
* @file regexp.h
|
3
|
+
*
|
4
|
+
* A regular expression parser.
|
5
|
+
*/
|
1
6
|
#ifndef PRISM_REGEXP_H
|
2
7
|
#define PRISM_REGEXP_H
|
3
8
|
|
@@ -12,8 +17,17 @@
|
|
12
17
|
#include <stddef.h>
|
13
18
|
#include <string.h>
|
14
19
|
|
15
|
-
|
16
|
-
|
20
|
+
/**
|
21
|
+
* Parse a regular expression and extract the names of all of the named capture
|
22
|
+
* groups.
|
23
|
+
*
|
24
|
+
* @param source The source code to parse.
|
25
|
+
* @param size The size of the source code.
|
26
|
+
* @param named_captures The list to add the names of the named capture groups.
|
27
|
+
* @param encoding_changed Whether or not the encoding changed from the default.
|
28
|
+
* @param encoding The encoding of the source code.
|
29
|
+
* @return Whether or not the parsing was successful.
|
30
|
+
*/
|
17
31
|
PRISM_EXPORTED_FUNCTION bool pm_regexp_named_capture_group_names(const uint8_t *source, size_t size, pm_string_list_t *named_captures, bool encoding_changed, pm_encoding_t *encoding);
|
18
32
|
|
19
33
|
#endif
|
@@ -1,3 +1,8 @@
|
|
1
|
+
/**
|
2
|
+
* @file pm_buffer.h
|
3
|
+
*
|
4
|
+
* A wrapper around a contiguous block of allocated memory.
|
5
|
+
*/
|
1
6
|
#ifndef PRISM_BUFFER_H
|
2
7
|
#define PRISM_BUFFER_H
|
3
8
|
|
@@ -9,52 +14,125 @@
|
|
9
14
|
#include <stdlib.h>
|
10
15
|
#include <string.h>
|
11
16
|
|
12
|
-
|
13
|
-
|
14
|
-
|
17
|
+
/**
|
18
|
+
* A pm_buffer_t is a simple memory buffer that stores data in a contiguous
|
19
|
+
* block of memory.
|
20
|
+
*/
|
15
21
|
typedef struct {
|
22
|
+
/** The length of the buffer in bytes. */
|
16
23
|
size_t length;
|
24
|
+
|
25
|
+
/** The capacity of the buffer in bytes that has been allocated. */
|
17
26
|
size_t capacity;
|
27
|
+
|
28
|
+
/** A pointer to the start of the buffer. */
|
18
29
|
char *value;
|
19
30
|
} pm_buffer_t;
|
20
31
|
|
21
|
-
|
32
|
+
/**
|
33
|
+
* Return the size of the pm_buffer_t struct.
|
34
|
+
*
|
35
|
+
* @returns The size of the pm_buffer_t struct.
|
36
|
+
*/
|
22
37
|
PRISM_EXPORTED_FUNCTION size_t pm_buffer_sizeof(void);
|
23
38
|
|
24
|
-
|
39
|
+
/**
|
40
|
+
* Initialize a pm_buffer_t with the given capacity.
|
41
|
+
*
|
42
|
+
* @param buffer The buffer to initialize.
|
43
|
+
* @param capacity The capacity of the buffer.
|
44
|
+
* @returns True if the buffer was initialized successfully, false otherwise.
|
45
|
+
*/
|
25
46
|
bool pm_buffer_init_capacity(pm_buffer_t *buffer, size_t capacity);
|
26
47
|
|
27
|
-
|
48
|
+
/**
|
49
|
+
* Initialize a pm_buffer_t with its default values.
|
50
|
+
*
|
51
|
+
* @param buffer The buffer to initialize.
|
52
|
+
* @returns True if the buffer was initialized successfully, false otherwise.
|
53
|
+
*/
|
28
54
|
PRISM_EXPORTED_FUNCTION bool pm_buffer_init(pm_buffer_t *buffer);
|
29
55
|
|
30
|
-
|
56
|
+
/**
|
57
|
+
* Return the value of the buffer.
|
58
|
+
*
|
59
|
+
* @param buffer The buffer to get the value of.
|
60
|
+
* @returns The value of the buffer.
|
61
|
+
*/
|
31
62
|
PRISM_EXPORTED_FUNCTION char * pm_buffer_value(pm_buffer_t *buffer);
|
32
63
|
|
33
|
-
|
64
|
+
/**
|
65
|
+
* Return the length of the buffer.
|
66
|
+
*
|
67
|
+
* @param buffer The buffer to get the length of.
|
68
|
+
* @returns The length of the buffer.
|
69
|
+
*/
|
34
70
|
PRISM_EXPORTED_FUNCTION size_t pm_buffer_length(pm_buffer_t *buffer);
|
35
71
|
|
36
|
-
|
72
|
+
/**
|
73
|
+
* Append the given amount of space as zeroes to the buffer.
|
74
|
+
*
|
75
|
+
* @param buffer The buffer to append to.
|
76
|
+
* @param length The amount of space to append and zero.
|
77
|
+
*/
|
37
78
|
void pm_buffer_append_zeroes(pm_buffer_t *buffer, size_t length);
|
38
79
|
|
39
|
-
|
80
|
+
/**
|
81
|
+
* Append a formatted string to the buffer.
|
82
|
+
*
|
83
|
+
* @param buffer The buffer to append to.
|
84
|
+
* @param format The format string to append.
|
85
|
+
* @param ... The arguments to the format string.
|
86
|
+
*/
|
40
87
|
void pm_buffer_append_format(pm_buffer_t *buffer, const char *format, ...) PRISM_ATTRIBUTE_FORMAT(2, 3);
|
41
88
|
|
42
|
-
|
89
|
+
/**
|
90
|
+
* Append a string to the buffer.
|
91
|
+
*
|
92
|
+
* @param buffer The buffer to append to.
|
93
|
+
* @param value The string to append.
|
94
|
+
* @param length The length of the string to append.
|
95
|
+
*/
|
43
96
|
void pm_buffer_append_string(pm_buffer_t *buffer, const char *value, size_t length);
|
44
97
|
|
45
|
-
|
98
|
+
/**
|
99
|
+
* Append a list of bytes to the buffer.
|
100
|
+
*
|
101
|
+
* @param buffer The buffer to append to.
|
102
|
+
* @param value The bytes to append.
|
103
|
+
* @param length The length of the bytes to append.
|
104
|
+
*/
|
46
105
|
void pm_buffer_append_bytes(pm_buffer_t *buffer, const uint8_t *value, size_t length);
|
47
106
|
|
48
|
-
|
107
|
+
/**
|
108
|
+
* Append a single byte to the buffer.
|
109
|
+
*
|
110
|
+
* @param buffer The buffer to append to.
|
111
|
+
* @param value The byte to append.
|
112
|
+
*/
|
49
113
|
void pm_buffer_append_byte(pm_buffer_t *buffer, uint8_t value);
|
50
114
|
|
51
|
-
|
115
|
+
/**
|
116
|
+
* Append a 32-bit unsigned integer to the buffer as a variable-length integer.
|
117
|
+
*
|
118
|
+
* @param buffer The buffer to append to.
|
119
|
+
* @param value The integer to append.
|
120
|
+
*/
|
52
121
|
void pm_buffer_append_varint(pm_buffer_t *buffer, uint32_t value);
|
53
122
|
|
54
|
-
|
123
|
+
/**
|
124
|
+
* Concatenate one buffer onto another.
|
125
|
+
*
|
126
|
+
* @param destination The buffer to concatenate onto.
|
127
|
+
* @param source The buffer to concatenate.
|
128
|
+
*/
|
55
129
|
void pm_buffer_concat(pm_buffer_t *destination, const pm_buffer_t *source);
|
56
130
|
|
57
|
-
|
131
|
+
/**
|
132
|
+
* Free the memory associated with the buffer.
|
133
|
+
*
|
134
|
+
* @param buffer The buffer to free.
|
135
|
+
*/
|
58
136
|
PRISM_EXPORTED_FUNCTION void pm_buffer_free(pm_buffer_t *buffer);
|
59
137
|
|
60
138
|
#endif
|
@@ -1,3 +1,8 @@
|
|
1
|
+
/**
|
2
|
+
* @file pm_char.h
|
3
|
+
*
|
4
|
+
* Functions for working with characters and strings.
|
5
|
+
*/
|
1
6
|
#ifndef PRISM_CHAR_H
|
2
7
|
#define PRISM_CHAR_H
|
3
8
|
|
@@ -7,85 +12,194 @@
|
|
7
12
|
#include <stdbool.h>
|
8
13
|
#include <stddef.h>
|
9
14
|
|
10
|
-
|
11
|
-
|
15
|
+
/**
|
16
|
+
* Returns the number of characters at the start of the string that are
|
17
|
+
* whitespace. Disallows searching past the given maximum number of characters.
|
18
|
+
*
|
19
|
+
* @param string The string to search.
|
20
|
+
* @param length The maximum number of characters to search.
|
21
|
+
* @return The number of characters at the start of the string that are
|
22
|
+
* whitespace.
|
23
|
+
*/
|
12
24
|
size_t pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length);
|
13
25
|
|
14
|
-
|
15
|
-
|
16
|
-
|
26
|
+
/**
|
27
|
+
* Returns the number of characters at the start of the string that are
|
28
|
+
* whitespace while also tracking the location of each newline. Disallows
|
29
|
+
* searching past the given maximum number of characters.
|
30
|
+
*
|
31
|
+
* @param string The string to search.
|
32
|
+
* @param length The maximum number of characters to search.
|
33
|
+
* @param newline_list The list of newlines to populate.
|
34
|
+
* @return The number of characters at the start of the string that are
|
35
|
+
* whitespace.
|
36
|
+
*/
|
17
37
|
size_t
|
18
38
|
pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list);
|
19
39
|
|
20
|
-
|
21
|
-
|
40
|
+
/**
|
41
|
+
* Returns the number of characters at the start of the string that are inline
|
42
|
+
* whitespace. Disallows searching past the given maximum number of characters.
|
43
|
+
*
|
44
|
+
* @param string The string to search.
|
45
|
+
* @param length The maximum number of characters to search.
|
46
|
+
* @return The number of characters at the start of the string that are inline
|
47
|
+
* whitespace.
|
48
|
+
*/
|
22
49
|
size_t pm_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length);
|
23
50
|
|
24
|
-
|
25
|
-
|
51
|
+
/**
|
52
|
+
* Returns the number of characters at the start of the string that are decimal
|
53
|
+
* digits. Disallows searching past the given maximum number of characters.
|
54
|
+
*
|
55
|
+
* @param string The string to search.
|
56
|
+
* @param length The maximum number of characters to search.
|
57
|
+
* @return The number of characters at the start of the string that are decimal
|
58
|
+
* digits.
|
59
|
+
*/
|
26
60
|
size_t pm_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length);
|
27
61
|
|
28
|
-
|
29
|
-
|
30
|
-
|
62
|
+
/**
|
63
|
+
* Returns the number of characters at the start of the string that are
|
64
|
+
* hexadecimal digits. Disallows searching past the given maximum number of
|
65
|
+
* characters.
|
66
|
+
*
|
67
|
+
* @param string The string to search.
|
68
|
+
* @param length The maximum number of characters to search.
|
69
|
+
* @return The number of characters at the start of the string that are
|
70
|
+
* hexadecimal digits.
|
71
|
+
*/
|
31
72
|
size_t pm_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length);
|
32
73
|
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
74
|
+
/**
|
75
|
+
* Returns the number of characters at the start of the string that are octal
|
76
|
+
* digits or underscores. Disallows searching past the given maximum number of
|
77
|
+
* characters.
|
78
|
+
*
|
79
|
+
* If multiple underscores are found in a row or if an underscore is
|
80
|
+
* found at the end of the number, then the invalid pointer is set to the index
|
81
|
+
* of the first invalid underscore.
|
82
|
+
*
|
83
|
+
* @param string The string to search.
|
84
|
+
* @param length The maximum number of characters to search.
|
85
|
+
* @param invalid The pointer to set to the index of the first invalid
|
86
|
+
* underscore.
|
87
|
+
* @return The number of characters at the start of the string that are octal
|
88
|
+
* digits or underscores.
|
89
|
+
*/
|
40
90
|
size_t pm_strspn_octal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
|
41
91
|
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
92
|
+
/**
|
93
|
+
* Returns the number of characters at the start of the string that are decimal
|
94
|
+
* digits or underscores. Disallows searching past the given maximum number of
|
95
|
+
* characters.
|
96
|
+
*
|
97
|
+
* If multiple underscores are found in a row or if an underscore is
|
98
|
+
* found at the end of the number, then the invalid pointer is set to the index
|
99
|
+
* of the first invalid underscore.
|
100
|
+
*
|
101
|
+
* @param string The string to search.
|
102
|
+
* @param length The maximum number of characters to search.
|
103
|
+
* @param invalid The pointer to set to the index of the first invalid
|
104
|
+
* underscore.
|
105
|
+
* @return The number of characters at the start of the string that are decimal
|
106
|
+
* digits or underscores.
|
107
|
+
*/
|
49
108
|
size_t pm_strspn_decimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
|
50
109
|
|
51
|
-
|
52
|
-
|
53
|
-
|
54
|
-
|
55
|
-
|
56
|
-
|
57
|
-
|
110
|
+
/**
|
111
|
+
* Returns the number of characters at the start of the string that are
|
112
|
+
* hexadecimal digits or underscores. Disallows searching past the given maximum
|
113
|
+
* number of characters.
|
114
|
+
*
|
115
|
+
* If multiple underscores are found in a row or if an underscore is
|
116
|
+
* found at the end of the number, then the invalid pointer is set to the index
|
117
|
+
* of the first invalid underscore.
|
118
|
+
*
|
119
|
+
* @param string The string to search.
|
120
|
+
* @param length The maximum number of characters to search.
|
121
|
+
* @param invalid The pointer to set to the index of the first invalid
|
122
|
+
* underscore.
|
123
|
+
* @return The number of characters at the start of the string that are
|
124
|
+
* hexadecimal digits or underscores.
|
125
|
+
*/
|
58
126
|
size_t pm_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
|
59
127
|
|
60
|
-
|
61
|
-
|
128
|
+
/**
|
129
|
+
* Returns the number of characters at the start of the string that are regexp
|
130
|
+
* options. Disallows searching past the given maximum number of characters.
|
131
|
+
*
|
132
|
+
* @param string The string to search.
|
133
|
+
* @param length The maximum number of characters to search.
|
134
|
+
* @return The number of characters at the start of the string that are regexp
|
135
|
+
* options.
|
136
|
+
*/
|
62
137
|
size_t pm_strspn_regexp_option(const uint8_t *string, ptrdiff_t length);
|
63
138
|
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
70
|
-
|
139
|
+
/**
|
140
|
+
* Returns the number of characters at the start of the string that are binary
|
141
|
+
* digits or underscores. Disallows searching past the given maximum number of
|
142
|
+
* characters.
|
143
|
+
*
|
144
|
+
* If multiple underscores are found in a row or if an underscore is
|
145
|
+
* found at the end of the number, then the invalid pointer is set to the index
|
146
|
+
* of the first invalid underscore.
|
147
|
+
*
|
148
|
+
* @param string The string to search.
|
149
|
+
* @param length The maximum number of characters to search.
|
150
|
+
* @param invalid The pointer to set to the index of the first invalid
|
151
|
+
* underscore.
|
152
|
+
* @return The number of characters at the start of the string that are binary
|
153
|
+
* digits or underscores.
|
154
|
+
*/
|
71
155
|
size_t pm_strspn_binary_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid);
|
72
156
|
|
73
|
-
|
157
|
+
/**
|
158
|
+
* Returns true if the given character is a whitespace character.
|
159
|
+
*
|
160
|
+
* @param b The character to check.
|
161
|
+
* @return True if the given character is a whitespace character.
|
162
|
+
*/
|
74
163
|
bool pm_char_is_whitespace(const uint8_t b);
|
75
164
|
|
76
|
-
|
165
|
+
/**
|
166
|
+
* Returns true if the given character is an inline whitespace character.
|
167
|
+
*
|
168
|
+
* @param b The character to check.
|
169
|
+
* @return True if the given character is an inline whitespace character.
|
170
|
+
*/
|
77
171
|
bool pm_char_is_inline_whitespace(const uint8_t b);
|
78
172
|
|
79
|
-
|
173
|
+
/**
|
174
|
+
* Returns true if the given character is a binary digit.
|
175
|
+
*
|
176
|
+
* @param b The character to check.
|
177
|
+
* @return True if the given character is a binary digit.
|
178
|
+
*/
|
80
179
|
bool pm_char_is_binary_digit(const uint8_t b);
|
81
180
|
|
82
|
-
|
181
|
+
/**
|
182
|
+
* Returns true if the given character is an octal digit.
|
183
|
+
*
|
184
|
+
* @param b The character to check.
|
185
|
+
* @return True if the given character is an octal digit.
|
186
|
+
*/
|
83
187
|
bool pm_char_is_octal_digit(const uint8_t b);
|
84
188
|
|
85
|
-
|
189
|
+
/**
|
190
|
+
* Returns true if the given character is a decimal digit.
|
191
|
+
*
|
192
|
+
* @param b The character to check.
|
193
|
+
* @return True if the given character is a decimal digit.
|
194
|
+
*/
|
86
195
|
bool pm_char_is_decimal_digit(const uint8_t b);
|
87
196
|
|
88
|
-
|
197
|
+
/**
|
198
|
+
* Returns true if the given character is a hexadecimal digit.
|
199
|
+
*
|
200
|
+
* @param b The character to check.
|
201
|
+
* @return True if the given character is a hexadecimal digit.
|
202
|
+
*/
|
89
203
|
bool pm_char_is_hexadecimal_digit(const uint8_t b);
|
90
204
|
|
91
205
|
#endif
|