yarp 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +29 -8
- data/CONTRIBUTING.md +2 -2
- data/Makefile +5 -5
- data/README.md +11 -12
- data/config.yml +6 -2
- data/docs/build_system.md +21 -21
- data/docs/building.md +4 -4
- data/docs/configuration.md +25 -21
- data/docs/design.md +2 -2
- data/docs/encoding.md +17 -17
- data/docs/fuzzing.md +4 -4
- data/docs/heredocs.md +3 -3
- data/docs/mapping.md +94 -94
- data/docs/ripper.md +4 -4
- data/docs/ruby_api.md +11 -11
- data/docs/serialization.md +17 -16
- data/docs/testing.md +6 -6
- data/ext/prism/api_node.c +4725 -0
- data/ext/{yarp → prism}/api_pack.c +82 -82
- data/ext/{yarp → prism}/extconf.rb +13 -13
- data/ext/{yarp → prism}/extension.c +175 -168
- data/ext/prism/extension.h +18 -0
- data/include/prism/ast.h +1932 -0
- data/include/prism/defines.h +45 -0
- data/include/prism/diagnostic.h +231 -0
- data/include/{yarp/enc/yp_encoding.h → prism/enc/pm_encoding.h} +40 -40
- data/include/prism/node.h +41 -0
- data/include/prism/pack.h +141 -0
- data/include/{yarp → prism}/parser.h +143 -142
- data/include/prism/regexp.h +19 -0
- data/include/prism/unescape.h +48 -0
- data/include/prism/util/pm_buffer.h +51 -0
- data/include/{yarp/util/yp_char.h → prism/util/pm_char.h} +20 -20
- data/include/{yarp/util/yp_constant_pool.h → prism/util/pm_constant_pool.h} +26 -22
- data/include/{yarp/util/yp_list.h → prism/util/pm_list.h} +21 -21
- data/include/prism/util/pm_memchr.h +14 -0
- data/include/{yarp/util/yp_newline_list.h → prism/util/pm_newline_list.h} +11 -11
- data/include/prism/util/pm_state_stack.h +24 -0
- data/include/{yarp/util/yp_string.h → prism/util/pm_string.h} +20 -20
- data/include/prism/util/pm_string_list.h +25 -0
- data/include/{yarp/util/yp_strpbrk.h → prism/util/pm_strpbrk.h} +7 -7
- data/include/prism/version.h +4 -0
- data/include/prism.h +82 -0
- data/lib/prism/compiler.rb +465 -0
- data/lib/prism/debug.rb +157 -0
- data/lib/{yarp/desugar_visitor.rb → prism/desugar_compiler.rb} +4 -2
- data/lib/prism/dispatcher.rb +2051 -0
- data/lib/prism/dsl.rb +750 -0
- data/lib/{yarp → prism}/ffi.rb +66 -67
- data/lib/{yarp → prism}/lex_compat.rb +40 -43
- data/lib/{yarp/mutation_visitor.rb → prism/mutation_compiler.rb} +3 -3
- data/lib/{yarp → prism}/node.rb +2012 -2593
- data/lib/prism/node_ext.rb +55 -0
- data/lib/prism/node_inspector.rb +68 -0
- data/lib/{yarp → prism}/pack.rb +1 -1
- data/lib/{yarp → prism}/parse_result/comments.rb +1 -1
- data/lib/{yarp → prism}/parse_result/newlines.rb +1 -1
- data/lib/prism/parse_result.rb +266 -0
- data/lib/{yarp → prism}/pattern.rb +14 -14
- data/lib/{yarp → prism}/ripper_compat.rb +5 -5
- data/lib/{yarp → prism}/serialize.rb +12 -7
- data/lib/prism/visitor.rb +470 -0
- data/lib/prism.rb +64 -0
- data/lib/yarp.rb +2 -614
- data/src/diagnostic.c +213 -208
- data/src/enc/pm_big5.c +52 -0
- data/src/enc/pm_euc_jp.c +58 -0
- data/src/enc/{yp_gbk.c → pm_gbk.c} +16 -16
- data/src/enc/pm_shift_jis.c +56 -0
- data/src/enc/{yp_tables.c → pm_tables.c} +69 -69
- data/src/enc/{yp_unicode.c → pm_unicode.c} +40 -40
- data/src/enc/pm_windows_31j.c +56 -0
- data/src/node.c +1293 -1233
- data/src/pack.c +247 -247
- data/src/prettyprint.c +1479 -1479
- data/src/{yarp.c → prism.c} +5205 -5083
- data/src/regexp.c +132 -132
- data/src/serialize.c +1121 -1121
- data/src/token_type.c +169 -167
- data/src/unescape.c +106 -87
- data/src/util/pm_buffer.c +103 -0
- data/src/util/{yp_char.c → pm_char.c} +72 -72
- data/src/util/{yp_constant_pool.c → pm_constant_pool.c} +85 -64
- data/src/util/{yp_list.c → pm_list.c} +10 -10
- data/src/util/{yp_memchr.c → pm_memchr.c} +6 -4
- data/src/util/{yp_newline_list.c → pm_newline_list.c} +21 -21
- data/src/util/{yp_state_stack.c → pm_state_stack.c} +4 -4
- data/src/util/{yp_string.c → pm_string.c} +38 -38
- data/src/util/pm_string_list.c +29 -0
- data/src/util/{yp_strncasecmp.c → pm_strncasecmp.c} +1 -1
- data/src/util/{yp_strpbrk.c → pm_strpbrk.c} +8 -8
- data/yarp.gemspec +68 -59
- metadata +70 -61
- data/ext/yarp/api_node.c +0 -4728
- data/ext/yarp/extension.h +0 -18
- data/include/yarp/ast.h +0 -1929
- data/include/yarp/defines.h +0 -45
- data/include/yarp/diagnostic.h +0 -226
- data/include/yarp/node.h +0 -42
- data/include/yarp/pack.h +0 -141
- data/include/yarp/regexp.h +0 -19
- data/include/yarp/unescape.h +0 -44
- data/include/yarp/util/yp_buffer.h +0 -51
- data/include/yarp/util/yp_memchr.h +0 -14
- data/include/yarp/util/yp_state_stack.h +0 -24
- data/include/yarp/util/yp_string_list.h +0 -25
- data/include/yarp/version.h +0 -4
- data/include/yarp.h +0 -82
- data/src/enc/yp_big5.c +0 -52
- data/src/enc/yp_euc_jp.c +0 -58
- data/src/enc/yp_shift_jis.c +0 -56
- data/src/enc/yp_windows_31j.c +0 -56
- data/src/util/yp_buffer.c +0 -101
- data/src/util/yp_string_list.c +0 -29
|
@@ -1,19 +1,19 @@
|
|
|
1
|
-
#include "
|
|
2
|
-
|
|
3
|
-
#define
|
|
4
|
-
#define
|
|
5
|
-
#define
|
|
6
|
-
|
|
7
|
-
#define
|
|
8
|
-
#define
|
|
9
|
-
#define
|
|
10
|
-
#define
|
|
11
|
-
#define
|
|
12
|
-
#define
|
|
13
|
-
#define
|
|
14
|
-
#define
|
|
15
|
-
|
|
16
|
-
static const uint8_t
|
|
1
|
+
#include "prism/util/pm_char.h"
|
|
2
|
+
|
|
3
|
+
#define PRISM_CHAR_BIT_WHITESPACE (1 << 0)
|
|
4
|
+
#define PRISM_CHAR_BIT_INLINE_WHITESPACE (1 << 1)
|
|
5
|
+
#define PRISM_CHAR_BIT_REGEXP_OPTION (1 << 2)
|
|
6
|
+
|
|
7
|
+
#define PRISM_NUMBER_BIT_BINARY_DIGIT (1 << 0)
|
|
8
|
+
#define PRISM_NUMBER_BIT_BINARY_NUMBER (1 << 1)
|
|
9
|
+
#define PRISM_NUMBER_BIT_OCTAL_DIGIT (1 << 2)
|
|
10
|
+
#define PRISM_NUMBER_BIT_OCTAL_NUMBER (1 << 3)
|
|
11
|
+
#define PRISM_NUMBER_BIT_DECIMAL_DIGIT (1 << 4)
|
|
12
|
+
#define PRISM_NUMBER_BIT_DECIMAL_NUMBER (1 << 5)
|
|
13
|
+
#define PRISM_NUMBER_BIT_HEXADECIMAL_DIGIT (1 << 6)
|
|
14
|
+
#define PRISM_NUMBER_BIT_HEXADECIMAL_NUMBER (1 << 7)
|
|
15
|
+
|
|
16
|
+
static const uint8_t pm_byte_table[256] = {
|
|
17
17
|
// 0 1 2 3 4 5 6 7 8 9 A B C D E F
|
|
18
18
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 1, 3, 3, 3, 0, 0, // 0x
|
|
19
19
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // 1x
|
|
@@ -33,7 +33,7 @@ static const uint8_t yp_byte_table[256] = {
|
|
|
33
33
|
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // Fx
|
|
34
34
|
};
|
|
35
35
|
|
|
36
|
-
static const uint8_t
|
|
36
|
+
static const uint8_t pm_number_table[256] = {
|
|
37
37
|
// 0 1 2 3 4 5 6 7 8 9 A B C D E F
|
|
38
38
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 0x
|
|
39
39
|
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, // 1x
|
|
@@ -54,36 +54,36 @@ static const uint8_t yp_number_table[256] = {
|
|
|
54
54
|
};
|
|
55
55
|
|
|
56
56
|
static inline size_t
|
|
57
|
-
|
|
57
|
+
pm_strspn_char_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) {
|
|
58
58
|
if (length <= 0) return 0;
|
|
59
59
|
|
|
60
60
|
size_t size = 0;
|
|
61
61
|
size_t maximum = (size_t) length;
|
|
62
62
|
|
|
63
|
-
while (size < maximum && (
|
|
63
|
+
while (size < maximum && (pm_byte_table[string[size]] & kind)) size++;
|
|
64
64
|
return size;
|
|
65
65
|
}
|
|
66
66
|
|
|
67
67
|
// Returns the number of characters at the start of the string that are
|
|
68
68
|
// whitespace. Disallows searching past the given maximum number of characters.
|
|
69
69
|
size_t
|
|
70
|
-
|
|
71
|
-
return
|
|
70
|
+
pm_strspn_whitespace(const uint8_t *string, ptrdiff_t length) {
|
|
71
|
+
return pm_strspn_char_kind(string, length, PRISM_CHAR_BIT_WHITESPACE);
|
|
72
72
|
}
|
|
73
73
|
|
|
74
74
|
// Returns the number of characters at the start of the string that are
|
|
75
75
|
// whitespace while also tracking the location of each newline. Disallows
|
|
76
76
|
// searching past the given maximum number of characters.
|
|
77
77
|
size_t
|
|
78
|
-
|
|
78
|
+
pm_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, pm_newline_list_t *newline_list) {
|
|
79
79
|
if (length <= 0) return 0;
|
|
80
80
|
|
|
81
81
|
size_t size = 0;
|
|
82
82
|
size_t maximum = (size_t) length;
|
|
83
83
|
|
|
84
|
-
while (size < maximum && (
|
|
84
|
+
while (size < maximum && (pm_byte_table[string[size]] & PRISM_CHAR_BIT_WHITESPACE)) {
|
|
85
85
|
if (string[size] == '\n') {
|
|
86
|
-
|
|
86
|
+
pm_newline_list_append(newline_list, string + size);
|
|
87
87
|
}
|
|
88
88
|
|
|
89
89
|
size++;
|
|
@@ -95,45 +95,45 @@ yp_strspn_whitespace_newlines(const uint8_t *string, ptrdiff_t length, yp_newlin
|
|
|
95
95
|
// Returns the number of characters at the start of the string that are inline
|
|
96
96
|
// whitespace. Disallows searching past the given maximum number of characters.
|
|
97
97
|
size_t
|
|
98
|
-
|
|
99
|
-
return
|
|
98
|
+
pm_strspn_inline_whitespace(const uint8_t *string, ptrdiff_t length) {
|
|
99
|
+
return pm_strspn_char_kind(string, length, PRISM_CHAR_BIT_INLINE_WHITESPACE);
|
|
100
100
|
}
|
|
101
101
|
|
|
102
102
|
// Returns the number of characters at the start of the string that are regexp
|
|
103
103
|
// options. Disallows searching past the given maximum number of characters.
|
|
104
104
|
size_t
|
|
105
|
-
|
|
106
|
-
return
|
|
105
|
+
pm_strspn_regexp_option(const uint8_t *string, ptrdiff_t length) {
|
|
106
|
+
return pm_strspn_char_kind(string, length, PRISM_CHAR_BIT_REGEXP_OPTION);
|
|
107
107
|
}
|
|
108
108
|
|
|
109
109
|
static inline bool
|
|
110
|
-
|
|
111
|
-
return (
|
|
110
|
+
pm_char_is_char_kind(const uint8_t b, uint8_t kind) {
|
|
111
|
+
return (pm_byte_table[b] & kind) != 0;
|
|
112
112
|
}
|
|
113
113
|
|
|
114
114
|
// Returns true if the given character is a whitespace character.
|
|
115
115
|
bool
|
|
116
|
-
|
|
117
|
-
return
|
|
116
|
+
pm_char_is_whitespace(const uint8_t b) {
|
|
117
|
+
return pm_char_is_char_kind(b, PRISM_CHAR_BIT_WHITESPACE);
|
|
118
118
|
}
|
|
119
119
|
|
|
120
120
|
// Returns true if the given character is an inline whitespace character.
|
|
121
121
|
bool
|
|
122
|
-
|
|
123
|
-
return
|
|
122
|
+
pm_char_is_inline_whitespace(const uint8_t b) {
|
|
123
|
+
return pm_char_is_char_kind(b, PRISM_CHAR_BIT_INLINE_WHITESPACE);
|
|
124
124
|
}
|
|
125
125
|
|
|
126
126
|
// Scan through the string and return the number of characters at the start of
|
|
127
127
|
// the string that match the given kind. Disallows searching past the given
|
|
128
128
|
// maximum number of characters.
|
|
129
129
|
static inline size_t
|
|
130
|
-
|
|
130
|
+
pm_strspn_number_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) {
|
|
131
131
|
if (length <= 0) return 0;
|
|
132
132
|
|
|
133
133
|
size_t size = 0;
|
|
134
134
|
size_t maximum = (size_t) length;
|
|
135
135
|
|
|
136
|
-
while (size < maximum && (
|
|
136
|
+
while (size < maximum && (pm_number_table[string[size]] & kind)) size++;
|
|
137
137
|
return size;
|
|
138
138
|
}
|
|
139
139
|
|
|
@@ -144,14 +144,14 @@ yp_strspn_number_kind(const uint8_t *string, ptrdiff_t length, uint8_t kind) {
|
|
|
144
144
|
// Additionally, report the location of the last invalid underscore character
|
|
145
145
|
// found in the string through the out invalid parameter.
|
|
146
146
|
static inline size_t
|
|
147
|
-
|
|
147
|
+
pm_strspn_number_kind_underscores(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid, uint8_t kind) {
|
|
148
148
|
if (length <= 0) return 0;
|
|
149
149
|
|
|
150
150
|
size_t size = 0;
|
|
151
151
|
size_t maximum = (size_t) length;
|
|
152
152
|
|
|
153
153
|
bool underscore = false;
|
|
154
|
-
while (size < maximum && (
|
|
154
|
+
while (size < maximum && (pm_number_table[string[size]] & kind)) {
|
|
155
155
|
if (string[size] == '_') {
|
|
156
156
|
if (underscore) *invalid = string + size;
|
|
157
157
|
underscore = true;
|
|
@@ -174,8 +174,8 @@ yp_strspn_number_kind_underscores(const uint8_t *string, ptrdiff_t length, const
|
|
|
174
174
|
// found at the end of the number, then the invalid pointer is set to the index
|
|
175
175
|
// of the first invalid underscore.
|
|
176
176
|
size_t
|
|
177
|
-
|
|
178
|
-
return
|
|
177
|
+
pm_strspn_binary_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid) {
|
|
178
|
+
return pm_strspn_number_kind_underscores(string, length, invalid, PRISM_NUMBER_BIT_BINARY_NUMBER);
|
|
179
179
|
}
|
|
180
180
|
|
|
181
181
|
// Returns the number of characters at the start of the string that are octal
|
|
@@ -186,15 +186,15 @@ yp_strspn_binary_number(const uint8_t *string, ptrdiff_t length, const uint8_t *
|
|
|
186
186
|
// found at the end of the number, then the invalid pointer is set to the index
|
|
187
187
|
// of the first invalid underscore.
|
|
188
188
|
size_t
|
|
189
|
-
|
|
190
|
-
return
|
|
189
|
+
pm_strspn_octal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid) {
|
|
190
|
+
return pm_strspn_number_kind_underscores(string, length, invalid, PRISM_NUMBER_BIT_OCTAL_NUMBER);
|
|
191
191
|
}
|
|
192
192
|
|
|
193
193
|
// Returns the number of characters at the start of the string that are decimal
|
|
194
194
|
// digits. Disallows searching past the given maximum number of characters.
|
|
195
195
|
size_t
|
|
196
|
-
|
|
197
|
-
return
|
|
196
|
+
pm_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length) {
|
|
197
|
+
return pm_strspn_number_kind(string, length, PRISM_NUMBER_BIT_DECIMAL_DIGIT);
|
|
198
198
|
}
|
|
199
199
|
|
|
200
200
|
// Returns the number of characters at the start of the string that are decimal
|
|
@@ -205,16 +205,16 @@ yp_strspn_decimal_digit(const uint8_t *string, ptrdiff_t length) {
|
|
|
205
205
|
// found at the end of the number, then the invalid pointer is set to the index
|
|
206
206
|
// of the first invalid underscore.
|
|
207
207
|
size_t
|
|
208
|
-
|
|
209
|
-
return
|
|
208
|
+
pm_strspn_decimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid) {
|
|
209
|
+
return pm_strspn_number_kind_underscores(string, length, invalid, PRISM_NUMBER_BIT_DECIMAL_NUMBER);
|
|
210
210
|
}
|
|
211
211
|
|
|
212
212
|
// Returns the number of characters at the start of the string that are
|
|
213
213
|
// hexadecimal digits. Disallows searching past the given maximum number of
|
|
214
214
|
// characters.
|
|
215
215
|
size_t
|
|
216
|
-
|
|
217
|
-
return
|
|
216
|
+
pm_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length) {
|
|
217
|
+
return pm_strspn_number_kind(string, length, PRISM_NUMBER_BIT_HEXADECIMAL_DIGIT);
|
|
218
218
|
}
|
|
219
219
|
|
|
220
220
|
// Returns the number of characters at the start of the string that are
|
|
@@ -225,48 +225,48 @@ yp_strspn_hexadecimal_digit(const uint8_t *string, ptrdiff_t length) {
|
|
|
225
225
|
// found at the end of the number, then the invalid pointer is set to the index
|
|
226
226
|
// of the first invalid underscore.
|
|
227
227
|
size_t
|
|
228
|
-
|
|
229
|
-
return
|
|
228
|
+
pm_strspn_hexadecimal_number(const uint8_t *string, ptrdiff_t length, const uint8_t **invalid) {
|
|
229
|
+
return pm_strspn_number_kind_underscores(string, length, invalid, PRISM_NUMBER_BIT_HEXADECIMAL_NUMBER);
|
|
230
230
|
}
|
|
231
231
|
|
|
232
232
|
static inline bool
|
|
233
|
-
|
|
234
|
-
return (
|
|
233
|
+
pm_char_is_number_kind(const uint8_t b, uint8_t kind) {
|
|
234
|
+
return (pm_number_table[b] & kind) != 0;
|
|
235
235
|
}
|
|
236
236
|
|
|
237
237
|
// Returns true if the given character is a binary digit.
|
|
238
238
|
bool
|
|
239
|
-
|
|
240
|
-
return
|
|
239
|
+
pm_char_is_binary_digit(const uint8_t b) {
|
|
240
|
+
return pm_char_is_number_kind(b, PRISM_NUMBER_BIT_BINARY_DIGIT);
|
|
241
241
|
}
|
|
242
242
|
|
|
243
243
|
// Returns true if the given character is an octal digit.
|
|
244
244
|
bool
|
|
245
|
-
|
|
246
|
-
return
|
|
245
|
+
pm_char_is_octal_digit(const uint8_t b) {
|
|
246
|
+
return pm_char_is_number_kind(b, PRISM_NUMBER_BIT_OCTAL_DIGIT);
|
|
247
247
|
}
|
|
248
248
|
|
|
249
249
|
// Returns true if the given character is a decimal digit.
|
|
250
250
|
bool
|
|
251
|
-
|
|
252
|
-
return
|
|
251
|
+
pm_char_is_decimal_digit(const uint8_t b) {
|
|
252
|
+
return pm_char_is_number_kind(b, PRISM_NUMBER_BIT_DECIMAL_DIGIT);
|
|
253
253
|
}
|
|
254
254
|
|
|
255
255
|
// Returns true if the given character is a hexadecimal digit.
|
|
256
256
|
bool
|
|
257
|
-
|
|
258
|
-
return
|
|
257
|
+
pm_char_is_hexadecimal_digit(const uint8_t b) {
|
|
258
|
+
return pm_char_is_number_kind(b, PRISM_NUMBER_BIT_HEXADECIMAL_DIGIT);
|
|
259
259
|
}
|
|
260
260
|
|
|
261
|
-
#undef
|
|
262
|
-
#undef
|
|
263
|
-
#undef
|
|
264
|
-
|
|
265
|
-
#undef
|
|
266
|
-
#undef
|
|
267
|
-
#undef
|
|
268
|
-
#undef
|
|
269
|
-
#undef
|
|
270
|
-
#undef
|
|
271
|
-
#undef
|
|
272
|
-
#undef
|
|
261
|
+
#undef PRISM_CHAR_BIT_WHITESPACE
|
|
262
|
+
#undef PRISM_CHAR_BIT_INLINE_WHITESPACE
|
|
263
|
+
#undef PRISM_CHAR_BIT_REGEXP_OPTION
|
|
264
|
+
|
|
265
|
+
#undef PRISM_NUMBER_BIT_BINARY_DIGIT
|
|
266
|
+
#undef PRISM_NUMBER_BIT_BINARY_NUMBER
|
|
267
|
+
#undef PRISM_NUMBER_BIT_OCTAL_DIGIT
|
|
268
|
+
#undef PRISM_NUMBER_BIT_OCTAL_NUMBER
|
|
269
|
+
#undef PRISM_NUMBER_BIT_DECIMAL_DIGIT
|
|
270
|
+
#undef PRISM_NUMBER_BIT_DECIMAL_NUMBER
|
|
271
|
+
#undef PRISM_NUMBER_BIT_HEXADECIMAL_NUMBER
|
|
272
|
+
#undef PRISM_NUMBER_BIT_HEXADECIMAL_DIGIT
|
|
@@ -1,8 +1,8 @@
|
|
|
1
|
-
#include "
|
|
1
|
+
#include "prism/util/pm_constant_pool.h"
|
|
2
2
|
|
|
3
3
|
// Initialize a list of constant ids.
|
|
4
4
|
void
|
|
5
|
-
|
|
5
|
+
pm_constant_id_list_init(pm_constant_id_list_t *list) {
|
|
6
6
|
list->ids = NULL;
|
|
7
7
|
list->size = 0;
|
|
8
8
|
list->capacity = 0;
|
|
@@ -11,10 +11,10 @@ yp_constant_id_list_init(yp_constant_id_list_t *list) {
|
|
|
11
11
|
// Append a constant id to a list of constant ids. Returns false if any
|
|
12
12
|
// potential reallocations fail.
|
|
13
13
|
bool
|
|
14
|
-
|
|
14
|
+
pm_constant_id_list_append(pm_constant_id_list_t *list, pm_constant_id_t id) {
|
|
15
15
|
if (list->size >= list->capacity) {
|
|
16
16
|
list->capacity = list->capacity == 0 ? 8 : list->capacity * 2;
|
|
17
|
-
list->ids = (
|
|
17
|
+
list->ids = (pm_constant_id_t *) realloc(list->ids, sizeof(pm_constant_id_t) * list->capacity);
|
|
18
18
|
if (list->ids == NULL) return false;
|
|
19
19
|
}
|
|
20
20
|
|
|
@@ -24,7 +24,7 @@ yp_constant_id_list_append(yp_constant_id_list_t *list, yp_constant_id_t id) {
|
|
|
24
24
|
|
|
25
25
|
// Checks if the current constant id list includes the given constant id.
|
|
26
26
|
bool
|
|
27
|
-
|
|
27
|
+
pm_constant_id_list_includes(pm_constant_id_list_t *list, pm_constant_id_t id) {
|
|
28
28
|
for (size_t index = 0; index < list->size; index++) {
|
|
29
29
|
if (list->ids[index] == id) return true;
|
|
30
30
|
}
|
|
@@ -33,13 +33,13 @@ yp_constant_id_list_includes(yp_constant_id_list_t *list, yp_constant_id_t id) {
|
|
|
33
33
|
|
|
34
34
|
// Get the memory size of a list of constant ids.
|
|
35
35
|
size_t
|
|
36
|
-
|
|
37
|
-
return sizeof(
|
|
36
|
+
pm_constant_id_list_memsize(pm_constant_id_list_t *list) {
|
|
37
|
+
return sizeof(pm_constant_id_list_t) + (list->capacity * sizeof(pm_constant_id_t));
|
|
38
38
|
}
|
|
39
39
|
|
|
40
40
|
// Free the memory associated with a list of constant ids.
|
|
41
41
|
void
|
|
42
|
-
|
|
42
|
+
pm_constant_id_list_free(pm_constant_id_list_t *list) {
|
|
43
43
|
if (list->ids != NULL) {
|
|
44
44
|
free(list->ids);
|
|
45
45
|
}
|
|
@@ -47,10 +47,10 @@ yp_constant_id_list_free(yp_constant_id_list_t *list) {
|
|
|
47
47
|
|
|
48
48
|
// A relatively simple hash function (djb2) that is used to hash strings. We are
|
|
49
49
|
// optimizing here for simplicity and speed.
|
|
50
|
-
static inline
|
|
51
|
-
|
|
50
|
+
static inline uint32_t
|
|
51
|
+
pm_constant_pool_hash(const uint8_t *start, size_t length) {
|
|
52
52
|
// This is a prime number used as the initial value for the hash function.
|
|
53
|
-
|
|
53
|
+
uint32_t value = 5381;
|
|
54
54
|
|
|
55
55
|
for (size_t index = 0; index < length; index++) {
|
|
56
56
|
value = ((value << 5) + value) + start[index];
|
|
@@ -60,8 +60,8 @@ yp_constant_pool_hash(const uint8_t *start, size_t length) {
|
|
|
60
60
|
}
|
|
61
61
|
|
|
62
62
|
// https://graphics.stanford.edu/~seander/bithacks.html#RoundUpPowerOf2
|
|
63
|
-
static
|
|
64
|
-
next_power_of_two(
|
|
63
|
+
static uint32_t
|
|
64
|
+
next_power_of_two(uint32_t v) {
|
|
65
65
|
// Avoid underflow in subtraction on next line.
|
|
66
66
|
if (v == 0) {
|
|
67
67
|
// 1 is the nearest power of 2 to 0 (2^0)
|
|
@@ -73,92 +73,107 @@ next_power_of_two(size_t v) {
|
|
|
73
73
|
v |= v >> 4;
|
|
74
74
|
v |= v >> 8;
|
|
75
75
|
v |= v >> 16;
|
|
76
|
-
#if defined(__LP64__) || defined(_WIN64)
|
|
77
|
-
v |= v >> 32;
|
|
78
|
-
#endif
|
|
79
76
|
v++;
|
|
80
77
|
return v;
|
|
81
78
|
}
|
|
82
79
|
|
|
83
80
|
#ifndef NDEBUG
|
|
84
81
|
static bool
|
|
85
|
-
is_power_of_two(
|
|
82
|
+
is_power_of_two(uint32_t size) {
|
|
86
83
|
return (size & (size - 1)) == 0;
|
|
87
84
|
}
|
|
88
85
|
#endif
|
|
89
86
|
|
|
90
87
|
// Resize a constant pool to a given capacity.
|
|
91
88
|
static inline bool
|
|
92
|
-
|
|
89
|
+
pm_constant_pool_resize(pm_constant_pool_t *pool) {
|
|
93
90
|
assert(is_power_of_two(pool->capacity));
|
|
94
|
-
|
|
91
|
+
|
|
92
|
+
uint32_t next_capacity = pool->capacity * 2;
|
|
95
93
|
if (next_capacity < pool->capacity) return false;
|
|
96
94
|
|
|
97
|
-
const
|
|
98
|
-
|
|
99
|
-
|
|
95
|
+
const uint32_t mask = next_capacity - 1;
|
|
96
|
+
const size_t element_size = sizeof(pm_constant_pool_bucket_t) + sizeof(pm_constant_t);
|
|
97
|
+
|
|
98
|
+
void *next = calloc(next_capacity, element_size);
|
|
99
|
+
if (next == NULL) return false;
|
|
100
100
|
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
101
|
+
pm_constant_pool_bucket_t *next_buckets = next;
|
|
102
|
+
pm_constant_t *next_constants = (void *)(((char *) next) + next_capacity * sizeof(pm_constant_pool_bucket_t));
|
|
103
|
+
|
|
104
|
+
// For each bucket in the current constant pool, find the index in the
|
|
105
|
+
// next constant pool, and insert it.
|
|
106
|
+
for (uint32_t index = 0; index < pool->capacity; index++) {
|
|
107
|
+
pm_constant_pool_bucket_t *bucket = &pool->buckets[index];
|
|
105
108
|
|
|
106
109
|
// If an id is set on this constant, then we know we have content here.
|
|
107
110
|
// In this case we need to insert it into the next constant pool.
|
|
108
|
-
if (
|
|
109
|
-
|
|
111
|
+
if (bucket->id != 0) {
|
|
112
|
+
uint32_t next_index = bucket->hash & mask;
|
|
110
113
|
|
|
111
114
|
// This implements linear scanning to find the next available slot
|
|
112
115
|
// in case this index is already taken. We don't need to bother
|
|
113
116
|
// comparing the values since we know that the hash is unique.
|
|
114
|
-
while (
|
|
117
|
+
while (next_buckets[next_index].id != 0) {
|
|
115
118
|
next_index = (next_index + 1) & mask;
|
|
116
119
|
}
|
|
117
120
|
|
|
118
|
-
// Here we copy over the entire
|
|
121
|
+
// Here we copy over the entire bucket, which includes the id so
|
|
119
122
|
// that they are consistent between resizes.
|
|
120
|
-
|
|
123
|
+
next_buckets[next_index] = *bucket;
|
|
121
124
|
}
|
|
122
125
|
}
|
|
123
126
|
|
|
124
|
-
|
|
127
|
+
// The constants are stable with respect to hash table resizes.
|
|
128
|
+
memcpy(next_constants, pool->constants, pool->size * sizeof(pm_constant_t));
|
|
129
|
+
|
|
130
|
+
// pool->constants and pool->buckets are allocated out of the same chunk
|
|
131
|
+
// of memory, with the buckets coming first.
|
|
132
|
+
free(pool->buckets);
|
|
125
133
|
pool->constants = next_constants;
|
|
134
|
+
pool->buckets = next_buckets;
|
|
126
135
|
pool->capacity = next_capacity;
|
|
127
136
|
return true;
|
|
128
137
|
}
|
|
129
138
|
|
|
130
139
|
// Initialize a new constant pool with a given capacity.
|
|
131
140
|
bool
|
|
132
|
-
|
|
133
|
-
const
|
|
134
|
-
if (capacity >= ((
|
|
141
|
+
pm_constant_pool_init(pm_constant_pool_t *pool, uint32_t capacity) {
|
|
142
|
+
const uint32_t maximum = (~((uint32_t) 0));
|
|
143
|
+
if (capacity >= ((maximum / 2) + 1)) return false;
|
|
135
144
|
|
|
136
145
|
capacity = next_power_of_two(capacity);
|
|
137
|
-
|
|
138
|
-
|
|
146
|
+
const size_t element_size = sizeof(pm_constant_pool_bucket_t) + sizeof(pm_constant_t);
|
|
147
|
+
void *memory = calloc(capacity, element_size);
|
|
148
|
+
if (memory == NULL) return false;
|
|
139
149
|
|
|
150
|
+
pool->buckets = memory;
|
|
151
|
+
pool->constants = (void *)(((char *)memory) + capacity * sizeof(pm_constant_pool_bucket_t));
|
|
140
152
|
pool->size = 0;
|
|
141
153
|
pool->capacity = capacity;
|
|
142
154
|
return true;
|
|
143
155
|
}
|
|
144
156
|
|
|
145
157
|
// Insert a constant into a constant pool and return its index in the pool.
|
|
146
|
-
static inline
|
|
147
|
-
|
|
158
|
+
static inline pm_constant_id_t
|
|
159
|
+
pm_constant_pool_insert(pm_constant_pool_t *pool, const uint8_t *start, size_t length, bool owned) {
|
|
148
160
|
if (pool->size >= (pool->capacity / 4 * 3)) {
|
|
149
|
-
if (!
|
|
161
|
+
if (!pm_constant_pool_resize(pool)) return 0;
|
|
150
162
|
}
|
|
151
163
|
|
|
152
164
|
assert(is_power_of_two(pool->capacity));
|
|
153
|
-
const
|
|
154
|
-
size_t hash = yp_constant_pool_hash(start, length);
|
|
155
|
-
size_t index = hash & mask;
|
|
156
|
-
yp_constant_t *constant;
|
|
165
|
+
const uint32_t mask = pool->capacity - 1;
|
|
157
166
|
|
|
158
|
-
|
|
167
|
+
uint32_t hash = pm_constant_pool_hash(start, length);
|
|
168
|
+
uint32_t index = hash & mask;
|
|
169
|
+
pm_constant_pool_bucket_t *bucket;
|
|
170
|
+
|
|
171
|
+
while (bucket = &pool->buckets[index], bucket->id != 0) {
|
|
159
172
|
// If there is a collision, then we need to check if the content is the
|
|
160
173
|
// same as the content we are trying to insert. If it is, then we can
|
|
161
174
|
// return the id of the existing constant.
|
|
175
|
+
pm_constant_t *constant = &pool->constants[bucket->id - 1];
|
|
176
|
+
|
|
162
177
|
if ((constant->length == length) && memcmp(constant->start, start, length) == 0) {
|
|
163
178
|
// Since we have found a match, we need to check if this is
|
|
164
179
|
// attempting to insert a shared or an owned constant. We want to
|
|
@@ -169,63 +184,69 @@ yp_constant_pool_insert(yp_constant_pool_t *pool, const uint8_t *start, size_t l
|
|
|
169
184
|
// memory. Either it's duplicated with the existing constant or
|
|
170
185
|
// it's not necessary because we have a shared version.
|
|
171
186
|
free((void *) start);
|
|
172
|
-
} else if (
|
|
187
|
+
} else if (bucket->owned) {
|
|
173
188
|
// If we're attempting to insert a shared constant and the
|
|
174
189
|
// existing constant is owned, then we can free the owned
|
|
175
190
|
// constant and replace it with the shared constant.
|
|
176
191
|
free((void *) constant->start);
|
|
177
192
|
constant->start = start;
|
|
178
|
-
|
|
193
|
+
bucket->owned = false;
|
|
179
194
|
}
|
|
180
195
|
|
|
181
|
-
return
|
|
196
|
+
return bucket->id;
|
|
182
197
|
}
|
|
183
198
|
|
|
184
199
|
index = (index + 1) & mask;
|
|
185
200
|
}
|
|
186
201
|
|
|
187
|
-
|
|
188
|
-
|
|
202
|
+
// IDs are allocated starting at 1, since the value 0 denotes a non-existant
|
|
203
|
+
// constant.
|
|
204
|
+
uint32_t id = ++pool->size;
|
|
205
|
+
assert(pool->size < ((uint32_t) (1 << 31)));
|
|
189
206
|
|
|
190
|
-
*
|
|
191
|
-
.id = (unsigned int) (
|
|
207
|
+
*bucket = (pm_constant_pool_bucket_t) {
|
|
208
|
+
.id = (unsigned int) (id & 0x7FFFFFFF),
|
|
192
209
|
.owned = owned,
|
|
210
|
+
.hash = hash
|
|
211
|
+
};
|
|
212
|
+
|
|
213
|
+
pool->constants[id - 1] = (pm_constant_t) {
|
|
193
214
|
.start = start,
|
|
194
215
|
.length = length,
|
|
195
|
-
.hash = hash
|
|
196
216
|
};
|
|
197
217
|
|
|
198
|
-
return
|
|
218
|
+
return id;
|
|
199
219
|
}
|
|
200
220
|
|
|
201
221
|
// Insert a constant into a constant pool. Returns the id of the constant, or 0
|
|
202
222
|
// if any potential calls to resize fail.
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
return
|
|
223
|
+
pm_constant_id_t
|
|
224
|
+
pm_constant_pool_insert_shared(pm_constant_pool_t *pool, const uint8_t *start, size_t length) {
|
|
225
|
+
return pm_constant_pool_insert(pool, start, length, false);
|
|
206
226
|
}
|
|
207
227
|
|
|
208
228
|
// Insert a constant into a constant pool from memory that is now owned by the
|
|
209
229
|
// constant pool. Returns the id of the constant, or 0 if any potential calls to
|
|
210
230
|
// resize fail.
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
return
|
|
231
|
+
pm_constant_id_t
|
|
232
|
+
pm_constant_pool_insert_owned(pm_constant_pool_t *pool, const uint8_t *start, size_t length) {
|
|
233
|
+
return pm_constant_pool_insert(pool, start, length, true);
|
|
214
234
|
}
|
|
215
235
|
|
|
216
236
|
// Free the memory associated with a constant pool.
|
|
217
237
|
void
|
|
218
|
-
|
|
238
|
+
pm_constant_pool_free(pm_constant_pool_t *pool) {
|
|
219
239
|
// For each constant in the current constant pool, free the contents if the
|
|
220
240
|
// contents are owned.
|
|
221
241
|
for (uint32_t index = 0; index < pool->capacity; index++) {
|
|
222
|
-
|
|
242
|
+
pm_constant_pool_bucket_t *bucket = &pool->buckets[index];
|
|
223
243
|
|
|
224
244
|
// If an id is set on this constant, then we know we have content here.
|
|
225
|
-
if (
|
|
245
|
+
if (bucket->id != 0 && bucket->owned) {
|
|
246
|
+
pm_constant_t *constant = &pool->constants[bucket->id - 1];
|
|
226
247
|
free((void *) constant->start);
|
|
227
248
|
}
|
|
228
249
|
}
|
|
229
250
|
|
|
230
|
-
free(pool->
|
|
251
|
+
free(pool->buckets);
|
|
231
252
|
}
|
|
@@ -1,20 +1,20 @@
|
|
|
1
|
-
#include "
|
|
1
|
+
#include "prism/util/pm_list.h"
|
|
2
2
|
|
|
3
3
|
// Returns true if the given list is empty.
|
|
4
|
-
|
|
5
|
-
|
|
4
|
+
PRISM_EXPORTED_FUNCTION bool
|
|
5
|
+
pm_list_empty_p(pm_list_t *list) {
|
|
6
6
|
return list->head == NULL;
|
|
7
7
|
}
|
|
8
8
|
|
|
9
9
|
// Returns the size of the list.
|
|
10
|
-
|
|
11
|
-
|
|
10
|
+
PRISM_EXPORTED_FUNCTION size_t
|
|
11
|
+
pm_list_size(pm_list_t *list) {
|
|
12
12
|
return list->size;
|
|
13
13
|
}
|
|
14
14
|
|
|
15
15
|
// Append a node to the given list.
|
|
16
16
|
void
|
|
17
|
-
|
|
17
|
+
pm_list_append(pm_list_t *list, pm_list_node_t *node) {
|
|
18
18
|
if (list->head == NULL) {
|
|
19
19
|
list->head = node;
|
|
20
20
|
} else {
|
|
@@ -26,10 +26,10 @@ yp_list_append(yp_list_t *list, yp_list_node_t *node) {
|
|
|
26
26
|
}
|
|
27
27
|
|
|
28
28
|
// Deallocate the internal state of the given list.
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
29
|
+
PRISM_EXPORTED_FUNCTION void
|
|
30
|
+
pm_list_free(pm_list_t *list) {
|
|
31
|
+
pm_list_node_t *node = list->head;
|
|
32
|
+
pm_list_node_t *next;
|
|
33
33
|
|
|
34
34
|
while (node != NULL) {
|
|
35
35
|
next = node->next;
|
|
@@ -1,13 +1,13 @@
|
|
|
1
|
-
#include "
|
|
1
|
+
#include "prism/util/pm_memchr.h"
|
|
2
2
|
|
|
3
|
-
#define
|
|
3
|
+
#define PRISM_MEMCHR_TRAILING_BYTE_MINIMUM 0x40
|
|
4
4
|
|
|
5
5
|
// We need to roll our own memchr to handle cases where the encoding changes and
|
|
6
6
|
// we need to search for a character in a buffer that could be the trailing byte
|
|
7
7
|
// of a multibyte character.
|
|
8
8
|
void *
|
|
9
|
-
|
|
10
|
-
if (encoding_changed && encoding->multibyte && character >=
|
|
9
|
+
pm_memchr(const void *memory, int character, size_t number, bool encoding_changed, pm_encoding_t *encoding) {
|
|
10
|
+
if (encoding_changed && encoding->multibyte && character >= PRISM_MEMCHR_TRAILING_BYTE_MINIMUM) {
|
|
11
11
|
const uint8_t *source = (const uint8_t *) memory;
|
|
12
12
|
size_t index = 0;
|
|
13
13
|
|
|
@@ -29,3 +29,5 @@ yp_memchr(const void *memory, int character, size_t number, bool encoding_change
|
|
|
29
29
|
return memchr(memory, character, number);
|
|
30
30
|
}
|
|
31
31
|
}
|
|
32
|
+
|
|
33
|
+
#undef PRISM_MEMCHR_TRAILING_BYTE_MINIMUM
|