prism 1.9.0 → 2.0.0.rc2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +88 -0
- data/CODE_OF_CONDUCT.md +1 -1
- data/LICENSE.md +1 -1
- data/Makefile +41 -12
- data/README.md +35 -97
- data/config.yml +174 -161
- data/docs/build_system.md +2 -2
- data/docs/configuration.md +5 -4
- data/docs/encoding.md +2 -2
- data/docs/fuzzing.md +4 -11
- data/docs/javascript.md +19 -4
- data/docs/mapping.md +1 -1
- data/docs/releasing.md +77 -5
- data/docs/relocation.md +2 -4
- data/docs/serialization.md +22 -24
- data/docs/testing.md +46 -29
- data/ext/prism/api_node.c +1461 -1401
- data/ext/prism/extconf.rb +3 -4
- data/ext/prism/extension.c +796 -422
- data/ext/prism/extension.h +3 -3
- data/include/prism/arena.h +37 -0
- data/include/prism/ast.h +2727 -610
- data/include/prism/buffer.h +52 -0
- data/include/prism/comments.h +55 -0
- data/include/prism/compiler/accel.h +26 -0
- data/include/prism/compiler/align.h +36 -0
- data/include/prism/compiler/assume.h +25 -0
- data/include/prism/compiler/bswap.h +46 -0
- data/include/prism/compiler/endianness.h +34 -0
- data/include/prism/compiler/exported.h +24 -0
- data/include/prism/compiler/fallthrough.h +22 -0
- data/include/prism/compiler/filesystem.h +32 -0
- data/include/prism/compiler/flex_array.h +19 -0
- data/include/prism/compiler/force_inline.h +21 -0
- data/include/prism/compiler/format.h +25 -0
- data/include/prism/compiler/inline.h +17 -0
- data/include/prism/compiler/nodiscard.h +22 -0
- data/include/prism/compiler/nonnull.h +18 -0
- data/include/prism/compiler/unused.h +18 -0
- data/include/prism/constant_pool.h +81 -0
- data/include/prism/diagnostic.h +38 -403
- data/include/prism/errors_format.h +53 -0
- data/include/prism/excludes.h +29 -0
- data/include/prism/integer.h +41 -0
- data/include/prism/internal/allocator.h +68 -0
- data/include/prism/internal/allocator_debug.h +88 -0
- data/include/prism/internal/arena.h +108 -0
- data/include/prism/internal/bit.h +42 -0
- data/include/prism/internal/buffer.h +94 -0
- data/include/prism/internal/char.h +145 -0
- data/include/prism/internal/comments.h +53 -0
- data/include/prism/internal/constant_pool.h +152 -0
- data/include/prism/internal/diagnostic.h +389 -0
- data/include/prism/{encoding.h → internal/encoding.h} +27 -68
- data/include/prism/internal/integer.h +68 -0
- data/include/prism/internal/isinf.h +16 -0
- data/include/prism/internal/line_offset_list.h +34 -0
- data/include/prism/{util/pm_list.h → internal/list.h} +11 -52
- data/include/prism/internal/magic_comments.h +23 -0
- data/include/prism/internal/memchr.h +15 -0
- data/include/prism/internal/node.h +38 -0
- data/include/prism/internal/options.h +232 -0
- data/include/prism/internal/parser.h +1004 -0
- data/include/prism/internal/regexp.h +26 -0
- data/include/prism/internal/serialize.h +39 -0
- data/include/prism/internal/source.h +72 -0
- data/include/prism/{static_literals.h → internal/static_literals.h} +26 -47
- data/include/prism/internal/stringy.h +35 -0
- data/include/prism/{util/pm_strncasecmp.h → internal/strncasecmp.h} +3 -17
- data/include/prism/{util/pm_strpbrk.h → internal/strpbrk.h} +7 -20
- data/include/prism/internal/tokens.h +11 -0
- data/include/prism/internal/xxh3.h +16 -0
- data/include/prism/json.h +32 -0
- data/include/prism/line_offset_list.h +61 -0
- data/include/prism/magic_comments.h +35 -0
- data/include/prism/node.h +18 -53
- data/include/prism/options.h +146 -280
- data/include/prism/parser.h +356 -867
- data/include/prism/prettyprint.h +6 -9
- data/include/prism/serialize.h +115 -0
- data/include/prism/source.h +155 -0
- data/include/prism/stream.h +28 -0
- data/include/prism/string_query.h +74 -0
- data/include/prism/stringy.h +72 -0
- data/include/prism/version.h +12 -3
- data/include/prism.h +43 -320
- data/lib/prism/compiler.rb +319 -305
- data/lib/prism/desugar_compiler.rb +105 -34
- data/lib/prism/dispatcher.rb +646 -473
- data/lib/prism/dot_visitor.rb +592 -511
- data/lib/prism/dsl.rb +391 -25
- data/lib/prism/ffi.rb +258 -90
- data/lib/prism/inspect_visitor.rb +357 -325
- data/lib/prism/lex_compat.rb +254 -212
- data/lib/prism/mutation_compiler.rb +313 -305
- data/lib/prism/node.rb +20966 -6992
- data/lib/prism/node_ext.rb +180 -187
- data/lib/prism/node_find.rb +215 -0
- data/lib/prism/parse_result/errors.rb +7 -1
- data/lib/prism/parse_result/newlines.rb +276 -8
- data/lib/prism/parse_result.rb +468 -139
- data/lib/prism/pattern.rb +65 -20
- data/lib/prism/reflection.rb +23 -12
- data/lib/prism/relocation.rb +149 -87
- data/lib/prism/serialize.rb +3460 -539
- data/lib/prism/string_query.rb +24 -1
- data/lib/prism/translation/parser/builder.rb +10 -2
- data/lib/prism/translation/parser/compiler.rb +44 -52
- data/lib/prism/translation/parser/lexer.rb +41 -92
- data/lib/prism/translation/parser.rb +1 -1
- data/lib/prism/translation/parser_current.rb +1 -1
- data/lib/prism/translation/ripper/lexer.rb +8 -10
- data/lib/prism/translation/ripper/sexp.rb +2 -10
- data/lib/prism/translation/ripper/shim.rb +2 -0
- data/lib/prism/translation/ripper.rb +967 -209
- data/lib/prism/translation/ruby_parser.rb +73 -316
- data/lib/prism/translation.rb +2 -0
- data/lib/prism/visitor.rb +632 -5
- data/lib/prism.rb +79 -9
- data/prism.gemspec +128 -62
- data/rbi/generated/prism/compiler.rbi +489 -0
- data/rbi/generated/prism/desugar_compiler.rbi +306 -0
- data/rbi/generated/prism/dispatcher.rbi +1138 -0
- data/rbi/generated/prism/dot_visitor.rbi +618 -0
- data/rbi/generated/prism/dsl.rbi +751 -0
- data/rbi/generated/prism/inspect_visitor.rbi +503 -0
- data/rbi/generated/prism/lex_compat.rbi +162 -0
- data/rbi/generated/prism/mutation_compiler.rbi +464 -0
- data/rbi/generated/prism/node.rbi +14407 -0
- data/rbi/generated/prism/node_ext.rbi +259 -0
- data/rbi/generated/prism/node_find.rbi +79 -0
- data/rbi/generated/prism/parse_result/errors.rbi +21 -0
- data/rbi/generated/prism/parse_result/newlines.rbi +208 -0
- data/rbi/generated/prism/parse_result.rbi +776 -0
- data/rbi/generated/prism/pattern.rbi +134 -0
- data/rbi/generated/prism/reflection.rbi +89 -0
- data/rbi/generated/prism/relocation.rbi +329 -0
- data/rbi/generated/prism/serialize.rbi +184 -0
- data/rbi/generated/prism/string_query.rbi +43 -0
- data/rbi/generated/prism/translation.rbi +7 -0
- data/rbi/generated/prism/visitor.rbi +650 -0
- data/rbi/generated/prism.rbi +86 -0
- data/rbi/prism/translation/parser_versions.rbi +5 -0
- data/rbi/rubyvm/node_find.rbi +14 -0
- data/sig/generated/prism/compiler.rbs +492 -0
- data/sig/generated/prism/desugar_compiler.rbs +305 -0
- data/sig/generated/prism/dispatcher.rbs +1292 -0
- data/sig/generated/prism/dot_visitor.rbs +635 -0
- data/sig/generated/prism/dsl.rbs +928 -0
- data/sig/generated/prism/inspect_visitor.rbs +509 -0
- data/sig/generated/prism/lex_compat.rbs +166 -0
- data/sig/{prism → generated/prism}/mutation_compiler.rbs +309 -4
- data/sig/generated/prism/node.rbs +20896 -0
- data/sig/generated/prism/node_ext.rbs +290 -0
- data/sig/generated/prism/node_find.rbs +90 -0
- data/sig/generated/prism/parse_result/errors.rbs +22 -0
- data/sig/generated/prism/parse_result/newlines.rbs +223 -0
- data/sig/generated/prism/parse_result.rbs +871 -0
- data/sig/generated/prism/pattern.rbs +156 -0
- data/sig/generated/prism/reflection.rbs +101 -0
- data/sig/generated/prism/relocation.rbs +428 -0
- data/sig/generated/prism/serialize.rbs +198 -0
- data/sig/generated/prism/string_query.rbs +43 -0
- data/sig/generated/prism/translation.rbs +8 -0
- data/sig/generated/prism/visitor.rbs +1111 -0
- data/sig/generated/prism.rbs +84 -0
- data/sig/manifest.yaml +2 -0
- data/src/arena.c +117 -0
- data/src/{util/pm_buffer.c → buffer.c} +53 -21
- data/src/{util/pm_char.c → char.c} +28 -57
- data/src/comments.c +454 -0
- data/src/constant_pool.c +406 -0
- data/src/diagnostic.c +97 -65
- data/src/encoding.c +8 -3
- data/src/errors_format.c +519 -0
- data/src/{util/pm_integer.c → integer.c} +33 -22
- data/src/json.c +5733 -0
- data/src/line_offset_list.c +100 -0
- data/src/list.c +24 -0
- data/src/{util/pm_memchr.c → memchr.c} +8 -6
- data/src/node.c +3504 -7674
- data/src/options.c +162 -49
- data/src/parser.c +367 -0
- data/src/prettyprint.c +421 -354
- data/src/prism.c +8402 -7161
- data/src/regexp.c +961 -45
- data/src/serialize.c +460 -424
- data/src/source.c +518 -0
- data/src/static_literals.c +111 -43
- data/src/string_query.c +180 -0
- data/src/stringy.c +91 -0
- data/src/{util/pm_strncasecmp.c → strncasecmp.c} +5 -4
- data/src/strpbrk.c +447 -0
- data/src/{token_type.c → tokens.c} +39 -18
- data/src/xxh3.c +292 -0
- metadata +128 -63
- data/ext/prism/api_pack.c +0 -276
- data/include/prism/defines.h +0 -260
- data/include/prism/pack.h +0 -163
- data/include/prism/regexp.h +0 -43
- data/include/prism/util/pm_buffer.h +0 -236
- data/include/prism/util/pm_char.h +0 -204
- data/include/prism/util/pm_constant_pool.h +0 -218
- data/include/prism/util/pm_integer.h +0 -130
- data/include/prism/util/pm_memchr.h +0 -29
- data/include/prism/util/pm_newline_list.h +0 -113
- data/include/prism/util/pm_string.h +0 -200
- data/lib/prism/pack.rb +0 -230
- data/lib/prism/parse_result/comments.rb +0 -188
- data/rbi/prism/compiler.rbi +0 -12
- data/rbi/prism/dsl.rbi +0 -524
- data/rbi/prism/inspect_visitor.rbi +0 -12
- data/rbi/prism/node.rbi +0 -8750
- data/rbi/prism/node_ext.rbi +0 -107
- data/rbi/prism/parse_result.rbi +0 -404
- data/rbi/prism/reflection.rbi +0 -58
- data/rbi/prism/string_query.rbi +0 -12
- data/rbi/prism/visitor.rbi +0 -473
- data/rbi/prism.rbi +0 -63
- data/sig/prism/compiler.rbs +0 -9
- data/sig/prism/dispatcher.rbs +0 -19
- data/sig/prism/dot_visitor.rbs +0 -6
- data/sig/prism/dsl.rbs +0 -351
- data/sig/prism/inspect_visitor.rbs +0 -22
- data/sig/prism/lex_compat.rbs +0 -10
- data/sig/prism/node.rbs +0 -4042
- data/sig/prism/node_ext.rbs +0 -149
- data/sig/prism/pack.rbs +0 -43
- data/sig/prism/parse_result/comments.rbs +0 -38
- data/sig/prism/parse_result.rbs +0 -197
- data/sig/prism/pattern.rbs +0 -13
- data/sig/prism/reflection.rbs +0 -50
- data/sig/prism/relocation.rbs +0 -185
- data/sig/prism/serialize.rbs +0 -8
- data/sig/prism/string_query.rbs +0 -11
- data/sig/prism/visitor.rbs +0 -169
- data/sig/prism.rbs +0 -272
- data/src/pack.c +0 -509
- data/src/util/pm_constant_pool.c +0 -342
- data/src/util/pm_list.c +0 -49
- data/src/util/pm_newline_list.c +0 -125
- data/src/util/pm_string.c +0 -381
- data/src/util/pm_strpbrk.c +0 -206
data/src/regexp.c
CHANGED
|
@@ -1,5 +1,20 @@
|
|
|
1
|
-
#include "prism/regexp.h"
|
|
2
|
-
|
|
1
|
+
#include "prism/internal/regexp.h"
|
|
2
|
+
|
|
3
|
+
#include "prism/compiler/inline.h"
|
|
4
|
+
#include "prism/compiler/fallthrough.h"
|
|
5
|
+
#include "prism/internal/buffer.h"
|
|
6
|
+
#include "prism/internal/char.h"
|
|
7
|
+
#include "prism/internal/diagnostic.h"
|
|
8
|
+
#include "prism/internal/encoding.h"
|
|
9
|
+
#include "prism/internal/memchr.h"
|
|
10
|
+
#include "prism/internal/parser.h"
|
|
11
|
+
#include "prism/internal/stringy.h"
|
|
12
|
+
#include "prism/internal/strncasecmp.h"
|
|
13
|
+
|
|
14
|
+
#include <assert.h>
|
|
15
|
+
#include <string.h>
|
|
16
|
+
|
|
17
|
+
/** The maximum depth of nested groups allowed in a regular expression. */
|
|
3
18
|
#define PM_REGEXP_PARSE_DEPTH_MAX 4096
|
|
4
19
|
|
|
5
20
|
/**
|
|
@@ -18,6 +33,54 @@ typedef struct {
|
|
|
18
33
|
/** A pointer to the end of the source that we are parsing. */
|
|
19
34
|
const uint8_t *end;
|
|
20
35
|
|
|
36
|
+
/** The encoding of the source. */
|
|
37
|
+
const pm_encoding_t *encoding;
|
|
38
|
+
|
|
39
|
+
/** The callback to call when a named capture group is found. */
|
|
40
|
+
pm_regexp_name_callback_t name_callback;
|
|
41
|
+
|
|
42
|
+
/** The data to pass to the name callback. */
|
|
43
|
+
void *name_data;
|
|
44
|
+
|
|
45
|
+
/** The start of the regexp node (for error locations). */
|
|
46
|
+
const uint8_t *node_start;
|
|
47
|
+
|
|
48
|
+
/** The end of the regexp node (for error locations). */
|
|
49
|
+
const uint8_t *node_end;
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* The explicit encoding determined by escape sequences. NULL if no
|
|
53
|
+
* encoding-setting escape has been seen, UTF-8 for `\u` escapes, or the
|
|
54
|
+
* source encoding for `\x` escapes.
|
|
55
|
+
*/
|
|
56
|
+
const pm_encoding_t *explicit_encoding;
|
|
57
|
+
|
|
58
|
+
/**
|
|
59
|
+
* Pointer to the first non-POSIX property name (for /n error messages).
|
|
60
|
+
* POSIX properties (Alnum, Alpha, etc.) work in all encodings.
|
|
61
|
+
* Script properties (Hiragana, Katakana, etc.) work in /e, /s, /u.
|
|
62
|
+
* Unicode-only properties (L, Ll, etc.) work only in /u.
|
|
63
|
+
*/
|
|
64
|
+
const uint8_t *property_name;
|
|
65
|
+
|
|
66
|
+
/** Length of the first non-POSIX property name found. */
|
|
67
|
+
size_t property_name_length;
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* Pointer to the first Unicode-only property name (for /e, /s error
|
|
71
|
+
* messages). NULL if only POSIX or script properties have been seen.
|
|
72
|
+
*/
|
|
73
|
+
const uint8_t *unicode_property_name;
|
|
74
|
+
|
|
75
|
+
/** Length of the first Unicode-only property name found. */
|
|
76
|
+
size_t unicode_property_name_length;
|
|
77
|
+
|
|
78
|
+
/** Buffer of hex escape byte values >= 0x80, separated by 0x00 sentinels. */
|
|
79
|
+
pm_buffer_t hex_escape_buffer;
|
|
80
|
+
|
|
81
|
+
/** Count of non-ASCII literal bytes (not from escapes). */
|
|
82
|
+
uint32_t non_ascii_literal_count;
|
|
83
|
+
|
|
21
84
|
/**
|
|
22
85
|
* Whether or not the regular expression currently being parsed is in
|
|
23
86
|
* extended mode, wherein whitespace is ignored and comments are allowed.
|
|
@@ -27,30 +90,68 @@ typedef struct {
|
|
|
27
90
|
/** Whether the encoding has changed from the default. */
|
|
28
91
|
bool encoding_changed;
|
|
29
92
|
|
|
30
|
-
/**
|
|
31
|
-
|
|
93
|
+
/**
|
|
94
|
+
* Whether syntax errors are reported. The named capture pass walks content
|
|
95
|
+
* that may be an owned copy of the source (so pointers into it have no
|
|
96
|
+
* offset in the source), and whose errors the validation pass has already
|
|
97
|
+
* reported, so it reports nothing.
|
|
98
|
+
*/
|
|
99
|
+
bool report_errors;
|
|
32
100
|
|
|
33
|
-
/**
|
|
34
|
-
|
|
101
|
+
/** Whether a `\u{...}` escape with value >= 0x80 was seen. */
|
|
102
|
+
bool has_unicode_escape;
|
|
35
103
|
|
|
36
|
-
/**
|
|
37
|
-
|
|
104
|
+
/** Whether a `\xNN` escape (or `\M-x`, etc.) with value >= 0x80 was seen. */
|
|
105
|
+
bool has_hex_escape;
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Tracks whether the last encoding-setting escape was `\u` (true) or `\x`
|
|
109
|
+
* (false). This matters for error messages when both types are mixed.
|
|
110
|
+
*/
|
|
111
|
+
bool last_escape_was_unicode;
|
|
112
|
+
|
|
113
|
+
/** Whether any `\p{...}` or `\P{...}` property escape was found. */
|
|
114
|
+
bool has_property_escape;
|
|
38
115
|
|
|
39
|
-
/**
|
|
40
|
-
|
|
116
|
+
/** Whether a Unicode-only property escape was found (not POSIX or script). */
|
|
117
|
+
bool has_unicode_property_escape;
|
|
41
118
|
|
|
42
|
-
/**
|
|
43
|
-
|
|
119
|
+
/** Whether a `\u` escape with invalid range (surrogate or > 0x10FFFF) was seen. */
|
|
120
|
+
bool invalid_unicode_range;
|
|
121
|
+
|
|
122
|
+
/** Whether we are accumulating consecutive hex escape bytes. */
|
|
123
|
+
bool hex_group_active;
|
|
124
|
+
|
|
125
|
+
/** Whether an invalid multibyte character was found during parsing. */
|
|
126
|
+
bool has_invalid_multibyte;
|
|
44
127
|
} pm_regexp_parser_t;
|
|
45
128
|
|
|
46
129
|
/**
|
|
47
|
-
* Append
|
|
130
|
+
* Append a syntax error to the parser's error list. Errors are only reported
|
|
131
|
+
* when report_errors is set, which is the case for the validation pass that
|
|
132
|
+
* walks the raw source, so start and end always point into the original source.
|
|
48
133
|
*/
|
|
49
|
-
static
|
|
50
|
-
pm_regexp_parse_error(pm_regexp_parser_t *
|
|
51
|
-
|
|
134
|
+
static PRISM_INLINE void
|
|
135
|
+
pm_regexp_parse_error(pm_regexp_parser_t *regexp_parser, const uint8_t *start, const uint8_t *end, const char *message) {
|
|
136
|
+
if (regexp_parser->report_errors) {
|
|
137
|
+
pm_parser_t *parser = regexp_parser->parser;
|
|
138
|
+
pm_diagnostic_list_append_format(&parser->metadata_arena, &parser->error_list, (uint32_t) (start - parser->start), (uint32_t) (end - start), PM_ERR_REGEXP_PARSE_ERROR, message);
|
|
139
|
+
}
|
|
52
140
|
}
|
|
53
141
|
|
|
142
|
+
/**
|
|
143
|
+
* Append a formatted syntax error to the parser's error list, following the
|
|
144
|
+
* same rules as pm_regexp_parse_error. This is a macro because we need
|
|
145
|
+
* variadic args for the format string.
|
|
146
|
+
*/
|
|
147
|
+
#define pm_regexp_parse_error_format(regexp_parser_, err_start_, err_end_, diag_id, ...) \
|
|
148
|
+
do { \
|
|
149
|
+
if ((regexp_parser_)->report_errors) { \
|
|
150
|
+
pm_parser_t *parser_ = (regexp_parser_)->parser; \
|
|
151
|
+
pm_diagnostic_list_append_format(&parser_->metadata_arena, &parser_->error_list, (uint32_t) ((err_start_) - parser_->start), (uint32_t) ((err_end_) - (err_start_)), diag_id, __VA_ARGS__); \
|
|
152
|
+
} \
|
|
153
|
+
} while (0)
|
|
154
|
+
|
|
54
155
|
/**
|
|
55
156
|
* This appends a new string to the list of named captures. This function
|
|
56
157
|
* assumes the caller has already checked the validity of the name callback.
|
|
@@ -59,14 +160,14 @@ static void
|
|
|
59
160
|
pm_regexp_parser_named_capture(pm_regexp_parser_t *parser, const uint8_t *start, const uint8_t *end) {
|
|
60
161
|
pm_string_t string;
|
|
61
162
|
pm_string_shared_init(&string, start, end);
|
|
62
|
-
parser->name_callback(&string, parser->name_data);
|
|
63
|
-
|
|
163
|
+
parser->name_callback(parser->parser, &string, parser->name_data);
|
|
164
|
+
pm_string_cleanup(&string);
|
|
64
165
|
}
|
|
65
166
|
|
|
66
167
|
/**
|
|
67
168
|
* Returns true if the next character is the end of the source.
|
|
68
169
|
*/
|
|
69
|
-
static
|
|
170
|
+
static PRISM_INLINE bool
|
|
70
171
|
pm_regexp_char_is_eof(pm_regexp_parser_t *parser) {
|
|
71
172
|
return parser->cursor >= parser->end;
|
|
72
173
|
}
|
|
@@ -74,7 +175,7 @@ pm_regexp_char_is_eof(pm_regexp_parser_t *parser) {
|
|
|
74
175
|
/**
|
|
75
176
|
* Optionally accept a char and consume it if it exists.
|
|
76
177
|
*/
|
|
77
|
-
static
|
|
178
|
+
static PRISM_INLINE bool
|
|
78
179
|
pm_regexp_char_accept(pm_regexp_parser_t *parser, uint8_t value) {
|
|
79
180
|
if (!pm_regexp_char_is_eof(parser) && *parser->cursor == value) {
|
|
80
181
|
parser->cursor++;
|
|
@@ -86,7 +187,7 @@ pm_regexp_char_accept(pm_regexp_parser_t *parser, uint8_t value) {
|
|
|
86
187
|
/**
|
|
87
188
|
* Expect a character to be present and consume it.
|
|
88
189
|
*/
|
|
89
|
-
static
|
|
190
|
+
static PRISM_INLINE bool
|
|
90
191
|
pm_regexp_char_expect(pm_regexp_parser_t *parser, uint8_t value) {
|
|
91
192
|
if (!pm_regexp_char_is_eof(parser) && *parser->cursor == value) {
|
|
92
193
|
parser->cursor++;
|
|
@@ -113,6 +214,47 @@ pm_regexp_char_find(pm_regexp_parser_t *parser, uint8_t value) {
|
|
|
113
214
|
return true;
|
|
114
215
|
}
|
|
115
216
|
|
|
217
|
+
/**
|
|
218
|
+
* Mark a group boundary in the hex escape byte buffer. When consecutive hex
|
|
219
|
+
* escape bytes >= 0x80 are followed by a non-hex-escape, this appends a 0x00
|
|
220
|
+
* sentinel to separate the groups for later multibyte validation.
|
|
221
|
+
*/
|
|
222
|
+
static PRISM_INLINE void
|
|
223
|
+
pm_regexp_hex_group_boundary(pm_regexp_parser_t *parser) {
|
|
224
|
+
if (parser->hex_group_active) {
|
|
225
|
+
pm_buffer_append_byte(&parser->hex_escape_buffer, 0x00);
|
|
226
|
+
parser->hex_group_active = false;
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* Track a hex escape byte value >= 0x80 for multibyte validation.
|
|
232
|
+
*/
|
|
233
|
+
static PRISM_INLINE void
|
|
234
|
+
pm_regexp_track_hex_escape(pm_regexp_parser_t *parser, uint8_t byte) {
|
|
235
|
+
if (byte >= 0x80) {
|
|
236
|
+
pm_buffer_append_byte(&parser->hex_escape_buffer, byte);
|
|
237
|
+
parser->hex_group_active = true;
|
|
238
|
+
parser->has_hex_escape = true;
|
|
239
|
+
|
|
240
|
+
parser->explicit_encoding = parser->encoding;
|
|
241
|
+
parser->last_escape_was_unicode = false;
|
|
242
|
+
} else {
|
|
243
|
+
pm_regexp_hex_group_boundary(parser);
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
/**
|
|
248
|
+
* Parse a hex digit character and return its value, or -1 if not a hex digit.
|
|
249
|
+
*/
|
|
250
|
+
static PRISM_INLINE int
|
|
251
|
+
pm_regexp_hex_digit_value(uint8_t byte) {
|
|
252
|
+
if (byte >= '0' && byte <= '9') return byte - '0';
|
|
253
|
+
if (byte >= 'a' && byte <= 'f') return byte - 'a' + 10;
|
|
254
|
+
if (byte >= 'A' && byte <= 'F') return byte - 'A' + 10;
|
|
255
|
+
return -1;
|
|
256
|
+
}
|
|
257
|
+
|
|
116
258
|
/**
|
|
117
259
|
* Range quantifiers are a special class of quantifiers that look like
|
|
118
260
|
*
|
|
@@ -121,13 +263,12 @@ pm_regexp_char_find(pm_regexp_parser_t *parser, uint8_t value) {
|
|
|
121
263
|
* * {digit,digit}
|
|
122
264
|
* * {,digit}
|
|
123
265
|
*
|
|
124
|
-
*
|
|
125
|
-
*
|
|
126
|
-
*
|
|
127
|
-
*
|
|
128
|
-
* return.
|
|
266
|
+
* If there are any spaces in between, then this just becomes a regular
|
|
267
|
+
* character match expression and we have to backtrack. So when this function
|
|
268
|
+
* first starts running, we'll create a "save" point and then attempt to parse
|
|
269
|
+
* the quantifier. If it fails, we'll restore the save point and return.
|
|
129
270
|
*
|
|
130
|
-
*
|
|
271
|
+
* To properly track everything, we're going to build a little state machine.
|
|
131
272
|
* It looks something like the following:
|
|
132
273
|
*
|
|
133
274
|
* +-------+ +---------+ ------------+
|
|
@@ -275,10 +416,392 @@ pm_regexp_parse_posix_class(pm_regexp_parser_t *parser) {
|
|
|
275
416
|
);
|
|
276
417
|
}
|
|
277
418
|
|
|
419
|
+
/**
|
|
420
|
+
* Property escape classification. Onigmo supports three tiers of property
|
|
421
|
+
* names depending on the encoding:
|
|
422
|
+
*
|
|
423
|
+
* - POSIX properties (Alnum, Alpha, ASCII, Blank, Cntrl, Digit, Graph, Lower,
|
|
424
|
+
* Print, Punct, Space, Upper, XDigit, Word): valid in all encodings.
|
|
425
|
+
* - Script properties (Hiragana, Katakana, Han, Latin, Greek, Cyrillic): valid
|
|
426
|
+
* in EUC-JP (/e), Windows-31J (/s), and UTF-8 (/u), but not ASCII-8BIT (/n).
|
|
427
|
+
* - Unicode-only properties (general categories like L, Ll, Lu, etc., plus
|
|
428
|
+
* Any, Assigned): valid only in UTF-8 (/u).
|
|
429
|
+
*/
|
|
430
|
+
typedef enum {
|
|
431
|
+
PM_REGEXP_PROPERTY_POSIX,
|
|
432
|
+
PM_REGEXP_PROPERTY_SCRIPT,
|
|
433
|
+
PM_REGEXP_PROPERTY_UNICODE
|
|
434
|
+
} pm_regexp_property_type_t;
|
|
435
|
+
|
|
436
|
+
/**
|
|
437
|
+
* Classify a property name. The name may start with '^' for negation, which
|
|
438
|
+
* is skipped before matching.
|
|
439
|
+
*/
|
|
440
|
+
static pm_regexp_property_type_t
|
|
441
|
+
pm_regexp_classify_property(const uint8_t *name, size_t length) {
|
|
442
|
+
// Skip leading '^' for negated properties like \p{^Hiragana}.
|
|
443
|
+
if (length > 0 && name[0] == '^') {
|
|
444
|
+
name++;
|
|
445
|
+
length--;
|
|
446
|
+
}
|
|
447
|
+
|
|
448
|
+
#define PM_REGEXP_CASECMP(str_) (pm_strncasecmp(name, (const uint8_t *) (str_), length) == 0)
|
|
449
|
+
|
|
450
|
+
switch (length) {
|
|
451
|
+
case 3:
|
|
452
|
+
if (PM_REGEXP_CASECMP("Han")) return PM_REGEXP_PROPERTY_SCRIPT;
|
|
453
|
+
break;
|
|
454
|
+
case 4:
|
|
455
|
+
if (PM_REGEXP_CASECMP("Word")) return PM_REGEXP_PROPERTY_POSIX;
|
|
456
|
+
break;
|
|
457
|
+
case 5:
|
|
458
|
+
/* Most properties are length 5, so dispatch on first character. */
|
|
459
|
+
switch (name[0] | 0x20) {
|
|
460
|
+
case 'a':
|
|
461
|
+
if (PM_REGEXP_CASECMP("Alnum")) return PM_REGEXP_PROPERTY_POSIX;
|
|
462
|
+
if (PM_REGEXP_CASECMP("Alpha")) return PM_REGEXP_PROPERTY_POSIX;
|
|
463
|
+
if (PM_REGEXP_CASECMP("ASCII")) return PM_REGEXP_PROPERTY_POSIX;
|
|
464
|
+
break;
|
|
465
|
+
case 'b':
|
|
466
|
+
if (PM_REGEXP_CASECMP("Blank")) return PM_REGEXP_PROPERTY_POSIX;
|
|
467
|
+
break;
|
|
468
|
+
case 'c':
|
|
469
|
+
if (PM_REGEXP_CASECMP("Cntrl")) return PM_REGEXP_PROPERTY_POSIX;
|
|
470
|
+
break;
|
|
471
|
+
case 'd':
|
|
472
|
+
if (PM_REGEXP_CASECMP("Digit")) return PM_REGEXP_PROPERTY_POSIX;
|
|
473
|
+
break;
|
|
474
|
+
case 'g':
|
|
475
|
+
if (PM_REGEXP_CASECMP("Graph")) return PM_REGEXP_PROPERTY_POSIX;
|
|
476
|
+
if (PM_REGEXP_CASECMP("Greek")) return PM_REGEXP_PROPERTY_SCRIPT;
|
|
477
|
+
break;
|
|
478
|
+
case 'l':
|
|
479
|
+
if (PM_REGEXP_CASECMP("Lower")) return PM_REGEXP_PROPERTY_POSIX;
|
|
480
|
+
if (PM_REGEXP_CASECMP("Latin")) return PM_REGEXP_PROPERTY_SCRIPT;
|
|
481
|
+
break;
|
|
482
|
+
case 'p':
|
|
483
|
+
if (PM_REGEXP_CASECMP("Print")) return PM_REGEXP_PROPERTY_POSIX;
|
|
484
|
+
if (PM_REGEXP_CASECMP("Punct")) return PM_REGEXP_PROPERTY_POSIX;
|
|
485
|
+
break;
|
|
486
|
+
case 's':
|
|
487
|
+
if (PM_REGEXP_CASECMP("Space")) return PM_REGEXP_PROPERTY_POSIX;
|
|
488
|
+
break;
|
|
489
|
+
case 'u':
|
|
490
|
+
if (PM_REGEXP_CASECMP("Upper")) return PM_REGEXP_PROPERTY_POSIX;
|
|
491
|
+
break;
|
|
492
|
+
}
|
|
493
|
+
break;
|
|
494
|
+
case 6:
|
|
495
|
+
if (PM_REGEXP_CASECMP("XDigit")) return PM_REGEXP_PROPERTY_POSIX;
|
|
496
|
+
break;
|
|
497
|
+
case 8:
|
|
498
|
+
if (PM_REGEXP_CASECMP("Hiragana")) return PM_REGEXP_PROPERTY_SCRIPT;
|
|
499
|
+
if (PM_REGEXP_CASECMP("Katakana")) return PM_REGEXP_PROPERTY_SCRIPT;
|
|
500
|
+
if (PM_REGEXP_CASECMP("Cyrillic")) return PM_REGEXP_PROPERTY_SCRIPT;
|
|
501
|
+
break;
|
|
502
|
+
}
|
|
503
|
+
|
|
504
|
+
#undef PM_REGEXP_CASECMP
|
|
505
|
+
|
|
506
|
+
// Everything else is Unicode-only (general categories, other scripts, etc.).
|
|
507
|
+
return PM_REGEXP_PROPERTY_UNICODE;
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
/**
|
|
511
|
+
* Check for and skip a `\p{...}` or `\P{...}` Unicode property escape. The
|
|
512
|
+
* cursor should be pointing at 'p' or 'P' when this is called. If a property
|
|
513
|
+
* escape is found, record it on the regexp parser and advance past the closing
|
|
514
|
+
* '}'.
|
|
515
|
+
*
|
|
516
|
+
* Properties are classified into three tiers (POSIX, script, Unicode-only) to
|
|
517
|
+
* determine which encoding modifiers they are valid with.
|
|
518
|
+
*/
|
|
519
|
+
static bool
|
|
520
|
+
pm_regexp_parse_property_escape(pm_regexp_parser_t *parser) {
|
|
521
|
+
assert(*parser->cursor == 'p' || *parser->cursor == 'P');
|
|
522
|
+
|
|
523
|
+
if (parser->cursor + 1 < parser->end && parser->cursor[1] == '{') {
|
|
524
|
+
const uint8_t *name_start = parser->cursor + 2;
|
|
525
|
+
const uint8_t *search = name_start;
|
|
526
|
+
|
|
527
|
+
while (search < parser->end && *search != '}') search++;
|
|
528
|
+
|
|
529
|
+
if (search < parser->end) {
|
|
530
|
+
size_t name_length = (size_t) (search - name_start);
|
|
531
|
+
parser->has_property_escape = true;
|
|
532
|
+
|
|
533
|
+
pm_regexp_property_type_t type = pm_regexp_classify_property(name_start, name_length);
|
|
534
|
+
|
|
535
|
+
// Track the first non-POSIX property name (for /n error messages).
|
|
536
|
+
if (type >= PM_REGEXP_PROPERTY_SCRIPT && parser->property_name == NULL) {
|
|
537
|
+
parser->property_name = name_start;
|
|
538
|
+
parser->property_name_length = name_length;
|
|
539
|
+
}
|
|
540
|
+
|
|
541
|
+
// Track the first Unicode-only property name (for /e, /s error messages).
|
|
542
|
+
if (type == PM_REGEXP_PROPERTY_UNICODE) {
|
|
543
|
+
parser->has_unicode_property_escape = true;
|
|
544
|
+
if (parser->unicode_property_name == NULL) {
|
|
545
|
+
parser->unicode_property_name = name_start;
|
|
546
|
+
parser->unicode_property_name_length = name_length;
|
|
547
|
+
}
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
parser->cursor = search + 1; // skip past '}'
|
|
551
|
+
return true;
|
|
552
|
+
}
|
|
553
|
+
}
|
|
554
|
+
|
|
555
|
+
// Not a property escape, just skip the single character after '\'.
|
|
556
|
+
parser->cursor++;
|
|
557
|
+
return false;
|
|
558
|
+
}
|
|
559
|
+
|
|
560
|
+
/**
|
|
561
|
+
* Validate and skip a \u escape sequence in a regular expression. The cursor
|
|
562
|
+
* should be pointing at the character after 'u' when this is called. This
|
|
563
|
+
* handles both the \u{NNNN MMMM} and \uNNNN forms. Also tracks encoding
|
|
564
|
+
* state for validation.
|
|
565
|
+
*/
|
|
566
|
+
static void
|
|
567
|
+
pm_regexp_parse_unicode_escape(pm_regexp_parser_t *parser) {
|
|
568
|
+
const uint8_t *escape_start = parser->cursor - 2; // points to '\'
|
|
569
|
+
|
|
570
|
+
if (pm_regexp_char_is_eof(parser)) {
|
|
571
|
+
pm_regexp_parse_error(parser, escape_start, parser->cursor, "invalid Unicode escape");
|
|
572
|
+
return;
|
|
573
|
+
}
|
|
574
|
+
|
|
575
|
+
if (*parser->cursor == '{') {
|
|
576
|
+
parser->cursor++; // skip '{'
|
|
577
|
+
|
|
578
|
+
// Skip leading whitespace.
|
|
579
|
+
while (!pm_regexp_char_is_eof(parser) && pm_char_is_whitespace(*parser->cursor)) {
|
|
580
|
+
parser->cursor++;
|
|
581
|
+
}
|
|
582
|
+
|
|
583
|
+
bool has_codepoint = false;
|
|
584
|
+
|
|
585
|
+
while (!pm_regexp_char_is_eof(parser) && *parser->cursor != '}') {
|
|
586
|
+
// Parse the hex digits to compute the codepoint value.
|
|
587
|
+
uint32_t value = 0;
|
|
588
|
+
size_t hex_count = 0;
|
|
589
|
+
|
|
590
|
+
int digit;
|
|
591
|
+
while (!pm_regexp_char_is_eof(parser) && (digit = pm_regexp_hex_digit_value(*parser->cursor)) >= 0) {
|
|
592
|
+
value = (value << 4) | (uint32_t) digit;
|
|
593
|
+
hex_count++;
|
|
594
|
+
parser->cursor++;
|
|
595
|
+
}
|
|
596
|
+
|
|
597
|
+
if (hex_count == 0) {
|
|
598
|
+
// Skip to '}' or end of regexp to find the full extent.
|
|
599
|
+
while (!pm_regexp_char_is_eof(parser) && *parser->cursor != '}') {
|
|
600
|
+
parser->cursor++;
|
|
601
|
+
}
|
|
602
|
+
|
|
603
|
+
const uint8_t *escape_end = parser->cursor;
|
|
604
|
+
if (!pm_regexp_char_is_eof(parser)) {
|
|
605
|
+
escape_end++;
|
|
606
|
+
parser->cursor++; // skip '}'
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
pm_regexp_parse_error_format(parser, escape_start, escape_end, PM_ERR_ESCAPE_INVALID_UNICODE_LIST, (int) (escape_end - escape_start), (const char *) escape_start);
|
|
610
|
+
return;
|
|
611
|
+
}
|
|
612
|
+
|
|
613
|
+
if (hex_count > 6) {
|
|
614
|
+
pm_regexp_parse_error(parser, escape_start, parser->cursor, "invalid Unicode range");
|
|
615
|
+
}
|
|
616
|
+
|
|
617
|
+
// Track encoding state for this codepoint.
|
|
618
|
+
if (value >= 0x80) {
|
|
619
|
+
parser->has_unicode_escape = true;
|
|
620
|
+
parser->explicit_encoding = PM_ENCODING_UTF_8_ENTRY;
|
|
621
|
+
parser->last_escape_was_unicode = true;
|
|
622
|
+
pm_regexp_hex_group_boundary(parser);
|
|
623
|
+
}
|
|
624
|
+
|
|
625
|
+
// Check for invalid Unicode range (surrogates or > 0x10FFFF).
|
|
626
|
+
if (value > 0x10FFFF || (value >= 0xD800 && value <= 0xDFFF)) {
|
|
627
|
+
parser->invalid_unicode_range = true;
|
|
628
|
+
}
|
|
629
|
+
|
|
630
|
+
has_codepoint = true;
|
|
631
|
+
|
|
632
|
+
// Skip whitespace between codepoints.
|
|
633
|
+
while (!pm_regexp_char_is_eof(parser) && pm_char_is_whitespace(*parser->cursor)) {
|
|
634
|
+
parser->cursor++;
|
|
635
|
+
}
|
|
636
|
+
}
|
|
637
|
+
|
|
638
|
+
if (pm_regexp_char_is_eof(parser)) {
|
|
639
|
+
pm_regexp_parse_error(parser, escape_start, parser->cursor, "unterminated Unicode escape");
|
|
640
|
+
} else {
|
|
641
|
+
if (!has_codepoint) {
|
|
642
|
+
pm_regexp_parse_error_format(parser, escape_start, parser->cursor + 1, PM_ERR_ESCAPE_INVALID_UNICODE_LIST, (int) (parser->cursor + 1 - escape_start), (const char *) escape_start);
|
|
643
|
+
}
|
|
644
|
+
parser->cursor++; // skip '}'
|
|
645
|
+
}
|
|
646
|
+
} else {
|
|
647
|
+
// \uNNNN form — need exactly 4 hex digits.
|
|
648
|
+
uint32_t value = 0;
|
|
649
|
+
size_t hex_count = 0;
|
|
650
|
+
|
|
651
|
+
int digit;
|
|
652
|
+
while (hex_count < 4 && !pm_regexp_char_is_eof(parser) && (digit = pm_regexp_hex_digit_value(*parser->cursor)) >= 0) {
|
|
653
|
+
value = (value << 4) | (uint32_t) digit;
|
|
654
|
+
hex_count++;
|
|
655
|
+
parser->cursor++;
|
|
656
|
+
}
|
|
657
|
+
|
|
658
|
+
if (hex_count < 4) {
|
|
659
|
+
pm_regexp_parse_error(parser, escape_start, parser->cursor, "invalid Unicode escape");
|
|
660
|
+
} else if (value >= 0x80) {
|
|
661
|
+
parser->has_unicode_escape = true;
|
|
662
|
+
parser->explicit_encoding = PM_ENCODING_UTF_8_ENTRY;
|
|
663
|
+
parser->last_escape_was_unicode = true;
|
|
664
|
+
pm_regexp_hex_group_boundary(parser);
|
|
665
|
+
}
|
|
666
|
+
|
|
667
|
+
// Check for invalid Unicode range.
|
|
668
|
+
if (hex_count == 4 && (value > 0x10FFFF || (value >= 0xD800 && value <= 0xDFFF))) {
|
|
669
|
+
parser->invalid_unicode_range = true;
|
|
670
|
+
}
|
|
671
|
+
}
|
|
672
|
+
}
|
|
673
|
+
|
|
278
674
|
// Forward declaration because character sets can be nested.
|
|
279
675
|
static bool
|
|
280
676
|
pm_regexp_parse_lbracket(pm_regexp_parser_t *parser, uint16_t depth);
|
|
281
677
|
|
|
678
|
+
/**
|
|
679
|
+
* Parse a \x escape and return the byte value. The cursor should be pointing
|
|
680
|
+
* at the character after 'x'. Returns -1 if no hex digits follow.
|
|
681
|
+
*/
|
|
682
|
+
static int
|
|
683
|
+
pm_regexp_parse_hex_escape(pm_regexp_parser_t *parser) {
|
|
684
|
+
int value = -1;
|
|
685
|
+
|
|
686
|
+
if (!pm_regexp_char_is_eof(parser)) {
|
|
687
|
+
int digit = pm_regexp_hex_digit_value(*parser->cursor);
|
|
688
|
+
if (digit >= 0) {
|
|
689
|
+
value = digit;
|
|
690
|
+
parser->cursor++;
|
|
691
|
+
|
|
692
|
+
if (!pm_regexp_char_is_eof(parser)) {
|
|
693
|
+
digit = pm_regexp_hex_digit_value(*parser->cursor);
|
|
694
|
+
if (digit >= 0) {
|
|
695
|
+
value = (value << 4) | digit;
|
|
696
|
+
parser->cursor++;
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
}
|
|
701
|
+
|
|
702
|
+
if (value >= 0) {
|
|
703
|
+
pm_regexp_track_hex_escape(parser, (uint8_t) value);
|
|
704
|
+
}
|
|
705
|
+
|
|
706
|
+
return value;
|
|
707
|
+
}
|
|
708
|
+
|
|
709
|
+
/**
|
|
710
|
+
* Parse a backslash escape sequence in a regexp, handling \u (unicode),
|
|
711
|
+
* \p/\P (property), \x (hex), and other single-character escapes. Also
|
|
712
|
+
* tracks encoding state for \M-x and \C-\M-x escapes.
|
|
713
|
+
*/
|
|
714
|
+
static void
|
|
715
|
+
pm_regexp_parse_backslash_escape(pm_regexp_parser_t *parser) {
|
|
716
|
+
if (pm_regexp_char_is_eof(parser)) return;
|
|
717
|
+
|
|
718
|
+
switch (*parser->cursor) {
|
|
719
|
+
case 'u':
|
|
720
|
+
parser->cursor++; // skip 'u'
|
|
721
|
+
pm_regexp_parse_unicode_escape(parser);
|
|
722
|
+
break;
|
|
723
|
+
case 'p':
|
|
724
|
+
case 'P':
|
|
725
|
+
pm_regexp_parse_property_escape(parser);
|
|
726
|
+
break;
|
|
727
|
+
case 'x':
|
|
728
|
+
parser->cursor++; // skip 'x'
|
|
729
|
+
pm_regexp_parse_hex_escape(parser);
|
|
730
|
+
break;
|
|
731
|
+
case 'M':
|
|
732
|
+
// \M-x produces (x | 0x80), always >= 0x80
|
|
733
|
+
if (parser->cursor + 2 < parser->end && parser->cursor[1] == '-') {
|
|
734
|
+
parser->cursor += 2; // skip 'M-'
|
|
735
|
+
if (!pm_regexp_char_is_eof(parser)) {
|
|
736
|
+
if (*parser->cursor == '\\') {
|
|
737
|
+
parser->cursor++;
|
|
738
|
+
// \M-\C-x or \M-\cx — the resulting byte is always >= 0x80
|
|
739
|
+
// We just need to track it as a hex escape >= 0x80.
|
|
740
|
+
pm_regexp_parse_backslash_escape(parser);
|
|
741
|
+
} else {
|
|
742
|
+
parser->cursor++;
|
|
743
|
+
}
|
|
744
|
+
// \M-x always produces a byte >= 0x80
|
|
745
|
+
pm_regexp_track_hex_escape(parser, 0x80);
|
|
746
|
+
}
|
|
747
|
+
} else {
|
|
748
|
+
parser->cursor++;
|
|
749
|
+
}
|
|
750
|
+
break;
|
|
751
|
+
case 'C':
|
|
752
|
+
// \C-x produces (x & 0x1F)
|
|
753
|
+
if (parser->cursor + 2 < parser->end && parser->cursor[1] == '-') {
|
|
754
|
+
parser->cursor += 2; // skip 'C-'
|
|
755
|
+
if (!pm_regexp_char_is_eof(parser)) {
|
|
756
|
+
if (*parser->cursor == '\\') {
|
|
757
|
+
parser->cursor++;
|
|
758
|
+
pm_regexp_parse_backslash_escape(parser);
|
|
759
|
+
} else {
|
|
760
|
+
parser->cursor++;
|
|
761
|
+
}
|
|
762
|
+
}
|
|
763
|
+
} else {
|
|
764
|
+
parser->cursor++;
|
|
765
|
+
}
|
|
766
|
+
break;
|
|
767
|
+
case 'c':
|
|
768
|
+
// \cx produces (x & 0x1F)
|
|
769
|
+
parser->cursor++; // skip 'c'
|
|
770
|
+
if (!pm_regexp_char_is_eof(parser)) {
|
|
771
|
+
if (*parser->cursor == '\\') {
|
|
772
|
+
parser->cursor++;
|
|
773
|
+
pm_regexp_parse_backslash_escape(parser);
|
|
774
|
+
} else {
|
|
775
|
+
parser->cursor++;
|
|
776
|
+
}
|
|
777
|
+
}
|
|
778
|
+
break;
|
|
779
|
+
default:
|
|
780
|
+
pm_regexp_hex_group_boundary(parser);
|
|
781
|
+
parser->cursor++;
|
|
782
|
+
break;
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
|
|
786
|
+
/**
|
|
787
|
+
* Check if a byte at the current position is a non-ASCII byte in a multibyte
|
|
788
|
+
* encoding that produces an invalid character. If so, emit an error at the
|
|
789
|
+
* byte location immediately.
|
|
790
|
+
*/
|
|
791
|
+
static void
|
|
792
|
+
pm_regexp_parse_invalid_multibyte(pm_regexp_parser_t *parser, const uint8_t *cursor) {
|
|
793
|
+
uint8_t byte = *cursor;
|
|
794
|
+
if (byte >= 0x80 && parser->encoding_changed && parser->encoding->multibyte) {
|
|
795
|
+
size_t width = parser->encoding->char_width(cursor, (ptrdiff_t) (parser->end - cursor));
|
|
796
|
+
if (width > 1) {
|
|
797
|
+
parser->cursor += width - 1;
|
|
798
|
+
} else if (width == 0) {
|
|
799
|
+
parser->has_invalid_multibyte = true;
|
|
800
|
+
pm_regexp_parse_error_format(parser, cursor, cursor + 1, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
|
|
801
|
+
}
|
|
802
|
+
}
|
|
803
|
+
}
|
|
804
|
+
|
|
282
805
|
/**
|
|
283
806
|
* match-char-set : '[' '^'? (match-range | match-char)* ']'
|
|
284
807
|
* ;
|
|
@@ -293,12 +816,16 @@ pm_regexp_parse_character_set(pm_regexp_parser_t *parser, uint16_t depth) {
|
|
|
293
816
|
pm_regexp_parse_lbracket(parser, (uint16_t) (depth + 1));
|
|
294
817
|
break;
|
|
295
818
|
case '\\':
|
|
296
|
-
|
|
297
|
-
parser->cursor++;
|
|
298
|
-
}
|
|
819
|
+
pm_regexp_parse_backslash_escape(parser);
|
|
299
820
|
break;
|
|
300
821
|
default:
|
|
301
|
-
//
|
|
822
|
+
// We've already advanced the cursor by one byte. If the byte
|
|
823
|
+
// was >= 0x80 in a multibyte encoding, we may need to consume
|
|
824
|
+
// additional continuation bytes and validate the character.
|
|
825
|
+
if (*(parser->cursor - 1) >= 0x80) {
|
|
826
|
+
parser->non_ascii_literal_count++;
|
|
827
|
+
}
|
|
828
|
+
pm_regexp_parse_invalid_multibyte(parser, parser->cursor - 1);
|
|
302
829
|
break;
|
|
303
830
|
}
|
|
304
831
|
}
|
|
@@ -354,8 +881,13 @@ typedef enum {
|
|
|
354
881
|
// These are the options that are configurable on the regular expression (or
|
|
355
882
|
// from within a group).
|
|
356
883
|
|
|
884
|
+
/** The minimum character value for a regexp option slot. */
|
|
357
885
|
#define PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM 'a'
|
|
886
|
+
|
|
887
|
+
/** The maximum character value for a regexp option slot. */
|
|
358
888
|
#define PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM 'x'
|
|
889
|
+
|
|
890
|
+
/** The number of regexp option slots. */
|
|
359
891
|
#define PRISM_REGEXP_OPTION_STATE_SLOTS (PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM + 1)
|
|
360
892
|
|
|
361
893
|
/**
|
|
@@ -498,7 +1030,15 @@ pm_regexp_parse_group(pm_regexp_parser_t *parser, uint16_t depth) {
|
|
|
498
1030
|
}
|
|
499
1031
|
|
|
500
1032
|
size_t width = parser->encoding->char_width(parser->cursor, (ptrdiff_t) (parser->end - parser->cursor));
|
|
501
|
-
if (width == 0)
|
|
1033
|
+
if (width == 0) {
|
|
1034
|
+
if (*parser->cursor >= 0x80) {
|
|
1035
|
+
parser->has_invalid_multibyte = true;
|
|
1036
|
+
pm_regexp_parse_error_format(parser, parser->cursor, parser->cursor + 1, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
|
|
1037
|
+
parser->cursor++;
|
|
1038
|
+
continue;
|
|
1039
|
+
}
|
|
1040
|
+
return false;
|
|
1041
|
+
}
|
|
502
1042
|
|
|
503
1043
|
escaped = (width == 1) && (*parser->cursor == '\\');
|
|
504
1044
|
parser->cursor += width;
|
|
@@ -686,9 +1226,7 @@ pm_regexp_parse_item(pm_regexp_parser_t *parser, uint16_t depth) {
|
|
|
686
1226
|
return pm_regexp_parse_quantifier(parser);
|
|
687
1227
|
case '\\':
|
|
688
1228
|
parser->cursor++;
|
|
689
|
-
|
|
690
|
-
parser->cursor++;
|
|
691
|
-
}
|
|
1229
|
+
pm_regexp_parse_backslash_escape(parser);
|
|
692
1230
|
return pm_regexp_parse_quantifier(parser);
|
|
693
1231
|
case '(':
|
|
694
1232
|
parser->cursor++;
|
|
@@ -720,9 +1258,30 @@ pm_regexp_parse_item(pm_regexp_parser_t *parser, uint16_t depth) {
|
|
|
720
1258
|
width = parser->encoding->char_width(parser->cursor, (ptrdiff_t) (parser->end - parser->cursor));
|
|
721
1259
|
}
|
|
722
1260
|
|
|
723
|
-
if (width == 0)
|
|
724
|
-
|
|
1261
|
+
if (width == 0) {
|
|
1262
|
+
if (*parser->cursor >= 0x80 && parser->encoding_changed) {
|
|
1263
|
+
if (parser->encoding->multibyte) {
|
|
1264
|
+
// Invalid multibyte character in a multibyte encoding.
|
|
1265
|
+
// Emit the error at the byte location immediately.
|
|
1266
|
+
parser->has_invalid_multibyte = true;
|
|
1267
|
+
pm_regexp_parse_error_format(parser, parser->cursor, parser->cursor + 1, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
|
|
1268
|
+
} else {
|
|
1269
|
+
// Non-ASCII byte in a single-byte encoding (e.g.,
|
|
1270
|
+
// US-ASCII). Count it for later error reporting.
|
|
1271
|
+
parser->non_ascii_literal_count++;
|
|
1272
|
+
}
|
|
1273
|
+
parser->cursor++;
|
|
1274
|
+
return pm_regexp_parse_quantifier(parser);
|
|
1275
|
+
}
|
|
1276
|
+
return false;
|
|
1277
|
+
}
|
|
725
1278
|
|
|
1279
|
+
// Count non-ASCII literal bytes.
|
|
1280
|
+
for (size_t i = 0; i < width; i++) {
|
|
1281
|
+
if (parser->cursor[i] >= 0x80) parser->non_ascii_literal_count++;
|
|
1282
|
+
}
|
|
1283
|
+
|
|
1284
|
+
parser->cursor += width;
|
|
726
1285
|
return pm_regexp_parse_quantifier(parser);
|
|
727
1286
|
}
|
|
728
1287
|
}
|
|
@@ -768,13 +1327,353 @@ pm_regexp_parse_pattern(pm_regexp_parser_t *parser) {
|
|
|
768
1327
|
return pm_regexp_char_is_eof(parser);
|
|
769
1328
|
}
|
|
770
1329
|
|
|
1330
|
+
// ---------------------------------------------------------------------------
|
|
1331
|
+
// Encoding validation
|
|
1332
|
+
// ---------------------------------------------------------------------------
|
|
1333
|
+
|
|
771
1334
|
/**
|
|
772
|
-
*
|
|
773
|
-
*
|
|
1335
|
+
* Validate that groups of hex escape bytes in the buffer form valid multibyte
|
|
1336
|
+
* characters in the given encoding. Groups are separated by 0x00 sentinels.
|
|
1337
|
+
*/
|
|
1338
|
+
static bool
|
|
1339
|
+
pm_regexp_validate_hex_escapes(const pm_encoding_t *encoding, const pm_buffer_t *buffer) {
|
|
1340
|
+
const uint8_t *data = (const uint8_t *) pm_buffer_value(buffer);
|
|
1341
|
+
size_t len = pm_buffer_length(buffer);
|
|
1342
|
+
size_t i = 0;
|
|
1343
|
+
|
|
1344
|
+
while (i < len) {
|
|
1345
|
+
size_t group_start = i;
|
|
1346
|
+
while (i < len && data[i] != 0x00) i++;
|
|
1347
|
+
|
|
1348
|
+
for (size_t j = group_start; j < i; ) {
|
|
1349
|
+
size_t width = encoding->char_width(data + j, (ptrdiff_t) (i - j));
|
|
1350
|
+
if (width == 0) return false;
|
|
1351
|
+
j += width;
|
|
1352
|
+
}
|
|
1353
|
+
|
|
1354
|
+
if (i < len) i++; // skip sentinel
|
|
1355
|
+
}
|
|
1356
|
+
|
|
1357
|
+
return true;
|
|
1358
|
+
}
|
|
1359
|
+
|
|
1360
|
+
/**
|
|
1361
|
+
* Format regexp source content for use in error messages, hex-escaping
|
|
1362
|
+
* non-ASCII bytes.
|
|
1363
|
+
*/
|
|
1364
|
+
static void
|
|
1365
|
+
pm_regexp_format_for_error(pm_buffer_t *buffer, const pm_encoding_t *encoding, const uint8_t *source, size_t length) {
|
|
1366
|
+
size_t index = 0;
|
|
1367
|
+
|
|
1368
|
+
if (encoding == PM_ENCODING_UTF_8_ENTRY) {
|
|
1369
|
+
pm_buffer_append_string(buffer, (const char *) source, length);
|
|
1370
|
+
return;
|
|
1371
|
+
}
|
|
1372
|
+
|
|
1373
|
+
while (index < length) {
|
|
1374
|
+
if (source[index] < 0x80) {
|
|
1375
|
+
pm_buffer_append_byte(buffer, source[index]);
|
|
1376
|
+
index++;
|
|
1377
|
+
} else if (encoding->multibyte) {
|
|
1378
|
+
size_t width = encoding->char_width(source + index, (ptrdiff_t) (length - index));
|
|
1379
|
+
|
|
1380
|
+
if (width > 1) {
|
|
1381
|
+
pm_buffer_append_string(buffer, "\\x{", 3);
|
|
1382
|
+
for (size_t i = 0; i < width; i++) {
|
|
1383
|
+
pm_buffer_append_format(buffer, "%02X", source[index + i]);
|
|
1384
|
+
}
|
|
1385
|
+
pm_buffer_append_byte(buffer, '}');
|
|
1386
|
+
index += width;
|
|
1387
|
+
} else {
|
|
1388
|
+
pm_buffer_append_format(buffer, "\\x%02X", source[index]);
|
|
1389
|
+
index++;
|
|
1390
|
+
}
|
|
1391
|
+
} else {
|
|
1392
|
+
pm_buffer_append_format(buffer, "\\x%02X", source[index]);
|
|
1393
|
+
index++;
|
|
1394
|
+
}
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
1397
|
+
|
|
1398
|
+
/**
|
|
1399
|
+
* Emit an encoding validation error on the regexp node.
|
|
1400
|
+
*/
|
|
1401
|
+
#define PM_REGEXP_ENCODING_ERROR(parser, diag_id, ...) \
|
|
1402
|
+
pm_diagnostic_list_append_format( \
|
|
1403
|
+
&(parser)->parser->metadata_arena, \
|
|
1404
|
+
&(parser)->parser->error_list, \
|
|
1405
|
+
(uint32_t) ((parser)->node_start - (parser)->parser->start), \
|
|
1406
|
+
(uint32_t) ((parser)->node_end - (parser)->node_start), \
|
|
1407
|
+
diag_id, __VA_ARGS__)
|
|
1408
|
+
|
|
1409
|
+
/**
|
|
1410
|
+
* Validate encoding for a regexp with an encoding modifier (/e, /s, /u, /n).
|
|
1411
|
+
*
|
|
1412
|
+
* The decision tree is:
|
|
1413
|
+
*
|
|
1414
|
+
* 1. No escape-set encoding (explicit_encoding == NULL):
|
|
1415
|
+
* a. ASCII-only content: validate property escapes, return forced US-ASCII
|
|
1416
|
+
* for /n or the modifier flags for others.
|
|
1417
|
+
* b. US-ASCII source with non-ASCII literals: emit per-byte errors.
|
|
1418
|
+
* c. Source encoding differs from modifier encoding: emit mismatch error.
|
|
1419
|
+
*
|
|
1420
|
+
* 2. Mixed \u and \x escapes: emit the appropriate conflict error depending
|
|
1421
|
+
* on the modifier and which escape type was last.
|
|
1422
|
+
*
|
|
1423
|
+
* 3. \u escape with non-/u modifier: incompatible encoding error.
|
|
1424
|
+
*
|
|
1425
|
+
* 4. Validate that hex escape byte sequences form valid multibyte characters
|
|
1426
|
+
* in the modifier's encoding.
|
|
1427
|
+
*/
|
|
1428
|
+
static pm_node_flags_t
|
|
1429
|
+
pm_regexp_validate_encoding_modifier(pm_regexp_parser_t *parser, bool ascii_only, pm_node_flags_t flags, char modifier, const pm_encoding_t *modifier_encoding, const char *source_start, int source_length) {
|
|
1430
|
+
|
|
1431
|
+
if (parser->explicit_encoding == NULL) {
|
|
1432
|
+
if (ascii_only) {
|
|
1433
|
+
// Check property escapes against the modifier's encoding tier.
|
|
1434
|
+
// /n (ASCII-8BIT): only POSIX properties are valid.
|
|
1435
|
+
// /e, /s: POSIX and script properties are valid.
|
|
1436
|
+
// /u: all properties are valid.
|
|
1437
|
+
if (modifier == 'n' && parser->property_name != NULL) {
|
|
1438
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INVALID_CHAR_PROPERTY,
|
|
1439
|
+
(int) parser->property_name_length, (const char *) parser->property_name,
|
|
1440
|
+
source_length, source_start);
|
|
1441
|
+
} else if (modifier != 'u' && parser->has_unicode_property_escape) {
|
|
1442
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INVALID_CHAR_PROPERTY,
|
|
1443
|
+
(int) parser->unicode_property_name_length, (const char *) parser->unicode_property_name,
|
|
1444
|
+
source_length, source_start);
|
|
1445
|
+
}
|
|
1446
|
+
return modifier == 'n' ? PM_REGULAR_EXPRESSION_FLAGS_FORCED_US_ASCII_ENCODING : flags;
|
|
1447
|
+
}
|
|
1448
|
+
|
|
1449
|
+
if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) {
|
|
1450
|
+
for (uint32_t i = 0; i < parser->non_ascii_literal_count; i++) {
|
|
1451
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
|
|
1452
|
+
}
|
|
1453
|
+
} else if (parser->encoding != modifier_encoding) {
|
|
1454
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_ENCODING_OPTION_MISMATCH, modifier, parser->encoding->name);
|
|
1455
|
+
|
|
1456
|
+
if (modifier == 'n' && !ascii_only) {
|
|
1457
|
+
pm_buffer_t formatted = { 0 };
|
|
1458
|
+
pm_regexp_format_for_error(&formatted, parser->encoding, (const uint8_t *) source_start, (size_t) source_length);
|
|
1459
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_NON_ESCAPED_MBC, (int) formatted.length, (const char *) formatted.value);
|
|
1460
|
+
pm_buffer_cleanup(&formatted);
|
|
1461
|
+
}
|
|
1462
|
+
}
|
|
1463
|
+
|
|
1464
|
+
return flags;
|
|
1465
|
+
}
|
|
1466
|
+
|
|
1467
|
+
// Mixed unicode + hex escapes.
|
|
1468
|
+
if (parser->has_unicode_escape && parser->has_hex_escape) {
|
|
1469
|
+
if (modifier == 'n') {
|
|
1470
|
+
if (parser->last_escape_was_unicode) {
|
|
1471
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP, source_length, source_start);
|
|
1472
|
+
} else {
|
|
1473
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_ESCAPED_NON_ASCII_IN_UTF8, source_length, source_start);
|
|
1474
|
+
}
|
|
1475
|
+
} else {
|
|
1476
|
+
if (!pm_regexp_validate_hex_escapes(modifier_encoding, &parser->hex_escape_buffer)) {
|
|
1477
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
|
|
1478
|
+
}
|
|
1479
|
+
}
|
|
1480
|
+
|
|
1481
|
+
return flags;
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
if (modifier != 'u' && parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
|
|
1485
|
+
if (parser->last_escape_was_unicode) {
|
|
1486
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING, source_length, source_start);
|
|
1487
|
+
} else if (parser->encoding != PM_ENCODING_UTF_8_ENTRY) {
|
|
1488
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING, source_length, source_start);
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
|
|
1492
|
+
if (modifier != 'n' && !pm_regexp_validate_hex_escapes(modifier_encoding, &parser->hex_escape_buffer)) {
|
|
1493
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
|
|
1494
|
+
}
|
|
1495
|
+
|
|
1496
|
+
return flags;
|
|
1497
|
+
}
|
|
1498
|
+
|
|
1499
|
+
/**
|
|
1500
|
+
* Validate encoding for a regexp without a modifier and compute the encoding
|
|
1501
|
+
* flags to set on the node.
|
|
1502
|
+
*
|
|
1503
|
+
* The decision tree is:
|
|
1504
|
+
*
|
|
1505
|
+
* 1. If a modifier (/n, /u, /e, /s) is present, delegate to
|
|
1506
|
+
* pm_regexp_validate_encoding_modifier.
|
|
1507
|
+
* 2. Invalid multibyte chars or unicode ranges: suppress further checks (errors
|
|
1508
|
+
* were already emitted during parsing).
|
|
1509
|
+
* 3. US-ASCII source with non-ASCII literals: emit per-byte errors.
|
|
1510
|
+
* 4. ASCII-only content: return forced US-ASCII (or forced UTF-8 if \p{...}).
|
|
1511
|
+
* 5. Escape-set encoding present: validate hex escapes against the target
|
|
1512
|
+
* encoding, handle mixed \u + \x conflicts, and return the appropriate
|
|
1513
|
+
* forced encoding flag.
|
|
1514
|
+
*/
|
|
1515
|
+
static pm_node_flags_t
|
|
1516
|
+
pm_regexp_validate_encoding(pm_regexp_parser_t *parser, bool ascii_only, pm_node_flags_t flags, const char *source_start, int source_length) {
|
|
1517
|
+
|
|
1518
|
+
// Invalid multibyte characters suppress further validation.
|
|
1519
|
+
// Errors were already emitted at the byte locations during parsing.
|
|
1520
|
+
if (parser->has_invalid_multibyte) {
|
|
1521
|
+
return flags;
|
|
1522
|
+
}
|
|
1523
|
+
|
|
1524
|
+
if (parser->invalid_unicode_range) {
|
|
1525
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INVALID_UNICODE_RANGE, source_length, source_start);
|
|
1526
|
+
return flags;
|
|
1527
|
+
}
|
|
1528
|
+
|
|
1529
|
+
// Check modifier flags first.
|
|
1530
|
+
if (flags & PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT) {
|
|
1531
|
+
return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags, 'n', PM_ENCODING_ASCII_8BIT_ENTRY, source_start, source_length);
|
|
1532
|
+
}
|
|
1533
|
+
if (flags & PM_REGULAR_EXPRESSION_FLAGS_UTF_8) {
|
|
1534
|
+
return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags, 'u', PM_ENCODING_UTF_8_ENTRY, source_start, source_length);
|
|
1535
|
+
}
|
|
1536
|
+
if (flags & PM_REGULAR_EXPRESSION_FLAGS_EUC_JP) {
|
|
1537
|
+
return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags, 'e', PM_ENCODING_EUC_JP_ENTRY, source_start, source_length);
|
|
1538
|
+
}
|
|
1539
|
+
if (flags & PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J) {
|
|
1540
|
+
return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags, 's', PM_ENCODING_WINDOWS_31J_ENTRY, source_start, source_length);
|
|
1541
|
+
}
|
|
1542
|
+
|
|
1543
|
+
// No modifier — check for non-ASCII literals in US-ASCII encoding.
|
|
1544
|
+
if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY && parser->explicit_encoding == NULL && !ascii_only) {
|
|
1545
|
+
for (uint32_t i = 0; i < parser->non_ascii_literal_count; i++) {
|
|
1546
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
|
|
1547
|
+
}
|
|
1548
|
+
}
|
|
1549
|
+
|
|
1550
|
+
// ASCII-only regexps get downgraded to US-ASCII, unless property escapes
|
|
1551
|
+
// force UTF-8.
|
|
1552
|
+
if (ascii_only) {
|
|
1553
|
+
if (parser->has_property_escape) {
|
|
1554
|
+
return PM_REGULAR_EXPRESSION_FLAGS_FORCED_UTF8_ENCODING;
|
|
1555
|
+
}
|
|
1556
|
+
return PM_REGULAR_EXPRESSION_FLAGS_FORCED_US_ASCII_ENCODING;
|
|
1557
|
+
}
|
|
1558
|
+
|
|
1559
|
+
// Check explicit encoding from escape sequences.
|
|
1560
|
+
if (parser->explicit_encoding != NULL) {
|
|
1561
|
+
// Mixed unicode + hex escapes without modifier.
|
|
1562
|
+
if (parser->has_unicode_escape && parser->has_hex_escape && parser->encoding != PM_ENCODING_UTF_8_ENTRY) {
|
|
1563
|
+
if (parser->encoding != PM_ENCODING_US_ASCII_ENTRY &&
|
|
1564
|
+
parser->encoding != PM_ENCODING_ASCII_8BIT_ENTRY &&
|
|
1565
|
+
!pm_regexp_validate_hex_escapes(parser->encoding, &parser->hex_escape_buffer)) {
|
|
1566
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
|
|
1567
|
+
} else if (parser->last_escape_was_unicode) {
|
|
1568
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP, source_length, source_start);
|
|
1569
|
+
} else {
|
|
1570
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_ESCAPED_NON_ASCII_IN_UTF8, source_length, source_start);
|
|
1571
|
+
}
|
|
1572
|
+
|
|
1573
|
+
return 0;
|
|
1574
|
+
}
|
|
1575
|
+
|
|
1576
|
+
if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
|
|
1577
|
+
if (!pm_regexp_validate_hex_escapes(parser->explicit_encoding, &parser->hex_escape_buffer)) {
|
|
1578
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
|
|
1579
|
+
}
|
|
1580
|
+
|
|
1581
|
+
return PM_REGULAR_EXPRESSION_FLAGS_FORCED_UTF8_ENCODING;
|
|
1582
|
+
} else if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) {
|
|
1583
|
+
return PM_REGULAR_EXPRESSION_FLAGS_FORCED_BINARY_ENCODING;
|
|
1584
|
+
} else {
|
|
1585
|
+
if (!pm_regexp_validate_hex_escapes(parser->explicit_encoding, &parser->hex_escape_buffer)) {
|
|
1586
|
+
PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
|
|
1587
|
+
}
|
|
1588
|
+
}
|
|
1589
|
+
}
|
|
1590
|
+
|
|
1591
|
+
return 0;
|
|
1592
|
+
}
|
|
1593
|
+
|
|
1594
|
+
/**
|
|
1595
|
+
* Parse a regular expression, validate its encoding, and optionally extract
|
|
1596
|
+
* named capture groups. Encoding validation walks the raw source (content_loc)
|
|
1597
|
+
* to distinguish escape-produced bytes from literal bytes. Named capture
|
|
1598
|
+
* extraction walks the unescaped content since escape sequences in group names
|
|
1599
|
+
* (e.g., line continuations) have already been processed by the lexer.
|
|
1600
|
+
*/
|
|
1601
|
+
pm_node_flags_t
|
|
1602
|
+
pm_regexp_parse(pm_parser_t *parser, pm_regular_expression_node_t *node, pm_regexp_name_callback_t name_callback, void *name_data) {
|
|
1603
|
+
const uint8_t *source = parser->start + node->content_loc.start;
|
|
1604
|
+
size_t size = node->content_loc.length;
|
|
1605
|
+
bool extended_mode = PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EXTENDED);
|
|
1606
|
+
pm_node_flags_t flags = PM_NODE_FLAGS(node);
|
|
1607
|
+
|
|
1608
|
+
const uint8_t *node_start = parser->start + node->base.location.start;
|
|
1609
|
+
const uint8_t *node_end = parser->start + node->base.location.start + node->base.location.length;
|
|
1610
|
+
|
|
1611
|
+
// First pass: walk raw source for encoding validation (no name extraction).
|
|
1612
|
+
pm_regexp_parser_t regexp_parser = {
|
|
1613
|
+
.parser = parser,
|
|
1614
|
+
.start = source,
|
|
1615
|
+
.cursor = source,
|
|
1616
|
+
.end = source + size,
|
|
1617
|
+
.extended_mode = extended_mode,
|
|
1618
|
+
.encoding_changed = parser->encoding_changed,
|
|
1619
|
+
.encoding = parser->encoding,
|
|
1620
|
+
.name_callback = NULL,
|
|
1621
|
+
.name_data = NULL,
|
|
1622
|
+
.report_errors = true,
|
|
1623
|
+
.node_start = node_start,
|
|
1624
|
+
.node_end = node_end,
|
|
1625
|
+
.has_unicode_escape = false,
|
|
1626
|
+
.has_hex_escape = false,
|
|
1627
|
+
.last_escape_was_unicode = false,
|
|
1628
|
+
.explicit_encoding = NULL,
|
|
1629
|
+
.has_property_escape = false,
|
|
1630
|
+
.has_unicode_property_escape = false,
|
|
1631
|
+
.property_name = NULL,
|
|
1632
|
+
.property_name_length = 0,
|
|
1633
|
+
.unicode_property_name = NULL,
|
|
1634
|
+
.unicode_property_name_length = 0,
|
|
1635
|
+
.non_ascii_literal_count = 0,
|
|
1636
|
+
.invalid_unicode_range = false,
|
|
1637
|
+
.hex_escape_buffer = { 0 },
|
|
1638
|
+
.hex_group_active = false,
|
|
1639
|
+
.has_invalid_multibyte = false,
|
|
1640
|
+
};
|
|
1641
|
+
|
|
1642
|
+
pm_regexp_parse_pattern(®exp_parser);
|
|
1643
|
+
|
|
1644
|
+
// Compute ascii_only from the regexp parser's tracked state. We cannot
|
|
1645
|
+
// use node->unescaped for this because regexp unescaped content preserves
|
|
1646
|
+
// escape text (e.g., \x80 is 4 ASCII chars), not the binary values.
|
|
1647
|
+
bool ascii_only = !regexp_parser.has_hex_escape && !regexp_parser.has_unicode_escape && regexp_parser.non_ascii_literal_count == 0;
|
|
1648
|
+
// Use the unescaped content for error messages to match CRuby's format,
|
|
1649
|
+
// where Ruby escapes like \M-\C-? are resolved to bytes but regexp escapes
|
|
1650
|
+
// like \u{80} are preserved as text.
|
|
1651
|
+
const char *error_source = (const char *) pm_string_source(&node->unescaped);
|
|
1652
|
+
int error_source_length = (int) pm_string_length(&node->unescaped);
|
|
1653
|
+
pm_node_flags_t encoding_flags = pm_regexp_validate_encoding(®exp_parser, ascii_only, flags, error_source, error_source_length);
|
|
1654
|
+
pm_buffer_cleanup(®exp_parser.hex_escape_buffer);
|
|
1655
|
+
|
|
1656
|
+
// Second pass: walk unescaped content for named capture extraction.
|
|
1657
|
+
if (name_callback != NULL) {
|
|
1658
|
+
pm_regexp_parse_named_captures(parser, pm_string_source(&node->unescaped), pm_string_length(&node->unescaped), extended_mode, name_callback, name_data);
|
|
1659
|
+
}
|
|
1660
|
+
|
|
1661
|
+
return encoding_flags;
|
|
1662
|
+
}
|
|
1663
|
+
|
|
1664
|
+
/**
|
|
1665
|
+
* Parse an interpolated regular expression for named capture groups only.
|
|
1666
|
+
* This is used for the =~ operator with interpolated regexps where we don't
|
|
1667
|
+
* have a pm_regular_expression_node_t. No encoding validation is performed.
|
|
1668
|
+
*
|
|
1669
|
+
* Note: The encoding-tracking fields (has_unicode_escape, has_hex_escape, etc.)
|
|
1670
|
+
* are initialized but not used for the result. They exist because the parsing
|
|
1671
|
+
* functions (pm_regexp_parse_backslash_escape, etc.) unconditionally update
|
|
1672
|
+
* them as they walk through the content.
|
|
774
1673
|
*/
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
1674
|
+
void
|
|
1675
|
+
pm_regexp_parse_named_captures(pm_parser_t *parser, const uint8_t *source, size_t size, bool extended_mode, pm_regexp_name_callback_t name_callback, void *name_data) {
|
|
1676
|
+
pm_regexp_parser_t regexp_parser = {
|
|
778
1677
|
.parser = parser,
|
|
779
1678
|
.start = source,
|
|
780
1679
|
.cursor = source,
|
|
@@ -784,7 +1683,24 @@ pm_regexp_parse(pm_parser_t *parser, const uint8_t *source, size_t size, bool ex
|
|
|
784
1683
|
.encoding = parser->encoding,
|
|
785
1684
|
.name_callback = name_callback,
|
|
786
1685
|
.name_data = name_data,
|
|
787
|
-
.
|
|
788
|
-
.
|
|
789
|
-
|
|
1686
|
+
.report_errors = false,
|
|
1687
|
+
.has_unicode_escape = false,
|
|
1688
|
+
.has_hex_escape = false,
|
|
1689
|
+
.last_escape_was_unicode = false,
|
|
1690
|
+
.explicit_encoding = NULL,
|
|
1691
|
+
.has_property_escape = false,
|
|
1692
|
+
.has_unicode_property_escape = false,
|
|
1693
|
+
.property_name = NULL,
|
|
1694
|
+
.property_name_length = 0,
|
|
1695
|
+
.unicode_property_name = NULL,
|
|
1696
|
+
.unicode_property_name_length = 0,
|
|
1697
|
+
.non_ascii_literal_count = 0,
|
|
1698
|
+
.invalid_unicode_range = false,
|
|
1699
|
+
.hex_escape_buffer = { 0 },
|
|
1700
|
+
.hex_group_active = false,
|
|
1701
|
+
.has_invalid_multibyte = false,
|
|
1702
|
+
};
|
|
1703
|
+
|
|
1704
|
+
pm_regexp_parse_pattern(®exp_parser);
|
|
1705
|
+
pm_buffer_cleanup(®exp_parser.hex_escape_buffer);
|
|
790
1706
|
}
|