prism 1.9.0 → 2.0.0.rc2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +88 -0
  3. data/CODE_OF_CONDUCT.md +1 -1
  4. data/LICENSE.md +1 -1
  5. data/Makefile +41 -12
  6. data/README.md +35 -97
  7. data/config.yml +174 -161
  8. data/docs/build_system.md +2 -2
  9. data/docs/configuration.md +5 -4
  10. data/docs/encoding.md +2 -2
  11. data/docs/fuzzing.md +4 -11
  12. data/docs/javascript.md +19 -4
  13. data/docs/mapping.md +1 -1
  14. data/docs/releasing.md +77 -5
  15. data/docs/relocation.md +2 -4
  16. data/docs/serialization.md +22 -24
  17. data/docs/testing.md +46 -29
  18. data/ext/prism/api_node.c +1461 -1401
  19. data/ext/prism/extconf.rb +3 -4
  20. data/ext/prism/extension.c +796 -422
  21. data/ext/prism/extension.h +3 -3
  22. data/include/prism/arena.h +37 -0
  23. data/include/prism/ast.h +2727 -610
  24. data/include/prism/buffer.h +52 -0
  25. data/include/prism/comments.h +55 -0
  26. data/include/prism/compiler/accel.h +26 -0
  27. data/include/prism/compiler/align.h +36 -0
  28. data/include/prism/compiler/assume.h +25 -0
  29. data/include/prism/compiler/bswap.h +46 -0
  30. data/include/prism/compiler/endianness.h +34 -0
  31. data/include/prism/compiler/exported.h +24 -0
  32. data/include/prism/compiler/fallthrough.h +22 -0
  33. data/include/prism/compiler/filesystem.h +32 -0
  34. data/include/prism/compiler/flex_array.h +19 -0
  35. data/include/prism/compiler/force_inline.h +21 -0
  36. data/include/prism/compiler/format.h +25 -0
  37. data/include/prism/compiler/inline.h +17 -0
  38. data/include/prism/compiler/nodiscard.h +22 -0
  39. data/include/prism/compiler/nonnull.h +18 -0
  40. data/include/prism/compiler/unused.h +18 -0
  41. data/include/prism/constant_pool.h +81 -0
  42. data/include/prism/diagnostic.h +38 -403
  43. data/include/prism/errors_format.h +53 -0
  44. data/include/prism/excludes.h +29 -0
  45. data/include/prism/integer.h +41 -0
  46. data/include/prism/internal/allocator.h +68 -0
  47. data/include/prism/internal/allocator_debug.h +88 -0
  48. data/include/prism/internal/arena.h +108 -0
  49. data/include/prism/internal/bit.h +42 -0
  50. data/include/prism/internal/buffer.h +94 -0
  51. data/include/prism/internal/char.h +145 -0
  52. data/include/prism/internal/comments.h +53 -0
  53. data/include/prism/internal/constant_pool.h +152 -0
  54. data/include/prism/internal/diagnostic.h +389 -0
  55. data/include/prism/{encoding.h → internal/encoding.h} +27 -68
  56. data/include/prism/internal/integer.h +68 -0
  57. data/include/prism/internal/isinf.h +16 -0
  58. data/include/prism/internal/line_offset_list.h +34 -0
  59. data/include/prism/{util/pm_list.h → internal/list.h} +11 -52
  60. data/include/prism/internal/magic_comments.h +23 -0
  61. data/include/prism/internal/memchr.h +15 -0
  62. data/include/prism/internal/node.h +38 -0
  63. data/include/prism/internal/options.h +232 -0
  64. data/include/prism/internal/parser.h +1004 -0
  65. data/include/prism/internal/regexp.h +26 -0
  66. data/include/prism/internal/serialize.h +39 -0
  67. data/include/prism/internal/source.h +72 -0
  68. data/include/prism/{static_literals.h → internal/static_literals.h} +26 -47
  69. data/include/prism/internal/stringy.h +35 -0
  70. data/include/prism/{util/pm_strncasecmp.h → internal/strncasecmp.h} +3 -17
  71. data/include/prism/{util/pm_strpbrk.h → internal/strpbrk.h} +7 -20
  72. data/include/prism/internal/tokens.h +11 -0
  73. data/include/prism/internal/xxh3.h +16 -0
  74. data/include/prism/json.h +32 -0
  75. data/include/prism/line_offset_list.h +61 -0
  76. data/include/prism/magic_comments.h +35 -0
  77. data/include/prism/node.h +18 -53
  78. data/include/prism/options.h +146 -280
  79. data/include/prism/parser.h +356 -867
  80. data/include/prism/prettyprint.h +6 -9
  81. data/include/prism/serialize.h +115 -0
  82. data/include/prism/source.h +155 -0
  83. data/include/prism/stream.h +28 -0
  84. data/include/prism/string_query.h +74 -0
  85. data/include/prism/stringy.h +72 -0
  86. data/include/prism/version.h +12 -3
  87. data/include/prism.h +43 -320
  88. data/lib/prism/compiler.rb +319 -305
  89. data/lib/prism/desugar_compiler.rb +105 -34
  90. data/lib/prism/dispatcher.rb +646 -473
  91. data/lib/prism/dot_visitor.rb +592 -511
  92. data/lib/prism/dsl.rb +391 -25
  93. data/lib/prism/ffi.rb +258 -90
  94. data/lib/prism/inspect_visitor.rb +357 -325
  95. data/lib/prism/lex_compat.rb +254 -212
  96. data/lib/prism/mutation_compiler.rb +313 -305
  97. data/lib/prism/node.rb +20966 -6992
  98. data/lib/prism/node_ext.rb +180 -187
  99. data/lib/prism/node_find.rb +215 -0
  100. data/lib/prism/parse_result/errors.rb +7 -1
  101. data/lib/prism/parse_result/newlines.rb +276 -8
  102. data/lib/prism/parse_result.rb +468 -139
  103. data/lib/prism/pattern.rb +65 -20
  104. data/lib/prism/reflection.rb +23 -12
  105. data/lib/prism/relocation.rb +149 -87
  106. data/lib/prism/serialize.rb +3460 -539
  107. data/lib/prism/string_query.rb +24 -1
  108. data/lib/prism/translation/parser/builder.rb +10 -2
  109. data/lib/prism/translation/parser/compiler.rb +44 -52
  110. data/lib/prism/translation/parser/lexer.rb +41 -92
  111. data/lib/prism/translation/parser.rb +1 -1
  112. data/lib/prism/translation/parser_current.rb +1 -1
  113. data/lib/prism/translation/ripper/lexer.rb +8 -10
  114. data/lib/prism/translation/ripper/sexp.rb +2 -10
  115. data/lib/prism/translation/ripper/shim.rb +2 -0
  116. data/lib/prism/translation/ripper.rb +967 -209
  117. data/lib/prism/translation/ruby_parser.rb +73 -316
  118. data/lib/prism/translation.rb +2 -0
  119. data/lib/prism/visitor.rb +632 -5
  120. data/lib/prism.rb +79 -9
  121. data/prism.gemspec +128 -62
  122. data/rbi/generated/prism/compiler.rbi +489 -0
  123. data/rbi/generated/prism/desugar_compiler.rbi +306 -0
  124. data/rbi/generated/prism/dispatcher.rbi +1138 -0
  125. data/rbi/generated/prism/dot_visitor.rbi +618 -0
  126. data/rbi/generated/prism/dsl.rbi +751 -0
  127. data/rbi/generated/prism/inspect_visitor.rbi +503 -0
  128. data/rbi/generated/prism/lex_compat.rbi +162 -0
  129. data/rbi/generated/prism/mutation_compiler.rbi +464 -0
  130. data/rbi/generated/prism/node.rbi +14407 -0
  131. data/rbi/generated/prism/node_ext.rbi +259 -0
  132. data/rbi/generated/prism/node_find.rbi +79 -0
  133. data/rbi/generated/prism/parse_result/errors.rbi +21 -0
  134. data/rbi/generated/prism/parse_result/newlines.rbi +208 -0
  135. data/rbi/generated/prism/parse_result.rbi +776 -0
  136. data/rbi/generated/prism/pattern.rbi +134 -0
  137. data/rbi/generated/prism/reflection.rbi +89 -0
  138. data/rbi/generated/prism/relocation.rbi +329 -0
  139. data/rbi/generated/prism/serialize.rbi +184 -0
  140. data/rbi/generated/prism/string_query.rbi +43 -0
  141. data/rbi/generated/prism/translation.rbi +7 -0
  142. data/rbi/generated/prism/visitor.rbi +650 -0
  143. data/rbi/generated/prism.rbi +86 -0
  144. data/rbi/prism/translation/parser_versions.rbi +5 -0
  145. data/rbi/rubyvm/node_find.rbi +14 -0
  146. data/sig/generated/prism/compiler.rbs +492 -0
  147. data/sig/generated/prism/desugar_compiler.rbs +305 -0
  148. data/sig/generated/prism/dispatcher.rbs +1292 -0
  149. data/sig/generated/prism/dot_visitor.rbs +635 -0
  150. data/sig/generated/prism/dsl.rbs +928 -0
  151. data/sig/generated/prism/inspect_visitor.rbs +509 -0
  152. data/sig/generated/prism/lex_compat.rbs +166 -0
  153. data/sig/{prism → generated/prism}/mutation_compiler.rbs +309 -4
  154. data/sig/generated/prism/node.rbs +20896 -0
  155. data/sig/generated/prism/node_ext.rbs +290 -0
  156. data/sig/generated/prism/node_find.rbs +90 -0
  157. data/sig/generated/prism/parse_result/errors.rbs +22 -0
  158. data/sig/generated/prism/parse_result/newlines.rbs +223 -0
  159. data/sig/generated/prism/parse_result.rbs +871 -0
  160. data/sig/generated/prism/pattern.rbs +156 -0
  161. data/sig/generated/prism/reflection.rbs +101 -0
  162. data/sig/generated/prism/relocation.rbs +428 -0
  163. data/sig/generated/prism/serialize.rbs +198 -0
  164. data/sig/generated/prism/string_query.rbs +43 -0
  165. data/sig/generated/prism/translation.rbs +8 -0
  166. data/sig/generated/prism/visitor.rbs +1111 -0
  167. data/sig/generated/prism.rbs +84 -0
  168. data/sig/manifest.yaml +2 -0
  169. data/src/arena.c +117 -0
  170. data/src/{util/pm_buffer.c → buffer.c} +53 -21
  171. data/src/{util/pm_char.c → char.c} +28 -57
  172. data/src/comments.c +454 -0
  173. data/src/constant_pool.c +406 -0
  174. data/src/diagnostic.c +97 -65
  175. data/src/encoding.c +8 -3
  176. data/src/errors_format.c +519 -0
  177. data/src/{util/pm_integer.c → integer.c} +33 -22
  178. data/src/json.c +5733 -0
  179. data/src/line_offset_list.c +100 -0
  180. data/src/list.c +24 -0
  181. data/src/{util/pm_memchr.c → memchr.c} +8 -6
  182. data/src/node.c +3504 -7674
  183. data/src/options.c +162 -49
  184. data/src/parser.c +367 -0
  185. data/src/prettyprint.c +421 -354
  186. data/src/prism.c +8402 -7161
  187. data/src/regexp.c +961 -45
  188. data/src/serialize.c +460 -424
  189. data/src/source.c +518 -0
  190. data/src/static_literals.c +111 -43
  191. data/src/string_query.c +180 -0
  192. data/src/stringy.c +91 -0
  193. data/src/{util/pm_strncasecmp.c → strncasecmp.c} +5 -4
  194. data/src/strpbrk.c +447 -0
  195. data/src/{token_type.c → tokens.c} +39 -18
  196. data/src/xxh3.c +292 -0
  197. metadata +128 -63
  198. data/ext/prism/api_pack.c +0 -276
  199. data/include/prism/defines.h +0 -260
  200. data/include/prism/pack.h +0 -163
  201. data/include/prism/regexp.h +0 -43
  202. data/include/prism/util/pm_buffer.h +0 -236
  203. data/include/prism/util/pm_char.h +0 -204
  204. data/include/prism/util/pm_constant_pool.h +0 -218
  205. data/include/prism/util/pm_integer.h +0 -130
  206. data/include/prism/util/pm_memchr.h +0 -29
  207. data/include/prism/util/pm_newline_list.h +0 -113
  208. data/include/prism/util/pm_string.h +0 -200
  209. data/lib/prism/pack.rb +0 -230
  210. data/lib/prism/parse_result/comments.rb +0 -188
  211. data/rbi/prism/compiler.rbi +0 -12
  212. data/rbi/prism/dsl.rbi +0 -524
  213. data/rbi/prism/inspect_visitor.rbi +0 -12
  214. data/rbi/prism/node.rbi +0 -8750
  215. data/rbi/prism/node_ext.rbi +0 -107
  216. data/rbi/prism/parse_result.rbi +0 -404
  217. data/rbi/prism/reflection.rbi +0 -58
  218. data/rbi/prism/string_query.rbi +0 -12
  219. data/rbi/prism/visitor.rbi +0 -473
  220. data/rbi/prism.rbi +0 -63
  221. data/sig/prism/compiler.rbs +0 -9
  222. data/sig/prism/dispatcher.rbs +0 -19
  223. data/sig/prism/dot_visitor.rbs +0 -6
  224. data/sig/prism/dsl.rbs +0 -351
  225. data/sig/prism/inspect_visitor.rbs +0 -22
  226. data/sig/prism/lex_compat.rbs +0 -10
  227. data/sig/prism/node.rbs +0 -4042
  228. data/sig/prism/node_ext.rbs +0 -149
  229. data/sig/prism/pack.rbs +0 -43
  230. data/sig/prism/parse_result/comments.rbs +0 -38
  231. data/sig/prism/parse_result.rbs +0 -197
  232. data/sig/prism/pattern.rbs +0 -13
  233. data/sig/prism/reflection.rbs +0 -50
  234. data/sig/prism/relocation.rbs +0 -185
  235. data/sig/prism/serialize.rbs +0 -8
  236. data/sig/prism/string_query.rbs +0 -11
  237. data/sig/prism/visitor.rbs +0 -169
  238. data/sig/prism.rbs +0 -272
  239. data/src/pack.c +0 -509
  240. data/src/util/pm_constant_pool.c +0 -342
  241. data/src/util/pm_list.c +0 -49
  242. data/src/util/pm_newline_list.c +0 -125
  243. data/src/util/pm_string.c +0 -381
  244. data/src/util/pm_strpbrk.c +0 -206
data/src/regexp.c CHANGED
@@ -1,5 +1,20 @@
1
- #include "prism/regexp.h"
2
-
1
+ #include "prism/internal/regexp.h"
2
+
3
+ #include "prism/compiler/inline.h"
4
+ #include "prism/compiler/fallthrough.h"
5
+ #include "prism/internal/buffer.h"
6
+ #include "prism/internal/char.h"
7
+ #include "prism/internal/diagnostic.h"
8
+ #include "prism/internal/encoding.h"
9
+ #include "prism/internal/memchr.h"
10
+ #include "prism/internal/parser.h"
11
+ #include "prism/internal/stringy.h"
12
+ #include "prism/internal/strncasecmp.h"
13
+
14
+ #include <assert.h>
15
+ #include <string.h>
16
+
17
+ /** The maximum depth of nested groups allowed in a regular expression. */
3
18
  #define PM_REGEXP_PARSE_DEPTH_MAX 4096
4
19
 
5
20
  /**
@@ -18,6 +33,54 @@ typedef struct {
18
33
  /** A pointer to the end of the source that we are parsing. */
19
34
  const uint8_t *end;
20
35
 
36
+ /** The encoding of the source. */
37
+ const pm_encoding_t *encoding;
38
+
39
+ /** The callback to call when a named capture group is found. */
40
+ pm_regexp_name_callback_t name_callback;
41
+
42
+ /** The data to pass to the name callback. */
43
+ void *name_data;
44
+
45
+ /** The start of the regexp node (for error locations). */
46
+ const uint8_t *node_start;
47
+
48
+ /** The end of the regexp node (for error locations). */
49
+ const uint8_t *node_end;
50
+
51
+ /**
52
+ * The explicit encoding determined by escape sequences. NULL if no
53
+ * encoding-setting escape has been seen, UTF-8 for `\u` escapes, or the
54
+ * source encoding for `\x` escapes.
55
+ */
56
+ const pm_encoding_t *explicit_encoding;
57
+
58
+ /**
59
+ * Pointer to the first non-POSIX property name (for /n error messages).
60
+ * POSIX properties (Alnum, Alpha, etc.) work in all encodings.
61
+ * Script properties (Hiragana, Katakana, etc.) work in /e, /s, /u.
62
+ * Unicode-only properties (L, Ll, etc.) work only in /u.
63
+ */
64
+ const uint8_t *property_name;
65
+
66
+ /** Length of the first non-POSIX property name found. */
67
+ size_t property_name_length;
68
+
69
+ /**
70
+ * Pointer to the first Unicode-only property name (for /e, /s error
71
+ * messages). NULL if only POSIX or script properties have been seen.
72
+ */
73
+ const uint8_t *unicode_property_name;
74
+
75
+ /** Length of the first Unicode-only property name found. */
76
+ size_t unicode_property_name_length;
77
+
78
+ /** Buffer of hex escape byte values >= 0x80, separated by 0x00 sentinels. */
79
+ pm_buffer_t hex_escape_buffer;
80
+
81
+ /** Count of non-ASCII literal bytes (not from escapes). */
82
+ uint32_t non_ascii_literal_count;
83
+
21
84
  /**
22
85
  * Whether or not the regular expression currently being parsed is in
23
86
  * extended mode, wherein whitespace is ignored and comments are allowed.
@@ -27,30 +90,68 @@ typedef struct {
27
90
  /** Whether the encoding has changed from the default. */
28
91
  bool encoding_changed;
29
92
 
30
- /** The encoding of the source. */
31
- const pm_encoding_t *encoding;
93
+ /**
94
+ * Whether syntax errors are reported. The named capture pass walks content
95
+ * that may be an owned copy of the source (so pointers into it have no
96
+ * offset in the source), and whose errors the validation pass has already
97
+ * reported, so it reports nothing.
98
+ */
99
+ bool report_errors;
32
100
 
33
- /** The callback to call when a named capture group is found. */
34
- pm_regexp_name_callback_t name_callback;
101
+ /** Whether a `\u{...}` escape with value >= 0x80 was seen. */
102
+ bool has_unicode_escape;
35
103
 
36
- /** The data to pass to the name callback. */
37
- void *name_data;
104
+ /** Whether a `\xNN` escape (or `\M-x`, etc.) with value >= 0x80 was seen. */
105
+ bool has_hex_escape;
106
+
107
+ /**
108
+ * Tracks whether the last encoding-setting escape was `\u` (true) or `\x`
109
+ * (false). This matters for error messages when both types are mixed.
110
+ */
111
+ bool last_escape_was_unicode;
112
+
113
+ /** Whether any `\p{...}` or `\P{...}` property escape was found. */
114
+ bool has_property_escape;
38
115
 
39
- /** The callback to call when a parse error is found. */
40
- pm_regexp_error_callback_t error_callback;
116
+ /** Whether a Unicode-only property escape was found (not POSIX or script). */
117
+ bool has_unicode_property_escape;
41
118
 
42
- /** The data to pass to the error callback. */
43
- void *error_data;
119
+ /** Whether a `\u` escape with invalid range (surrogate or > 0x10FFFF) was seen. */
120
+ bool invalid_unicode_range;
121
+
122
+ /** Whether we are accumulating consecutive hex escape bytes. */
123
+ bool hex_group_active;
124
+
125
+ /** Whether an invalid multibyte character was found during parsing. */
126
+ bool has_invalid_multibyte;
44
127
  } pm_regexp_parser_t;
45
128
 
46
129
  /**
47
- * Append an error to the parser.
130
+ * Append a syntax error to the parser's error list. Errors are only reported
131
+ * when report_errors is set, which is the case for the validation pass that
132
+ * walks the raw source, so start and end always point into the original source.
48
133
  */
49
- static inline void
50
- pm_regexp_parse_error(pm_regexp_parser_t *parser, const uint8_t *start, const uint8_t *end, const char *message) {
51
- parser->error_callback(start, end, message, parser->error_data);
134
+ static PRISM_INLINE void
135
+ pm_regexp_parse_error(pm_regexp_parser_t *regexp_parser, const uint8_t *start, const uint8_t *end, const char *message) {
136
+ if (regexp_parser->report_errors) {
137
+ pm_parser_t *parser = regexp_parser->parser;
138
+ pm_diagnostic_list_append_format(&parser->metadata_arena, &parser->error_list, (uint32_t) (start - parser->start), (uint32_t) (end - start), PM_ERR_REGEXP_PARSE_ERROR, message);
139
+ }
52
140
  }
53
141
 
142
+ /**
143
+ * Append a formatted syntax error to the parser's error list, following the
144
+ * same rules as pm_regexp_parse_error. This is a macro because we need
145
+ * variadic args for the format string.
146
+ */
147
+ #define pm_regexp_parse_error_format(regexp_parser_, err_start_, err_end_, diag_id, ...) \
148
+ do { \
149
+ if ((regexp_parser_)->report_errors) { \
150
+ pm_parser_t *parser_ = (regexp_parser_)->parser; \
151
+ pm_diagnostic_list_append_format(&parser_->metadata_arena, &parser_->error_list, (uint32_t) ((err_start_) - parser_->start), (uint32_t) ((err_end_) - (err_start_)), diag_id, __VA_ARGS__); \
152
+ } \
153
+ } while (0)
154
+
54
155
  /**
55
156
  * This appends a new string to the list of named captures. This function
56
157
  * assumes the caller has already checked the validity of the name callback.
@@ -59,14 +160,14 @@ static void
59
160
  pm_regexp_parser_named_capture(pm_regexp_parser_t *parser, const uint8_t *start, const uint8_t *end) {
60
161
  pm_string_t string;
61
162
  pm_string_shared_init(&string, start, end);
62
- parser->name_callback(&string, parser->name_data);
63
- pm_string_free(&string);
163
+ parser->name_callback(parser->parser, &string, parser->name_data);
164
+ pm_string_cleanup(&string);
64
165
  }
65
166
 
66
167
  /**
67
168
  * Returns true if the next character is the end of the source.
68
169
  */
69
- static inline bool
170
+ static PRISM_INLINE bool
70
171
  pm_regexp_char_is_eof(pm_regexp_parser_t *parser) {
71
172
  return parser->cursor >= parser->end;
72
173
  }
@@ -74,7 +175,7 @@ pm_regexp_char_is_eof(pm_regexp_parser_t *parser) {
74
175
  /**
75
176
  * Optionally accept a char and consume it if it exists.
76
177
  */
77
- static inline bool
178
+ static PRISM_INLINE bool
78
179
  pm_regexp_char_accept(pm_regexp_parser_t *parser, uint8_t value) {
79
180
  if (!pm_regexp_char_is_eof(parser) && *parser->cursor == value) {
80
181
  parser->cursor++;
@@ -86,7 +187,7 @@ pm_regexp_char_accept(pm_regexp_parser_t *parser, uint8_t value) {
86
187
  /**
87
188
  * Expect a character to be present and consume it.
88
189
  */
89
- static inline bool
190
+ static PRISM_INLINE bool
90
191
  pm_regexp_char_expect(pm_regexp_parser_t *parser, uint8_t value) {
91
192
  if (!pm_regexp_char_is_eof(parser) && *parser->cursor == value) {
92
193
  parser->cursor++;
@@ -113,6 +214,47 @@ pm_regexp_char_find(pm_regexp_parser_t *parser, uint8_t value) {
113
214
  return true;
114
215
  }
115
216
 
217
+ /**
218
+ * Mark a group boundary in the hex escape byte buffer. When consecutive hex
219
+ * escape bytes >= 0x80 are followed by a non-hex-escape, this appends a 0x00
220
+ * sentinel to separate the groups for later multibyte validation.
221
+ */
222
+ static PRISM_INLINE void
223
+ pm_regexp_hex_group_boundary(pm_regexp_parser_t *parser) {
224
+ if (parser->hex_group_active) {
225
+ pm_buffer_append_byte(&parser->hex_escape_buffer, 0x00);
226
+ parser->hex_group_active = false;
227
+ }
228
+ }
229
+
230
+ /**
231
+ * Track a hex escape byte value >= 0x80 for multibyte validation.
232
+ */
233
+ static PRISM_INLINE void
234
+ pm_regexp_track_hex_escape(pm_regexp_parser_t *parser, uint8_t byte) {
235
+ if (byte >= 0x80) {
236
+ pm_buffer_append_byte(&parser->hex_escape_buffer, byte);
237
+ parser->hex_group_active = true;
238
+ parser->has_hex_escape = true;
239
+
240
+ parser->explicit_encoding = parser->encoding;
241
+ parser->last_escape_was_unicode = false;
242
+ } else {
243
+ pm_regexp_hex_group_boundary(parser);
244
+ }
245
+ }
246
+
247
+ /**
248
+ * Parse a hex digit character and return its value, or -1 if not a hex digit.
249
+ */
250
+ static PRISM_INLINE int
251
+ pm_regexp_hex_digit_value(uint8_t byte) {
252
+ if (byte >= '0' && byte <= '9') return byte - '0';
253
+ if (byte >= 'a' && byte <= 'f') return byte - 'a' + 10;
254
+ if (byte >= 'A' && byte <= 'F') return byte - 'A' + 10;
255
+ return -1;
256
+ }
257
+
116
258
  /**
117
259
  * Range quantifiers are a special class of quantifiers that look like
118
260
  *
@@ -121,13 +263,12 @@ pm_regexp_char_find(pm_regexp_parser_t *parser, uint8_t value) {
121
263
  * * {digit,digit}
122
264
  * * {,digit}
123
265
  *
124
- * Unfortunately, if there are any spaces in between, then this just becomes a
125
- * regular character match expression and we have to backtrack. So when this
126
- * function first starts running, we'll create a "save" point and then attempt
127
- * to parse the quantifier. If it fails, we'll restore the save point and
128
- * return.
266
+ * If there are any spaces in between, then this just becomes a regular
267
+ * character match expression and we have to backtrack. So when this function
268
+ * first starts running, we'll create a "save" point and then attempt to parse
269
+ * the quantifier. If it fails, we'll restore the save point and return.
129
270
  *
130
- * The properly track everything, we're going to build a little state machine.
271
+ * To properly track everything, we're going to build a little state machine.
131
272
  * It looks something like the following:
132
273
  *
133
274
  * +-------+ +---------+ ------------+
@@ -275,10 +416,392 @@ pm_regexp_parse_posix_class(pm_regexp_parser_t *parser) {
275
416
  );
276
417
  }
277
418
 
419
+ /**
420
+ * Property escape classification. Onigmo supports three tiers of property
421
+ * names depending on the encoding:
422
+ *
423
+ * - POSIX properties (Alnum, Alpha, ASCII, Blank, Cntrl, Digit, Graph, Lower,
424
+ * Print, Punct, Space, Upper, XDigit, Word): valid in all encodings.
425
+ * - Script properties (Hiragana, Katakana, Han, Latin, Greek, Cyrillic): valid
426
+ * in EUC-JP (/e), Windows-31J (/s), and UTF-8 (/u), but not ASCII-8BIT (/n).
427
+ * - Unicode-only properties (general categories like L, Ll, Lu, etc., plus
428
+ * Any, Assigned): valid only in UTF-8 (/u).
429
+ */
430
+ typedef enum {
431
+ PM_REGEXP_PROPERTY_POSIX,
432
+ PM_REGEXP_PROPERTY_SCRIPT,
433
+ PM_REGEXP_PROPERTY_UNICODE
434
+ } pm_regexp_property_type_t;
435
+
436
+ /**
437
+ * Classify a property name. The name may start with '^' for negation, which
438
+ * is skipped before matching.
439
+ */
440
+ static pm_regexp_property_type_t
441
+ pm_regexp_classify_property(const uint8_t *name, size_t length) {
442
+ // Skip leading '^' for negated properties like \p{^Hiragana}.
443
+ if (length > 0 && name[0] == '^') {
444
+ name++;
445
+ length--;
446
+ }
447
+
448
+ #define PM_REGEXP_CASECMP(str_) (pm_strncasecmp(name, (const uint8_t *) (str_), length) == 0)
449
+
450
+ switch (length) {
451
+ case 3:
452
+ if (PM_REGEXP_CASECMP("Han")) return PM_REGEXP_PROPERTY_SCRIPT;
453
+ break;
454
+ case 4:
455
+ if (PM_REGEXP_CASECMP("Word")) return PM_REGEXP_PROPERTY_POSIX;
456
+ break;
457
+ case 5:
458
+ /* Most properties are length 5, so dispatch on first character. */
459
+ switch (name[0] | 0x20) {
460
+ case 'a':
461
+ if (PM_REGEXP_CASECMP("Alnum")) return PM_REGEXP_PROPERTY_POSIX;
462
+ if (PM_REGEXP_CASECMP("Alpha")) return PM_REGEXP_PROPERTY_POSIX;
463
+ if (PM_REGEXP_CASECMP("ASCII")) return PM_REGEXP_PROPERTY_POSIX;
464
+ break;
465
+ case 'b':
466
+ if (PM_REGEXP_CASECMP("Blank")) return PM_REGEXP_PROPERTY_POSIX;
467
+ break;
468
+ case 'c':
469
+ if (PM_REGEXP_CASECMP("Cntrl")) return PM_REGEXP_PROPERTY_POSIX;
470
+ break;
471
+ case 'd':
472
+ if (PM_REGEXP_CASECMP("Digit")) return PM_REGEXP_PROPERTY_POSIX;
473
+ break;
474
+ case 'g':
475
+ if (PM_REGEXP_CASECMP("Graph")) return PM_REGEXP_PROPERTY_POSIX;
476
+ if (PM_REGEXP_CASECMP("Greek")) return PM_REGEXP_PROPERTY_SCRIPT;
477
+ break;
478
+ case 'l':
479
+ if (PM_REGEXP_CASECMP("Lower")) return PM_REGEXP_PROPERTY_POSIX;
480
+ if (PM_REGEXP_CASECMP("Latin")) return PM_REGEXP_PROPERTY_SCRIPT;
481
+ break;
482
+ case 'p':
483
+ if (PM_REGEXP_CASECMP("Print")) return PM_REGEXP_PROPERTY_POSIX;
484
+ if (PM_REGEXP_CASECMP("Punct")) return PM_REGEXP_PROPERTY_POSIX;
485
+ break;
486
+ case 's':
487
+ if (PM_REGEXP_CASECMP("Space")) return PM_REGEXP_PROPERTY_POSIX;
488
+ break;
489
+ case 'u':
490
+ if (PM_REGEXP_CASECMP("Upper")) return PM_REGEXP_PROPERTY_POSIX;
491
+ break;
492
+ }
493
+ break;
494
+ case 6:
495
+ if (PM_REGEXP_CASECMP("XDigit")) return PM_REGEXP_PROPERTY_POSIX;
496
+ break;
497
+ case 8:
498
+ if (PM_REGEXP_CASECMP("Hiragana")) return PM_REGEXP_PROPERTY_SCRIPT;
499
+ if (PM_REGEXP_CASECMP("Katakana")) return PM_REGEXP_PROPERTY_SCRIPT;
500
+ if (PM_REGEXP_CASECMP("Cyrillic")) return PM_REGEXP_PROPERTY_SCRIPT;
501
+ break;
502
+ }
503
+
504
+ #undef PM_REGEXP_CASECMP
505
+
506
+ // Everything else is Unicode-only (general categories, other scripts, etc.).
507
+ return PM_REGEXP_PROPERTY_UNICODE;
508
+ }
509
+
510
+ /**
511
+ * Check for and skip a `\p{...}` or `\P{...}` Unicode property escape. The
512
+ * cursor should be pointing at 'p' or 'P' when this is called. If a property
513
+ * escape is found, record it on the regexp parser and advance past the closing
514
+ * '}'.
515
+ *
516
+ * Properties are classified into three tiers (POSIX, script, Unicode-only) to
517
+ * determine which encoding modifiers they are valid with.
518
+ */
519
+ static bool
520
+ pm_regexp_parse_property_escape(pm_regexp_parser_t *parser) {
521
+ assert(*parser->cursor == 'p' || *parser->cursor == 'P');
522
+
523
+ if (parser->cursor + 1 < parser->end && parser->cursor[1] == '{') {
524
+ const uint8_t *name_start = parser->cursor + 2;
525
+ const uint8_t *search = name_start;
526
+
527
+ while (search < parser->end && *search != '}') search++;
528
+
529
+ if (search < parser->end) {
530
+ size_t name_length = (size_t) (search - name_start);
531
+ parser->has_property_escape = true;
532
+
533
+ pm_regexp_property_type_t type = pm_regexp_classify_property(name_start, name_length);
534
+
535
+ // Track the first non-POSIX property name (for /n error messages).
536
+ if (type >= PM_REGEXP_PROPERTY_SCRIPT && parser->property_name == NULL) {
537
+ parser->property_name = name_start;
538
+ parser->property_name_length = name_length;
539
+ }
540
+
541
+ // Track the first Unicode-only property name (for /e, /s error messages).
542
+ if (type == PM_REGEXP_PROPERTY_UNICODE) {
543
+ parser->has_unicode_property_escape = true;
544
+ if (parser->unicode_property_name == NULL) {
545
+ parser->unicode_property_name = name_start;
546
+ parser->unicode_property_name_length = name_length;
547
+ }
548
+ }
549
+
550
+ parser->cursor = search + 1; // skip past '}'
551
+ return true;
552
+ }
553
+ }
554
+
555
+ // Not a property escape, just skip the single character after '\'.
556
+ parser->cursor++;
557
+ return false;
558
+ }
559
+
560
+ /**
561
+ * Validate and skip a \u escape sequence in a regular expression. The cursor
562
+ * should be pointing at the character after 'u' when this is called. This
563
+ * handles both the \u{NNNN MMMM} and \uNNNN forms. Also tracks encoding
564
+ * state for validation.
565
+ */
566
+ static void
567
+ pm_regexp_parse_unicode_escape(pm_regexp_parser_t *parser) {
568
+ const uint8_t *escape_start = parser->cursor - 2; // points to '\'
569
+
570
+ if (pm_regexp_char_is_eof(parser)) {
571
+ pm_regexp_parse_error(parser, escape_start, parser->cursor, "invalid Unicode escape");
572
+ return;
573
+ }
574
+
575
+ if (*parser->cursor == '{') {
576
+ parser->cursor++; // skip '{'
577
+
578
+ // Skip leading whitespace.
579
+ while (!pm_regexp_char_is_eof(parser) && pm_char_is_whitespace(*parser->cursor)) {
580
+ parser->cursor++;
581
+ }
582
+
583
+ bool has_codepoint = false;
584
+
585
+ while (!pm_regexp_char_is_eof(parser) && *parser->cursor != '}') {
586
+ // Parse the hex digits to compute the codepoint value.
587
+ uint32_t value = 0;
588
+ size_t hex_count = 0;
589
+
590
+ int digit;
591
+ while (!pm_regexp_char_is_eof(parser) && (digit = pm_regexp_hex_digit_value(*parser->cursor)) >= 0) {
592
+ value = (value << 4) | (uint32_t) digit;
593
+ hex_count++;
594
+ parser->cursor++;
595
+ }
596
+
597
+ if (hex_count == 0) {
598
+ // Skip to '}' or end of regexp to find the full extent.
599
+ while (!pm_regexp_char_is_eof(parser) && *parser->cursor != '}') {
600
+ parser->cursor++;
601
+ }
602
+
603
+ const uint8_t *escape_end = parser->cursor;
604
+ if (!pm_regexp_char_is_eof(parser)) {
605
+ escape_end++;
606
+ parser->cursor++; // skip '}'
607
+ }
608
+
609
+ pm_regexp_parse_error_format(parser, escape_start, escape_end, PM_ERR_ESCAPE_INVALID_UNICODE_LIST, (int) (escape_end - escape_start), (const char *) escape_start);
610
+ return;
611
+ }
612
+
613
+ if (hex_count > 6) {
614
+ pm_regexp_parse_error(parser, escape_start, parser->cursor, "invalid Unicode range");
615
+ }
616
+
617
+ // Track encoding state for this codepoint.
618
+ if (value >= 0x80) {
619
+ parser->has_unicode_escape = true;
620
+ parser->explicit_encoding = PM_ENCODING_UTF_8_ENTRY;
621
+ parser->last_escape_was_unicode = true;
622
+ pm_regexp_hex_group_boundary(parser);
623
+ }
624
+
625
+ // Check for invalid Unicode range (surrogates or > 0x10FFFF).
626
+ if (value > 0x10FFFF || (value >= 0xD800 && value <= 0xDFFF)) {
627
+ parser->invalid_unicode_range = true;
628
+ }
629
+
630
+ has_codepoint = true;
631
+
632
+ // Skip whitespace between codepoints.
633
+ while (!pm_regexp_char_is_eof(parser) && pm_char_is_whitespace(*parser->cursor)) {
634
+ parser->cursor++;
635
+ }
636
+ }
637
+
638
+ if (pm_regexp_char_is_eof(parser)) {
639
+ pm_regexp_parse_error(parser, escape_start, parser->cursor, "unterminated Unicode escape");
640
+ } else {
641
+ if (!has_codepoint) {
642
+ pm_regexp_parse_error_format(parser, escape_start, parser->cursor + 1, PM_ERR_ESCAPE_INVALID_UNICODE_LIST, (int) (parser->cursor + 1 - escape_start), (const char *) escape_start);
643
+ }
644
+ parser->cursor++; // skip '}'
645
+ }
646
+ } else {
647
+ // \uNNNN form — need exactly 4 hex digits.
648
+ uint32_t value = 0;
649
+ size_t hex_count = 0;
650
+
651
+ int digit;
652
+ while (hex_count < 4 && !pm_regexp_char_is_eof(parser) && (digit = pm_regexp_hex_digit_value(*parser->cursor)) >= 0) {
653
+ value = (value << 4) | (uint32_t) digit;
654
+ hex_count++;
655
+ parser->cursor++;
656
+ }
657
+
658
+ if (hex_count < 4) {
659
+ pm_regexp_parse_error(parser, escape_start, parser->cursor, "invalid Unicode escape");
660
+ } else if (value >= 0x80) {
661
+ parser->has_unicode_escape = true;
662
+ parser->explicit_encoding = PM_ENCODING_UTF_8_ENTRY;
663
+ parser->last_escape_was_unicode = true;
664
+ pm_regexp_hex_group_boundary(parser);
665
+ }
666
+
667
+ // Check for invalid Unicode range.
668
+ if (hex_count == 4 && (value > 0x10FFFF || (value >= 0xD800 && value <= 0xDFFF))) {
669
+ parser->invalid_unicode_range = true;
670
+ }
671
+ }
672
+ }
673
+
278
674
  // Forward declaration because character sets can be nested.
279
675
  static bool
280
676
  pm_regexp_parse_lbracket(pm_regexp_parser_t *parser, uint16_t depth);
281
677
 
678
+ /**
679
+ * Parse a \x escape and return the byte value. The cursor should be pointing
680
+ * at the character after 'x'. Returns -1 if no hex digits follow.
681
+ */
682
+ static int
683
+ pm_regexp_parse_hex_escape(pm_regexp_parser_t *parser) {
684
+ int value = -1;
685
+
686
+ if (!pm_regexp_char_is_eof(parser)) {
687
+ int digit = pm_regexp_hex_digit_value(*parser->cursor);
688
+ if (digit >= 0) {
689
+ value = digit;
690
+ parser->cursor++;
691
+
692
+ if (!pm_regexp_char_is_eof(parser)) {
693
+ digit = pm_regexp_hex_digit_value(*parser->cursor);
694
+ if (digit >= 0) {
695
+ value = (value << 4) | digit;
696
+ parser->cursor++;
697
+ }
698
+ }
699
+ }
700
+ }
701
+
702
+ if (value >= 0) {
703
+ pm_regexp_track_hex_escape(parser, (uint8_t) value);
704
+ }
705
+
706
+ return value;
707
+ }
708
+
709
+ /**
710
+ * Parse a backslash escape sequence in a regexp, handling \u (unicode),
711
+ * \p/\P (property), \x (hex), and other single-character escapes. Also
712
+ * tracks encoding state for \M-x and \C-\M-x escapes.
713
+ */
714
+ static void
715
+ pm_regexp_parse_backslash_escape(pm_regexp_parser_t *parser) {
716
+ if (pm_regexp_char_is_eof(parser)) return;
717
+
718
+ switch (*parser->cursor) {
719
+ case 'u':
720
+ parser->cursor++; // skip 'u'
721
+ pm_regexp_parse_unicode_escape(parser);
722
+ break;
723
+ case 'p':
724
+ case 'P':
725
+ pm_regexp_parse_property_escape(parser);
726
+ break;
727
+ case 'x':
728
+ parser->cursor++; // skip 'x'
729
+ pm_regexp_parse_hex_escape(parser);
730
+ break;
731
+ case 'M':
732
+ // \M-x produces (x | 0x80), always >= 0x80
733
+ if (parser->cursor + 2 < parser->end && parser->cursor[1] == '-') {
734
+ parser->cursor += 2; // skip 'M-'
735
+ if (!pm_regexp_char_is_eof(parser)) {
736
+ if (*parser->cursor == '\\') {
737
+ parser->cursor++;
738
+ // \M-\C-x or \M-\cx — the resulting byte is always >= 0x80
739
+ // We just need to track it as a hex escape >= 0x80.
740
+ pm_regexp_parse_backslash_escape(parser);
741
+ } else {
742
+ parser->cursor++;
743
+ }
744
+ // \M-x always produces a byte >= 0x80
745
+ pm_regexp_track_hex_escape(parser, 0x80);
746
+ }
747
+ } else {
748
+ parser->cursor++;
749
+ }
750
+ break;
751
+ case 'C':
752
+ // \C-x produces (x & 0x1F)
753
+ if (parser->cursor + 2 < parser->end && parser->cursor[1] == '-') {
754
+ parser->cursor += 2; // skip 'C-'
755
+ if (!pm_regexp_char_is_eof(parser)) {
756
+ if (*parser->cursor == '\\') {
757
+ parser->cursor++;
758
+ pm_regexp_parse_backslash_escape(parser);
759
+ } else {
760
+ parser->cursor++;
761
+ }
762
+ }
763
+ } else {
764
+ parser->cursor++;
765
+ }
766
+ break;
767
+ case 'c':
768
+ // \cx produces (x & 0x1F)
769
+ parser->cursor++; // skip 'c'
770
+ if (!pm_regexp_char_is_eof(parser)) {
771
+ if (*parser->cursor == '\\') {
772
+ parser->cursor++;
773
+ pm_regexp_parse_backslash_escape(parser);
774
+ } else {
775
+ parser->cursor++;
776
+ }
777
+ }
778
+ break;
779
+ default:
780
+ pm_regexp_hex_group_boundary(parser);
781
+ parser->cursor++;
782
+ break;
783
+ }
784
+ }
785
+
786
+ /**
787
+ * Check if a byte at the current position is a non-ASCII byte in a multibyte
788
+ * encoding that produces an invalid character. If so, emit an error at the
789
+ * byte location immediately.
790
+ */
791
+ static void
792
+ pm_regexp_parse_invalid_multibyte(pm_regexp_parser_t *parser, const uint8_t *cursor) {
793
+ uint8_t byte = *cursor;
794
+ if (byte >= 0x80 && parser->encoding_changed && parser->encoding->multibyte) {
795
+ size_t width = parser->encoding->char_width(cursor, (ptrdiff_t) (parser->end - cursor));
796
+ if (width > 1) {
797
+ parser->cursor += width - 1;
798
+ } else if (width == 0) {
799
+ parser->has_invalid_multibyte = true;
800
+ pm_regexp_parse_error_format(parser, cursor, cursor + 1, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
801
+ }
802
+ }
803
+ }
804
+
282
805
  /**
283
806
  * match-char-set : '[' '^'? (match-range | match-char)* ']'
284
807
  * ;
@@ -293,12 +816,16 @@ pm_regexp_parse_character_set(pm_regexp_parser_t *parser, uint16_t depth) {
293
816
  pm_regexp_parse_lbracket(parser, (uint16_t) (depth + 1));
294
817
  break;
295
818
  case '\\':
296
- if (!pm_regexp_char_is_eof(parser)) {
297
- parser->cursor++;
298
- }
819
+ pm_regexp_parse_backslash_escape(parser);
299
820
  break;
300
821
  default:
301
- // do nothing, we've already advanced the cursor
822
+ // We've already advanced the cursor by one byte. If the byte
823
+ // was >= 0x80 in a multibyte encoding, we may need to consume
824
+ // additional continuation bytes and validate the character.
825
+ if (*(parser->cursor - 1) >= 0x80) {
826
+ parser->non_ascii_literal_count++;
827
+ }
828
+ pm_regexp_parse_invalid_multibyte(parser, parser->cursor - 1);
302
829
  break;
303
830
  }
304
831
  }
@@ -354,8 +881,13 @@ typedef enum {
354
881
  // These are the options that are configurable on the regular expression (or
355
882
  // from within a group).
356
883
 
884
+ /** The minimum character value for a regexp option slot. */
357
885
  #define PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM 'a'
886
+
887
+ /** The maximum character value for a regexp option slot. */
358
888
  #define PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM 'x'
889
+
890
+ /** The number of regexp option slots. */
359
891
  #define PRISM_REGEXP_OPTION_STATE_SLOTS (PRISM_REGEXP_OPTION_STATE_SLOT_MAXIMUM - PRISM_REGEXP_OPTION_STATE_SLOT_MINIMUM + 1)
360
892
 
361
893
  /**
@@ -498,7 +1030,15 @@ pm_regexp_parse_group(pm_regexp_parser_t *parser, uint16_t depth) {
498
1030
  }
499
1031
 
500
1032
  size_t width = parser->encoding->char_width(parser->cursor, (ptrdiff_t) (parser->end - parser->cursor));
501
- if (width == 0) return false;
1033
+ if (width == 0) {
1034
+ if (*parser->cursor >= 0x80) {
1035
+ parser->has_invalid_multibyte = true;
1036
+ pm_regexp_parse_error_format(parser, parser->cursor, parser->cursor + 1, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
1037
+ parser->cursor++;
1038
+ continue;
1039
+ }
1040
+ return false;
1041
+ }
502
1042
 
503
1043
  escaped = (width == 1) && (*parser->cursor == '\\');
504
1044
  parser->cursor += width;
@@ -686,9 +1226,7 @@ pm_regexp_parse_item(pm_regexp_parser_t *parser, uint16_t depth) {
686
1226
  return pm_regexp_parse_quantifier(parser);
687
1227
  case '\\':
688
1228
  parser->cursor++;
689
- if (!pm_regexp_char_is_eof(parser)) {
690
- parser->cursor++;
691
- }
1229
+ pm_regexp_parse_backslash_escape(parser);
692
1230
  return pm_regexp_parse_quantifier(parser);
693
1231
  case '(':
694
1232
  parser->cursor++;
@@ -720,9 +1258,30 @@ pm_regexp_parse_item(pm_regexp_parser_t *parser, uint16_t depth) {
720
1258
  width = parser->encoding->char_width(parser->cursor, (ptrdiff_t) (parser->end - parser->cursor));
721
1259
  }
722
1260
 
723
- if (width == 0) return false; // TODO: add appropriate error
724
- parser->cursor += width;
1261
+ if (width == 0) {
1262
+ if (*parser->cursor >= 0x80 && parser->encoding_changed) {
1263
+ if (parser->encoding->multibyte) {
1264
+ // Invalid multibyte character in a multibyte encoding.
1265
+ // Emit the error at the byte location immediately.
1266
+ parser->has_invalid_multibyte = true;
1267
+ pm_regexp_parse_error_format(parser, parser->cursor, parser->cursor + 1, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
1268
+ } else {
1269
+ // Non-ASCII byte in a single-byte encoding (e.g.,
1270
+ // US-ASCII). Count it for later error reporting.
1271
+ parser->non_ascii_literal_count++;
1272
+ }
1273
+ parser->cursor++;
1274
+ return pm_regexp_parse_quantifier(parser);
1275
+ }
1276
+ return false;
1277
+ }
725
1278
 
1279
+ // Count non-ASCII literal bytes.
1280
+ for (size_t i = 0; i < width; i++) {
1281
+ if (parser->cursor[i] >= 0x80) parser->non_ascii_literal_count++;
1282
+ }
1283
+
1284
+ parser->cursor += width;
726
1285
  return pm_regexp_parse_quantifier(parser);
727
1286
  }
728
1287
  }
@@ -768,13 +1327,353 @@ pm_regexp_parse_pattern(pm_regexp_parser_t *parser) {
768
1327
  return pm_regexp_char_is_eof(parser);
769
1328
  }
770
1329
 
1330
+ // ---------------------------------------------------------------------------
1331
+ // Encoding validation
1332
+ // ---------------------------------------------------------------------------
1333
+
771
1334
  /**
772
- * Parse a regular expression and extract the names of all of the named capture
773
- * groups.
1335
+ * Validate that groups of hex escape bytes in the buffer form valid multibyte
1336
+ * characters in the given encoding. Groups are separated by 0x00 sentinels.
1337
+ */
1338
+ static bool
1339
+ pm_regexp_validate_hex_escapes(const pm_encoding_t *encoding, const pm_buffer_t *buffer) {
1340
+ const uint8_t *data = (const uint8_t *) pm_buffer_value(buffer);
1341
+ size_t len = pm_buffer_length(buffer);
1342
+ size_t i = 0;
1343
+
1344
+ while (i < len) {
1345
+ size_t group_start = i;
1346
+ while (i < len && data[i] != 0x00) i++;
1347
+
1348
+ for (size_t j = group_start; j < i; ) {
1349
+ size_t width = encoding->char_width(data + j, (ptrdiff_t) (i - j));
1350
+ if (width == 0) return false;
1351
+ j += width;
1352
+ }
1353
+
1354
+ if (i < len) i++; // skip sentinel
1355
+ }
1356
+
1357
+ return true;
1358
+ }
1359
+
1360
+ /**
1361
+ * Format regexp source content for use in error messages, hex-escaping
1362
+ * non-ASCII bytes.
1363
+ */
1364
+ static void
1365
+ pm_regexp_format_for_error(pm_buffer_t *buffer, const pm_encoding_t *encoding, const uint8_t *source, size_t length) {
1366
+ size_t index = 0;
1367
+
1368
+ if (encoding == PM_ENCODING_UTF_8_ENTRY) {
1369
+ pm_buffer_append_string(buffer, (const char *) source, length);
1370
+ return;
1371
+ }
1372
+
1373
+ while (index < length) {
1374
+ if (source[index] < 0x80) {
1375
+ pm_buffer_append_byte(buffer, source[index]);
1376
+ index++;
1377
+ } else if (encoding->multibyte) {
1378
+ size_t width = encoding->char_width(source + index, (ptrdiff_t) (length - index));
1379
+
1380
+ if (width > 1) {
1381
+ pm_buffer_append_string(buffer, "\\x{", 3);
1382
+ for (size_t i = 0; i < width; i++) {
1383
+ pm_buffer_append_format(buffer, "%02X", source[index + i]);
1384
+ }
1385
+ pm_buffer_append_byte(buffer, '}');
1386
+ index += width;
1387
+ } else {
1388
+ pm_buffer_append_format(buffer, "\\x%02X", source[index]);
1389
+ index++;
1390
+ }
1391
+ } else {
1392
+ pm_buffer_append_format(buffer, "\\x%02X", source[index]);
1393
+ index++;
1394
+ }
1395
+ }
1396
+ }
1397
+
1398
+ /**
1399
+ * Emit an encoding validation error on the regexp node.
1400
+ */
1401
+ #define PM_REGEXP_ENCODING_ERROR(parser, diag_id, ...) \
1402
+ pm_diagnostic_list_append_format( \
1403
+ &(parser)->parser->metadata_arena, \
1404
+ &(parser)->parser->error_list, \
1405
+ (uint32_t) ((parser)->node_start - (parser)->parser->start), \
1406
+ (uint32_t) ((parser)->node_end - (parser)->node_start), \
1407
+ diag_id, __VA_ARGS__)
1408
+
1409
+ /**
1410
+ * Validate encoding for a regexp with an encoding modifier (/e, /s, /u, /n).
1411
+ *
1412
+ * The decision tree is:
1413
+ *
1414
+ * 1. No escape-set encoding (explicit_encoding == NULL):
1415
+ * a. ASCII-only content: validate property escapes, return forced US-ASCII
1416
+ * for /n or the modifier flags for others.
1417
+ * b. US-ASCII source with non-ASCII literals: emit per-byte errors.
1418
+ * c. Source encoding differs from modifier encoding: emit mismatch error.
1419
+ *
1420
+ * 2. Mixed \u and \x escapes: emit the appropriate conflict error depending
1421
+ * on the modifier and which escape type was last.
1422
+ *
1423
+ * 3. \u escape with non-/u modifier: incompatible encoding error.
1424
+ *
1425
+ * 4. Validate that hex escape byte sequences form valid multibyte characters
1426
+ * in the modifier's encoding.
1427
+ */
1428
+ static pm_node_flags_t
1429
+ pm_regexp_validate_encoding_modifier(pm_regexp_parser_t *parser, bool ascii_only, pm_node_flags_t flags, char modifier, const pm_encoding_t *modifier_encoding, const char *source_start, int source_length) {
1430
+
1431
+ if (parser->explicit_encoding == NULL) {
1432
+ if (ascii_only) {
1433
+ // Check property escapes against the modifier's encoding tier.
1434
+ // /n (ASCII-8BIT): only POSIX properties are valid.
1435
+ // /e, /s: POSIX and script properties are valid.
1436
+ // /u: all properties are valid.
1437
+ if (modifier == 'n' && parser->property_name != NULL) {
1438
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INVALID_CHAR_PROPERTY,
1439
+ (int) parser->property_name_length, (const char *) parser->property_name,
1440
+ source_length, source_start);
1441
+ } else if (modifier != 'u' && parser->has_unicode_property_escape) {
1442
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INVALID_CHAR_PROPERTY,
1443
+ (int) parser->unicode_property_name_length, (const char *) parser->unicode_property_name,
1444
+ source_length, source_start);
1445
+ }
1446
+ return modifier == 'n' ? PM_REGULAR_EXPRESSION_FLAGS_FORCED_US_ASCII_ENCODING : flags;
1447
+ }
1448
+
1449
+ if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) {
1450
+ for (uint32_t i = 0; i < parser->non_ascii_literal_count; i++) {
1451
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
1452
+ }
1453
+ } else if (parser->encoding != modifier_encoding) {
1454
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_ENCODING_OPTION_MISMATCH, modifier, parser->encoding->name);
1455
+
1456
+ if (modifier == 'n' && !ascii_only) {
1457
+ pm_buffer_t formatted = { 0 };
1458
+ pm_regexp_format_for_error(&formatted, parser->encoding, (const uint8_t *) source_start, (size_t) source_length);
1459
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_NON_ESCAPED_MBC, (int) formatted.length, (const char *) formatted.value);
1460
+ pm_buffer_cleanup(&formatted);
1461
+ }
1462
+ }
1463
+
1464
+ return flags;
1465
+ }
1466
+
1467
+ // Mixed unicode + hex escapes.
1468
+ if (parser->has_unicode_escape && parser->has_hex_escape) {
1469
+ if (modifier == 'n') {
1470
+ if (parser->last_escape_was_unicode) {
1471
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP, source_length, source_start);
1472
+ } else {
1473
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_ESCAPED_NON_ASCII_IN_UTF8, source_length, source_start);
1474
+ }
1475
+ } else {
1476
+ if (!pm_regexp_validate_hex_escapes(modifier_encoding, &parser->hex_escape_buffer)) {
1477
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1478
+ }
1479
+ }
1480
+
1481
+ return flags;
1482
+ }
1483
+
1484
+ if (modifier != 'u' && parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
1485
+ if (parser->last_escape_was_unicode) {
1486
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING, source_length, source_start);
1487
+ } else if (parser->encoding != PM_ENCODING_UTF_8_ENTRY) {
1488
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INCOMPAT_CHAR_ENCODING, source_length, source_start);
1489
+ }
1490
+ }
1491
+
1492
+ if (modifier != 'n' && !pm_regexp_validate_hex_escapes(modifier_encoding, &parser->hex_escape_buffer)) {
1493
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1494
+ }
1495
+
1496
+ return flags;
1497
+ }
1498
+
1499
+ /**
1500
+ * Validate encoding for a regexp without a modifier and compute the encoding
1501
+ * flags to set on the node.
1502
+ *
1503
+ * The decision tree is:
1504
+ *
1505
+ * 1. If a modifier (/n, /u, /e, /s) is present, delegate to
1506
+ * pm_regexp_validate_encoding_modifier.
1507
+ * 2. Invalid multibyte chars or unicode ranges: suppress further checks (errors
1508
+ * were already emitted during parsing).
1509
+ * 3. US-ASCII source with non-ASCII literals: emit per-byte errors.
1510
+ * 4. ASCII-only content: return forced US-ASCII (or forced UTF-8 if \p{...}).
1511
+ * 5. Escape-set encoding present: validate hex escapes against the target
1512
+ * encoding, handle mixed \u + \x conflicts, and return the appropriate
1513
+ * forced encoding flag.
1514
+ */
1515
+ static pm_node_flags_t
1516
+ pm_regexp_validate_encoding(pm_regexp_parser_t *parser, bool ascii_only, pm_node_flags_t flags, const char *source_start, int source_length) {
1517
+
1518
+ // Invalid multibyte characters suppress further validation.
1519
+ // Errors were already emitted at the byte locations during parsing.
1520
+ if (parser->has_invalid_multibyte) {
1521
+ return flags;
1522
+ }
1523
+
1524
+ if (parser->invalid_unicode_range) {
1525
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_INVALID_UNICODE_RANGE, source_length, source_start);
1526
+ return flags;
1527
+ }
1528
+
1529
+ // Check modifier flags first.
1530
+ if (flags & PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT) {
1531
+ return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags, 'n', PM_ENCODING_ASCII_8BIT_ENTRY, source_start, source_length);
1532
+ }
1533
+ if (flags & PM_REGULAR_EXPRESSION_FLAGS_UTF_8) {
1534
+ return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags, 'u', PM_ENCODING_UTF_8_ENTRY, source_start, source_length);
1535
+ }
1536
+ if (flags & PM_REGULAR_EXPRESSION_FLAGS_EUC_JP) {
1537
+ return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags, 'e', PM_ENCODING_EUC_JP_ENTRY, source_start, source_length);
1538
+ }
1539
+ if (flags & PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J) {
1540
+ return pm_regexp_validate_encoding_modifier(parser, ascii_only, flags, 's', PM_ENCODING_WINDOWS_31J_ENTRY, source_start, source_length);
1541
+ }
1542
+
1543
+ // No modifier — check for non-ASCII literals in US-ASCII encoding.
1544
+ if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY && parser->explicit_encoding == NULL && !ascii_only) {
1545
+ for (uint32_t i = 0; i < parser->non_ascii_literal_count; i++) {
1546
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_CHAR, parser->encoding->name);
1547
+ }
1548
+ }
1549
+
1550
+ // ASCII-only regexps get downgraded to US-ASCII, unless property escapes
1551
+ // force UTF-8.
1552
+ if (ascii_only) {
1553
+ if (parser->has_property_escape) {
1554
+ return PM_REGULAR_EXPRESSION_FLAGS_FORCED_UTF8_ENCODING;
1555
+ }
1556
+ return PM_REGULAR_EXPRESSION_FLAGS_FORCED_US_ASCII_ENCODING;
1557
+ }
1558
+
1559
+ // Check explicit encoding from escape sequences.
1560
+ if (parser->explicit_encoding != NULL) {
1561
+ // Mixed unicode + hex escapes without modifier.
1562
+ if (parser->has_unicode_escape && parser->has_hex_escape && parser->encoding != PM_ENCODING_UTF_8_ENTRY) {
1563
+ if (parser->encoding != PM_ENCODING_US_ASCII_ENTRY &&
1564
+ parser->encoding != PM_ENCODING_ASCII_8BIT_ENTRY &&
1565
+ !pm_regexp_validate_hex_escapes(parser->encoding, &parser->hex_escape_buffer)) {
1566
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1567
+ } else if (parser->last_escape_was_unicode) {
1568
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_UTF8_CHAR_NON_UTF8_REGEXP, source_length, source_start);
1569
+ } else {
1570
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_REGEXP_ESCAPED_NON_ASCII_IN_UTF8, source_length, source_start);
1571
+ }
1572
+
1573
+ return 0;
1574
+ }
1575
+
1576
+ if (parser->explicit_encoding == PM_ENCODING_UTF_8_ENTRY) {
1577
+ if (!pm_regexp_validate_hex_escapes(parser->explicit_encoding, &parser->hex_escape_buffer)) {
1578
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1579
+ }
1580
+
1581
+ return PM_REGULAR_EXPRESSION_FLAGS_FORCED_UTF8_ENCODING;
1582
+ } else if (parser->encoding == PM_ENCODING_US_ASCII_ENTRY) {
1583
+ return PM_REGULAR_EXPRESSION_FLAGS_FORCED_BINARY_ENCODING;
1584
+ } else {
1585
+ if (!pm_regexp_validate_hex_escapes(parser->explicit_encoding, &parser->hex_escape_buffer)) {
1586
+ PM_REGEXP_ENCODING_ERROR(parser, PM_ERR_INVALID_MULTIBYTE_ESCAPE, source_length, source_start);
1587
+ }
1588
+ }
1589
+ }
1590
+
1591
+ return 0;
1592
+ }
1593
+
1594
+ /**
1595
+ * Parse a regular expression, validate its encoding, and optionally extract
1596
+ * named capture groups. Encoding validation walks the raw source (content_loc)
1597
+ * to distinguish escape-produced bytes from literal bytes. Named capture
1598
+ * extraction walks the unescaped content since escape sequences in group names
1599
+ * (e.g., line continuations) have already been processed by the lexer.
1600
+ */
1601
+ pm_node_flags_t
1602
+ pm_regexp_parse(pm_parser_t *parser, pm_regular_expression_node_t *node, pm_regexp_name_callback_t name_callback, void *name_data) {
1603
+ const uint8_t *source = parser->start + node->content_loc.start;
1604
+ size_t size = node->content_loc.length;
1605
+ bool extended_mode = PM_NODE_FLAG_P(node, PM_REGULAR_EXPRESSION_FLAGS_EXTENDED);
1606
+ pm_node_flags_t flags = PM_NODE_FLAGS(node);
1607
+
1608
+ const uint8_t *node_start = parser->start + node->base.location.start;
1609
+ const uint8_t *node_end = parser->start + node->base.location.start + node->base.location.length;
1610
+
1611
+ // First pass: walk raw source for encoding validation (no name extraction).
1612
+ pm_regexp_parser_t regexp_parser = {
1613
+ .parser = parser,
1614
+ .start = source,
1615
+ .cursor = source,
1616
+ .end = source + size,
1617
+ .extended_mode = extended_mode,
1618
+ .encoding_changed = parser->encoding_changed,
1619
+ .encoding = parser->encoding,
1620
+ .name_callback = NULL,
1621
+ .name_data = NULL,
1622
+ .report_errors = true,
1623
+ .node_start = node_start,
1624
+ .node_end = node_end,
1625
+ .has_unicode_escape = false,
1626
+ .has_hex_escape = false,
1627
+ .last_escape_was_unicode = false,
1628
+ .explicit_encoding = NULL,
1629
+ .has_property_escape = false,
1630
+ .has_unicode_property_escape = false,
1631
+ .property_name = NULL,
1632
+ .property_name_length = 0,
1633
+ .unicode_property_name = NULL,
1634
+ .unicode_property_name_length = 0,
1635
+ .non_ascii_literal_count = 0,
1636
+ .invalid_unicode_range = false,
1637
+ .hex_escape_buffer = { 0 },
1638
+ .hex_group_active = false,
1639
+ .has_invalid_multibyte = false,
1640
+ };
1641
+
1642
+ pm_regexp_parse_pattern(&regexp_parser);
1643
+
1644
+ // Compute ascii_only from the regexp parser's tracked state. We cannot
1645
+ // use node->unescaped for this because regexp unescaped content preserves
1646
+ // escape text (e.g., \x80 is 4 ASCII chars), not the binary values.
1647
+ bool ascii_only = !regexp_parser.has_hex_escape && !regexp_parser.has_unicode_escape && regexp_parser.non_ascii_literal_count == 0;
1648
+ // Use the unescaped content for error messages to match CRuby's format,
1649
+ // where Ruby escapes like \M-\C-? are resolved to bytes but regexp escapes
1650
+ // like \u{80} are preserved as text.
1651
+ const char *error_source = (const char *) pm_string_source(&node->unescaped);
1652
+ int error_source_length = (int) pm_string_length(&node->unescaped);
1653
+ pm_node_flags_t encoding_flags = pm_regexp_validate_encoding(&regexp_parser, ascii_only, flags, error_source, error_source_length);
1654
+ pm_buffer_cleanup(&regexp_parser.hex_escape_buffer);
1655
+
1656
+ // Second pass: walk unescaped content for named capture extraction.
1657
+ if (name_callback != NULL) {
1658
+ pm_regexp_parse_named_captures(parser, pm_string_source(&node->unescaped), pm_string_length(&node->unescaped), extended_mode, name_callback, name_data);
1659
+ }
1660
+
1661
+ return encoding_flags;
1662
+ }
1663
+
1664
+ /**
1665
+ * Parse an interpolated regular expression for named capture groups only.
1666
+ * This is used for the =~ operator with interpolated regexps where we don't
1667
+ * have a pm_regular_expression_node_t. No encoding validation is performed.
1668
+ *
1669
+ * Note: The encoding-tracking fields (has_unicode_escape, has_hex_escape, etc.)
1670
+ * are initialized but not used for the result. They exist because the parsing
1671
+ * functions (pm_regexp_parse_backslash_escape, etc.) unconditionally update
1672
+ * them as they walk through the content.
774
1673
  */
775
- PRISM_EXPORTED_FUNCTION void
776
- pm_regexp_parse(pm_parser_t *parser, const uint8_t *source, size_t size, bool extended_mode, pm_regexp_name_callback_t name_callback, void *name_data, pm_regexp_error_callback_t error_callback, void *error_data) {
777
- pm_regexp_parse_pattern(&(pm_regexp_parser_t) {
1674
+ void
1675
+ pm_regexp_parse_named_captures(pm_parser_t *parser, const uint8_t *source, size_t size, bool extended_mode, pm_regexp_name_callback_t name_callback, void *name_data) {
1676
+ pm_regexp_parser_t regexp_parser = {
778
1677
  .parser = parser,
779
1678
  .start = source,
780
1679
  .cursor = source,
@@ -784,7 +1683,24 @@ pm_regexp_parse(pm_parser_t *parser, const uint8_t *source, size_t size, bool ex
784
1683
  .encoding = parser->encoding,
785
1684
  .name_callback = name_callback,
786
1685
  .name_data = name_data,
787
- .error_callback = error_callback,
788
- .error_data = error_data
789
- });
1686
+ .report_errors = false,
1687
+ .has_unicode_escape = false,
1688
+ .has_hex_escape = false,
1689
+ .last_escape_was_unicode = false,
1690
+ .explicit_encoding = NULL,
1691
+ .has_property_escape = false,
1692
+ .has_unicode_property_escape = false,
1693
+ .property_name = NULL,
1694
+ .property_name_length = 0,
1695
+ .unicode_property_name = NULL,
1696
+ .unicode_property_name_length = 0,
1697
+ .non_ascii_literal_count = 0,
1698
+ .invalid_unicode_range = false,
1699
+ .hex_escape_buffer = { 0 },
1700
+ .hex_group_active = false,
1701
+ .has_invalid_multibyte = false,
1702
+ };
1703
+
1704
+ pm_regexp_parse_pattern(&regexp_parser);
1705
+ pm_buffer_cleanup(&regexp_parser.hex_escape_buffer);
790
1706
  }