corvus_json_schema 0.1.3 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +2 -12
  3. data/README.md +10 -1
  4. data/THIRD-PARTY-NOTICES.txt +95 -0
  5. data/VERSIONHISTORY.md +24 -0
  6. data/ext/corvus_json_schema/Cargo.toml +2 -2
  7. data/ext/corvus_json_schema/clippy.toml +3 -4
  8. data/ext/corvus_json_schema/src/lib.rs +42 -19
  9. data/ext/corvus_json_schema/vendor/corvus-json-schema/Cargo.toml +9 -4
  10. data/ext/corvus_json_schema/vendor/corvus-json-schema/README.md +87 -4
  11. data/ext/corvus_json_schema/vendor/corvus-json-schema/THIRD-PARTY-NOTICES.txt +93 -0
  12. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/compiler.rs +88 -96
  13. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/document.rs +2 -2
  14. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/charset.rs +271 -0
  15. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/compile.rs +941 -0
  16. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/dfa.rs +366 -0
  17. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/fold.rs +138 -0
  18. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/mod.rs +329 -0
  19. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/parse.rs +1128 -0
  20. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/re2.rs +129 -0
  21. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/tables_gen.rs +6685 -0
  22. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/tests.rs +1672 -0
  23. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/ucd.rs +309 -0
  24. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/units.rs +165 -0
  25. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/vm.rs +981 -0
  26. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/eval/plan/fused.rs +13 -9
  27. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/eval/plan.rs +12 -11
  28. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/eval.rs +105 -73
  29. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/formats.rs +95 -1
  30. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/lib.rs +165 -30
  31. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/loader.rs +19 -21
  32. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/numbers.rs +1 -1
  33. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/options.rs +199 -4
  34. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/pattern.rs +758 -79
  35. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/uri.rs +4 -4
  36. data/lib/corvus_json_schema/version.rb +1 -1
  37. data/lib/corvus_json_schema.rb +13 -3
  38. metadata +16 -2
@@ -0,0 +1,1672 @@
1
+ //! The tests of the engine, ported from the tests of the Go package. The answers they expect are V8's, in the files
2
+ //! of the Go package's `testdata` directory, which the Object Pascal and OCaml ports read too. So the tests run from
3
+ //! a checkout of the repository, as the rest of the crate's tests do.
4
+
5
+ use std::collections::{BTreeSet, HashMap};
6
+ use std::path::{Path, PathBuf};
7
+ use std::time::{Duration, Instant};
8
+
9
+ use serde_json::Value;
10
+
11
+ use super::charset::{CharSet, property_set};
12
+ use super::compile::{MAX_MEMO_LOOPS, NO_MATCH, Op, Program, compile_program, needs, quicks};
13
+ use super::fold::canonicalize;
14
+ use super::parse::{self, Limit, MAX_NESTING, MAX_PATTERN_LENGTH, MAX_SIZE};
15
+ use super::units::{has_supplementary, unit_at, unit_before};
16
+ use super::vm::{MatchError, set_memo_from_the_start};
17
+ use super::{Error, Regexp, Rune, check, compile, ucd, valid};
18
+
19
+ const NODE_SIZE: usize = 32;
20
+ const RANGE_SIZE: usize = 8;
21
+
22
+ /// A time no test should come near, for the bounds that must not depend on a quiet machine.
23
+ const GENEROUS: Duration = Duration::from_secs(20);
24
+
25
+ fn go_file(name: &str) -> PathBuf {
26
+ Path::new(env!("CARGO_MANIFEST_DIR")).join("../../src-go/corvus-json-schema/internal").join(name)
27
+ }
28
+
29
+ fn testdata(name: &str) -> Value {
30
+ let path = go_file("ecmaregex/testdata").join(name);
31
+ let text = std::fs::read_to_string(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display()));
32
+ serde_json::from_str(&text).unwrap_or_else(|e| panic!("{}: {e}", path.display()))
33
+ }
34
+
35
+ /// One pattern of `v8_cases.json` with what V8 says of it: whether it is valid with the u flag and with no flag,
36
+ /// and whether it matches each text. `gen_cases.js` of the Go package writes the file. Where V8 departs from
37
+ /// ECMA-262 the answers are ECMA-262's, and `ecma262` names the departure.
38
+ struct V8Case {
39
+ p: String,
40
+ u: bool,
41
+ l: bool,
42
+ ecma262: String,
43
+ tests: Vec<(String, bool)>,
44
+ }
45
+
46
+ fn cases(set: &str) -> Vec<V8Case> {
47
+ let all = testdata("v8_cases.json");
48
+ all[set]
49
+ .as_array()
50
+ .unwrap_or_else(|| panic!("v8_cases.json has no set {set}"))
51
+ .iter()
52
+ .map(|c| V8Case {
53
+ p: c["p"].as_str().unwrap().to_string(),
54
+ u: c["u"].as_bool().unwrap(),
55
+ l: c["l"].as_bool().unwrap(),
56
+ ecma262: c["ecma262"].as_str().unwrap_or("").to_string(),
57
+ tests: c["tests"]
58
+ .as_array()
59
+ .unwrap()
60
+ .iter()
61
+ .map(|t| (t["t"].as_str().unwrap().to_string(), t["want"].as_bool().unwrap()))
62
+ .collect(),
63
+ })
64
+ .collect()
65
+ }
66
+
67
+ /// Compares the validity of a pattern with V8's. It returns the pattern compiled, or `None` when the pattern is
68
+ /// not valid.
69
+ fn check_verdict(p: &str, valid_unicode: bool, valid_legacy: bool) -> Option<Regexp> {
70
+ assert_eq!(valid(p), valid_unicode, "valid({p:?})");
71
+ let re = compile(p);
72
+ assert_eq!(re.is_ok(), valid_unicode || valid_legacy, "compile({p:?}): {:?}", re.as_ref().err());
73
+ let re = re.ok()?;
74
+ assert_eq!(re.unicode(), valid_unicode, "compile({p:?}).unicode()");
75
+ assert_eq!(check(p).map(|facts| (facts.unicode, facts.size)), Ok((valid_unicode, re.size())), "check({p:?})");
76
+ assert_eq!(re.as_str(), p);
77
+ Some(re)
78
+ }
79
+
80
+ /// Compares the answers of the pattern with V8's on one text: through the automaton when the pattern has one, on
81
+ /// the backtracking matcher alone, and with a limit of a nanosecond, where a match that reaches its first check is
82
+ /// abandoned and says nothing. A match has `limit`, and one that does not answer within it fails the test.
83
+ fn check_answer(re: &Regexp, text: &str, want: bool, limit: Duration) {
84
+ let p = re.as_str();
85
+ assert_eq!(re.match_limit(text, Some(limit)), Ok(want), "{p:?} on {text:?}");
86
+ assert_eq!(re.match_backtrack(text, Some(limit)), Ok(want), "{p:?} on {text:?} (the backtracking matcher)");
87
+ if let Ok(got) = re.match_limit(text, Some(Duration::from_nanos(1))) {
88
+ assert_eq!(got, want, "{p:?} on {text:?} (with a limit)");
89
+ }
90
+ }
91
+
92
+ struct Counts {
93
+ answers: usize,
94
+ matched: usize,
95
+ legacy: usize,
96
+ invalid: usize,
97
+ }
98
+
99
+ /// Compares the validity of each pattern and its answers with V8's. A pattern has `limit` to answer all its texts.
100
+ fn check_cases(cases: &[V8Case], limit: Duration) -> Counts {
101
+ let mut counts = Counts { answers: 0, matched: 0, legacy: 0, invalid: 0 };
102
+ for c in cases {
103
+ let Some(re) = check_verdict(&c.p, c.u, c.l) else {
104
+ counts.invalid += 1;
105
+ continue;
106
+ };
107
+ if !c.u {
108
+ counts.legacy += 1;
109
+ }
110
+ let start = Instant::now();
111
+ for (text, want) in &c.tests {
112
+ check_answer(&re, text, *want, limit);
113
+ counts.answers += 1;
114
+ counts.matched += usize::from(*want);
115
+ }
116
+ assert!(start.elapsed() < limit, "{:?} did not answer its {} texts in {limit:?}", c.p, c.tests.len());
117
+ }
118
+ counts
119
+ }
120
+
121
+ /// Runs a test that reads patterns nested as deeply as the limits allow, and walks their trees, on a thread with
122
+ /// the stack for it. `compile`, `check` and `valid` see to that themselves. The parser and the compiler, which
123
+ /// these tests call directly, do not.
124
+ fn deep(test: impl FnOnce() + Send + 'static) {
125
+ std::thread::Builder::new().stack_size(64 << 20).spawn(test).unwrap().join().unwrap();
126
+ }
127
+
128
+ /// The start of a pattern, for a message about a long one.
129
+ fn short(pattern: &str) -> String {
130
+ pattern.chars().take(30).collect()
131
+ }
132
+
133
+ fn parse_u(pattern: &str) -> parse::Node {
134
+ parse::parse(pattern, true).unwrap_or_else(|e| panic!("parse({}): {e:?}", short(pattern))).0
135
+ }
136
+
137
+ fn program(pattern: &str) -> Program {
138
+ compile_program(parse_u(pattern), true)
139
+ }
140
+
141
+ /// Matching by UTF-16 code unit, against V8. A pattern that is valid only with no flag is matched a code unit at a
142
+ /// time. A character beyond the Basic Multilingual Plane is two units to the dot, to a class, to a quantifier, to a
143
+ /// group and to a backreference, a match can start and end between them, and such a character in the pattern is two
144
+ /// atoms. A class range between two of them is an error in both grammars.
145
+ #[test]
146
+ fn code_units() {
147
+ let units = cases("units");
148
+ let c = check_cases(&units, Duration::from_secs(60));
149
+ assert!(c.legacy >= 90 && c.invalid >= 3 && c.matched >= 500, "the cases are not the ones this test is about");
150
+ }
151
+
152
+ /// Checks one of the sets of what is a pattern: which patterns are valid, in each grammar, and what the valid ones
153
+ /// match. The expectations are V8's, and ECMA-262's where V8 13.6 departs from it. It returns the number of
154
+ /// patterns that are not valid, and the number marked for each departure.
155
+ fn check_syntax(set: &str) -> (usize, usize, HashMap<String, usize>) {
156
+ let cases = cases(set);
157
+ let c = check_cases(&cases, Duration::from_secs(10));
158
+ let mut departures = HashMap::new();
159
+ for case in &cases {
160
+ if !case.ecma262.is_empty() {
161
+ *departures.entry(case.ecma262.clone()).or_insert(0) += 1;
162
+ }
163
+ }
164
+ (cases.len(), c.invalid, departures)
165
+ }
166
+
167
+ /// `\k` in a class. In a pattern with no named group and no u flag it is the letter k. In a pattern with a named
168
+ /// group it is an error, in a class as outside one.
169
+ #[test]
170
+ fn escaped_k_in_a_class() {
171
+ let (patterns, invalid, _) = check_syntax("classK");
172
+ assert!(patterns >= 30 && invalid >= 9);
173
+ }
174
+
175
+ /// The two counts of a quantifier. A first count greater than the second is an error whatever the size of the
176
+ /// numbers. V8 13.6 keeps a count as a number that stops at 2^31 - 1 and so takes `a{2147483648,2147483647}`. Those
177
+ /// cases have ECMA-262's answer.
178
+ #[test]
179
+ fn counts_are_compared_as_written() {
180
+ let (_, invalid, departures) = check_syntax("counts");
181
+ assert!(invalid >= 20 && departures.get("count-order").copied().unwrap_or(0) >= 10);
182
+ }
183
+
184
+ /// The names and values of `\p{...}` where V8 13.6 departs from ECMA-262. WSpace is not a name ECMA-262 lists.
185
+ /// Katakana_Or_Hiragana (Hrkt) is a Script value, which no character has.
186
+ #[test]
187
+ fn property_names_and_values() {
188
+ let (_, _, departures) = check_syntax("properties");
189
+ assert!(departures.get("property-name").copied().unwrap_or(0) >= 3);
190
+ assert!(departures.get("script-value").copied().unwrap_or(0) >= 20);
191
+ }
192
+
193
+ /// A name on two groups. It is valid only when the groups are in separate alternatives of one disjunction, so that
194
+ /// no match goes through both. V8 13.6 takes some patterns where both can take part. Those cases have ECMA-262's
195
+ /// answer.
196
+ #[test]
197
+ fn a_group_name_used_twice() {
198
+ let (_, invalid, departures) = check_syntax("names");
199
+ assert!(invalid >= 15 && departures.get("duplicate-name").copied().unwrap_or(0) >= 4);
200
+ }
201
+
202
+ /// Repetitions whose body can match only the empty string, and backreferences inside the group they name, against
203
+ /// V8. Run as written, the first patterns of the set take time exponential in their counts.
204
+ #[test]
205
+ fn repetitions_that_cannot_consume() {
206
+ check_cases(&cases("loops"), Duration::from_secs(60));
207
+ }
208
+
209
+ /// The two rewrites of `simplify`, by the counted loops and the backreferences of the program.
210
+ #[test]
211
+ fn simplify() {
212
+ for (pattern, loops, backrefs) in [
213
+ (r"(?:\b|(?=a)|){4,5}", 0, 0),
214
+ (r"(?:(?=(a))|){2}\1", 0, 1),
215
+ (r"(?:(?=a))*b", 0, 0),
216
+ (r"(?:a|(?=b)){2,3}", 1, 0),
217
+ (r"(a\1)+", 0, 0),
218
+ (r"(?:(a)|b\1){2}", 1, 1),
219
+ (r"((\1|\2|)){3}", 0, 0),
220
+ (r"(?<n>a\k<n>)", 0, 0),
221
+ (r"(?:(?<n>a)|(?<n>b\k<n>)){2}", 1, 2),
222
+ (r"(?:(\1){3}|(?!a)){2,}(?<=b)", 0, 0),
223
+ (r"(?:\1(a)){2}", 1, 1),
224
+ ] {
225
+ let p = program(pattern);
226
+ let found = p.insts.iter().filter(|i| i.op == Op::Backref).count();
227
+ assert_eq!((p.loops, found), (loops, backrefs), "{pattern:?}");
228
+ }
229
+ }
230
+
231
+ /// Counted loops whose iterations can be empty, before a rest of the pattern that needs more text than is left,
232
+ /// against V8. Without the needs, the first three patterns of the set do not finish.
233
+ #[test]
234
+ fn loops_give_up_when_the_rest_cannot_fit() {
235
+ let needs = cases("needs");
236
+ let c = check_cases(&needs, Duration::from_secs(60));
237
+ assert!(needs.len() >= 20 && c.matched >= 100);
238
+ }
239
+
240
+ /// The least number of bytes a match consumes, which `set_needs` works out for a program and for each of its
241
+ /// counted loops, and the least one iteration of each loop consumes.
242
+ #[test]
243
+ fn needs_of_a_program() {
244
+ const NONE: i64 = NO_MATCH;
245
+ let table: &[(&str, i64, &[i64], &[i64])] = &[
246
+ (r"abc", 3, &[], &[]),
247
+ (r"a+b{2,}c?", 3, &[], &[]),
248
+ (r"(?:a|bc)d", 2, &[], &[]),
249
+ (r"(?=abc)a", 1, &[], &[]),
250
+ (r"(?<=abc)d", 1, &[], &[]),
251
+ (r"\u00e9{2}\u{10400}", 3, &[], &[]),
252
+ (r"(?:a|){3}xy$", 2, &[2], &[0]),
253
+ (r"(?:a|){3}", 0, &[0], &[0]),
254
+ (r"(?:(?:a|){2}b){2}c", 3, &[1, 2], &[1, 0]),
255
+ (r"(?=(?:a|){2}xyz)(?:b|){2}c", 1, &[3, 1], &[0, 0]),
256
+ (r"(?<=(?:ab|){2}c)(?:d|){2}ef", 2, &[0, 2], &[0, 0]),
257
+ (r"(?!(?:a|){2}xx)(?:(x)|){2}\1y", 1, &[2, 1], &[0, 0]),
258
+ (r"(?:a|){2}[]", NONE, &[NONE], &[0]),
259
+ (r"(?:(?:a|){2}|bcd)e", 1, &[1], &[0]),
260
+ // The least count of a loop times the least of its body.
261
+ (r"(?:a|b){5}c", 6, &[1], &[1]),
262
+ (r"(?:ab|cde){3,}?x", 7, &[1], &[2]),
263
+ (r"(?:a{1000}){1000}", 1000000, &[0], &[1000]),
264
+ (r"(?:\u00e9|xyz){2}", 4, &[0], &[2]),
265
+ (r"(?:a|b){2147483647}c", NONE, &[1], &[1]),
266
+ (r"(?:(?:a|bc){2}d){3}", 9, &[0, 1], &[3, 1]),
267
+ // A loop that cannot reach a match, with a count times a body over the cap: the sum of the two is capped
268
+ // too, and does not wrap to a negative need.
269
+ (r"(?:ab){2147483647}[]", NONE, &[NONE], &[2]),
270
+ (r"(?:(?:ab){2147483647}[]|c)d", 2, &[NONE], &[2]),
271
+ // The body of a lookbehind is matched backwards and counts nothing.
272
+ (r"(?<=(?:ab|c){2}d)e", 1, &[0], &[0]),
273
+ (r"(?:a|(?=(?:b|c){3}))x{2}", 2, &[0], &[1]),
274
+ ];
275
+ for &(pattern, need, loops, iters) in table {
276
+ let p = program(pattern);
277
+ let found: Vec<i64> = p.insts.iter().filter(|i| i.op == Op::Loop).map(|i| i.need as i64).collect();
278
+ let found_iters: Vec<i64> = p.insts.iter().filter(|i| i.op == Op::Loop).map(|i| i.iter as i64).collect();
279
+ assert_eq!((p.need as i64, &found[..], &found_iters[..]), (need, loops, iters), "{pattern:?}");
280
+ }
281
+ }
282
+
283
+ /// The matcher uses the least count of a counted loop: a pattern that needs a million characters, or two thousand
284
+ /// million, answers a text of a hundred thousand at once. The limit of the match here is a small part of what the
285
+ /// work would take, so a match that does the work is abandoned and the test fails on its error.
286
+ #[test]
287
+ fn a_loop_gives_up_when_its_iterations_cannot_fit() {
288
+ let text = "a".repeat(100000);
289
+ for pattern in
290
+ [r"(?:a{1000}){1000}", r"(?:a|b){2147483647}c", r"(?=.)(?:(?:ab|a){500}){500}", r"x?(?:[ab]c?){100001}$"]
291
+ {
292
+ let re = compile(pattern).unwrap();
293
+ assert_eq!(re.match_backtrack(&text, Some(Duration::from_millis(200))), Ok(false), "{pattern:?}");
294
+ }
295
+ // A text that is long enough still matches.
296
+ let re = compile(r"^(?:a{10}){100}$").unwrap();
297
+ assert_eq!(re.match_backtrack(&text[..1000], Some(Duration::from_secs(60))), Ok(true));
298
+ assert_eq!(re.match_backtrack(&text[..999], Some(Duration::from_secs(60))), Ok(false));
299
+ }
300
+
301
+ /// The two readers of a code unit on every position of a text, forwards and backwards.
302
+ #[test]
303
+ fn code_unit_positions() {
304
+ let text = "a\u{e9}\u{20ac}\u{10400}\u{10FFFF}z".as_bytes();
305
+ let want: [Rune; 8] = [0x61, 0xE9, 0x20AC, 0xD801, 0xDC00, 0xDBFF, 0xDFFF, 0x7A];
306
+ let mut ends = Vec::new();
307
+ let mut pos = 0;
308
+ for unit in want {
309
+ let (got, w) = unit_at(text, pos);
310
+ assert_eq!(got, unit, "unit_at({pos})");
311
+ pos += w;
312
+ ends.push(pos);
313
+ }
314
+ assert_eq!(pos, text.len());
315
+ for k in (0..want.len()).rev() {
316
+ let (got, w) = unit_before(text, ends[k]);
317
+ assert_eq!(got, want[k], "unit_before({})", ends[k]);
318
+ if k > 0 {
319
+ assert_eq!(ends[k] - w, ends[k - 1], "unit_before({}) spans {w} bytes", ends[k]);
320
+ }
321
+ }
322
+ assert!(has_supplementary(text) && !has_supplementary("a\u{e9}\u{20ac}\u{ffff}".as_bytes()));
323
+ // Text that is not UTF-8 is read a byte at a time, as it is when matching by code point.
324
+ for bad in [&b"\xf0\x9f"[..], b"\xf0\x9f\x98", b"\x80\x80\x80", b"\xf0a\x80\x80", b"\xf8\x88\x80\x80\x80"] {
325
+ let mut pos = 0;
326
+ while pos < bad.len() {
327
+ let (r, w) = unit_at(bad, pos);
328
+ assert!(w == 1 && (r == 0xFFFD || r == bad[pos] as Rune), "unit_at({bad:?}, {pos}) = {r:#x}, {w}");
329
+ pos += w;
330
+ }
331
+ let mut pos = bad.len();
332
+ while pos > 0 {
333
+ let (r, w) = unit_before(bad, pos);
334
+ assert!(w == 1 && (r == 0xFFFD || r == bad[pos - 1] as Rune), "unit_before({bad:?}, {pos}) = {r:#x}, {w}");
335
+ pos -= w;
336
+ }
337
+ }
338
+ }
339
+
340
+ // ---------------------------------------------------------------------------------------------------------------------
341
+ // The oracle: what V8 answers for a set of patterns and texts. gen_oracle.js of the Go package writes it.
342
+
343
+ struct Oracle {
344
+ inputs: Vec<String>,
345
+ short_inputs: Vec<String>,
346
+ verdicts: String,
347
+ random_matches: Vec<String>,
348
+ /// The pattern, whether it is valid with the u flag and with no flag, and one bit per input, set for a match,
349
+ /// written as hexadecimal digits of four bits each.
350
+ hand: Vec<(String, bool, bool, String)>,
351
+ /// The pattern, the flags it was matched with (out of i, m and s), whether it is valid with the u flag, and
352
+ /// the bits.
353
+ flagged: Vec<(String, String, bool, String)>,
354
+ }
355
+
356
+ fn oracle() -> Oracle {
357
+ let o = testdata("v8_oracle.json");
358
+ let strings =
359
+ |v: &Value| -> Vec<String> { v.as_array().unwrap().iter().map(|s| s.as_str().unwrap().into()).collect() };
360
+ // A pattern that is not valid has no answers.
361
+ let text = |v: &Value| v.as_str().unwrap_or("").to_string();
362
+ Oracle {
363
+ inputs: strings(&o["inputs"]),
364
+ short_inputs: strings(&o["shortInputs"]),
365
+ verdicts: text(&o["verdicts"]),
366
+ random_matches: strings(&o["randomMatches"]),
367
+ hand: o["hand"]
368
+ .as_array()
369
+ .unwrap()
370
+ .iter()
371
+ .map(|h| (text(&h["p"]), h["u"].as_bool().unwrap(), h["l"].as_bool().unwrap(), text(&h["m"])))
372
+ .collect(),
373
+ flagged: o["flagged"]
374
+ .as_array()
375
+ .unwrap()
376
+ .iter()
377
+ .map(|f| (text(&f["p"]), text(&f["f"]), f["u"].as_bool().unwrap(), text(&f["m"])))
378
+ .collect(),
379
+ }
380
+ }
381
+
382
+ /// Bit `k` of a string of the oracle's bits.
383
+ fn bit(hex: &str, k: usize) -> bool {
384
+ let digit = (hex.as_bytes()[k / 4] as char).to_digit(16).unwrap();
385
+ (digit >> (3 - k % 4)) & 1 == 1
386
+ }
387
+
388
+ /// The patterns for which V8 13.6 answers a case-insensitive modifier group differently from the same pattern with
389
+ /// the i flag. ECMA-262 gives the two the same meaning. Only the validity of these patterns is taken from the
390
+ /// oracle. `oracle_flags` checks the behaviour of each through the i flag, where V8 agrees with the engine.
391
+ const V8_MODIFIER_BUGS: &[&str] =
392
+ &[r"^(?i:\u212a)$", r"^(?i:\u017f)$", r"^(?i:\u1e9e)$", r"^(?i:\u03bc)$", r"^(?i:\u03bc)\&?$", r"(?i:\Bs)"];
393
+
394
+ /// Compares the answers of the pattern with the oracle's on every text, the texts with a character beyond the Basic
395
+ /// Multilingual Plane among them. It returns the number of answers compared.
396
+ fn check_answers(re: &Regexp, inputs: &[String], bits: &str) -> usize {
397
+ for (i, s) in inputs.iter().enumerate() {
398
+ check_answer(re, s, bit(bits, i), GENEROUS);
399
+ }
400
+ inputs.len()
401
+ }
402
+
403
+ /// The hand-written patterns (the Java port's validator cases, the Rust port's pattern cases and the Go package's
404
+ /// own) against V8, for validity and for the answer on every text.
405
+ #[test]
406
+ fn oracle_hand_written() {
407
+ let o = oracle();
408
+ let mut answers = 0;
409
+ for (p, u, l, m) in &o.hand {
410
+ let Some(re) = check_verdict(p, *u, *l) else { continue };
411
+ if !V8_MODIFIER_BUGS.contains(&p.as_str()) {
412
+ answers += check_answers(&re, &o.inputs, m);
413
+ }
414
+ }
415
+ assert!(o.hand.len() > 300 && answers > 50000, "{} patterns, {answers} answers", o.hand.len());
416
+ }
417
+
418
+ /// Case-insensitive, multiline and dot-all matching. V8 matched each pattern with flags set for the whole pattern,
419
+ /// and here the pattern is wrapped in the modifier group that means the same.
420
+ #[test]
421
+ fn oracle_flags() {
422
+ let o = oracle();
423
+ let mut answers = 0;
424
+ for (p, f, u, m) in &o.flagged {
425
+ let p = format!("(?{f}:{p})");
426
+ if let Some(re) = check_verdict(&p, *u, !*u) {
427
+ answers += check_answers(&re, &o.inputs, m);
428
+ }
429
+ }
430
+ assert!(answers > 5000, "{answers} answers");
431
+ }
432
+
433
+ /// The generator `gen_oracle.js` uses, so the test builds the patterns the oracle's verdicts are about.
434
+ struct Xorshift(u32);
435
+
436
+ impl Xorshift {
437
+ fn next(&mut self) -> u32 {
438
+ let mut s = self.0;
439
+ s ^= s << 13;
440
+ s ^= s >> 17;
441
+ s ^= s << 5;
442
+ self.0 = s;
443
+ s
444
+ }
445
+ }
446
+
447
+ /// Random patterns over an alphabet of syntax characters (the Java port's generated test). Each one's validity is
448
+ /// checked against V8, and the first few thousand are also matched.
449
+ #[test]
450
+ fn oracle_generated() {
451
+ let o = oracle();
452
+ let alphabet = br"ab0\\^$.*+?()[]{}|-,:=!<>dwsbpkuxc1L";
453
+ let mut seed = Xorshift(0x9e3779b9);
454
+ let mut counts = [0; 3];
455
+ let mut answers = 0;
456
+ for (i, verdict) in o.verdicts.bytes().enumerate() {
457
+ let n = 1 + seed.next() % 8;
458
+ let p: String = (0..n).map(|_| alphabet[(seed.next() % alphabet.len() as u32) as usize] as char).collect();
459
+ counts[(verdict - b'0') as usize] += 1;
460
+ let Some(re) = check_verdict(&p, verdict == b'1', verdict == b'2') else { continue };
461
+ if let Some(bits) = o.random_matches.get(i) {
462
+ answers += check_answers(&re, &o.short_inputs, bits);
463
+ }
464
+ }
465
+ assert!(counts.iter().all(|&c| c > 100) && answers > 10000, "{counts:?}, {answers} answers");
466
+ }
467
+
468
+ // ---------------------------------------------------------------------------------------------------------------------
469
+ // The size of a pattern.
470
+
471
+ /// The size the parser counts for a pattern under one grammar, or -1 when the grammar does not accept it.
472
+ fn size_of(pattern: &str, unicode: bool) -> i64 {
473
+ parse::parse(pattern, unicode).map_or(-1, |(_, ps)| ps.size as i64)
474
+ }
475
+
476
+ /// The size the parser counts for each pattern of `size_cases.json`, which the Go, Object Pascal and OCaml ports
477
+ /// read too. The size of a pattern decides whether it is refused as too large, so the ports must count the same
478
+ /// figure for every pattern, or a schema one port compiles would be refused by another.
479
+ #[test]
480
+ fn the_size_of_a_pattern() {
481
+ let cases = testdata("size_cases.json");
482
+ let cases = cases.as_array().unwrap();
483
+ let (mut valid, mut legacy, mut invalid) = (0, 0, 0);
484
+ for c in cases {
485
+ let (p, u, l) = (c["p"].as_str().unwrap(), c["u"].as_i64().unwrap(), c["l"].as_i64().unwrap());
486
+ assert_eq!((size_of(p, true), size_of(p, false)), (u, l), "the size of {p:?} with the u flag and without");
487
+ if u >= 0 {
488
+ valid += 1;
489
+ } else if l >= 0 {
490
+ legacy += 1;
491
+ } else {
492
+ invalid += 1;
493
+ }
494
+ }
495
+ assert_eq!(cases.len(), 122);
496
+ assert!(valid >= 90 && legacy >= 10 && invalid >= 5, "the cases are not the ones this test is about");
497
+ }
498
+
499
+ // ---------------------------------------------------------------------------------------------------------------------
500
+ // The next character.
501
+
502
+ /// The patterns of the quick set against V8. They have a choice, a loop or a lookaround where the next character of
503
+ /// the text rules a way out, with everything a way can end with or go through without a character, which is where
504
+ /// ruling a way out wrongly would show.
505
+ #[test]
506
+ fn the_next_character_rules_out_a_way() {
507
+ let quick = cases("quick");
508
+ let c = check_cases(&quick, Duration::from_secs(60));
509
+ assert!(quick.len() >= 75 && c.matched >= 2000);
510
+ }
511
+
512
+ /// What `set_quick` works out for the first instruction of a program: the ASCII characters a match can start with,
513
+ /// whether it can start with one that is not ASCII, and whether it can end without one while text is left and at
514
+ /// the end of the text.
515
+ #[test]
516
+ fn quick_of_a_program() {
517
+ for (pattern, ascii, other, mid, end) in [
518
+ (r"abc", "a", false, false, false),
519
+ (r"[a-c]x", "abc", false, false, false),
520
+ (r"a|b|\u00e9", "ab", true, false, false),
521
+ (r"a?b", "ab", false, false, false),
522
+ (r"a*$", "a", false, false, true),
523
+ (r"(?:a|)b", "ab", false, false, false),
524
+ (r"$", "", false, false, true),
525
+ (r"(?:a|$)b", "a", false, false, false),
526
+ (r"(?=a)[a-c]", "a", false, false, false),
527
+ (r"(?=[ab])(?:b|x)", "b", false, false, false),
528
+ (r"(?!a)[a-c]", "abc", false, false, false),
529
+ (r"(?=a|$)(?:a|)$", "a", false, false, true),
530
+ (r"(?<=a)b", "b", false, false, false),
531
+ (r"(?<=a|b)$", "", false, false, true),
532
+ (r"\bx", "x", false, false, false),
533
+ (r"^a", "a", false, false, false),
534
+ (r"(?:a|b)*c", "abc", false, false, false),
535
+ (r"(?:a?b?)*c", "abc", false, false, false),
536
+ (r"(?:a|){2}x", "ax", false, false, false),
537
+ (r"[^a]", "bcd", true, false, false),
538
+ (r"(?=)a", "a", false, false, false),
539
+ (r"[]|a", "a", false, false, false),
540
+ (r"(?!)[]", "", false, false, false),
541
+ ] {
542
+ let q = program(pattern).quick[0];
543
+ let has = |b: u8| (q.ascii[(b >> 6) as usize] >> (b & 63)) & 1 != 0;
544
+ let ok = (q.other, q.mid, q.end) == (other, mid, end)
545
+ && ascii.bytes().all(has)
546
+ // A character the pattern cannot start with: none of the cases starts with a digit.
547
+ && has(b'0') == (pattern == r"[^a]");
548
+ assert!(ok, "{pattern:?}: quick {q:?}, want {ascii:?}, {other}, {mid}, {end}");
549
+ }
550
+ // A backreference and the body of a lookbehind allow anything.
551
+ for pattern in [r"(a)\1x", r"(?<=ab)"] {
552
+ let p = program(pattern);
553
+ let mut found = false;
554
+ for (pc, i) in p.insts.iter().enumerate() {
555
+ if i.op == Op::Backref || i.op == Op::CharBack {
556
+ found = true;
557
+ let q = p.quick[pc];
558
+ assert!(q.ascii == [!0, !0] && q.other && q.mid && q.end, "{pattern:?}: instruction {pc}");
559
+ }
560
+ }
561
+ assert!(found, "{pattern:?} has no instruction this is about");
562
+ }
563
+ }
564
+
565
+ // ---------------------------------------------------------------------------------------------------------------------
566
+ // The tables of the Go package's own tests.
567
+
568
+ /// The Java port's hand-written patterns: the pattern, whether it is valid with the u flag, and whether it is valid
569
+ /// at all.
570
+ #[test]
571
+ fn java_hand_written_patterns() {
572
+ let table: &[(&str, bool, bool)] = &[
573
+ ("", true, true),
574
+ ("a", true, true),
575
+ ("^a$", true, true),
576
+ ("a|b", true, true),
577
+ ("(a)", true, true),
578
+ ("(?:a)", true, true),
579
+ ("(?=a)", true, true),
580
+ ("(?!a)", true, true),
581
+ ("(?<=a)", true, true),
582
+ ("(?<!a)", true, true),
583
+ (r"(?<n>a)\k<n>", true, true),
584
+ (r"\k<n>", false, true),
585
+ ("(?<n>a)(?<n>b)", false, false),
586
+ ("(?<$x_1>a)", true, true),
587
+ ("(?<1a>a)", false, false),
588
+ ("a{2}", true, true),
589
+ ("a{2,}", true, true),
590
+ ("a{2,3}", true, true),
591
+ ("a{3,2}", false, false),
592
+ ("a{", false, true),
593
+ ("a{,2}", false, true),
594
+ ("{", false, true),
595
+ ("}", false, true),
596
+ ("]", false, true),
597
+ ("[", false, false),
598
+ ("[]", true, true),
599
+ ("[^]", true, true),
600
+ ("[a-z]", true, true),
601
+ ("[z-a]", false, false),
602
+ (r"[\d-z]", false, true),
603
+ (r"[a-\d]", false, true),
604
+ (r"[\b]", true, true),
605
+ (r"[\-]", true, true),
606
+ (r"\-", false, true),
607
+ (r"\a", false, true),
608
+ (r"\c", false, true),
609
+ (r"\cA", true, true),
610
+ (r"\c1", false, true),
611
+ (r"\0", true, true),
612
+ (r"\01", false, true),
613
+ (r"\1", false, true),
614
+ (r"(a)\1", true, true),
615
+ (r"(a)\2", false, true),
616
+ (r"\x4", false, true),
617
+ (r"\x41", true, true),
618
+ (r"\u004", false, true),
619
+ (r"\u0041", true, true),
620
+ (r"\u{1F600}", true, true),
621
+ (r"\u{110000}", false, true),
622
+ ("\u{1F600}", true, true),
623
+ ("[\u{1F600}-\u{1F64F}]", true, true),
624
+ (r"\p{L}", true, true),
625
+ (r"\p{Letter}", true, true),
626
+ (r"\p{digit}", true, true),
627
+ (r"\p{Nope}", false, true),
628
+ (r"\p{gc=Lu}", true, true),
629
+ (r"\p{Script=Greek}", true, true),
630
+ (r"\p{sc=Grek}", true, true),
631
+ (r"\p{Script=Nope}", false, true),
632
+ (r"\P{ASCII}", true, true),
633
+ (r"\p", false, true),
634
+ (r"\p{", false, true),
635
+ ("a**", false, false),
636
+ ("a+?", true, true),
637
+ ("*a", false, false),
638
+ ("(?=a)*", false, true),
639
+ ("^*", false, false),
640
+ ("$+", false, false),
641
+ (r"\b+", false, false),
642
+ ("a)", false, false),
643
+ ("(a", false, false),
644
+ ("(?a)", false, false),
645
+ (r"\/", true, true),
646
+ (r"\.", true, true),
647
+ ("a\\", false, false),
648
+ ("[a", false, false),
649
+ ("x{1}{2}", false, false),
650
+ (r"\w+@\w+\.\w+", true, true),
651
+ ("^[a-z][a-z0-9_]*$", true, true),
652
+ (r"(?<a>.)\k<a>", true, true),
653
+ (r"[\p{L}\d]", true, true),
654
+ (r"\s\S\w\W\d\D", true, true),
655
+ ];
656
+ for &(pattern, unicode, is_valid) in table {
657
+ assert_eq!(valid(pattern), unicode, "valid({pattern:?})");
658
+ let re = compile(pattern);
659
+ assert_eq!(re.is_ok(), is_valid, "compile({pattern:?}): {:?}", re.as_ref().err());
660
+ if let Ok(re) = re {
661
+ assert_eq!(re.unicode(), unicode, "compile({pattern:?}).unicode()");
662
+ }
663
+ }
664
+ }
665
+
666
+ /// Every string of the JSON-Schema-Test-Suite that is a pattern, a key of patternProperties or an instance.
667
+ fn suite_corpus() -> Option<BTreeSet<String>> {
668
+ fn walk(v: &Value, seen: &mut BTreeSet<String>) {
669
+ match v {
670
+ Value::Array(items) => items.iter().for_each(|item| walk(item, seen)),
671
+ Value::Object(object) => {
672
+ for (name, value) in object {
673
+ if let Value::String(s) = value
674
+ && (name == "pattern" || name == "data")
675
+ {
676
+ seen.insert(s.clone());
677
+ }
678
+ if let Value::Object(patterns) = value
679
+ && name == "patternProperties"
680
+ {
681
+ seen.extend(patterns.keys().cloned());
682
+ }
683
+ walk(value, seen);
684
+ }
685
+ }
686
+ _ => {}
687
+ }
688
+ }
689
+ fn files(dir: &Path, seen: &mut BTreeSet<String>) {
690
+ let Ok(entries) = std::fs::read_dir(dir) else { return };
691
+ for entry in entries.flatten() {
692
+ let path = entry.path();
693
+ if path.is_dir() {
694
+ files(&path, seen);
695
+ } else if path.extension().is_some_and(|e| e == "json")
696
+ && let Ok(text) = std::fs::read_to_string(&path)
697
+ && let Ok(v) = serde_json::from_str::<Value>(&text)
698
+ {
699
+ walk(&v, seen);
700
+ }
701
+ }
702
+ }
703
+ let root = std::env::var_os("JSON_SCHEMA_TEST_SUITE")
704
+ .map(PathBuf::from)
705
+ .unwrap_or_else(|| Path::new(env!("CARGO_MANIFEST_DIR")).join("../../JSON-Schema-Test-Suite"));
706
+ if !root.join("tests").is_dir() {
707
+ eprintln!("JSON-Schema-Test-Suite not found at {}; skipping", root.display());
708
+ return None;
709
+ }
710
+ let mut seen = BTreeSet::new();
711
+ files(&root.join("tests"), &mut seen);
712
+ Some(seen)
713
+ }
714
+
715
+ /// `valid` and `compile` agree on every string of the JSON-Schema-Test-Suite.
716
+ #[test]
717
+ fn suite_corpus_is_read_the_same_by_valid_and_compile() {
718
+ let Some(corpus) = suite_corpus() else { return };
719
+ let (mut valid_count, mut compiled) = (0, 0);
720
+ for p in &corpus {
721
+ let ok = valid(p);
722
+ let re = compile(p);
723
+ if ok {
724
+ valid_count += 1;
725
+ assert!(re.as_ref().is_ok_and(Regexp::unicode), "valid({p:?}) but compile gives {:?}", re.as_ref().err());
726
+ }
727
+ if let Ok(re) = re {
728
+ compiled += 1;
729
+ assert!(ok || !re.unicode(), "compile({p:?}) used the u flag grammar but valid rejects it");
730
+ }
731
+ }
732
+ assert!(valid_count > 500 && compiled >= valid_count, "{} strings, {valid_count} valid", corpus.len());
733
+ }
734
+
735
+ fn differential_inputs(n: usize) -> Vec<String> {
736
+ let alphabet = [
737
+ "a",
738
+ "b",
739
+ "c",
740
+ "z",
741
+ "A",
742
+ "X",
743
+ "Z",
744
+ "0",
745
+ "1",
746
+ "5",
747
+ "9",
748
+ "_",
749
+ "-",
750
+ ".",
751
+ ":",
752
+ "/",
753
+ "@",
754
+ "#",
755
+ "$",
756
+ "*",
757
+ "!",
758
+ "{",
759
+ "}",
760
+ "|",
761
+ " ",
762
+ "\n",
763
+ "\r",
764
+ "\t",
765
+ "\u{a0}",
766
+ "\u{e9}",
767
+ "\u{b5}",
768
+ "\u{1F600}",
769
+ "\u{1F432}",
770
+ "\u{2028}",
771
+ "\u{FFFD}",
772
+ "x-",
773
+ "es",
774
+ "ES",
775
+ "ms",
776
+ "txt",
777
+ "Au",
778
+ "to",
779
+ "No",
780
+ "ne",
781
+ "2015",
782
+ "Co",
783
+ "re",
784
+ "20",
785
+ "15",
786
+ "22",
787
+ "2",
788
+ ",",
789
+ "a,",
790
+ "ab,",
791
+ "a-",
792
+ "foo",
793
+ "bar",
794
+ "\u{3b1}",
795
+ "\u{3a9}",
796
+ "\u{663}",
797
+ "\\",
798
+ "&",
799
+ "(",
800
+ ")",
801
+ "[",
802
+ "]",
803
+ "^",
804
+ "+",
805
+ "?",
806
+ "<",
807
+ ">",
808
+ "=",
809
+ "%",
810
+ ];
811
+ let mut seed = Xorshift(0x2545f491);
812
+ let mut inputs: Vec<String> = std::iter::once("").chain(alphabet).map(String::from).collect();
813
+ while inputs.len() < n {
814
+ let k = 1 + seed.next() % 8;
815
+ inputs.push((0..k).map(|_| alphabet[(seed.next() % alphabet.len() as u32) as usize]).collect());
816
+ }
817
+ inputs
818
+ }
819
+
820
+ /// The automaton and the backtracking matcher give the same answer, for every pattern of the oracle and of the
821
+ /// JSON-Schema-Test-Suite that has an automaton, on random texts. (The Go test compares the regexp package too, and
822
+ /// has texts that are not UTF-8, which a `&str` cannot hold.)
823
+ #[test]
824
+ fn the_automaton_and_the_matcher_agree() {
825
+ let o = oracle();
826
+ let mut patterns: BTreeSet<String> = o.hand.iter().map(|h| h.0.clone()).collect();
827
+ patterns.extend(o.flagged.iter().map(|(p, f, _, _)| format!("(?{f}:{p})")));
828
+ patterns.extend(
829
+ [
830
+ "\u{FFFD}",
831
+ "^[^\u{FFFD}]+$",
832
+ "^\u{FFFD}+$",
833
+ "^.$",
834
+ "^..$",
835
+ "^[^a]$",
836
+ r"\P{Any}",
837
+ r"^\p{Any}*$",
838
+ r"^[\s\S]{2,4}$",
839
+ r"^(?:\p{L}|\p{N})+$",
840
+ r"\b\w+\b",
841
+ r"\B",
842
+ "^$",
843
+ "$",
844
+ "^",
845
+ "(?:)",
846
+ "^(?:a{2}){2,3}$",
847
+ "^(?:a|b|ab){1,4}?c",
848
+ "^.{0,300}$",
849
+ r"^(?:[a-z]{1,3}\d?){2,5}$",
850
+ "^[\\u0000-\u{FFFF}]+$",
851
+ r"^[\u{10000}-\u{10ffff}]+$",
852
+ "(?:a{50}){30}",
853
+ "^a{1001}$",
854
+ "^(?:a?){1001}$",
855
+ r"\x{41}",
856
+ r"^\u{2}$",
857
+ ]
858
+ .map(String::from),
859
+ );
860
+ patterns.extend(suite_corpus().unwrap_or_default());
861
+ let inputs = differential_inputs(1500);
862
+ let (mut regular, mut automata, mut decided) = (0, 0, 0);
863
+ for p in &patterns {
864
+ let Ok(re) = compile(p) else { continue };
865
+ if !re.regular {
866
+ continue;
867
+ }
868
+ regular += 1;
869
+ for s in &inputs {
870
+ let (a, b) = (re.match_limit(s, None), re.match_backtrack(s, None));
871
+ assert_eq!(a, b, "{p:?} on {s:?}: the automaton and the backtracking matcher");
872
+ if let Some(Some(dfa)) = re.dfa.get()
873
+ && dfa.is_match(s.as_bytes()).is_some()
874
+ {
875
+ decided += 1;
876
+ }
877
+ }
878
+ automata += usize::from(matches!(re.dfa.get(), Some(Some(_))));
879
+ }
880
+ assert!(regular > 200 && automata > 100 && decided > 100000, "{regular} patterns, {automata} automata, {decided}");
881
+ }
882
+
883
+ #[test]
884
+ fn backtracking_semantics() {
885
+ for (pattern, text, want) in [
886
+ (r"^\1(a)$", "a", true),
887
+ (r"^(a)?\1b$", "b", true),
888
+ (r"^(?:(a)|b)\1$", "b", true),
889
+ (r"^(?:(a)|b)*\1$", "ab", true),
890
+ (r"^(?:(a)|b)*\1$", "aba", false),
891
+ (r"^(?:(a)|b)*\1$", "baa", true),
892
+ (r"^(?:a*)*$(?<=a)", "aaa", true),
893
+ (r"^(?:a*)*b(?<=b)", "aaa", false),
894
+ (r"^(?:(?=(a)))?\1b", "b", true),
895
+ (r"^(?:(?=(a)))?\1a", "a", true),
896
+ (r"^(?:a?){3,5}b(?<=b)", "aab", true),
897
+ (r"^(?:|a)+b(?<=b)", "aab", true),
898
+ (r"^(?=(a+))a*b\1$", "aaaba", false),
899
+ (r"^(?=(a+))a*b\1$", "aaabaaa", true),
900
+ (r"(?!(a))\1b", "b", true),
901
+ (r"(?<=(\d+)(\d+))$", "1053", true),
902
+ (r"(?<=\1(a))b", "aab", true),
903
+ (r"(?<=\1(a))b", "ab", false),
904
+ (r"(?<=^a*)b", "aab", true),
905
+ (r"(?<!^a*)b", "aab", false),
906
+ (r"(?<=a|bc)x", "bcx", true),
907
+ (r"^(?<n>.)\k<n>$", "\u{1F432}\u{1F432}", true),
908
+ (r"^(?<n>.)\k<n>$", "\u{1F432}\u{1F409}", false),
909
+ (r"(?<=^.)$", "\u{1F432}", true),
910
+ ("(?<=[\u{1F600}-\u{1F60E}])x", "\u{1F603}x", true),
911
+ (r"(?<!\u{1F600})x", "\u{1F600}x", false),
912
+ (r"^(?:a|ab)+?b$(?<=b)", "aabb", true),
913
+ (r"^(?:(a)|b){2,}?\1$", "abbaa", true),
914
+ (r"^(?:(a)|b){2,}?\1$", "abba", false),
915
+ (r"^(.+?)\1+$", "abcabcabc", true),
916
+ (r"^(.+?)\1+$", "abcabcab", false),
917
+ (r"^(?:(\w)(?!\1))+$", "abab", true),
918
+ (r"^(?:(\w)(?!\1))+$", "abba", false),
919
+ (r"^(?!.*(.).*\1)[a-c]+$", "abc", true),
920
+ (r"^(?!.*(.).*\1)[a-c]+$", "abca", false),
921
+ ] {
922
+ let re = compile(pattern).unwrap();
923
+ assert!(!re.regular, "{pattern:?} needs no backtracking");
924
+ assert_eq!(re.is_match(text), want, "{pattern:?} on {text:?}");
925
+ }
926
+ }
927
+
928
+ /// A long text does not overflow the stack of the thread: the choices are on a stack of the matcher's own.
929
+ #[test]
930
+ fn long_input() {
931
+ let long = "ab".repeat(1 << 20);
932
+ let longer = |tail: &str| format!("{long}{tail}");
933
+ for (pattern, text, want) in [
934
+ (r"^(?:a|b)*$(?<=b)", long.clone(), true),
935
+ (r"^(?:a|b)*c(?<=c)", long.clone(), false),
936
+ (r"^(?:ab)+(?!.)", long.clone(), true),
937
+ (r"^(?=a)(?:\w\w)*?$", long.clone(), true),
938
+ (r"^(a|b)*\1$", longer("b"), true),
939
+ (r"(?<=^(?:ab)*)c", longer("c"), true),
940
+ (r"^(?!b).*b$", long.clone(), true),
941
+ (r"^[ab]{2097152}(?<=b)$", long.clone(), true),
942
+ (r"^(?:[ab]c?){2097152}(?<=b)$", long.clone(), true),
943
+ ] {
944
+ let re = compile(pattern).unwrap();
945
+ assert!(!re.regular, "{pattern:?} needs no backtracking");
946
+ assert_eq!(re.is_match(&text), want, "{pattern:?} on {} bytes", text.len());
947
+ }
948
+ }
949
+
950
+ /// A compiled pattern is shared between threads, each of which matches with a working memory of its own.
951
+ #[test]
952
+ fn concurrent_use() {
953
+ for p in [r"^(?:[a-z]+\d)+$", r"^(?:([a-z])\1?\d)+(?<=\d)$"] {
954
+ let re = std::sync::Arc::new(compile(p).unwrap());
955
+ let threads: Vec<_> = (0..16)
956
+ .map(|g| {
957
+ let re = re.clone();
958
+ std::thread::spawn(move || {
959
+ let yes = "aa1b2".repeat(50 + g);
960
+ let no = format!("{yes}!");
961
+ (0..500).all(|_| re.is_match(&yes) && !re.is_match(&no))
962
+ })
963
+ })
964
+ .collect();
965
+ for thread in threads {
966
+ assert!(thread.join().unwrap(), "{p:?} gave a wrong answer under concurrent use");
967
+ }
968
+ }
969
+ }
970
+
971
+ /// The classes of equivalent characters: every member of a class has the canonical form of the class, and every
972
+ /// two characters with one canonical form are in a class.
973
+ #[test]
974
+ fn fold_tables() {
975
+ for unicode_mode in [true, false] {
976
+ // Every character with a case variant, with the closure of the set that holds it alone.
977
+ let mut seen: HashMap<Rune, Rune> = HashMap::new();
978
+ for r in 0..=ucd::MAX_RUNE {
979
+ let key = canonicalize(r, unicode_mode);
980
+ let other = seen.get(&key).copied().or((key != r).then_some(key));
981
+ if let Some(other) = other {
982
+ for (a, b) in [(r, other), (other, r)] {
983
+ let (closed, variants) = super::fold::fold_closure(&CharSet::shared(&[a, a], false), unicode_mode);
984
+ assert!(variants == 1 && closed.contains(b), "u flag {unicode_mode}: U+{a:04X} and U+{b:04X}");
985
+ }
986
+ }
987
+ seen.insert(key, r);
988
+ }
989
+ // The closure of everything holds nothing more, and every member of a closure has one canonical form.
990
+ for r in [0x4B, 0x6B, 0x212A, 0x53, 0x17F, 0xDF, 0x1E9E, 0x3A3, 0x3C2, 0x3C3, 0x130, 0x131, 0x1C4, 0x10400] {
991
+ let (closed, _) = super::fold::fold_closure(&CharSet::shared(&[r, r], false), unicode_mode);
992
+ for &[lo, hi] in closed.ranges.as_chunks::<2>().0 {
993
+ for member in lo..=hi {
994
+ assert_eq!(canonicalize(member, unicode_mode), canonicalize(r, unicode_mode), "U+{member:04X}");
995
+ }
996
+ }
997
+ }
998
+ }
999
+ }
1000
+
1001
+ #[test]
1002
+ fn modifiers() {
1003
+ for (pattern, text, want) in [
1004
+ (r"^(?i:abc)$", "aBc", true),
1005
+ (r"^a(?i:b)c$", "aBc", true),
1006
+ (r"^a(?i:b)c$", "ABc", false),
1007
+ (r"^(?i:a(?-i:b)c)$", "AbC", true),
1008
+ (r"^(?i:a(?-i:b)c)$", "ABC", false),
1009
+ (r"^(?i:k)$", "\u{212a}", true),
1010
+ (r"^(?i:k)\&?$", "\u{212a}", false),
1011
+ (r"^(?i:s)$", "\u{17f}", true),
1012
+ (r"^(?i:s)\&?$", "\u{17f}", false),
1013
+ (r"^(?i:\w)$", "\u{17f}", true),
1014
+ (r"^(?i:\W)$", "\u{17f}", false),
1015
+ (r"(?i:\b)\u212a", "\u{212a}", true),
1016
+ (r"\b\u212a", "\u{212a}", false),
1017
+ (r"^(?i:[^a-z])$", "K", false),
1018
+ (r"^(?i:[^a-z])$", "1", true),
1019
+ ("^(?i:\u{df})$", "\u{1e9e}", true),
1020
+ ("^(?i:\u{df})$", "SS", false),
1021
+ (r"^(.)(?i:\1)$", "aA", true),
1022
+ (r"^(.)\1$", "aA", false),
1023
+ (r"^(.)(?i:\1)$", "k\u{212a}", true),
1024
+ (r"(?<=(?i:\1)(.))x", "Aax", true),
1025
+ (r"^(?s:.)$", "\n", true),
1026
+ (r"^.$", "\n", false),
1027
+ (r"^(?s:a.)b.$", "a\nb\n", false),
1028
+ (r"(?m:^)b", "a\nb", true),
1029
+ (r"^b", "a\nb", false),
1030
+ (r"a(?m:$)", "a\u{2028}b", true),
1031
+ (r"a(?m:$)", "ab", false),
1032
+ (r"(?m:^)b", "a\rb", true),
1033
+ (r"(?m:^)b", "a\u{2029}b", true),
1034
+ (r"^(?:(?<a>x)|(?<a>y))\k<a>$", "yy", true),
1035
+ (r"^(?:(?<a>x)|(?<a>y))\k<a>$", "xy", false),
1036
+ (r"^(?:(?<a>x)|(?<a>y))+\k<a>$", "xyy", true),
1037
+ (r"^(?:(?<a>x)|(?<a>y))+\k<a>$", "xyx", false),
1038
+ (r"^(?<\u0061b>.)\k<ab>$", "zz", true),
1039
+ ] {
1040
+ let re = compile(pattern).unwrap();
1041
+ assert_eq!(re.is_match(text), want, "{pattern:?} on {text:?}");
1042
+ assert_eq!(re.match_backtrack(text, None), Ok(want), "{pattern:?} on {text:?} (the backtracking matcher)");
1043
+ }
1044
+ for p in [
1045
+ r"(?i)a",
1046
+ r"(?-:a)",
1047
+ r"(?ii:a)",
1048
+ r"(?i-i:a)",
1049
+ r"(?x:a)",
1050
+ r"(?i-m-s:a)",
1051
+ r"(?<a>x)(?<a>y)",
1052
+ r"(?<a>x|(?<a>y))",
1053
+ r"(?:(?<a>x)|b)(?:(?<a>y)|c)",
1054
+ r"(?<a\u0020>x)",
1055
+ r"(?<\u0030>x)",
1056
+ ] {
1057
+ assert!(!valid(p), "valid({p:?})");
1058
+ assert!(compile(p).is_err(), "compile({p:?}) gives no error");
1059
+ }
1060
+ }
1061
+
1062
+ #[test]
1063
+ fn properties() {
1064
+ let mut expressions = 0;
1065
+ let mut resolve = |expr: &str| {
1066
+ expressions += 1;
1067
+ assert!(property_set(expr).is_some(), "\\p{{{expr}}} does not resolve");
1068
+ };
1069
+ for name in ucd::binary_names() {
1070
+ resolve(name);
1071
+ }
1072
+ for name in
1073
+ ["ASCII", "Any", "Assigned", "Alpha", "AHex", "space", "XIDS", "L", "Letter", "LC", "digit", "punct", "Cn"]
1074
+ {
1075
+ resolve(name);
1076
+ }
1077
+ for name in
1078
+ ["L", "Lu", "Uppercase_Letter", "Cased_Letter", "Mark", "Combining_Mark", "digit", "punct", "cntrl", "Cn"]
1079
+ {
1080
+ resolve(&format!("gc={name}"));
1081
+ resolve(&format!("General_Category={name}"));
1082
+ }
1083
+ for names in ucd::script_names() {
1084
+ for value in *names {
1085
+ for property in ["sc", "Script", "scx", "Script_Extensions"] {
1086
+ resolve(&format!("{property}={value}"));
1087
+ }
1088
+ }
1089
+ }
1090
+ for value in ["Unknown", "Zzzz", "Katakana_Or_Hiragana", "Hrkt"] {
1091
+ resolve(&format!("sc={value}"));
1092
+ resolve(&format!("scx={value}"));
1093
+ }
1094
+ assert!(expressions > 1000, "{expressions} property expressions");
1095
+ for (pattern, text, want) in [
1096
+ (r"^\p{L}$", "\u{e9}", true),
1097
+ (r"^\p{L}$", "1", false),
1098
+ (r"^\p{Lu}$", "\u{c9}", true),
1099
+ (r"^\P{Lu}$", "\u{c9}", false),
1100
+ (r"^\p{Nd}$", "\u{9ea}", true),
1101
+ (r"^\p{digit}$", "\u{9ea}", true),
1102
+ (r"^\p{sc=Grek}$", "\u{3b1}", true),
1103
+ (r"^\p{Script=Greek}$", "a", false),
1104
+ (r"^\p{sc=Grek}$", "\u{342}", false),
1105
+ (r"^\p{scx=Grek}$", "\u{342}", true),
1106
+ (r"^\p{sc=Zinh}$", "\u{342}", true),
1107
+ (r"^\p{Emoji}$", "\u{1F432}", true),
1108
+ (r"^\p{Emoji_Presentation}$", "#", false),
1109
+ (r"^\p{Emoji}$", "#", true),
1110
+ (r"^\p{Alphabetic}$", "\u{2177}", true),
1111
+ (r"^\p{Uppercase}$", "\u{24b6}", true),
1112
+ (r"^\p{ID_Start}$", "_", false),
1113
+ (r"^\p{ID_Continue}$", "_", true),
1114
+ (r"^\p{White_Space}$", "\u{85}", true),
1115
+ (r"^\s$", "\u{85}", false),
1116
+ (r"^\p{White_Space}$", "\u{feff}", false),
1117
+ (r"^\s$", "\u{feff}", true),
1118
+ (r"^\p{Any}$", "\u{10ffff}", true),
1119
+ (r"^\p{Assigned}$", "\u{10ffff}", false),
1120
+ (r"^\p{Script=Unknown}$", "\u{e0080}", true),
1121
+ (r"^\p{ASCII}$", "\u{e9}", false),
1122
+ (r"^[\p{Lu}\p{Nd}]+$", "A1\u{c9}\u{9ea}", true),
1123
+ (r"^[^\p{L}]+$", "1 -", true),
1124
+ (r"^[^\p{L}]+$", "1a", false),
1125
+ ] {
1126
+ let re = compile(pattern).unwrap();
1127
+ assert_eq!(re.is_match(text), want, "{pattern:?} on {text:?}");
1128
+ }
1129
+ for p in
1130
+ [r"\p{Nope}", r"\p{L=x}", r"\p{gc=Greek}", r"\p{sc=L}", r"\p{}", r"\p{Lu", r"\pL", r"\p{sc=}", r"\p{ascii}"]
1131
+ {
1132
+ assert!(!valid(p), "valid({p:?})");
1133
+ }
1134
+ }
1135
+
1136
+ #[test]
1137
+ fn automaton() {
1138
+ for (p, want) in [
1139
+ (r"^[a-z][a-z0-9_]*$", true),
1140
+ (r"(base64key|awskms)://(.*)", true),
1141
+ (r"^[Ee][Ss]20(1[5-9]|2[0-2])(\.[a-z]+)?$", true),
1142
+ (r"^$", true),
1143
+ (r"$", true),
1144
+ (r"a*", true),
1145
+ (r"^(?:a?){40}$", true),
1146
+ ("\u{e9}+", true),
1147
+ (r"\bfoo", false),
1148
+ (r"^a{600}$", false),
1149
+ (r"(?:a{50}){12}", false),
1150
+ (r"^(?:[a-z]{1,3}\d?){2,90}$", false),
1151
+ (r"(?=a)b", false),
1152
+ (r"(a)\1", false),
1153
+ ] {
1154
+ let re = compile(p).unwrap();
1155
+ re.is_match("");
1156
+ assert_eq!(matches!(re.dfa.get(), Some(Some(_))), want, "{p:?}: automaton");
1157
+ }
1158
+ for (p, s, want) in [
1159
+ (r"^$", "", true),
1160
+ (r"^$", "a", false),
1161
+ (r"$", "abc", true),
1162
+ (r"^", "abc", true),
1163
+ (r"$^", "", true),
1164
+ (r"$^", "a", false),
1165
+ (r"a$", "ba", true),
1166
+ (r"a$", "ab", false),
1167
+ (r"^a", "ab", true),
1168
+ (r"^a", "ba", false),
1169
+ (r"b", "aaab", true),
1170
+ (r"^(a|ab)(c|bcd)$", "abcd", true),
1171
+ (r"^(a|ab)(c|bcd)$", "abc", true),
1172
+ (r"^(a|ab)(c|bcd)$", "abcc", false),
1173
+ (r"x{2,3}", "axxb", true),
1174
+ (r"^x{2,3}$", "xxxx", false),
1175
+ (r"^.$", "\n", false),
1176
+ (r"^[^a]$", "\n", true),
1177
+ ("\u{e9}", "caf", false),
1178
+ (r"^a*$", "", true),
1179
+ (r"^(a$|b)c?$", "a", true),
1180
+ (r"^(a$|b)c?$", "ac", false),
1181
+ ] {
1182
+ let re = compile(p).unwrap();
1183
+ assert_eq!(re.is_match(s), want, "{p:?} on {s:?}");
1184
+ assert_eq!(re.match_backtrack(s, None), Ok(want), "{p:?} on {s:?} (the backtracking matcher)");
1185
+ let Some(Some(dfa)) = re.dfa.get() else { panic!("{p:?}: no automaton") };
1186
+ assert_eq!(dfa.is_match(s.as_bytes()), Some(want), "{p:?} on {s:?}: the automaton");
1187
+ }
1188
+ let re = compile(r"^[a-z]+$").unwrap();
1189
+ assert!(re.is_match("abc") && !re.is_match("caf\u{e9}"));
1190
+ assert_eq!(re.dfa.get().unwrap().as_ref().unwrap().is_match("caf\u{e9}".as_bytes()), None);
1191
+ let re = compile(r"^\p{L}+$").unwrap();
1192
+ assert!(re.is_match("caf\u{e9}") && !re.is_match("caf\u{e9} 1") && re.is_match("cafe"));
1193
+ }
1194
+
1195
+ // ---------------------------------------------------------------------------------------------------------------------
1196
+ // The limit of a match.
1197
+
1198
+ /// Patterns and texts whose match takes longer than any test waits: a group read back inside a repetition, which
1199
+ /// the memory of failed places does not cover.
1200
+ fn forever() -> Vec<(&'static str, String)> {
1201
+ vec![
1202
+ (r"^(?:(a+)\1*)+$", "a".repeat(60) + "b"),
1203
+ (r"(a*)*\1b", "a".repeat(60)),
1204
+ (r"^(?:(\w+)\s?\1?)+$", "ab ".repeat(30) + "!"),
1205
+ (r"(?=(?:(a+)\1*)+b)a", "a".repeat(60)),
1206
+ ]
1207
+ }
1208
+
1209
+ #[test]
1210
+ fn a_match_is_abandoned_at_its_limit() {
1211
+ for limit in [Duration::from_millis(20), Duration::from_millis(300)] {
1212
+ for (pattern, text) in forever() {
1213
+ let re = compile(pattern).unwrap();
1214
+ let start = Instant::now();
1215
+ let result = re.match_limit(&text, Some(limit));
1216
+ let took = start.elapsed();
1217
+ assert_eq!(result, Err(MatchError::Time), "{pattern:?} with a limit of {limit:?}");
1218
+ assert!(
1219
+ took >= limit && took < limit + GENEROUS,
1220
+ "{pattern:?} with {limit:?} was abandoned after {took:?}"
1221
+ );
1222
+ }
1223
+ }
1224
+ }
1225
+
1226
+ #[test]
1227
+ fn a_match_with_no_limit_is_not_abandoned() {
1228
+ let re = compile(r"^(?:(a+)\1*)+$").unwrap();
1229
+ let text = "a".repeat(14) + "b";
1230
+ assert_eq!(re.match_limit(&text, Some(Duration::from_nanos(1))), Err(MatchError::Time));
1231
+ assert_eq!(re.match_limit(&text, None), Ok(false));
1232
+ // A limit of zero is no limit.
1233
+ assert_eq!(re.match_limit(&text, Some(Duration::ZERO)), Ok(false));
1234
+ assert!(!re.is_match(&text));
1235
+ assert_eq!(re.match_limit(&text[..14], Some(GENEROUS)), Ok(true));
1236
+ }
1237
+
1238
+ #[test]
1239
+ fn a_match_is_abandoned_at_its_memory_limit() {
1240
+ for (pattern, text) in [
1241
+ (r"(?:a?){2147483647}", String::new()),
1242
+ (r"(?:a?){100000000}", "a".repeat(100)),
1243
+ (r"(?:(?:(?:a?){1000}){1000}){1000}", "a".repeat(20)),
1244
+ (r"(?:(a)|b|){2147483647}\1", String::new()),
1245
+ ] {
1246
+ let re = compile(pattern).unwrap();
1247
+ let start = Instant::now();
1248
+ let result = re.match_limit(&text, Some(Duration::from_secs(3600)));
1249
+ assert_eq!(result, Err(MatchError::Memory), "{pattern:?}");
1250
+ assert!(start.elapsed() < GENEROUS, "{pattern:?} was abandoned after {:?}", start.elapsed());
1251
+ }
1252
+ }
1253
+
1254
+ #[test]
1255
+ fn a_limit_does_not_change_an_answer() {
1256
+ let long = "ab".repeat(50000);
1257
+ let accented = format!("{long}\u{e9}");
1258
+ for (pattern, text, want) in [
1259
+ (r"^[a-z]+$", "abc", true),
1260
+ (r"^[a-z]+$", "abc1", false),
1261
+ (r"^\p{L}+$", "caf\u{e9}", true),
1262
+ (r"^\p{L}+$", "caf\u{e9} ", false),
1263
+ (r"^(?=.*\d)\w+$", "abc1", true),
1264
+ (r"^(?=.*\d)\w+$", "abcd", false),
1265
+ (r"^(a+)\1$", "aaaa", true),
1266
+ (r"^(a+)\1$", "aaa", false),
1267
+ (r"(?<=\$)\d+", "cost $42", true),
1268
+ (r"^(?=.)(?:a|b)*$", &long, true),
1269
+ (r"^(?=.)(?:a|b)*c$", &long, false),
1270
+ (r"^[ab]*$", &long, true),
1271
+ (r"^[ab]*c$", &long, false),
1272
+ ("^[ab\u{e9}]*$", &accented, true),
1273
+ ("^[ab\u{e9}]*c$", &accented, false),
1274
+ ] {
1275
+ let re = compile(pattern).unwrap();
1276
+ for limit in [None, Some(Duration::from_secs(60))] {
1277
+ assert_eq!(re.match_limit(text, limit), Ok(want), "{pattern:?} with a limit of {limit:?}");
1278
+ assert_eq!(re.match_backtrack(text, limit), Ok(want), "{pattern:?} with a limit of {limit:?}");
1279
+ }
1280
+ }
1281
+ }
1282
+
1283
+ // ---------------------------------------------------------------------------------------------------------------------
1284
+ // The memory of failed places.
1285
+
1286
+ struct MemoFromTheStart;
1287
+
1288
+ impl MemoFromTheStart {
1289
+ fn on() -> MemoFromTheStart {
1290
+ set_memo_from_the_start(true);
1291
+ MemoFromTheStart
1292
+ }
1293
+ }
1294
+
1295
+ impl Drop for MemoFromTheStart {
1296
+ fn drop(&mut self) {
1297
+ set_memo_from_the_start(false);
1298
+ }
1299
+ }
1300
+
1301
+ /// The patterns of the memo set against V8, and then every set and the hand-written patterns of the oracle with the
1302
+ /// memory in use from the first step.
1303
+ #[test]
1304
+ fn failed_places_are_remembered() {
1305
+ let memo = cases("memo");
1306
+ let c = check_cases(&memo, Duration::from_secs(60));
1307
+ let with_points = memo.iter().filter(|m| !program(&m.p).points.is_empty()).count();
1308
+ assert!(memo.len() >= 50 && with_points >= 45 && c.matched >= 500, "{with_points} with memo points");
1309
+ let _memo = MemoFromTheStart::on();
1310
+ let mut total = 0;
1311
+ for set in ["memo", "quick", "loops", "needs", "units", "fast"] {
1312
+ total += check_cases(&cases(set), Duration::from_secs(120)).answers;
1313
+ }
1314
+ let o = oracle();
1315
+ for (p, u, l, m) in &o.hand {
1316
+ let Some(re) = check_verdict(p, *u, *l) else { continue };
1317
+ if !V8_MODIFIER_BUGS.contains(&p.as_str()) {
1318
+ total += check_answers(&re, &o.inputs, m);
1319
+ }
1320
+ }
1321
+ assert!(total > 60000, "{total} answers with the memory in use from the first step");
1322
+ }
1323
+
1324
+ #[test]
1325
+ fn a_repetition_in_a_repetition_is_answered() {
1326
+ let a = "a".repeat(60);
1327
+ let ab = "ab".repeat(30);
1328
+ for (pattern, text, want) in [
1329
+ (r"(?=a)(a*)*b", a.clone(), false),
1330
+ (r"(?=a)(a*)*b", a.clone() + "b", true),
1331
+ (r"^(?=.)(?:a+)+$", a.clone() + "x", false),
1332
+ (r"^(?=.)(?:a+)+$", a.clone(), true),
1333
+ (r"^(?=.)(?:a|aa)+$", a.clone() + "b", false),
1334
+ (r"^(?=.)(?:a|aa)+$", a.clone(), true),
1335
+ (r"^(?=.)(?:a?b?)*x", ab.clone(), false),
1336
+ (r"^(?=.)(?:a?b?)*x", ab.clone() + "x", true),
1337
+ (r"^(?=.)(?:(?:a?b?)*x?)*$", ab.clone() + "!", false),
1338
+ (r"^(?=.)(?:(?:a*)+b?)+$", a.clone() + "!", false),
1339
+ (r"^(?=.)(?:(?:a*)+b?)+$", a.clone() + "b", true),
1340
+ (r"^(?=.)(?:(?:a?)+?b?)*?x", a.clone(), false),
1341
+ (r"(?=.)(?:a*?b*?)*?x$", a.clone() + "b", false),
1342
+ (r"(?i:\b(?:\w+\W?)+\b!)", "ab ".repeat(40), false),
1343
+ (r"(?i:\b(?:\w+\W?)+\b!)", "ab ".repeat(40) + "ab!", true),
1344
+ (r"(?m:^(?:\w+\s?)+$)", "a ".repeat(30) + "!", false),
1345
+ (r"^(?!(?:a+)+x)a*b", a.clone() + "b", true),
1346
+ (r"^(?!(?:a+)+x)a*b", a.clone(), false),
1347
+ (r"(?<!(?:a|aa)+b)x$", a.clone() + "x", true),
1348
+ (r"(?=.).*.*.*x", "a".repeat(3000), false),
1349
+ (r"(?=.)a*?a*?a*?b", "a".repeat(3000), false),
1350
+ (r"(?=.)(?:a|b)*c", "ab".repeat(50000), false),
1351
+ (r"(?=.)(?:a|\u00e9)*c", "a\u{e9}".repeat(30000), false),
1352
+ ] {
1353
+ let re = compile(pattern).unwrap();
1354
+ assert!(!re.regular && !re.prog.points.is_empty(), "{pattern:?} has {} memo points", re.prog.points.len());
1355
+ for limit in [Some(GENEROUS), None] {
1356
+ assert_eq!(re.match_limit(&text, limit), Ok(want), "{pattern:?} on {} characters, {limit:?}", text.len());
1357
+ }
1358
+ }
1359
+ }
1360
+
1361
+ #[test]
1362
+ fn a_backreference_turns_the_memory_off() {
1363
+ for pattern in [r"(a*)*\1b", r"^(?:(a+)\1*)+$", r"(a)(?:b|c)*\1", r"(?<n>a)(?:b+)+\k<n>"] {
1364
+ assert!(program(pattern).points.is_empty(), "{pattern:?} has memo points");
1365
+ }
1366
+ let re = compile(r"(a*)*\1b").unwrap();
1367
+ assert_eq!(re.match_limit(&"a".repeat(60), Some(Duration::from_millis(50))), Err(MatchError::Time));
1368
+ }
1369
+
1370
+ #[test]
1371
+ fn memo_points() {
1372
+ let table: &[(&str, &[&str])] = &[
1373
+ (r"(?:a|b)*c", &["split", "split"]),
1374
+ (r"(?:a?b?)*c", &["loop", "rep 0", "rep 0"]),
1375
+ (r"(?:(?:a?)*b)*", &["split", "loop", "rep 0"]),
1376
+ (r"(?:(?:a?)+b?)+x", &["loop", "loop 0", "rep 0 1", "rep 0"]),
1377
+ (r"(?:(?:(?:a?)*b?)*x?)*", &["loop", "loop 0", "loop 0 1", "rep 0 1", "rep 0"]),
1378
+ (r"(?:a|){2,3}(?:b|c)", &["split"]),
1379
+ (r"(?:(?:a|b){2}x?)*", &["split", "rep"]),
1380
+ (r"(?:(?:a?b?)*?(?:c|d)){3}", &[]),
1381
+ (r"(?:a?(?=(?:b|c)*x))*", &["loop", "rep 0", "split", "split"]),
1382
+ (r"(?:(?:a|){2}(?=b|c))*", &["loop", "split"]),
1383
+ (r"(?<=(?:a?b?)*)x", &["loop", "split 0", "split 0"]),
1384
+ (r"a*b*?c{2,3}", &["rep", "lazy", "rep"]),
1385
+ (r"(?:a+)+$", &["rep", "split"]),
1386
+ (r"(a)\1(?:b|c)", &[]),
1387
+ ];
1388
+ for &(pattern, want) in table {
1389
+ let p = program(pattern);
1390
+ let mut points = Vec::new();
1391
+ for i in &p.insts {
1392
+ if i.m == 0 {
1393
+ continue;
1394
+ }
1395
+ let mut s = match i.op {
1396
+ Op::Split => "split",
1397
+ Op::Loop => "loop",
1398
+ Op::Rep => "rep",
1399
+ Op::RepLazy => "lazy",
1400
+ _ => "?",
1401
+ }
1402
+ .to_string();
1403
+ for l in p.points[(i.m - 1) as usize].loops {
1404
+ if l >= 0 {
1405
+ s += &format!(" {l}");
1406
+ }
1407
+ }
1408
+ points.push(s);
1409
+ }
1410
+ assert_eq!(points, want, "{pattern:?}");
1411
+ assert_eq!(p.points.len(), want.len(), "{pattern:?}");
1412
+ }
1413
+ assert_eq!(MAX_MEMO_LOOPS, 2);
1414
+ }
1415
+
1416
+ /// A match too large for the memory goes without it, and ends in an answer or at its limit.
1417
+ #[test]
1418
+ fn a_match_too_large_for_the_memory_goes_without_it() {
1419
+ let alternatives: Vec<String> = (0..400).map(|k| format!("a{}", "b".repeat(k % 7))).collect();
1420
+ let re = compile(&format!("^(?=.)(?:{})+$", alternatives.join("|"))).unwrap();
1421
+ let result = re.match_limit(&("ab".repeat(50000) + "x"), Some(Duration::from_millis(100)));
1422
+ assert!(matches!(result, Ok(false) | Err(MatchError::Time)), "{result:?}");
1423
+ }
1424
+
1425
+ // ---------------------------------------------------------------------------------------------------------------------
1426
+ // The limits of size.
1427
+
1428
+ fn ranges_of(name: &str) -> usize {
1429
+ property_set(name).unwrap().ranges.len() / 2
1430
+ }
1431
+
1432
+ /// Patterns just under the limit of size, of every shape that is costly for its size.
1433
+ fn at_the_limit() -> Vec<(&'static str, String)> {
1434
+ let fit = |each: usize| (MAX_SIZE - 4096) / each;
1435
+ let l_ranges = ranges_of("L");
1436
+ let nest = |open: &str, body: &str, close: &str, depth: usize| open.repeat(depth) + body + &close.repeat(depth);
1437
+ let named: Vec<String> = (0..400).map(|k| format!("(?<n>{k})")).collect();
1438
+ vec![
1439
+ ("literal", "a".repeat(fit(2 * NODE_SIZE))),
1440
+ ("alternatives", "a|".repeat(fit(3 * NODE_SIZE)).trim_end_matches('|').to_string()),
1441
+ ("groups", "(a)".repeat(fit(4 * NODE_SIZE))),
1442
+ ("groups read back", "(a)".repeat(4000) + &r"\1".repeat(4000) + &r"\4000".repeat(300)),
1443
+ ("nested groups", nest("(", &"a".repeat(fit(2 * NODE_SIZE) - 2 * MAX_NESTING), ")", MAX_NESTING - 1)),
1444
+ ("nested counted loops", nest("(?:", &"a?b".repeat(2000), "){2,3}", MAX_NESTING - 1)),
1445
+ ("nested lookbehinds", nest("(?<=", &"a".repeat(2000), ")", MAX_NESTING - 1)),
1446
+ ("counted loops", "(?:a|){2}".repeat(fit(7 * NODE_SIZE))),
1447
+ ("lazy loops", "(?:a|b)*?".repeat(fit(7 * NODE_SIZE))),
1448
+ ("properties", r"\p{L}".repeat(fit(2 * NODE_SIZE))),
1449
+ ("negated properties", r"\P{L}".repeat(fit(2 * NODE_SIZE + RANGE_SIZE * (l_ranges + 1)))),
1450
+ ("classes of properties", r"[\p{L}\p{N}]".repeat(fit(2 * NODE_SIZE + RANGE_SIZE * (l_ranges + 200)))),
1451
+ ("folded negated classes", format!("(?i:{})", "[^a]".repeat(fit(2 * NODE_SIZE + 32)))),
1452
+ ("folded dots", format!("(?i:{})", ".".repeat(fit(2 * NODE_SIZE)))),
1453
+ ("folded wide classes", format!("(?i:{})", r"[\u0000-\u{10FFFF}]".repeat(fit(3200)))),
1454
+ ("folded wide negations", format!("(?i:{})", r"[^\u0100-\u{10FFFF}]".repeat(fit(3300)))),
1455
+ ("folded properties", format!("(?i:{})", r"\p{L}".repeat(fit(8600)))),
1456
+ ("folded literal", format!("(?i:{})", "k".repeat(fit(2 * NODE_SIZE)))),
1457
+ ("one large class", format!("[{}]", "\u{100}\u{102}".repeat(fit(2 * RANGE_SIZE)))),
1458
+ ("groups of one name", named.join("|")),
1459
+ ("named groups, nested", nest("(?:x|", &"(?<a>b)|(?<c>d)".repeat(40), ")", 600)),
1460
+ ("counted, for regexp", "(?:a?){1000}".repeat(fit(4 * NODE_SIZE))),
1461
+ ("quantified everything", "a{2,5}?".repeat(fit(2 * NODE_SIZE))),
1462
+ ("long escapes", r"\u{0000000000000000000000000000000000000000000041}".repeat(fit(2 * NODE_SIZE))),
1463
+ ]
1464
+ }
1465
+
1466
+ /// A pattern at the limit of size compiles, and matches for the first time, in a time far below the one a test
1467
+ /// waits. The thread has the stack of a thread the standard library starts, so the deepest nesting the limits allow
1468
+ /// is also shown not to overflow it.
1469
+ #[test]
1470
+ fn a_pattern_at_the_limit_of_size_is_cheap() {
1471
+ for (name, pattern) in at_the_limit() {
1472
+ let start = Instant::now();
1473
+ let re = compile(&pattern).unwrap_or_else(|e| panic!("{name} ({} bytes): {e}", pattern.len()));
1474
+ let compiled = start.elapsed();
1475
+ let start = Instant::now();
1476
+ let result = re.match_limit("abc", Some(Duration::from_secs(60)));
1477
+ let matched = start.elapsed();
1478
+ eprintln!("{name:<24} {:>7} bytes: compile {compiled:?}, first match {matched:?} ({result:?})", pattern.len());
1479
+ assert!(
1480
+ compiled < GENEROUS && matched < GENEROUS,
1481
+ "{name} took {compiled:?} to compile and {matched:?} to match"
1482
+ );
1483
+ }
1484
+ }
1485
+
1486
+ /// The deepest nesting the limits allow is read, compiled, matched and dropped by a thread with a small stack: the
1487
+ /// work that goes a level deeper for each level of groups is done on a stack of its own.
1488
+ #[test]
1489
+ fn the_deepest_nesting_does_not_overflow_a_small_stack() {
1490
+ let small = std::thread::Builder::new().stack_size(128 << 10);
1491
+ let test = small.spawn(|| {
1492
+ let nest =
1493
+ |open: &str, body: &str, close: &str| open.repeat(MAX_NESTING - 1) + body + &close.repeat(MAX_NESTING - 1);
1494
+ for (pattern, text, want) in [
1495
+ (nest("(", "a", ")"), "a", true),
1496
+ (nest("(?:", "a?b", "){1,2}"), "ab", true),
1497
+ (nest("(?=", "a", ")") + "b", "a", false),
1498
+ (nest("(?:a|", "b", ")"), "b", true),
1499
+ (nest("(?:x|", "a", ")?") + "y$", "xxay", true),
1500
+ ] {
1501
+ assert!(valid(&pattern), "{}", short(&pattern));
1502
+ assert!(check(&pattern).is_ok());
1503
+ let re = compile(&pattern).unwrap();
1504
+ assert_eq!(re.match_limit(text, Some(GENEROUS)), Ok(want), "{}", short(&pattern));
1505
+ assert_eq!(re.match_backtrack(text, Some(GENEROUS)), Ok(want), "{}", short(&pattern));
1506
+ }
1507
+ // One level more is the limit of nesting, and that many open groups with nothing to close them is an error.
1508
+ assert!(matches!(compile(&nest("((", "a", "))")), Err(Error::TooLarge(_))));
1509
+ assert!(matches!(compile(&"(".repeat(900)), Err(Error::Invalid(_))));
1510
+ assert!(!valid(&"(".repeat(100000)));
1511
+ });
1512
+ test.unwrap().join().unwrap();
1513
+ }
1514
+
1515
+ #[test]
1516
+ fn a_pattern_over_a_limit_of_size_is_too_large() {
1517
+ let nest = |depth: usize| "(?:".repeat(depth) + "a" + &")".repeat(depth);
1518
+ let terms = MAX_SIZE / (2 * NODE_SIZE);
1519
+ let escape = |zeros: usize| format!("\\u{{{}41}}", "0".repeat(zeros));
1520
+ for (name, at, over, limit, value) in [
1521
+ ("length", escape(MAX_PATTERN_LENGTH - 6), escape(MAX_PATTERN_LENGTH - 5), Limit::Length, MAX_PATTERN_LENGTH),
1522
+ ("nesting", nest(MAX_NESTING - 1), nest(MAX_NESTING), Limit::Nesting, MAX_NESTING),
1523
+ ("size of terms", "a".repeat(terms), "a".repeat(terms + 1), Limit::Size, MAX_SIZE),
1524
+ (
1525
+ "size of classes",
1526
+ format!("[{}]", "\u{100}\u{102}".repeat(60000)),
1527
+ format!("[{}]", "\u{100}\u{102}".repeat(70000)),
1528
+ Limit::Size,
1529
+ MAX_SIZE,
1530
+ ),
1531
+ ("size without the u flag", r"\-".repeat(terms), r"\-".repeat(terms + 1), Limit::Size, MAX_SIZE),
1532
+ (
1533
+ "nesting without the u flag",
1534
+ format!(r"\-{}", nest(MAX_NESTING - 1)),
1535
+ format!(r"\-{}", nest(MAX_NESTING)),
1536
+ Limit::Nesting,
1537
+ MAX_NESTING,
1538
+ ),
1539
+ ("nesting, and not valid", nest(MAX_NESTING - 1) + "x", nest(MAX_NESTING) + "*+)", Limit::Nesting, MAX_NESTING),
1540
+ ("size, and not valid", "a".repeat(terms), "a".repeat(terms + 1) + "(", Limit::Size, MAX_SIZE),
1541
+ ] {
1542
+ assert!(compile(&at).is_ok(), "{name}: the pattern at the limit ({} bytes)", at.len());
1543
+ assert!(check(&at).is_ok(), "{name}: check of the pattern at the limit");
1544
+ for result in [compile(&over).map(|_| ()), check(&over).map(|_| ())] {
1545
+ match result {
1546
+ Err(Error::TooLarge(large)) => {
1547
+ assert_eq!((large.limit, large.value), (limit, value), "{name}");
1548
+ assert_eq!(large.limit.name(), ["length", "nesting", "size"][limit as usize]);
1549
+ assert!(large.to_string().contains(&value.to_string()), "{large}");
1550
+ }
1551
+ other => panic!("{name}: the pattern over the limit ({} bytes): {other:?}", over.len()),
1552
+ }
1553
+ }
1554
+ assert!(!valid(&over), "{name}: valid of the pattern over the limit");
1555
+ }
1556
+ for pattern in ["(".to_string(), "a**".into(), r"(?<n>a)\k<m>".into(), "[b-a]".into(), nest(MAX_NESTING - 1) + ")"]
1557
+ {
1558
+ assert!(matches!(compile(&pattern), Err(Error::Invalid(_))), "compile({})", short(&pattern));
1559
+ assert!(matches!(check(&pattern), Err(Error::Invalid(_))), "check({})", short(&pattern));
1560
+ }
1561
+ }
1562
+
1563
+ /// The needs settle in two passes over the instructions and the next characters in three, however deep the loops
1564
+ /// nest. A pass for each loop of a nest would be a thousand passes for loops a thousand deep.
1565
+ #[test]
1566
+ fn the_needs_settle_in_few_passes() {
1567
+ deep(needs_settle);
1568
+ }
1569
+
1570
+ fn needs_settle() {
1571
+ let nest = |open: &str, body: &str, close: &str, depth: usize| open.repeat(depth) + body + &close.repeat(depth);
1572
+ for pattern in [
1573
+ "abc".to_string(),
1574
+ "(?:a|b)*c".into(),
1575
+ "(?:a?b?)*c".into(),
1576
+ "(?:(?:a+)+b?)*".into(),
1577
+ nest("(?:", "a?b", "){2,3}", 999),
1578
+ nest("(?:", "a?b", ")*", 999),
1579
+ nest("(?:", "a?b?", ")*", 999),
1580
+ nest("(?:", "a?b", ")+?", 999),
1581
+ nest("(?:x|", "a?b", ")*y?", 500),
1582
+ nest("(?:(?=", "a", ")b?)*", 400),
1583
+ nest("(?:", "a|b", "){0,5}?c", 999),
1584
+ ] {
1585
+ let mut p = program(&pattern);
1586
+ let (_, need_passes) = needs(&mut p.insts);
1587
+ let (_, quick_passes) = quicks(&p.insts);
1588
+ assert!(need_passes <= 2 && quick_passes <= 3, "{}: {need_passes} and {quick_passes} passes", short(&pattern));
1589
+ }
1590
+ }
1591
+
1592
+ // ---------------------------------------------------------------------------------------------------------------------
1593
+ // The tables.
1594
+
1595
+ /// The tables of `tables_gen.rs` are those of the Go module: every table of `tables_gen.go`, by its name, has the
1596
+ /// same ranges here, and the two case mappings have the same rows. `tools/gen-ucd-tables.ps1` writes one file from
1597
+ /// the other, and this is what fails when one of them is generated again and the other is not.
1598
+ #[test]
1599
+ fn tables_are_those_of_the_go_module() {
1600
+ let go = std::fs::read_to_string(go_file("ucd/tables_gen.go")).unwrap();
1601
+ assert!(go.contains(&format!("const Version = \"{}\"", ucd::VERSION)));
1602
+ let number = |s: &str| -> i64 {
1603
+ let s = s.trim();
1604
+ match s.strip_prefix("0x") {
1605
+ Some(hex) => i64::from_str_radix(hex, 16).unwrap(),
1606
+ None => s.parse().unwrap(),
1607
+ }
1608
+ };
1609
+ // The tables, by the name of the Go variable.
1610
+ let mut tables: HashMap<&str, Vec<Rune>> = HashMap::new();
1611
+ let mut cases: HashMap<&str, Vec<[i64; 4]>> = HashMap::new();
1612
+ let mut lookups: Vec<(&str, Vec<&str>, &str)> = Vec::new();
1613
+ let (mut table, mut lookup, mut names) = ("", "", Vec::new());
1614
+ for line in go.lines() {
1615
+ if let Some(rest) = line.strip_prefix("var ") {
1616
+ table = rest.split(' ').next().unwrap();
1617
+ } else if let Some(rest) = line.strip_prefix("func ") {
1618
+ lookup = rest.split('(').next().unwrap();
1619
+ } else if let Some(rest) = line.strip_prefix("\tcase ") {
1620
+ names = rest.trim_end_matches(':').split(", ").map(|n| n.trim_matches('"')).collect();
1621
+ } else if let Some(rest) = line.strip_prefix("\t\treturn &") {
1622
+ lookups.push((lookup, std::mem::take(&mut names), rest));
1623
+ } else if let Some(row) = line.strip_prefix("\t{").and_then(|r| r.strip_suffix("},"))
1624
+ && (table == "caseFolds" || table == "upperCases")
1625
+ {
1626
+ let row: Vec<i64> = row.split(',').map(number).collect();
1627
+ cases.entry(table).or_default().push([row[0], row[1], row[2], row[3]]);
1628
+ } else if line.starts_with("\t\t0x") {
1629
+ tables
1630
+ .entry(table)
1631
+ .or_default()
1632
+ .extend(line.split(',').filter(|s| !s.trim().is_empty()).map(|s| number(s) as Rune));
1633
+ }
1634
+ }
1635
+ assert!(tables.len() > 300 && lookups.len() > 400, "{} tables, {} lookups", tables.len(), lookups.len());
1636
+ for (lookup, names, table) in lookups {
1637
+ // The ranges of the Go table, the part up to U+FFFF and the part beyond it joined where they touch.
1638
+ let mut want: Vec<Rune> = Vec::new();
1639
+ for &[lo, hi] in tables.get(table).map_or(&[][..], |t| &t[..]).as_chunks::<2>().0 {
1640
+ match want.last_mut() {
1641
+ Some(last) if *last == 0xFFFF && lo == 0x10000 => *last = hi,
1642
+ _ => want.extend_from_slice(&[lo, hi]),
1643
+ }
1644
+ }
1645
+ for name in names {
1646
+ let found = match lookup {
1647
+ "categoryTable" => ucd::category(name),
1648
+ "binaryTable" => ucd::binary(name),
1649
+ "scriptTable" => ucd::script(name),
1650
+ "scriptExtensionsTable" => ucd::script_extensions(name),
1651
+ other => panic!("an unknown lookup {other}"),
1652
+ };
1653
+ assert_eq!(
1654
+ found.map(|t| t.pairs()),
1655
+ Some(want.clone()),
1656
+ "{lookup}({name:?}) is not {table} of the Go module"
1657
+ );
1658
+ }
1659
+ }
1660
+ // The case mappings, by what they do to every code point.
1661
+ let map = |rows: &[[i64; 4]], r: i64| -> i64 {
1662
+ let row = rows.iter().rev().find(|row| row[0] <= r);
1663
+ match row {
1664
+ Some(row) if r - row[0] < row[2] && (r - row[0]) & row[3] == 0 => r + row[1],
1665
+ _ => r,
1666
+ }
1667
+ };
1668
+ for r in 0x80..=ucd::MAX_RUNE {
1669
+ assert_eq!(ucd::fold(r) as i64, map(&cases["caseFolds"], r as i64), "the folding of U+{r:04X}");
1670
+ assert_eq!(ucd::to_upper(r) as i64, map(&cases["upperCases"], r as i64), "the uppercase of U+{r:04X}");
1671
+ }
1672
+ }