corvus_json_schema 0.1.3 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +2 -12
  3. data/README.md +10 -1
  4. data/THIRD-PARTY-NOTICES.txt +95 -0
  5. data/VERSIONHISTORY.md +24 -0
  6. data/ext/corvus_json_schema/Cargo.toml +2 -2
  7. data/ext/corvus_json_schema/clippy.toml +3 -4
  8. data/ext/corvus_json_schema/src/lib.rs +42 -19
  9. data/ext/corvus_json_schema/vendor/corvus-json-schema/Cargo.toml +9 -4
  10. data/ext/corvus_json_schema/vendor/corvus-json-schema/README.md +87 -4
  11. data/ext/corvus_json_schema/vendor/corvus-json-schema/THIRD-PARTY-NOTICES.txt +93 -0
  12. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/compiler.rs +88 -96
  13. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/document.rs +2 -2
  14. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/charset.rs +271 -0
  15. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/compile.rs +941 -0
  16. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/dfa.rs +366 -0
  17. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/fold.rs +138 -0
  18. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/mod.rs +329 -0
  19. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/parse.rs +1128 -0
  20. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/re2.rs +129 -0
  21. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/tables_gen.rs +6685 -0
  22. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/tests.rs +1672 -0
  23. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/ucd.rs +309 -0
  24. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/units.rs +165 -0
  25. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/vm.rs +981 -0
  26. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/eval/plan/fused.rs +13 -9
  27. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/eval/plan.rs +12 -11
  28. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/eval.rs +105 -73
  29. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/formats.rs +95 -1
  30. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/lib.rs +165 -30
  31. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/loader.rs +19 -21
  32. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/numbers.rs +1 -1
  33. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/options.rs +199 -4
  34. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/pattern.rs +758 -79
  35. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/uri.rs +4 -4
  36. data/lib/corvus_json_schema/version.rb +1 -1
  37. data/lib/corvus_json_schema.rb +13 -3
  38. metadata +16 -2
@@ -1,4 +1,6 @@
1
- //! `pattern`/`patternProperties` matching with ECMA-262 semantics (the `u` flag), as JSON Schema specifies.
1
+ //! `pattern`/`patternProperties` matching with ECMA-262 semantics (the `u` flag), as JSON Schema specifies. A
2
+ //! pattern that is valid only without the flag is accepted too, and matched as ECMA-262 says for it: by UTF-16
3
+ //! code unit.
2
4
  //!
3
5
  //! A pattern gets the cheapest matcher that decides it exactly:
4
6
  //!
@@ -11,19 +13,31 @@
11
13
  //! - patterns within the common subset of ECMA-262 and the `regex` crate's syntax, translated with ECMA semantics
12
14
  //! (`\d`/`\w` ASCII-only, ECMA's `\s` and `.`) and run by `regex`, which searches in linear time without
13
15
  //! allocating per match;
14
- //! - anything else (lookarounds, backreferences, word boundaries, Unicode properties) by `regress`, a backtracking
15
- //! ECMA-262 engine.
16
+ //! - anything else (lookarounds, backreferences, `\B`, modifier groups, and every pattern `regex` is not given) by
17
+ //! the crate's own ECMA-262 engine (`ecmaregex`), a port of the engine of the Go module. It answers every pattern
18
+ //! as ECMA-262 does, as the pattern is written.
19
+ //!
20
+ //! Whether a pattern is valid is the engine's parser's answer, whichever matcher runs it, and so are the limits of
21
+ //! size: a pattern over one is refused before any matcher reads it.
16
22
  //!
17
23
  //! Patterns compile once per process (a pattern is immutable, so identical patterns share one matcher).
18
24
 
19
- use std::collections::HashMap;
25
+ use std::collections::{HashMap, VecDeque};
20
26
  use std::fmt;
21
- use std::sync::{Arc, LazyLock, Mutex};
27
+ use std::sync::{Arc, LazyLock, Mutex, OnceLock};
28
+ use std::time::Duration;
29
+
30
+ use crate::ecmaregex;
31
+ use crate::options::{
32
+ PatternMatchLimitError, PatternTooLargeError, SchemaCompilationError, SchemaCompilationErrorKind,
33
+ };
22
34
 
23
35
  /// A compiled `pattern`.
24
36
  pub(crate) struct Pattern {
25
37
  pub source: String,
26
38
  matcher: Matcher,
39
+ /// What the pattern is counted as in the cache of compiled patterns, in bytes (see [`Cache`]).
40
+ retained: usize,
27
41
  }
28
42
 
29
43
  enum Matcher {
@@ -55,12 +69,14 @@ enum Matcher {
55
69
  set: CharSet,
56
70
  bangs: bool,
57
71
  },
58
- Regex(regex::Regex),
72
+ Regex(Linear),
59
73
  /// A pattern that is valid only without the `u` flag (an identity escape such as `\&`), so matching UTF-16 code
60
74
  /// units: translated for `regex`, which decides it exactly on strings with no character beyond the Basic
61
- /// Multilingual Plane (where code units are characters); regress decides the rest.
62
- RegexBmp(regex::Regex, regress::Regex),
63
- Regress(regress::Regex),
75
+ /// Multilingual Plane (where code units are characters). The engine decides the rest, by code unit.
76
+ RegexBmp(Linear),
77
+ /// The crate's ECMA-262 engine: by character for a pattern with the `u` flag, and by UTF-16 code unit for one
78
+ /// that is valid only without it.
79
+ Engine(ecmaregex::Regexp),
64
80
  }
65
81
 
66
82
  impl fmt::Debug for Pattern {
@@ -69,9 +85,110 @@ impl fmt::Debug for Pattern {
69
85
  }
70
86
  }
71
87
 
88
+ /// A pattern compiled by the `regex` crate, which matches in time proportional to its program times the string and
89
+ /// cannot be stopped, with the engine for the strings it is not given.
90
+ struct Linear {
91
+ re: regex::Regex,
92
+ /// The number of instructions estimated for the program (`ecmaregex::Facts::linear_cost`), and one at least.
93
+ cost: u64,
94
+ /// The engine for the pattern: compiled with the pattern when it is valid only without the `u` flag, and
95
+ /// otherwise for the first string that is too long for `regex` to answer within the limit of a match.
96
+ engine: OnceLock<Option<ecmaregex::Regexp>>,
97
+ }
98
+
99
+ impl Linear {
100
+ /// Whether `regex` answers the string within the limit of a match: its program times the string, at
101
+ /// [`LINEAR_STEP`] for each, is within the limit. With no limit every string is.
102
+ #[inline(always)]
103
+ fn fits(&self, s: &str, budget: &PatternBudget) -> bool {
104
+ (s.len() as u64).saturating_mul(self.cost) <= budget.limit.linear
105
+ }
106
+
107
+ #[cold]
108
+ fn engine(&self, source: &str) -> Option<&ecmaregex::Regexp> {
109
+ self.engine.get_or_init(|| ecmaregex::compile(source).ok()).as_ref()
110
+ }
111
+ }
112
+
113
+ /// The time the `regex` crate can take for one instruction of its program and one byte of the string, when every
114
+ /// instruction has a thread on it (it then runs its simulation of the automaton, which is its slowest matcher). A
115
+ /// match on `regex` cannot be stopped, so a match with a limit goes there only when the size of the program times
116
+ /// the length of the string times this is within the limit. Any other goes to the engine, which reads the clock.
117
+ /// The figure is the Go module's for the regexp package of Go. For the `regex` crate the most that was measured,
118
+ /// with patterns that defeat its lazy automaton on a text of 100,000 bytes, was 3.6 ns.
119
+ const LINEAR_STEP: Duration = Duration::from_nanos(16);
120
+
121
+ /// The limit of a match, as a validator holds it.
122
+ #[derive(Clone, Copy, Debug)]
123
+ pub(crate) struct PatternLimit {
124
+ /// The time one match may take, or `None` for no limit.
125
+ limit: Option<Duration>,
126
+ /// The program of a `regex` matcher times the bytes of a string that the limit allows (see [`LINEAR_STEP`]).
127
+ linear: u64,
128
+ }
129
+
130
+ impl PatternLimit {
131
+ /// A limit of `limit` for each match. `None` and zero are no limit.
132
+ pub(crate) fn new(limit: Option<Duration>) -> PatternLimit {
133
+ let limit = limit.filter(|l| !l.is_zero());
134
+ let linear =
135
+ limit.map_or(u64::MAX, |l| u64::try_from(l.as_nanos() / LINEAR_STEP.as_nanos()).unwrap_or(u64::MAX));
136
+ PatternLimit { limit, linear }
137
+ }
138
+ }
139
+
140
+ /// What one evaluation, or the compilation of one schema, gives every match of a pattern: the time each may take.
141
+ /// It notes the first match that reached a limit, after which every match is abandoned at once.
142
+ pub(crate) struct PatternBudget {
143
+ limit: PatternLimit,
144
+ /// The match that reached a limit. It is boxed so that an evaluation, which holds a budget, stays small.
145
+ failure: Option<Box<PatternMatchLimitError>>,
146
+ }
147
+
148
+ impl PatternBudget {
149
+ #[inline(always)]
150
+ pub(crate) fn new(limit: PatternLimit) -> PatternBudget {
151
+ PatternBudget { limit, failure: None }
152
+ }
153
+
154
+ /// Whether a match has reached a limit, so that what the evaluation came to is not an answer.
155
+ #[inline(always)]
156
+ pub(crate) fn failed(&self) -> bool {
157
+ self.failure.is_some()
158
+ }
159
+
160
+ /// The error of the match that reached a limit, if one did.
161
+ pub(crate) fn take_failure(&mut self) -> Option<PatternMatchLimitError> {
162
+ self.failure.take().map(|e| *e)
163
+ }
164
+
165
+ /// Matches the string on the engine within the budget. When a limit is reached the answer is false, and it is
166
+ /// not an answer: the caller of the evaluation gets the error.
167
+ fn engine_match(&mut self, engine: Option<&ecmaregex::Regexp>, source: &str, s: &str) -> bool {
168
+ if self.failure.is_some() {
169
+ return false;
170
+ }
171
+ let result = match engine {
172
+ Some(engine) => engine.match_limit(s, self.limit.limit),
173
+ // No engine could be made for the pattern (see `ecmaregex::compile`), so there is no answer.
174
+ None => Err(ecmaregex::MatchError::Memory),
175
+ };
176
+ result.unwrap_or_else(|e| {
177
+ self.failure = Some(Box::new(PatternMatchLimitError::new(
178
+ source,
179
+ self.limit.limit.unwrap_or_default(),
180
+ e == ecmaregex::MatchError::Memory,
181
+ )));
182
+ false
183
+ })
184
+ }
185
+ }
186
+
72
187
  impl Pattern {
188
+ /// Whether the pattern matches somewhere in the string. The engine matches within the budget. Every other
189
+ /// matcher reads the string once, or a bounded number of times, and has no limit.
73
190
  #[inline]
74
- pub fn is_match(&self, s: &str) -> bool {
191
+ pub fn is_match(&self, s: &str, budget: &mut PatternBudget) -> bool {
75
192
  match &self.matcher {
76
193
  Matcher::Everything => true,
77
194
  Matcher::Literal { text, whole } => {
@@ -115,20 +232,46 @@ impl Pattern {
115
232
  }
116
233
  word
117
234
  }
118
- Matcher::Regex(re) => re.is_match(s),
235
+ Matcher::Regex(linear) => {
236
+ if linear.fits(s, budget) {
237
+ linear.re.is_match(s)
238
+ } else {
239
+ budget.engine_match(linear.engine(&self.source), &self.source, s)
240
+ }
241
+ }
119
242
  // A UTF-8 lead byte of 0xF0 or above starts a character beyond the BMP.
120
- Matcher::RegexBmp(re, fallback) => {
121
- if s.bytes().any(|b| b >= 0xF0) {
122
- fallback.find(s).is_some()
243
+ Matcher::RegexBmp(linear) => {
244
+ if linear.fits(s, budget) && !ecmaregex::has_supplementary(s.as_bytes()) {
245
+ linear.re.is_match(s)
123
246
  } else {
124
- re.is_match(s)
247
+ budget.engine_match(linear.engine(&self.source), &self.source, s)
125
248
  }
126
249
  }
127
- Matcher::Regress(re) => re.find(s).is_some(),
250
+ Matcher::Engine(engine) => budget.engine_match(Some(engine), &self.source, s),
128
251
  }
129
252
  }
130
253
  }
131
254
 
255
+ /// The kind of matcher a pattern has, for the tables of which pattern goes where.
256
+ #[cfg(test)]
257
+ pub(crate) fn kind(p: &Pattern) -> &'static str {
258
+ match &p.matcher {
259
+ Matcher::Regex(_) => "regex",
260
+ Matcher::RegexBmp(..) => "regex-bmp",
261
+ Matcher::Engine(engine) if engine.unicode() => "engine",
262
+ Matcher::Engine(_) => "engine-units",
263
+ Matcher::Everything => "everything",
264
+ Matcher::Literal { .. } => "literal",
265
+ Matcher::Sequence(_) => "sequence",
266
+ Matcher::SeparatedList(_) => "list",
267
+ Matcher::HasContent { .. } => "has-content",
268
+ Matcher::Line { .. } => "line",
269
+ Matcher::Literals(_) => "literals",
270
+ Matcher::Alternatives(_) => "alternatives",
271
+ Matcher::ExcludedClassWithWord { .. } => "excluded-class",
272
+ }
273
+ }
274
+
132
275
  /// ECMA-262 `LineTerminator`, which `.` does not match.
133
276
  #[inline]
134
277
  fn is_line_terminator(c: char) -> bool {
@@ -237,44 +380,195 @@ impl Alternative {
237
380
  }
238
381
  }
239
382
 
240
- /// Compiles a pattern with the `u` flag, or (as many validators accept them) without it; the flag is returned.
241
- fn compile_regress(pattern: &str) -> Option<(regress::Regex, bool)> {
242
- match regress::Regex::with_flags(pattern, "u") {
243
- Ok(re) => Some((re, true)),
244
- Err(_) => regress::Regex::new(pattern).ok().map(|re| (re, false)),
383
+ /// The most patterns the process-wide cache holds.
384
+ const MAX_CACHED_PATTERNS: usize = 1024;
385
+
386
+ /// The most the patterns of the cache come to, in the bytes counted for each: 64 MiB. A pattern is counted as its
387
+ /// source, the size the engine's parser counted for it four times over (its program is proportional to that), and
388
+ /// for a pattern on the `regex` crate the memory estimated for that crate's program (`ecmaregex::Facts::linear_size`,
389
+ /// which is an estimate: the crate does not say what it holds), up to three times [`REGEX_SIZE_LIMIT`].
390
+ const MAX_CACHED_BYTES: usize = 64 << 20;
391
+
392
+ /// A pattern counted as more than this is compiled for the schema that has it and not kept in the cache.
393
+ const MAX_CACHED_PATTERN_BYTES: usize = MAX_CACHED_BYTES / 32;
394
+
395
+ /// The largest program the `regex` crate compiles for a pattern, in bytes: 2 MiB. (Its own default is 10 MiB.) A
396
+ /// pattern that needs more is the engine's, which holds a class once however often the pattern repeats it.
397
+ const REGEX_SIZE_LIMIT: usize = 2 << 20;
398
+
399
+ /// The most the `regex` crate's lazy automaton caches for one pattern and one thread, in bytes: 2 MiB, which is
400
+ /// that crate's default, said here so that it stays so.
401
+ const REGEX_DFA_SIZE_LIMIT: usize = 2 << 20;
402
+
403
+ /// The compiled patterns of the process, so that identical patterns share one matcher. It is bounded in number and
404
+ /// in size, since the patterns of a schema may come from outside: when it is over either bound the patterns that
405
+ /// were compiled first go, oldest first. A validator holds the patterns of its schema itself, so a pattern that has
406
+ /// left the cache is still matched by the validators that have it, and is compiled again only for a schema compiled
407
+ /// later.
408
+ #[derive(Default)]
409
+ struct Cache {
410
+ patterns: HashMap<Arc<str>, Arc<Pattern>>,
411
+ /// The sources in the order they were compiled.
412
+ order: VecDeque<Arc<str>>,
413
+ /// The sum of what the patterns are counted as.
414
+ bytes: usize,
415
+ }
416
+
417
+ impl Cache {
418
+ fn get(&self, source: &str) -> Option<Arc<Pattern>> {
419
+ self.patterns.get(source).cloned()
420
+ }
421
+
422
+ /// Keeps a pattern, and returns the one to use: the pattern itself, or the one another thread kept for the same
423
+ /// source meanwhile.
424
+ fn insert(&mut self, pattern: Arc<Pattern>) -> Arc<Pattern> {
425
+ if pattern.retained > MAX_CACHED_PATTERN_BYTES {
426
+ return pattern;
427
+ }
428
+ if let Some(kept) = self.patterns.get(pattern.source.as_str()) {
429
+ return kept.clone();
430
+ }
431
+ let source: Arc<str> = Arc::from(pattern.source.as_str());
432
+ self.bytes += pattern.retained;
433
+ self.patterns.insert(source.clone(), pattern.clone());
434
+ self.order.push_back(source);
435
+ while self.patterns.len() > MAX_CACHED_PATTERNS || self.bytes > MAX_CACHED_BYTES {
436
+ let Some(oldest) = self.order.pop_front() else { break };
437
+ if let Some(evicted) = self.patterns.remove(&oldest) {
438
+ self.bytes -= evicted.retained;
439
+ }
440
+ }
441
+ pattern
442
+ }
443
+ }
444
+
445
+ static CACHE: LazyLock<Mutex<Cache>> = LazyLock::new(|| Mutex::new(Cache::default()));
446
+
447
+ fn cache() -> std::sync::MutexGuard<'static, Cache> {
448
+ CACHE.lock().unwrap_or_else(|e| e.into_inner())
449
+ }
450
+
451
+ /// Why a pattern does not compile.
452
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
453
+ pub(crate) enum PatternError {
454
+ /// It is not a valid ECMA-262 regular expression.
455
+ Invalid,
456
+ /// It is over a limit of size. It was not read to its end, so nothing is known of whether it is valid.
457
+ TooLarge(ecmaregex::TooLargeError),
458
+ }
459
+
460
+ impl PatternError {
461
+ /// The error of a schema whose keyword `keyword` holds the pattern `pattern`.
462
+ pub(crate) fn of(self, pattern: &str, keyword: &'static str) -> SchemaCompilationError {
463
+ match self {
464
+ PatternError::Invalid => SchemaCompilationError::of_kind(
465
+ format!("Invalid regular expression '{pattern}' in {keyword}."),
466
+ SchemaCompilationErrorKind::InvalidPattern,
467
+ ),
468
+ PatternError::TooLarge(large) => {
469
+ let error =
470
+ PatternTooLargeError::new(pattern, keyword, large.limit.name(), large.value, large.to_string());
471
+ SchemaCompilationError::of_kind(error.to_string(), SchemaCompilationErrorKind::PatternTooLarge(error))
472
+ }
473
+ }
245
474
  }
246
475
  }
247
476
 
248
- static CACHE: LazyLock<Mutex<HashMap<String, Arc<Pattern>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
477
+ impl From<ecmaregex::Error> for PatternError {
478
+ fn from(e: ecmaregex::Error) -> PatternError {
479
+ match e {
480
+ ecmaregex::Error::TooLarge(large) => PatternError::TooLarge(large),
481
+ ecmaregex::Error::Invalid(_) => PatternError::Invalid,
482
+ }
483
+ }
484
+ }
485
+
486
+ /// A pattern in quotes for a message, or its first 120 bytes and its length when it is longer.
487
+ pub(crate) fn shown_pattern(pattern: &str) -> String {
488
+ const SHOWN: usize = 120;
489
+ if pattern.len() <= SHOWN {
490
+ return format!("'{pattern}'");
491
+ }
492
+ let mut cut = SHOWN;
493
+ while !pattern.is_char_boundary(cut) {
494
+ cut -= 1;
495
+ }
496
+ format!("'{}...' ({} bytes)", &pattern[..cut], pattern.len())
497
+ }
249
498
 
250
- /// Compiles (or fetches from the process-wide cache) a pattern; `None` when it is not a valid ECMA-262 regex.
251
- pub(crate) fn compile(pattern: &str) -> Option<Arc<Pattern>> {
252
- if let Some(p) = CACHE.lock().unwrap().get(pattern) {
253
- return Some(p.clone());
499
+ /// Compiles (or fetches from the process-wide cache) a pattern.
500
+ pub(crate) fn compile(pattern: &str) -> Result<Arc<Pattern>, PatternError> {
501
+ if let Some(p) = cache().get(pattern) {
502
+ return Ok(p);
254
503
  }
255
- // Validity is ECMA-262's: a pattern regress rejects is an error, whichever matcher would run it.
256
- let (regress, unicode) = compile_regress(pattern)?;
257
- let matcher = match choose(pattern, unicode) {
504
+ // The limits of size come first: nothing below reads a pattern that is over one. Validity is ECMA-262's: a
505
+ // pattern the engine's parser rejects is an error, whichever matcher would run it.
506
+ let facts = ecmaregex::check(pattern)?;
507
+ let matcher = match choose(pattern, &facts) {
258
508
  Some(m) => m,
259
- None if !unicode => match translate(pattern, false).and_then(|t| regex::Regex::new(&t).ok()) {
260
- Some(re) => Matcher::RegexBmp(re, regress),
261
- None => Matcher::Regress(regress),
509
+ // Without the flag the pattern matches UTF-16 code units. `regex` matches characters, which is the same
510
+ // thing on a string with no character beyond the Basic Multilingual Plane.
511
+ None => match (!facts.unicode).then(|| linear(pattern, &facts)).flatten() {
512
+ Some(linear) => Matcher::RegexBmp(linear),
513
+ None => Matcher::Engine(ecmaregex::compile(pattern)?),
262
514
  },
263
- None => Matcher::Regress(regress),
264
515
  };
265
- let p = Arc::new(Pattern { source: pattern.to_string(), matcher });
266
- CACHE.lock().unwrap().insert(pattern.to_string(), p.clone());
267
- Some(p)
516
+ // What the cache counts for the pattern (see `MAX_CACHED_BYTES`).
517
+ let mut retained = pattern.len() + 256;
518
+ if matches!(matcher, Matcher::Regex(_) | Matcher::RegexBmp(_) | Matcher::Engine(_)) {
519
+ retained += 4 * facts.size;
520
+ }
521
+ if matches!(matcher, Matcher::Regex(_) | Matcher::RegexBmp(_)) {
522
+ retained += (facts.linear_size as usize).min(3 * REGEX_SIZE_LIMIT);
523
+ }
524
+ Ok(cache().insert(Arc::new(Pattern { source: pattern.to_string(), matcher, retained })))
525
+ }
526
+
527
+ /// The pattern translated for the `regex` crate and compiled by it, when it is in the syntax both have and its
528
+ /// program is not large (see `ecmaregex::MAX_RE2_COST`: `regex` writes a counted repetition out, and takes time
529
+ /// proportional to its program to compile and, for every character of a text, to match).
530
+ fn linear(pattern: &str, facts: &ecmaregex::Facts) -> Option<Linear> {
531
+ if facts.re2_cost > ecmaregex::MAX_RE2_COST {
532
+ return None;
533
+ }
534
+ let translation = translate(pattern, facts.unicode)?;
535
+ let re = if ecmaregex::shallow(pattern) {
536
+ regex::RegexBuilder::new(&translation)
537
+ .size_limit(REGEX_SIZE_LIMIT)
538
+ .dfa_size_limit(REGEX_DFA_SIZE_LIMIT)
539
+ .build()
540
+ .ok()?
541
+ } else {
542
+ // `regex` reads 250 levels of nesting unless it is told otherwise, because what it does with a pattern goes
543
+ // a level deeper for each. A pattern that opens many groups is compiled on a thread with a stack for the
544
+ // 1,000 levels the engine's own limit allows (a group, its quantifier and a class inside it are three
545
+ // levels to `regex`), so that it is not the engine's for its nesting alone.
546
+ ecmaregex::deep(|| {
547
+ regex::RegexBuilder::new(&translation)
548
+ .size_limit(REGEX_SIZE_LIMIT)
549
+ .dfa_size_limit(REGEX_DFA_SIZE_LIMIT)
550
+ .nest_limit(4 * ecmaregex::MAX_NESTING as u32)
551
+ .build()
552
+ .ok()
553
+ })??
554
+ };
555
+ let engine = OnceLock::new();
556
+ if !facts.unicode || !ecmaregex::shallow(pattern) {
557
+ // The engine has the strings with a character beyond the Basic Multilingual Plane, so it is compiled now.
558
+ // So is that of a pattern that nests deeply, which is compiled on a stack of its own.
559
+ let _ = engine.set(Some(ecmaregex::compile(pattern).ok()?));
560
+ }
561
+ Some(Linear { re, cost: facts.linear_cost.max(1) as u64, engine })
268
562
  }
269
563
 
270
- fn choose(pattern: &str, unicode: bool) -> Option<Matcher> {
564
+ fn choose(pattern: &str, facts: &ecmaregex::Facts) -> Option<Matcher> {
271
565
  // Unanchored (or start-anchored) `.*` finds an empty match in any string; `^.*$` does not (`.` stops at a line
272
566
  // terminator), so it is not listed.
273
567
  if matches!(pattern, "" | ".*" | "^.*" | ".*$" | "(.*)" | "^(.*)" | "[\\s\\S]*" | "^[\\s\\S]*" | "^[\\s\\S]*$") {
274
568
  return Some(Matcher::Everything);
275
569
  }
276
- // Without the `u` flag, a pattern matches UTF-16 code units rather than characters: only regress has that.
277
- if !unicode {
570
+ // Without the `u` flag, a pattern matches UTF-16 code units rather than characters: only the engine has that.
571
+ if !facts.unicode {
278
572
  return None;
279
573
  }
280
574
  match pattern {
@@ -288,7 +582,9 @@ fn choose(pattern: &str, unicode: bool) -> Option<Matcher> {
288
582
  && rest.len() > 1
289
583
  && !ends_with_escape(rest)
290
584
  && !rest.ends_with(['*', '+', '?', '}', '|', '(', '^'])
291
- && let Some(m) = choose(rest, unicode)
585
+ && let Ok(rest_facts) = ecmaregex::check(rest)
586
+ && rest_facts.unicode
587
+ && let Some(m) = choose(rest, &rest_facts)
292
588
  {
293
589
  return Some(m);
294
590
  }
@@ -313,8 +609,7 @@ fn choose(pattern: &str, unicode: bool) -> Option<Matcher> {
313
609
  None => Matcher::Sequence(seq),
314
610
  });
315
611
  }
316
- let translated = translate(pattern, true)?;
317
- regex::Regex::new(&translated).ok().map(Matcher::Regex)
612
+ linear(pattern, facts).map(Matcher::Regex)
318
613
  }
319
614
 
320
615
  /// `^.{m,n}$` (and `^.*$`, `^.+$`, `^.{m}$`, `^.{m,}$`, each also as `^(.…)$`): the bounds on the length of a line.
@@ -393,12 +688,18 @@ const MAX_ALTERNATIVES: usize = 64;
393
688
  /// `regex` crate's single automaton pass (measured on jsconfig's case-folded `lib` names).
394
689
  const MAX_SEQUENCE_ALTERNATIVES: usize = 4;
395
690
 
691
+ /// The widest class sequence, in characters, and the longest literal, in bytes, that is searched for at every
692
+ /// position of a text. The search for a sequence takes the text times this, and these matchers have no limit, so a
693
+ /// wider one is left to the engine, which has the limit of a match. (The rule is the Go module's, so that the ports
694
+ /// give a pattern the same kind of matcher.)
695
+ const MAX_ANYWHERE_WIDTH: usize = 64;
696
+
396
697
  /// Two or more alternatives once groups of alternatives (and optional groups) are expanded into whole alternatives
397
698
  /// (`^a|b|c$`, `^([a|A]uto)|([n|N]one)$`, `^[Ee][Ss]2015(\.([Cc]ore|[Pp]roxy))?$`), each a literal or a class
398
699
  /// sequence anchored where its own `^` and `$` say.
399
700
  fn alternatives(p: &str) -> Option<Box<[Alternative]>> {
400
701
  let c: Vec<char> = p.chars().collect();
401
- let (parts, end) = expand(&c, 0)?;
702
+ let (parts, end) = expand(&c, 0, 0)?;
402
703
  // A single alternative is only new here when a group was expanded (Sequence::parse takes the rest).
403
704
  if end != c.len() || parts.is_empty() || (parts.len() == 1 && parts[0] == p) {
404
705
  return None;
@@ -412,6 +713,10 @@ fn alternatives(p: &str) -> Option<Box<[Alternative]>> {
412
713
  _ => (false, part),
413
714
  };
414
715
  if let Some(text) = literal_text(body) {
716
+ // A literal that is anchored at neither end is searched for at every position of the text.
717
+ if !start && !end && text.len() > MAX_ANYWHERE_WIDTH {
718
+ return None;
719
+ }
415
720
  return Some(Alternative::Literal { text: text.into(), start, end });
416
721
  }
417
722
  let seq = Sequence::parse(&format!("^{body}{}", if end { "$" } else { "" }))?;
@@ -420,7 +725,11 @@ fn alternatives(p: &str) -> Option<Box<[Alternative]>> {
420
725
  }
421
726
  let width =
422
727
  seq.items.iter().all(|i| i.min == i.max).then(|| seq.items.iter().map(|i| i.min as usize).sum())?;
423
- Some(if end { Alternative::End { seq, width } } else { Alternative::Anywhere { seq, width } })
728
+ if end {
729
+ return Some(Alternative::End { seq, width });
730
+ }
731
+ // A sequence that is not anchored is tried at every position of the text, and reads its width at each.
732
+ (width <= MAX_ANYWHERE_WIDTH).then_some(Alternative::Anywhere { seq, width })
424
733
  })
425
734
  .collect::<Option<_>>()?;
426
735
  let sequences = alternatives.iter().any(|a| !matches!(a, Alternative::Literal { .. }));
@@ -432,10 +741,18 @@ fn ends_with_escape(t: &str) -> bool {
432
741
  t.bytes().rev().take_while(|&b| b == b'\\').count() % 2 == 1
433
742
  }
434
743
 
744
+ /// The deepest nesting of groups that is multiplied out. `expand` goes a level deeper for each, and a pattern may
745
+ /// nest a thousand deep, which the stack of a thread cannot be relied on to hold. A pattern that nests deeper than
746
+ /// this is left to the matchers after this one.
747
+ const MAX_EXPANDED_NESTING: usize = 32;
748
+
435
749
  /// Expands the alternatives from `i` up to an unmatched `)` or the end: groups of alternatives multiply out, a group
436
750
  /// quantified by `?` also contributes the empty alternative, and everything else is copied. `None` for lookarounds,
437
- /// other quantified groups, or too many alternatives.
438
- fn expand(c: &[char], mut i: usize) -> Option<(Vec<String>, usize)> {
751
+ /// other quantified groups, too many alternatives, or groups nested deeper than `MAX_EXPANDED_NESTING`.
752
+ fn expand(c: &[char], mut i: usize, depth: usize) -> Option<(Vec<String>, usize)> {
753
+ if depth > MAX_EXPANDED_NESTING {
754
+ return None;
755
+ }
439
756
  let mut all = Vec::new();
440
757
  let mut branch = vec![String::new()];
441
758
  while i < c.len() {
@@ -454,7 +771,7 @@ fn expand(c: &[char], mut i: usize) -> Option<(Vec<String>, usize)> {
454
771
  }
455
772
  i += 2;
456
773
  }
457
- let (mut inner, next) = expand(c, i)?;
774
+ let (mut inner, next) = expand(c, i, depth + 1)?;
458
775
  if c.get(next) != Some(&')') {
459
776
  return None;
460
777
  }
@@ -550,9 +867,10 @@ fn excluded_class_with_word(p: &str) -> Option<(CharSet, bool)> {
550
867
  .then_some((excluded, bangs))
551
868
  }
552
869
 
553
- /// The `regex` format: a valid ECMA-262 regular expression (with the `u` flag).
870
+ /// The `regex` format: a valid ECMA-262 regular expression (with the `u` flag). A string over a limit of size is
871
+ /// not read to its end and is not valid.
554
872
  pub(crate) fn is_valid_ecma_regex(s: &str) -> bool {
555
- regress::Regex::with_flags(s, "u").is_ok()
873
+ ecmaregex::valid(s)
556
874
  }
557
875
 
558
876
  // ---------------------------------------------------------------------------------------------------------------------
@@ -848,8 +1166,14 @@ impl SeparatedList {
848
1166
  }
849
1167
  }
850
1168
  let final_ = self.final_.as_ref().expect("a separated list has a first or a final item");
1169
+ // Whether the text is ASCII is worked out once, for the whole of it. The final sequence is tried after
1170
+ // every item, and reading what is left of the text each time to see whether it is ASCII made the match take
1171
+ // time quadratic in the number of items.
1172
+ let ascii = s.is_ascii();
851
1173
  loop {
852
- if repeats >= self.min_repeats && final_.is_match(rest) {
1174
+ if repeats >= self.min_repeats
1175
+ && (if ascii { final_.is_match_ascii(rest.as_bytes()) } else { final_.is_match_chars(rest) })
1176
+ {
853
1177
  return true;
854
1178
  }
855
1179
  let Some(n) = self.repeated.consume(rest) else { return false };
@@ -895,7 +1219,9 @@ impl SeparatedList {
895
1219
  let group = strip_group(&body[open..=close])?;
896
1220
  let before = &body[..open];
897
1221
  let after = &body[close + 2..];
898
- let items = |t: &str| Sequence::parse(&format!("^{t}")).map(|s| s.items.into_vec());
1222
+ // The items of a text that is a class sequence. A `$` at the end of the text is not an item, and the list has
1223
+ // no place for it: `^x(,a$)*$` is not `^x(,a)*$`. Such a text is not a part of a list.
1224
+ let items = |t: &str| Sequence::parse(&format!("^{t}")).filter(|s| !s.to_end).map(|s| s.items.into_vec());
899
1225
  let seq = |items: &[Item], to_end| Sequence { items: items.into(), to_end };
900
1226
  let fixed = |items: &[Item]| !items.is_empty() && items.iter().all(|i| i.min == i.max && i.min > 0);
901
1227
  let g = items(group)?;
@@ -923,7 +1249,10 @@ impl SeparatedList {
923
1249
  let final_ = Sequence::parse(&format!("^{}$", unwrap_group(after)?))?;
924
1250
  (1..g.len()).find_map(|k| {
925
1251
  let (repeated, separator) = g.split_at(k);
926
- (fixed(separator) && Sequence::greedy_before(repeated, separator[0].set)).then(|| SeparatedList {
1252
+ (fixed(separator)
1253
+ && Sequence::greedy_before(repeated, separator[0].set)
1254
+ && final_stops_at(&final_, separator[0].set))
1255
+ .then(|| SeparatedList {
927
1256
  first: None,
928
1257
  repeated: seq(repeated, false),
929
1258
  separator: seq(separator, false),
@@ -937,6 +1266,14 @@ impl SeparatedList {
937
1266
  }
938
1267
  }
939
1268
 
1269
+ /// Whether the final sequence of a list cannot read past the separator that follows an item: no item of it holds a
1270
+ /// character the first item of the separator holds. Tried at the start of an item, it then fails at that separator
1271
+ /// at the latest, so trying it after every item reads the text a bounded number of times. A final sequence that can
1272
+ /// read through separators would read to the end of the text from every item.
1273
+ fn final_stops_at(final_: &Sequence, separator: CharSet) -> bool {
1274
+ final_.items.iter().all(|item| item.set.disjoint(separator))
1275
+ }
1276
+
940
1277
  /// The inside of `(...)` or `(?:...)`; `None` for other groups.
941
1278
  fn strip_group(g: &str) -> Option<&str> {
942
1279
  let inner = g.strip_prefix('(')?.strip_suffix(')')?;
@@ -1102,6 +1439,11 @@ const ECMA_SPACE: &str =
1102
1439
  /// escapes of any non-alphanumeric character), and the translation has its meaning on strings within the BMP.
1103
1440
  fn translate(p: &str, unicode: bool) -> Option<String> {
1104
1441
  let c: Vec<char> = p.chars().collect();
1442
+ // Without the flag a character beyond the BMP is two code units, and what is around it (a quantifier, the ends
1443
+ // of a range) applies to one of them. That has no translation.
1444
+ if !unicode && c.iter().any(|&ch| ch as u32 > 0xFFFF) {
1445
+ return None;
1446
+ }
1105
1447
  let mut out = String::with_capacity(p.len() * 2);
1106
1448
  let mut i = 0;
1107
1449
  while i < c.len() {
@@ -1225,23 +1567,29 @@ fn escape(c: &[char], i: usize, in_class: bool, unicode: bool) -> Option<(String
1225
1567
  out.push_str("(?-u:\\b)");
1226
1568
  i + 1
1227
1569
  }
1228
- 'B' => {
1229
- out.push_str("(?-u:\\B)");
1230
- i + 1
1231
- }
1232
- // Unicode properties (validated as ECMA-262 names by regress) have the same names in the `regex` crate.
1233
- // Without the `u` flag `\p` is the letter.
1570
+ // `\B` has no translation. `(?-u:\B)` also holds between the bytes of a character that takes several, where
1571
+ // no match may start or end, and `regex` 1.13.1 then says there is no match where it finds one: for
1572
+ // `s*?(?-u:\B)|..$` on `k\u{17f}k`, `is_match` is false and `find` is not. The engine has it.
1573
+ 'B' => return None,
1574
+ // A Unicode property is written as the ranges of the engine's own tables, and never by its name: the
1575
+ // `regex` crate has tables of its own, of the version of Unicode its release carries, and a name would
1576
+ // match what that version says. Without the `u` flag `\p` is the letter.
1234
1577
  'p' | 'P' if unicode && c.get(i + 1) == Some(&'{') => {
1235
1578
  let end = i + 1 + c[i + 1..].iter().position(|&x| x == '}')?;
1236
1579
  let body: String = c[i + 2..end].iter().collect();
1237
- if body.is_empty() || !body.chars().all(|x| x.is_ascii_alphanumeric() || x == '_' || x == '=') {
1238
- return None;
1580
+ let set = ecmaregex::property_set(&body)?;
1581
+ let negated = e == 'P';
1582
+ if set.ranges.is_empty() {
1583
+ // No character has the property: nothing, or with `\P` everything.
1584
+ out.push_str(if negated { r"[\x{0}-\x{10FFFF}]" } else { r"[^\x{0}-\x{10FFFF}]" });
1585
+ } else if in_class && !negated {
1586
+ push_ranges(&mut out, &set);
1587
+ } else {
1588
+ // A class of its own, which is also how a class holds the complement of a property.
1589
+ out.push_str(if negated { "[^" } else { "[" });
1590
+ push_ranges(&mut out, &set);
1591
+ out.push(']');
1239
1592
  }
1240
- out.push('\\');
1241
- out.push(e);
1242
- out.push('{');
1243
- out.push_str(&body);
1244
- out.push('}');
1245
1593
  end + 1
1246
1594
  }
1247
1595
  'x' => {
@@ -1252,6 +1600,10 @@ fn escape(c: &[char], i: usize, in_class: bool, unicode: bool) -> Option<(String
1252
1600
  // Without the `u` flag, `\u{...}` is not a code point escape.
1253
1601
  'u' if unicode || c.get(i + 1) != Some(&'{') => {
1254
1602
  let (code, next) = unicode_escape(c, i + 1)?;
1603
+ // Without the flag, two escapes that are a surrogate pair are two code units.
1604
+ if !unicode && code > 0xFFFF {
1605
+ return None;
1606
+ }
1255
1607
  push_literal(&mut out, char::from_u32(code)?);
1256
1608
  next
1257
1609
  }
@@ -1271,6 +1623,42 @@ fn escape(c: &[char], i: usize, in_class: bool, unicode: bool) -> Option<(String
1271
1623
  Some((out, next))
1272
1624
  }
1273
1625
 
1626
+ /// The ranges of a set, each end spelled as an escape. A surrogate is no character to the `regex` crate, and no
1627
+ /// string holds one, so a range is written without the surrogates it spans. Two ranges that then touch, one ending
1628
+ /// at U+D7FF and the next starting at U+E000, are written as one: see [`class`] for what `regex` makes of the two.
1629
+ fn push_ranges(out: &mut String, set: &ecmaregex::CharSet) {
1630
+ // The range that has been read and not yet written.
1631
+ let mut pending: Option<(u32, u32)> = None;
1632
+ let write = |out: &mut String, (lo, hi): (u32, u32)| {
1633
+ if let (Some(lo), Some(hi)) = (char::from_u32(lo), char::from_u32(hi)) {
1634
+ push_literal(out, lo);
1635
+ if lo != hi {
1636
+ out.push('-');
1637
+ push_literal(out, hi);
1638
+ }
1639
+ }
1640
+ };
1641
+ for &[lo, hi] in set.ranges.as_chunks::<2>().0 {
1642
+ // The ends that are surrogates move out of the surrogates. A range of surrogates alone is nothing.
1643
+ let lo = if (0xD800..=0xDFFF).contains(&lo) { 0xE000 } else { lo as u32 };
1644
+ let hi = if (0xD800..=0xDFFF).contains(&hi) { 0xD7FF } else { hi as u32 };
1645
+ if lo > hi {
1646
+ continue;
1647
+ }
1648
+ match pending {
1649
+ Some((first, 0xD7FF)) if lo == 0xE000 => pending = Some((first, hi)),
1650
+ Some(range) => {
1651
+ write(out, range);
1652
+ pending = Some((lo, hi));
1653
+ }
1654
+ None => pending = Some((lo, hi)),
1655
+ }
1656
+ }
1657
+ if let Some(range) = pending {
1658
+ write(out, range);
1659
+ }
1660
+ }
1661
+
1274
1662
  /// `\u` escapes after the `u`: `XXXX` (joining a surrogate pair) or `{X...}`.
1275
1663
  fn unicode_escape(c: &[char], i: usize) -> Option<(u32, usize)> {
1276
1664
  if c.get(i) == Some(&'{') {
@@ -1354,6 +1742,13 @@ fn class(c: &[char], mut i: usize, unicode: bool) -> Option<(String, usize)> {
1354
1742
  if members == 0 {
1355
1743
  return None; // `[]` and `[^]` have no counterpart.
1356
1744
  }
1745
+ // `regex` 1.13.1 (regex-syntax 0.8.11) negates wrongly a class with a range that ends at U+D7FF and another
1746
+ // that starts at U+E000, the characters on either side of the surrogates: the complement takes both in, so
1747
+ // `[^\x{D7FF}\x{E000}]` matches every character. A negated class that names both (every literal here is
1748
+ // spelled `\x{...}`, so a `^` is a negation) is left to the engine.
1749
+ if out.contains('^') && out.contains(r"\x{D7FF}") && out.contains(r"\x{E000}") {
1750
+ return None;
1751
+ }
1357
1752
  out.push(']');
1358
1753
  Some((out, i))
1359
1754
  }
@@ -1484,6 +1879,20 @@ mod tests {
1484
1879
  "^(é,)*a$",
1485
1880
  "^(?=!+[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$",
1486
1881
  "^(?=!+[^a]+$)(?=(.*\\w)).+$",
1882
+ // Lists with an anchor or an assertion inside them, and with an escaped anchor.
1883
+ "^x(,a$)*$",
1884
+ "^a(,a$)+$",
1885
+ "^x$(,a)*$",
1886
+ "^(a$)(,a)*$",
1887
+ "^$(,a)*$",
1888
+ "^(a,$)*b$",
1889
+ "^([a-z]+,$)+a$",
1890
+ "^a(^,a)*$",
1891
+ "^(^a,)*b$",
1892
+ "^a(,a\\b)*$",
1893
+ "^a(,a\\$)*$",
1894
+ "^a\\$(,a)*$",
1895
+ "^(a\\$,)*b$",
1487
1896
  // Valid only without the `u` flag (identity escapes), so matching code units.
1488
1897
  "^\\/[^\\*\\?\\&\\%]*(\\/\\*)?$",
1489
1898
  "^[\\&\\@\\_]+$",
@@ -1493,9 +1902,10 @@ mod tests {
1493
1902
  "\\uD83D\\uDE00|\\&",
1494
1903
  ];
1495
1904
 
1496
- /// Every matcher agrees with regress on strings over an alphabet that exercises classes, anchors and non-ASCII.
1905
+ /// Every matcher agrees with the engine's backtracking matcher on strings over an alphabet that exercises
1906
+ /// classes, anchors and non-ASCII.
1497
1907
  #[test]
1498
- fn matchers_agree_with_regress() {
1908
+ fn matchers_agree_with_the_engine() {
1499
1909
  let alphabet = [
1500
1910
  "a", "b", "z", "A", "X", "Z", "0", "1", "5", "9", "_", "-", ".", ":", "/", "@", "#", "$", "*", "!", "{",
1501
1911
  "}", "|", " ", "\n", "\u{a0}", "é", "µ", "😀", "\u{2028}", "x-", "es", "ES", "ms", "txt", "Au", "to", "No",
@@ -1509,12 +1919,21 @@ mod tests {
1509
1919
  seed
1510
1920
  };
1511
1921
  for &p in PATTERNS {
1512
- let (reference, _) = compile_regress(p).unwrap();
1922
+ let reference = ecmaregex::compile(p).unwrap();
1513
1923
  let compiled = compile(p).unwrap();
1514
1924
  for _ in 0..4000 {
1515
1925
  let len = next() % 9;
1516
1926
  let s: String = (0..len).map(|_| alphabet[(next() % alphabet.len() as u64) as usize]).collect();
1517
- assert_eq!(compiled.is_match(&s), reference.find(&s).is_some(), "{p:?} on {s:?}");
1927
+ // Without the `u` flag the engine matches the UTF-16 code units of the string.
1928
+ let found = reference.match_backtrack(&s, None).unwrap();
1929
+ for limit in [None, Some(Duration::from_secs(60)), Some(Duration::from_nanos(1))] {
1930
+ let mut budget = PatternBudget::new(PatternLimit::new(limit));
1931
+ let got = compiled.is_match(&s, &mut budget);
1932
+ // A limit of a nanosecond gives `regex` no string, so this is the engine for every pattern on
1933
+ // it. A match that reached its first check has no answer.
1934
+ assert!(got == found || budget.failed(), "{p:?} on {s:?} with a limit of {limit:?}");
1935
+ assert!(limit.is_some_and(|l| l.as_nanos() == 1) || !budget.failed(), "{p:?} on {s:?}");
1936
+ }
1518
1937
  }
1519
1938
  }
1520
1939
  }
@@ -1555,9 +1974,34 @@ mod tests {
1555
1974
  "^([a-zA-Z0-9]{2,3})(-[a-zA-Z0-9]{1,6})*$",
1556
1975
  "^([a-zA-Z_$][a-zA-Z0-9_$]{0,39}\\.)*([a-zA-Z_$][a-zA-Z0-9_$]{0,39})$",
1557
1976
  "^([a-z_$][a-z0-9_$]{0,39}\\.)*([a-zA-Z_$][a-zA-Z0-9_$]{0,39})$",
1977
+ // An escaped `$` or `^` is a literal character of the list.
1978
+ "^x(,a\\$)*$",
1979
+ "^x\\$(,a)*$",
1980
+ "^(a\\$,)*b$",
1981
+ "^x(,\\^a)*$",
1982
+ "^(a,)*b\\$$",
1983
+ "^x(,[$^]a)*$",
1558
1984
  ] {
1559
1985
  assert!(matches!(compile(p).unwrap().matcher, Matcher::SeparatedList(_)), "{p}");
1560
1986
  }
1987
+ // A list with an anchor or an assertion inside it has no shape.
1988
+ for p in [
1989
+ "^x(,a$)*$",
1990
+ "^x(?:,a$)+$",
1991
+ "^x$(,a)*$",
1992
+ "^(x$)(,a)*$",
1993
+ "^$(,a)*$",
1994
+ "^(a,$)*b$",
1995
+ "^x(^,a)*$",
1996
+ "^(^a,)*b$",
1997
+ "^x(,(a$))*$",
1998
+ "^x(,a\\b)*$",
1999
+ "^x\\B(,a)*$",
2000
+ "^x(,a(?=,|$))*$",
2001
+ "^x(?=,)(,a)*$",
2002
+ ] {
2003
+ assert!(matches!(compile(p).unwrap().matcher, Matcher::Regex(_) | Matcher::Engine(_)), "{p}");
2004
+ }
1561
2005
  assert!(matches!(
1562
2006
  compile("^(?=[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$").unwrap().matcher,
1563
2007
  Matcher::ExcludedClassWithWord { .. }
@@ -1568,8 +2012,240 @@ mod tests {
1568
2012
  for p in ["^\\/[^\\*\\?\\&\\%]*(\\/\\*)?$", "a\\&.b", "^\\-a"] {
1569
2013
  assert!(matches!(compile(p).unwrap().matcher, Matcher::RegexBmp(..)), "{p}");
1570
2014
  }
1571
- for p in ["^((\\.(?!\\.)\\/)?\\w+\\/?)+$", "^\\1(a)", "^\\p{L}\\&"] {
1572
- assert!(matches!(compile(p).unwrap().matcher, Matcher::Regress(_)), "{p}");
2015
+ for p in ["^((\\.(?!\\.)\\/)?\\w+\\/?)+$", "^\\1(a)", "\\Bs", "(?i:a)", "^[]", "\\cA"] {
2016
+ assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
2017
+ }
2018
+ // Valid only without the `u` flag, and not in the syntax `regex` takes: the engine, by code unit.
2019
+ for p in ["^\\p{L}\\&", "^\\-?\u{10400}*$", "^\\-(?=a)"] {
2020
+ let compiled = compile(p).unwrap();
2021
+ assert!(matches!(&compiled.matcher, Matcher::Engine(engine) if !engine.unicode()), "{p}");
2022
+ }
2023
+ }
2024
+ }
2025
+
2026
+ #[cfg(test)]
2027
+ mod limits {
2028
+ use super::*;
2029
+
2030
+ /// Which matcher a pattern gets (the Go module's `TestAWideSearchIsLeftToTheEngine`). A sequence or a literal
2031
+ /// that would be searched for at every position of the text keeps its matcher up to 64 characters, and beyond
2032
+ /// that is left to a matcher that takes time proportional to the text or has the limit of a match.
2033
+ #[test]
2034
+ fn a_wide_search_is_left_to_the_engine() {
2035
+ let a = |n: usize| "a".repeat(n);
2036
+ for (pattern, alternatives) in [
2037
+ (r"[a-z]{64}[0-9]{0}|x".to_string(), true),
2038
+ (r"[a-z]{63}[0-9]|x".to_string(), true),
2039
+ (r"[a-z]{64}[0-9]|x".to_string(), false),
2040
+ (r"[a-z]{50000}[0-9]|x".to_string(), false),
2041
+ (r"^[a-z]{50000}[0-9]|x".to_string(), true),
2042
+ (r"[a-z]{50000}[0-9]$|^x".to_string(), true),
2043
+ (format!("{}|^x", a(64)), true),
2044
+ (format!("{}|^x", a(65)), false),
2045
+ (format!("^{}|^x", a(65)), true),
2046
+ (format!("{}$|^x", a(65)), true),
2047
+ ] {
2048
+ let found = kind(&compile(&pattern).unwrap());
2049
+ assert_eq!(found == "alternatives", alternatives, "{pattern:.30} has the matcher {found}");
2050
+ assert!(alternatives || matches!(found, "regex" | "engine"), "{pattern:.30} has the matcher {found}");
2051
+ }
2052
+ // The final sequence of a list must stop at the separator.
2053
+ for (pattern, list) in [
2054
+ (r"^(?:[a-z]+,)*[a-z]{2}[0-9]$", true),
2055
+ (r"^(?:[a-z]+,)*[a-z,]+[0-9]$", false),
2056
+ (r"^(?:[a-z]+,)*[0-9]{1,3}$", true),
2057
+ ] {
2058
+ let found = kind(&compile(pattern).unwrap());
2059
+ assert_eq!(found == "list", list, "{pattern} has the matcher {found}");
2060
+ }
2061
+ }
2062
+
2063
+ /// The `regex` crate is given no large program (the Go module's `TestTheRegexpPackageIsGivenNoLargeProgram`):
2064
+ /// it writes a counted repetition out, so the pattern goes to the engine when the copies are many.
2065
+ #[test]
2066
+ fn regex_is_given_no_large_program() {
2067
+ for (pattern, want) in [
2068
+ (r"a{1000}".to_string(), "regex"),
2069
+ (r"(?:a|b|ab)*c".to_string(), "regex"),
2070
+ (r"\p{L}{1000}".repeat(90), "engine"),
2071
+ (r"(?:[ab]{1000}){50}c".to_string(), "engine"),
2072
+ (r"(?:a{100}){101}".to_string(), "engine"),
2073
+ (r"(?:(?:a?){100}){100}b".to_string(), "engine"),
2074
+ ] {
2075
+ assert_eq!(kind(&compile(&pattern).unwrap()), want, "{pattern:.40}");
2076
+ }
2077
+ }
2078
+
2079
+ /// A string too long for `regex` to answer within the limit of a match goes to the engine, which can be
2080
+ /// stopped (the Go module's `TestALongTextIsNotGivenToTheRegexpPackage`).
2081
+ #[test]
2082
+ fn a_long_string_is_not_given_to_regex() {
2083
+ let compiled = compile(r"^(?:[a-c\u00e9]{0,500}x)*$").unwrap();
2084
+ let Matcher::Regex(linear) = &compiled.matcher else { panic!("the pattern is on {}", kind(&compiled)) };
2085
+ assert!(linear.cost >= 500, "a program of {} instructions", linear.cost);
2086
+ let short = "\u{e9}abcx".to_string();
2087
+ let long = "\u{e9}abcx".repeat(100000);
2088
+ let second = PatternLimit::new(Some(Duration::from_secs(1)));
2089
+ let mut budget = PatternBudget::new(second);
2090
+ assert!(linear.fits(&short, &budget) && !linear.fits(&long, &budget), "the strings are not on each side");
2091
+ assert!(compiled.is_match(&short, &mut budget) && linear.engine.get().is_none(), "a short string");
2092
+ assert!(compiled.is_match(&long, &mut budget) && !budget.failed(), "a long string");
2093
+ assert!(matches!(linear.engine.get(), Some(Some(_))), "the engine was not made for the long string");
2094
+ // With no limit every string is `regex`'s.
2095
+ let mut unlimited = PatternBudget::new(PatternLimit::new(None));
2096
+ assert!(linear.fits(&long, &unlimited) && !compiled.is_match(&format!("{long}y"), &mut unlimited));
2097
+ }
2098
+
2099
+ /// The cache of compiled patterns is bounded in number and in size, and a pattern that has left it is still
2100
+ /// matched by whoever holds it.
2101
+ #[test]
2102
+ fn the_cache_of_patterns_is_bounded() {
2103
+ let mut budget = PatternBudget::new(PatternLimit::new(None));
2104
+ let first = compile("^cache-bound-0+$").unwrap();
2105
+ let held: Vec<Arc<Pattern>> =
2106
+ (0..3 * MAX_CACHED_PATTERNS).map(|n| compile(&format!("^cache-bound-{n}(?=x)x$")).unwrap()).collect();
2107
+ {
2108
+ let cache = cache();
2109
+ assert!(cache.patterns.len() <= MAX_CACHED_PATTERNS && cache.bytes <= MAX_CACHED_BYTES);
2110
+ assert_eq!(cache.patterns.len(), cache.order.len());
2111
+ assert_eq!(cache.bytes, cache.patterns.values().map(|p| p.retained).sum::<usize>());
2112
+ assert!(cache.get("^cache-bound-0+$").is_none(), "the oldest pattern is still in the cache");
2113
+ }
2114
+ assert!(first.is_match("cache-bound-000", &mut budget));
2115
+ assert!(held[0].is_match("cache-bound-0x", &mut budget) && !held[7].is_match("cache-bound-0x", &mut budget));
2116
+ // The same source compiles again, to a matcher of its own.
2117
+ let again = compile("^cache-bound-0+$").unwrap();
2118
+ assert!(!Arc::ptr_eq(&first, &again) && again.is_match("cache-bound-0", &mut budget));
2119
+ // Large patterns leave the cache by size, well before their number is the bound.
2120
+ let class = r"[\u0100\u0102\u0104\u0106]".repeat(1500);
2121
+ let large: Vec<Arc<Pattern>> = (0..200).map(|n| compile(&format!("^(?=large-{n}){class}$")).unwrap()).collect();
2122
+ assert!(large[0].retained > 256 << 10, "a large pattern is counted as {} bytes", large[0].retained);
2123
+ assert!(cache().bytes <= MAX_CACHED_BYTES);
2124
+ // A pattern too large for the cache is not kept at all.
2125
+ let huge = compile(&format!("^(?=huge){}$", "a".repeat(15000))).unwrap();
2126
+ assert!(huge.retained > MAX_CACHED_PATTERN_BYTES && cache().get(&huge.source).is_none());
2127
+ }
2128
+
2129
+ /// The `regex` crate compiles no program over its limit of size: a class of a Unicode property repeated often
2130
+ /// is the engine's, which holds the class once.
2131
+ #[test]
2132
+ fn regex_compiles_no_program_over_its_limit() {
2133
+ for (pattern, want) in [
2134
+ (r"^\p{L}+$", "regex"),
2135
+ (r"^[\p{L}\p{N}_-]{1,8}$", "regex"),
2136
+ (r"^[\p{L}\p{N}_-]{1,64}$", "engine"),
2137
+ (r"^[\p{Alphabetic}\p{Nd}]{1,255}$", "engine"),
2138
+ ] {
2139
+ let compiled = compile(pattern).unwrap();
2140
+ assert_eq!(kind(&compiled), want, "{pattern}");
2141
+ let mut budget = PatternBudget::new(PatternLimit::new(Some(Duration::from_secs(1))));
2142
+ assert!(compiled.is_match("caf\u{e9}", &mut budget) && !compiled.is_match("caf\u{e9}!", &mut budget));
2143
+ }
2144
+ }
2145
+
2146
+ /// Alternatives of one character each are one class to a linear matcher, so a long alternation is a small
2147
+ /// program, and `regex` keeps a long string for it. And a pattern that nests more deeply than `regex` reads by
2148
+ /// default is still compiled by it, on a thread with a stack for the nesting.
2149
+ #[test]
2150
+ fn regex_keeps_a_long_alternation_and_a_deep_nest() {
2151
+ let alternatives: Vec<String> = (0..495u32).map(|k| char::from_u32(0x4E00 + k).unwrap().to_string()).collect();
2152
+ let compiled = compile(&format!("(?:{})*z", alternatives.join("|"))).unwrap();
2153
+ let Matcher::Regex(linear) = &compiled.matcher else { panic!("the alternation is on {}", kind(&compiled)) };
2154
+ assert!(linear.cost <= 4, "a program of {} instructions", linear.cost);
2155
+ let text = alternatives.concat().repeat(70);
2156
+ assert!(text.len() >= 100_000);
2157
+ let mut budget = PatternBudget::new(PatternLimit::new(Some(Duration::from_secs(1))));
2158
+ assert!(linear.fits(&text, &budget) && !compiled.is_match(&text, &mut budget) && !budget.failed());
2159
+ assert!(compiled.is_match(&format!("{text}z"), &mut budget) && linear.engine.get().is_none());
2160
+
2161
+ for depth in [300, 999] {
2162
+ let nested = format!("^{}a{}b$", "(".repeat(depth), "*)".repeat(depth));
2163
+ let compiled = compile(&nested).unwrap();
2164
+ assert_eq!(kind(&compiled), "regex", "{depth} nested groups");
2165
+ let start = std::time::Instant::now();
2166
+ assert!(
2167
+ compiled.is_match("aaaab", &mut budget)
2168
+ && !compiled.is_match("\u{e9}aaaaaaaaaaaaaaaaaaaaaaaaa", &mut budget)
2169
+ );
2170
+ assert!(!budget.failed() && start.elapsed() < Duration::from_secs(20), "{depth} nested groups");
2171
+ }
2172
+ }
2173
+
2174
+ /// The translation for `regex` names no Unicode class: every property is written as the ranges of the engine's
2175
+ /// own tables, so the answer does not depend on the tables of the `regex` crate.
2176
+ #[test]
2177
+ fn the_translation_carries_no_unicode_class() {
2178
+ for p in [
2179
+ r"^\p{L}+$",
2180
+ r"\P{Lu}",
2181
+ r"[\p{Script=Greek}\p{Nd}]",
2182
+ r"[^\p{scx=Hira}]",
2183
+ r"\d\w\s.",
2184
+ r"\p{Any}\p{Assigned}\p{ASCII}",
2185
+ ] {
2186
+ let compiled = compile(p).unwrap();
2187
+ assert_eq!(kind(&compiled), "regex", "{p}");
2188
+ let translated = translate(p, true).unwrap();
2189
+ for bad in [r"\p", r"\P", "(?i", "[[:", r"\d", r"\w", r"\s"] {
2190
+ assert!(!translated.contains(bad), "the translation of {p} contains {bad}");
2191
+ }
2192
+ }
2193
+ // The four scripts Unicode 17 added are in the tables, whatever the `regex` crate's own tables hold.
2194
+ let mut budget = PatternBudget::new(PatternLimit::new(None));
2195
+ for (p, text) in [
2196
+ (r"^\p{sc=Sidt}$", "\u{10940}"),
2197
+ (r"^\p{sc=Tols}$", "\u{11DB0}"),
2198
+ (r"^\p{sc=Berf}$", "\u{16EA0}"),
2199
+ (r"^\p{sc=Tayo}$", "\u{1E6C0}"),
2200
+ (r"^[\p{sc=Sidt}a]$", "\u{10940}"),
2201
+ (r"^[^\P{sc=Sidt}]$", "\u{10940}"),
2202
+ ] {
2203
+ let compiled = compile(p).unwrap();
2204
+ assert_eq!(kind(&compiled), "regex", "{p}");
2205
+ assert!(compiled.is_match(text, &mut budget) && !compiled.is_match("a!", &mut budget), "{p}");
2206
+ }
2207
+ // The complement of a set whose ranges meet at the surrogates, which `regex` gets wrong when it is given
2208
+ // the two ranges: Unknown holds U+D7FC to U+D7FF and U+E000 to U+F8FF. And the same written by hand, which
2209
+ // is the engine's.
2210
+ for (p, want) in [
2211
+ (r"^\P{sc=Unknown}$", "regex"),
2212
+ (r"^[^\p{scx=Unknown}]$", "regex"),
2213
+ (r"^[^\p{Script=Unknown}a]$", "regex"),
2214
+ (r"^[^\ud7ff\ue000]$", "engine"),
2215
+ (r"^[^\ud7fc-\ud7ff\ue000-\uf8ff]$", "engine"),
2216
+ (r"^[^\p{Co}\ud7ff]$", "engine"),
2217
+ (r"^[^\p{Cn}\ue000]$", "engine"),
2218
+ ] {
2219
+ let compiled = compile(p).unwrap();
2220
+ assert_eq!(kind(&compiled), want, "{p}");
2221
+ let engine = ecmaregex::compile(p).unwrap();
2222
+ for text in ["\u{D7FB}", "\u{D7FC}", "\u{D7FF}", "\u{E000}", "\u{E001}", "\u{F8FF}", "\u{F900}", "a", "b"] {
2223
+ assert_eq!(
2224
+ compiled.is_match(text, &mut budget),
2225
+ engine.match_backtrack(text, None).unwrap(),
2226
+ "{p} on {text:?}"
2227
+ );
2228
+ }
2229
+ assert!(!compiled.is_match("\u{E000}", &mut budget), "{p} matches U+E000");
2230
+ }
2231
+ // A set with a range on each side of the surrogates and none of them is written as one range, and a range
2232
+ // of surrogates alone as nothing.
2233
+ let mut written = String::new();
2234
+ push_ranges(&mut written, &ecmaregex::CharSet::new(&[0x61, 0x7A, 0xD700, 0xD7FF, 0xE000, 0xE0FF], false));
2235
+ assert_eq!(written, r"\x{61}-\x{7A}\x{D700}-\x{E0FF}");
2236
+ written.clear();
2237
+ push_ranges(&mut written, &ecmaregex::CharSet::new(&[0xD7FF, 0xDBFF, 0xDC00, 0xDFFF, 0xDFFF, 0xE000], false));
2238
+ assert_eq!(written, r"\x{D7FF}-\x{E000}");
2239
+ written.clear();
2240
+ push_ranges(&mut written, &ecmaregex::CharSet::new(&[0xD800, 0xDFFF], false));
2241
+ assert_eq!(written, "");
2242
+ // A property no character has, and its complement.
2243
+ for (p, yes, no) in
2244
+ [(r"^\P{sc=Hrkt}$", "a", "aa"), (r"^[\p{sc=Hrkt}a]$", "a", "b"), (r"^[^\P{sc=Hrkt}]?$", "", "a")]
2245
+ {
2246
+ let compiled = compile(p).unwrap();
2247
+ assert_eq!(kind(&compiled), "regex", "{p}");
2248
+ assert!(compiled.is_match(yes, &mut budget) && !compiled.is_match(no, &mut budget), "{p}");
1573
2249
  }
1574
2250
  }
1575
2251
  }
@@ -1583,14 +2259,17 @@ mod corpus_patterns {
1583
2259
  let Ok(path) = std::env::var("CORVUS_PATTERNS") else { return };
1584
2260
  for line in std::fs::read_to_string(path).unwrap().lines() {
1585
2261
  let (corpus, p) = line.split_once('\t').unwrap();
1586
- let kind = super::compile(p)
1587
- .map(|c| match &c.matcher {
1588
- super::Matcher::Regex(_) => "regex",
1589
- super::Matcher::Regress(_) => "regress",
1590
- _ => "fast",
1591
- })
1592
- .unwrap_or("invalid");
1593
- println!("{kind}\t{corpus}\t{p}");
2262
+ let kind = super::compile(p).map(|c| super::kind(&c)).unwrap_or("invalid");
2263
+ // What the `regex` crate is given: the translation.
2264
+ let unicode = crate::ecmaregex::valid(p);
2265
+ let given = match kind {
2266
+ "regex" | "regex-bmp" => super::translate(p, unicode).unwrap_or_default(),
2267
+ _ => String::new(),
2268
+ };
2269
+ // And what the cache counts for the pattern, which for a pattern on `regex` holds the estimate of
2270
+ // that crate's program.
2271
+ let retained = super::compile(p).map_or(0, |c| c.retained);
2272
+ println!("{kind}\t{corpus}\t{p}\t{given}\t{retained}");
1594
2273
  }
1595
2274
  }
1596
2275
  }