corvus_json_schema 0.1.4 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (30) hide show
  1. checksums.yaml +4 -4
  2. data/Cargo.lock +13 -2
  3. data/README.md +6 -4
  4. data/THIRD-PARTY-NOTICES.txt +15 -14
  5. data/VERSIONHISTORY.md +9 -0
  6. data/ext/corvus_json_schema/Cargo.toml +2 -2
  7. data/ext/corvus_json_schema/src/lib.rs +29 -2
  8. data/ext/corvus_json_schema/vendor/corvus-json-schema/Cargo.toml +4 -9
  9. data/ext/corvus_json_schema/vendor/corvus-json-schema/README.md +30 -49
  10. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/compiler.rs +5 -3
  11. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/formats.rs +7 -6
  12. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/lib.rs +4 -4
  13. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/options.rs +12 -0
  14. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/pattern.rs +344 -974
  15. data/lib/corvus_json_schema/version.rb +1 -1
  16. data/lib/corvus_json_schema.rb +8 -3
  17. metadata +2 -15
  18. data/ext/corvus_json_schema/vendor/corvus-json-schema/THIRD-PARTY-NOTICES.txt +0 -93
  19. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/charset.rs +0 -271
  20. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/compile.rs +0 -941
  21. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/dfa.rs +0 -366
  22. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/fold.rs +0 -138
  23. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/mod.rs +0 -329
  24. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/parse.rs +0 -1128
  25. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/re2.rs +0 -129
  26. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/tables_gen.rs +0 -6685
  27. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/tests.rs +0 -1672
  28. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/ucd.rs +0 -309
  29. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/units.rs +0 -165
  30. data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/vm.rs +0 -981
@@ -8,33 +8,75 @@
8
8
  //! - anchored sequences of quantified ASCII character classes and literals (`^[a-z][a-z0-9_]{0,29}$`, `^x-`,
9
9
  //! `^[@$_#]`), matched in one pass over the string (the TypeScript evaluator's class-sequence fast path);
10
10
  //! - alternatives of such sequences once groups are multiplied out (`^([a|A]uto)|([n|N]one)$`,
11
- //! `^[Ee][Ss]2015(\.([Cc]ore|[Pp]roxy))?$`), sets of literals, and separated lists (`^([a-z]+)(\.[a-z]+)*$`),
12
- //! as the C# evaluator's pattern matchers do;
13
- //! - patterns within the common subset of ECMA-262 and the `regex` crate's syntax, translated with ECMA semantics
14
- //! (`\d`/`\w` ASCII-only, ECMA's `\s` and `.`) and run by `regex`, which searches in linear time without
15
- //! allocating per match;
16
- //! - anything else (lookarounds, backreferences, `\B`, modifier groups, and every pattern `regex` is not given) by
17
- //! the crate's own ECMA-262 engine (`ecmaregex`), a port of the engine of the Go module. It answers every pattern
18
- //! as ECMA-262 does, as the pattern is written.
11
+ //! `^[Ee][Ss]2015(\.([Cc]ore|[Pp]roxy))?$`) and sets of literals, as the C# evaluator's pattern matchers do,
12
+ //! where they were measured faster than the engine (see [`MOST_LITERALS`]);
13
+ //! - anything else by [Corvus.Regex.Ecma262](https://docs.rs/corvus-regex-ecma262), an ECMA-262 engine that answers
14
+ //! every pattern as V8 does, as the pattern is written, and matches a pattern with no backreference in time
15
+ //! proportional to the pattern's size times the string.
19
16
  //!
20
- //! Whether a pattern is valid is the engine's parser's answer, whichever matcher runs it, and so are the limits of
21
- //! size: a pattern over one is refused before any matcher reads it.
17
+ //! Whether a pattern is valid is the engine's answer, whichever matcher runs it, in the grammar of the edition the
18
+ //! dialect reads patterns with (ECMAScript 2020 for draft 2019-09 and draft 2020-12, today's for the others; see
19
+ //! [`Grammar`]), and so are the limits of size: a pattern over one is refused before any matcher reads it.
22
20
  //!
23
- //! Patterns compile once per process (a pattern is immutable, so identical patterns share one matcher).
21
+ //! Patterns compile once per process for each grammar (a pattern is immutable, so identical patterns share one
22
+ //! matcher).
24
23
 
25
24
  use std::collections::{HashMap, VecDeque};
26
25
  use std::fmt;
27
- use std::sync::{Arc, LazyLock, Mutex, OnceLock};
26
+ use std::sync::{Arc, LazyLock, Mutex};
28
27
  use std::time::Duration;
29
28
 
30
- use crate::ecmaregex;
29
+ use corvus_regex_ecma262::{Edition, Flags, MatchError, Regex, RegexBuilder, UnicodeVersion};
30
+
31
+ use crate::dialect::Dialect;
31
32
  use crate::options::{
32
- PatternMatchLimitError, PatternTooLargeError, SchemaCompilationError, SchemaCompilationErrorKind,
33
+ CompileOptions, PatternMatchLimitError, PatternTooLargeError, SchemaCompilationError, SchemaCompilationErrorKind,
33
34
  };
34
35
 
36
+ /// The grammar a pattern is read in: the edition of ECMA-262 and the version of Unicode.
37
+ #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
38
+ pub(crate) struct Grammar {
39
+ pub edition: Edition,
40
+ pub unicode_version: UnicodeVersion,
41
+ }
42
+
43
+ impl Grammar {
44
+ /// The grammar of a dialect's patterns: the edition the options give, or by default ECMAScript 2020 for draft
45
+ /// 2019-09 and draft 2020-12, which cite it, and today's for the earlier drafts; and the options' version of
46
+ /// Unicode.
47
+ pub(crate) fn of(dialect: Dialect, options: &CompileOptions) -> Grammar {
48
+ let edition = options.regex_edition.unwrap_or(if dialect >= Dialect::Draft201909 {
49
+ Edition::Es2020
50
+ } else {
51
+ Edition::Current
52
+ });
53
+ Grammar { edition, unicode_version: options.regex_unicode_version }
54
+ }
55
+
56
+ /// The flags of the grammar of the `u` flag, or of no flag.
57
+ fn flags(unicode: bool) -> Flags {
58
+ Flags { unicode, ..Flags::default() }
59
+ }
60
+
61
+ /// The pattern compiled in this grammar, with the `u` flag or with none.
62
+ fn build(self, pattern: &str, unicode: bool) -> Result<Regex, corvus_regex_ecma262::Error> {
63
+ RegexBuilder::new(pattern)
64
+ .with_flags(Grammar::flags(unicode))
65
+ .edition(self.edition)
66
+ .unicode_version(self.unicode_version)
67
+ .build()
68
+ }
69
+
70
+ /// Whether the pattern is valid in this grammar with the `u` flag.
71
+ fn is_valid_unicode(self, pattern: &str) -> bool {
72
+ Regex::is_valid_with(pattern, Grammar::flags(true), self.edition, self.unicode_version)
73
+ }
74
+ }
75
+
35
76
  /// A compiled `pattern`.
36
77
  pub(crate) struct Pattern {
37
78
  pub source: String,
79
+ grammar: Grammar,
38
80
  matcher: Matcher,
39
81
  /// What the pattern is counted as in the cache of compiled patterns, in bytes (see [`Cache`]).
40
82
  retained: usize,
@@ -49,7 +91,6 @@ enum Matcher {
49
91
  whole: bool,
50
92
  },
51
93
  Sequence(Sequence),
52
- SeparatedList(SeparatedList),
53
94
  /// `.+` (`^.+` with `start`): some (the first) character is not a line terminator.
54
95
  HasContent {
55
96
  start: bool,
@@ -69,14 +110,9 @@ enum Matcher {
69
110
  set: CharSet,
70
111
  bangs: bool,
71
112
  },
72
- Regex(Linear),
73
- /// A pattern that is valid only without the `u` flag (an identity escape such as `\&`), so matching UTF-16 code
74
- /// units: translated for `regex`, which decides it exactly on strings with no character beyond the Basic
75
- /// Multilingual Plane (where code units are characters). The engine decides the rest, by code unit.
76
- RegexBmp(Linear),
77
- /// The crate's ECMA-262 engine: by character for a pattern with the `u` flag, and by UTF-16 code unit for one
78
- /// that is valid only without it.
79
- Engine(ecmaregex::Regexp),
113
+ /// The `corvus-regex-ecma262` engine: by character for a pattern with the `u` flag, and by UTF-16 code unit for
114
+ /// one that is valid only without it.
115
+ Engine(Box<Regex>),
80
116
  }
81
117
 
82
118
  impl fmt::Debug for Pattern {
@@ -85,55 +121,17 @@ impl fmt::Debug for Pattern {
85
121
  }
86
122
  }
87
123
 
88
- /// A pattern compiled by the `regex` crate, which matches in time proportional to its program times the string and
89
- /// cannot be stopped, with the engine for the strings it is not given.
90
- struct Linear {
91
- re: regex::Regex,
92
- /// The number of instructions estimated for the program (`ecmaregex::Facts::linear_cost`), and one at least.
93
- cost: u64,
94
- /// The engine for the pattern: compiled with the pattern when it is valid only without the `u` flag, and
95
- /// otherwise for the first string that is too long for `regex` to answer within the limit of a match.
96
- engine: OnceLock<Option<ecmaregex::Regexp>>,
97
- }
98
-
99
- impl Linear {
100
- /// Whether `regex` answers the string within the limit of a match: its program times the string, at
101
- /// [`LINEAR_STEP`] for each, is within the limit. With no limit every string is.
102
- #[inline(always)]
103
- fn fits(&self, s: &str, budget: &PatternBudget) -> bool {
104
- (s.len() as u64).saturating_mul(self.cost) <= budget.limit.linear
105
- }
106
-
107
- #[cold]
108
- fn engine(&self, source: &str) -> Option<&ecmaregex::Regexp> {
109
- self.engine.get_or_init(|| ecmaregex::compile(source).ok()).as_ref()
110
- }
111
- }
112
-
113
- /// The time the `regex` crate can take for one instruction of its program and one byte of the string, when every
114
- /// instruction has a thread on it (it then runs its simulation of the automaton, which is its slowest matcher). A
115
- /// match on `regex` cannot be stopped, so a match with a limit goes there only when the size of the program times
116
- /// the length of the string times this is within the limit. Any other goes to the engine, which reads the clock.
117
- /// The figure is the Go module's for the regexp package of Go. For the `regex` crate the most that was measured,
118
- /// with patterns that defeat its lazy automaton on a text of 100,000 bytes, was 3.6 ns.
119
- const LINEAR_STEP: Duration = Duration::from_nanos(16);
120
-
121
124
  /// The limit of a match, as a validator holds it.
122
125
  #[derive(Clone, Copy, Debug)]
123
126
  pub(crate) struct PatternLimit {
124
127
  /// The time one match may take, or `None` for no limit.
125
128
  limit: Option<Duration>,
126
- /// The program of a `regex` matcher times the bytes of a string that the limit allows (see [`LINEAR_STEP`]).
127
- linear: u64,
128
129
  }
129
130
 
130
131
  impl PatternLimit {
131
132
  /// A limit of `limit` for each match. `None` and zero are no limit.
132
133
  pub(crate) fn new(limit: Option<Duration>) -> PatternLimit {
133
- let limit = limit.filter(|l| !l.is_zero());
134
- let linear =
135
- limit.map_or(u64::MAX, |l| u64::try_from(l.as_nanos() / LINEAR_STEP.as_nanos()).unwrap_or(u64::MAX));
136
- PatternLimit { limit, linear }
134
+ PatternLimit { limit: limit.filter(|l| !l.is_zero()) }
137
135
  }
138
136
  }
139
137
 
@@ -164,20 +162,19 @@ impl PatternBudget {
164
162
 
165
163
  /// Matches the string on the engine within the budget. When a limit is reached the answer is false, and it is
166
164
  /// not an answer: the caller of the evaluation gets the error.
167
- fn engine_match(&mut self, engine: Option<&ecmaregex::Regexp>, source: &str, s: &str) -> bool {
165
+ fn engine_match(&mut self, engine: &Regex, source: &str, s: &str) -> bool {
168
166
  if self.failure.is_some() {
169
167
  return false;
170
168
  }
171
- let result = match engine {
172
- Some(engine) => engine.match_limit(s, self.limit.limit),
173
- // No engine could be made for the pattern (see `ecmaregex::compile`), so there is no answer.
174
- None => Err(ecmaregex::MatchError::Memory),
169
+ let result = match self.limit.limit {
170
+ Some(limit) => engine.is_match_within(s, limit),
171
+ None => engine.is_match(s),
175
172
  };
176
173
  result.unwrap_or_else(|e| {
177
174
  self.failure = Some(Box::new(PatternMatchLimitError::new(
178
175
  source,
179
176
  self.limit.limit.unwrap_or_default(),
180
- e == ecmaregex::MatchError::Memory,
177
+ e == MatchError::Memory,
181
178
  )));
182
179
  false
183
180
  })
@@ -199,7 +196,6 @@ impl Pattern {
199
196
  }
200
197
  }
201
198
  Matcher::Sequence(seq) => seq.is_match(s),
202
- Matcher::SeparatedList(list) => list.is_match(s),
203
199
  Matcher::HasContent { start } => {
204
200
  if *start {
205
201
  s.chars().next().is_some_and(|c| !is_line_terminator(c))
@@ -232,22 +228,7 @@ impl Pattern {
232
228
  }
233
229
  word
234
230
  }
235
- Matcher::Regex(linear) => {
236
- if linear.fits(s, budget) {
237
- linear.re.is_match(s)
238
- } else {
239
- budget.engine_match(linear.engine(&self.source), &self.source, s)
240
- }
241
- }
242
- // A UTF-8 lead byte of 0xF0 or above starts a character beyond the BMP.
243
- Matcher::RegexBmp(linear) => {
244
- if linear.fits(s, budget) && !ecmaregex::has_supplementary(s.as_bytes()) {
245
- linear.re.is_match(s)
246
- } else {
247
- budget.engine_match(linear.engine(&self.source), &self.source, s)
248
- }
249
- }
250
- Matcher::Engine(engine) => budget.engine_match(Some(engine), &self.source, s),
231
+ Matcher::Engine(engine) => budget.engine_match(engine, &self.source, s),
251
232
  }
252
233
  }
253
234
  }
@@ -256,14 +237,11 @@ impl Pattern {
256
237
  #[cfg(test)]
257
238
  pub(crate) fn kind(p: &Pattern) -> &'static str {
258
239
  match &p.matcher {
259
- Matcher::Regex(_) => "regex",
260
- Matcher::RegexBmp(..) => "regex-bmp",
261
- Matcher::Engine(engine) if engine.unicode() => "engine",
240
+ Matcher::Engine(engine) if engine.flags().unicode => "engine",
262
241
  Matcher::Engine(_) => "engine-units",
263
242
  Matcher::Everything => "everything",
264
243
  Matcher::Literal { .. } => "literal",
265
244
  Matcher::Sequence(_) => "sequence",
266
- Matcher::SeparatedList(_) => "list",
267
245
  Matcher::HasContent { .. } => "has-content",
268
246
  Matcher::Line { .. } => "line",
269
247
  Matcher::Literals(_) => "literals",
@@ -294,30 +272,34 @@ fn line_length(s: &str) -> Option<usize> {
294
272
  Some(n)
295
273
  }
296
274
 
297
- /// A set of strings: compared in turn when there are few, hashed otherwise.
298
- enum Literals {
299
- Few(Box<[Box<str>]>),
300
- Many(std::collections::HashSet<Box<str>>),
301
- }
275
+ /// A set of at most [`MOST_LITERALS`] strings, compared in turn.
276
+ struct Literals(Box<[Box<str>]>);
302
277
 
303
278
  impl Literals {
304
279
  fn new(texts: Vec<String>) -> Literals {
305
- if texts.len() <= 8 {
306
- Literals::Few(texts.into_iter().map(String::into_boxed_str).collect())
307
- } else {
308
- Literals::Many(texts.into_iter().map(String::into_boxed_str).collect())
309
- }
280
+ Literals(texts.into_iter().map(String::into_boxed_str).collect())
310
281
  }
311
282
 
312
283
  #[inline]
313
284
  fn contains(&self, s: &str) -> bool {
314
- match self {
315
- Literals::Few(texts) => texts.iter().any(|t| **t == *s),
316
- Literals::Many(set) => set.contains(s),
317
- }
285
+ self.0.iter().any(|t| **t == *s)
318
286
  }
319
287
  }
320
288
 
289
+ // What the dedicated matchers are kept for: each was timed against the engine, pattern by pattern, over the strings of
290
+ // the jsonschema-benchmark corpora (`corpus_patterns::time_corpus_patterns`), and a form is kept only where its matcher
291
+ // was the faster. Four literals compared in turn took 0.78 of the engine's time, and ten 3.2 times it. Two alternatives
292
+ // of which neither is anchored at both ends (`^([a|A]uto)|([n|N]one)$`) took 0.21 of it, since the engine searches for
293
+ // the unanchored one; three such took 1.4 to 1.6 times it, and every alternative anchored at both ends was faster on
294
+ // the engine (by 1.1 to 2.7 times), which reads the whole string once whatever the alternatives. A separated list was
295
+ // no faster than the engine (the same total time over the seven in the corpora), so it has no matcher of its own.
296
+
297
+ /// The most literals of [`Matcher::Literals`].
298
+ const MOST_LITERALS: usize = 4;
299
+
300
+ /// The most alternatives of [`Matcher::Alternatives`].
301
+ const MOST_ALTERNATIVES: usize = 2;
302
+
321
303
  /// One alternative of [`Matcher::Alternatives`]: a literal anchored at either end (or neither), a class sequence
322
304
  /// anchored at the start, or a fixed-width class sequence anchored at the end.
323
305
  enum Alternative {
@@ -384,22 +366,12 @@ impl Alternative {
384
366
  const MAX_CACHED_PATTERNS: usize = 1024;
385
367
 
386
368
  /// The most the patterns of the cache come to, in the bytes counted for each: 64 MiB. A pattern is counted as its
387
- /// source, the size the engine's parser counted for it four times over (its program is proportional to that), and
388
- /// for a pattern on the `regex` crate the memory estimated for that crate's program (`ecmaregex::Facts::linear_size`,
389
- /// which is an estimate: the crate does not say what it holds), up to three times [`REGEX_SIZE_LIMIT`].
369
+ /// source and an estimate of what its matcher holds (see [`compile`]).
390
370
  const MAX_CACHED_BYTES: usize = 64 << 20;
391
371
 
392
372
  /// A pattern counted as more than this is compiled for the schema that has it and not kept in the cache.
393
373
  const MAX_CACHED_PATTERN_BYTES: usize = MAX_CACHED_BYTES / 32;
394
374
 
395
- /// The largest program the `regex` crate compiles for a pattern, in bytes: 2 MiB. (Its own default is 10 MiB.) A
396
- /// pattern that needs more is the engine's, which holds a class once however often the pattern repeats it.
397
- const REGEX_SIZE_LIMIT: usize = 2 << 20;
398
-
399
- /// The most the `regex` crate's lazy automaton caches for one pattern and one thread, in bytes: 2 MiB, which is
400
- /// that crate's default, said here so that it stays so.
401
- const REGEX_DFA_SIZE_LIMIT: usize = 2 << 20;
402
-
403
375
  /// The compiled patterns of the process, so that identical patterns share one matcher. It is bounded in number and
404
376
  /// in size, since the patterns of a schema may come from outside: when it is over either bound the patterns that
405
377
  /// were compiled first go, oldest first. A validator holds the patterns of its schema itself, so a pattern that has
@@ -407,16 +379,16 @@ const REGEX_DFA_SIZE_LIMIT: usize = 2 << 20;
407
379
  /// later.
408
380
  #[derive(Default)]
409
381
  struct Cache {
410
- patterns: HashMap<Arc<str>, Arc<Pattern>>,
411
- /// The sources in the order they were compiled.
412
- order: VecDeque<Arc<str>>,
382
+ patterns: HashMap<(Arc<str>, Grammar), Arc<Pattern>>,
383
+ /// The sources and their grammars in the order they were compiled.
384
+ order: VecDeque<(Arc<str>, Grammar)>,
413
385
  /// The sum of what the patterns are counted as.
414
386
  bytes: usize,
415
387
  }
416
388
 
417
389
  impl Cache {
418
- fn get(&self, source: &str) -> Option<Arc<Pattern>> {
419
- self.patterns.get(source).cloned()
390
+ fn get(&self, source: &str, grammar: Grammar) -> Option<Arc<Pattern>> {
391
+ self.patterns.get(&(Arc::from(source), grammar)).cloned()
420
392
  }
421
393
 
422
394
  /// Keeps a pattern, and returns the one to use: the pattern itself, or the one another thread kept for the same
@@ -425,13 +397,13 @@ impl Cache {
425
397
  if pattern.retained > MAX_CACHED_PATTERN_BYTES {
426
398
  return pattern;
427
399
  }
428
- if let Some(kept) = self.patterns.get(pattern.source.as_str()) {
400
+ let key: (Arc<str>, Grammar) = (Arc::from(pattern.source.as_str()), pattern.grammar);
401
+ if let Some(kept) = self.patterns.get(&key) {
429
402
  return kept.clone();
430
403
  }
431
- let source: Arc<str> = Arc::from(pattern.source.as_str());
432
404
  self.bytes += pattern.retained;
433
- self.patterns.insert(source.clone(), pattern.clone());
434
- self.order.push_back(source);
405
+ self.patterns.insert(key.clone(), pattern.clone());
406
+ self.order.push_back(key);
435
407
  while self.patterns.len() > MAX_CACHED_PATTERNS || self.bytes > MAX_CACHED_BYTES {
436
408
  let Some(oldest) = self.order.pop_front() else { break };
437
409
  if let Some(evicted) = self.patterns.remove(&oldest) {
@@ -454,7 +426,7 @@ pub(crate) enum PatternError {
454
426
  /// It is not a valid ECMA-262 regular expression.
455
427
  Invalid,
456
428
  /// It is over a limit of size. It was not read to its end, so nothing is known of whether it is valid.
457
- TooLarge(ecmaregex::TooLargeError),
429
+ TooLarge(corvus_regex_ecma262::TooLargeError),
458
430
  }
459
431
 
460
432
  impl PatternError {
@@ -474,11 +446,12 @@ impl PatternError {
474
446
  }
475
447
  }
476
448
 
477
- impl From<ecmaregex::Error> for PatternError {
478
- fn from(e: ecmaregex::Error) -> PatternError {
449
+ impl From<corvus_regex_ecma262::Error> for PatternError {
450
+ fn from(e: corvus_regex_ecma262::Error) -> PatternError {
479
451
  match e {
480
- ecmaregex::Error::TooLarge(large) => PatternError::TooLarge(large),
481
- ecmaregex::Error::Invalid(_) => PatternError::Invalid,
452
+ corvus_regex_ecma262::Error::TooLarge(large) => PatternError::TooLarge(large),
453
+ // Not valid in the grammar, or (not given here) flags that are not.
454
+ _ => PatternError::Invalid,
482
455
  }
483
456
  }
484
457
  }
@@ -496,79 +469,44 @@ pub(crate) fn shown_pattern(pattern: &str) -> String {
496
469
  format!("'{}...' ({} bytes)", &pattern[..cut], pattern.len())
497
470
  }
498
471
 
499
- /// Compiles (or fetches from the process-wide cache) a pattern.
500
- pub(crate) fn compile(pattern: &str) -> Result<Arc<Pattern>, PatternError> {
501
- if let Some(p) = cache().get(pattern) {
472
+ /// Compiles (or fetches from the process-wide cache) a pattern, in the grammar of its dialect.
473
+ pub(crate) fn compile(pattern: &str, grammar: Grammar) -> Result<Arc<Pattern>, PatternError> {
474
+ if let Some(p) = cache().get(pattern, grammar) {
502
475
  return Ok(p);
503
476
  }
504
- // The limits of size come first: nothing below reads a pattern that is over one. Validity is ECMA-262's: a
505
- // pattern the engine's parser rejects is an error, whichever matcher would run it.
506
- let facts = ecmaregex::check(pattern)?;
507
- let matcher = match choose(pattern, &facts) {
508
- Some(m) => m,
509
- // Without the flag the pattern matches UTF-16 code units. `regex` matches characters, which is the same
510
- // thing on a string with no character beyond the Basic Multilingual Plane.
511
- None => match (!facts.unicode).then(|| linear(pattern, &facts)).flatten() {
512
- Some(linear) => Matcher::RegexBmp(linear),
513
- None => Matcher::Engine(ecmaregex::compile(pattern)?),
477
+ // Validity is ECMA-262's: a pattern the engine refuses is an error, whichever matcher would run it. A pattern is
478
+ // read with the u flag, and failing that with no flag, whose grammar (Annex B) many schemas need. A pattern valid
479
+ // with the flag that a dedicated matcher takes is read and never built: building the engine's program and
480
+ // automata costs far more than reading. The engine is built only for the patterns it matches, and for those over
481
+ // a limit of size, which it refuses (the reading refuses them too, without saying why).
482
+ let matcher = match grammar.is_valid_unicode(pattern).then(|| choose(pattern, true, grammar)).flatten() {
483
+ Some(shape) => shape,
484
+ None => match grammar.build(pattern, true) {
485
+ Ok(engine) => Matcher::Engine(Box::new(engine)),
486
+ Err(corvus_regex_ecma262::Error::TooLarge(large)) => return Err(PatternError::TooLarge(large)),
487
+ Err(_) => {
488
+ let engine = grammar.build(pattern, false)?;
489
+ choose(pattern, false, grammar).unwrap_or_else(|| Matcher::Engine(Box::new(engine)))
490
+ }
514
491
  },
515
492
  };
516
- // What the cache counts for the pattern (see `MAX_CACHED_BYTES`).
493
+ // What the cache counts for the pattern (see `MAX_CACHED_BYTES`): its source, and for the engine an estimate of
494
+ // its program and automata, which grow with the pattern.
517
495
  let mut retained = pattern.len() + 256;
518
- if matches!(matcher, Matcher::Regex(_) | Matcher::RegexBmp(_) | Matcher::Engine(_)) {
519
- retained += 4 * facts.size;
520
- }
521
- if matches!(matcher, Matcher::Regex(_) | Matcher::RegexBmp(_)) {
522
- retained += (facts.linear_size as usize).min(3 * REGEX_SIZE_LIMIT);
523
- }
524
- Ok(cache().insert(Arc::new(Pattern { source: pattern.to_string(), matcher, retained })))
525
- }
526
-
527
- /// The pattern translated for the `regex` crate and compiled by it, when it is in the syntax both have and its
528
- /// program is not large (see `ecmaregex::MAX_RE2_COST`: `regex` writes a counted repetition out, and takes time
529
- /// proportional to its program to compile and, for every character of a text, to match).
530
- fn linear(pattern: &str, facts: &ecmaregex::Facts) -> Option<Linear> {
531
- if facts.re2_cost > ecmaregex::MAX_RE2_COST {
532
- return None;
496
+ if matches!(matcher, Matcher::Engine(_)) {
497
+ retained += 16 * pattern.len();
533
498
  }
534
- let translation = translate(pattern, facts.unicode)?;
535
- let re = if ecmaregex::shallow(pattern) {
536
- regex::RegexBuilder::new(&translation)
537
- .size_limit(REGEX_SIZE_LIMIT)
538
- .dfa_size_limit(REGEX_DFA_SIZE_LIMIT)
539
- .build()
540
- .ok()?
541
- } else {
542
- // `regex` reads 250 levels of nesting unless it is told otherwise, because what it does with a pattern goes
543
- // a level deeper for each. A pattern that opens many groups is compiled on a thread with a stack for the
544
- // 1,000 levels the engine's own limit allows (a group, its quantifier and a class inside it are three
545
- // levels to `regex`), so that it is not the engine's for its nesting alone.
546
- ecmaregex::deep(|| {
547
- regex::RegexBuilder::new(&translation)
548
- .size_limit(REGEX_SIZE_LIMIT)
549
- .dfa_size_limit(REGEX_DFA_SIZE_LIMIT)
550
- .nest_limit(4 * ecmaregex::MAX_NESTING as u32)
551
- .build()
552
- .ok()
553
- })??
554
- };
555
- let engine = OnceLock::new();
556
- if !facts.unicode || !ecmaregex::shallow(pattern) {
557
- // The engine has the strings with a character beyond the Basic Multilingual Plane, so it is compiled now.
558
- // So is that of a pattern that nests deeply, which is compiled on a stack of its own.
559
- let _ = engine.set(Some(ecmaregex::compile(pattern).ok()?));
560
- }
561
- Some(Linear { re, cost: facts.linear_cost.max(1) as u64, engine })
499
+ Ok(cache().insert(Arc::new(Pattern { source: pattern.to_string(), grammar, matcher, retained })))
562
500
  }
563
501
 
564
- fn choose(pattern: &str, facts: &ecmaregex::Facts) -> Option<Matcher> {
502
+ fn choose(pattern: &str, unicode: bool, grammar: Grammar) -> Option<Matcher> {
565
503
  // Unanchored (or start-anchored) `.*` finds an empty match in any string; `^.*$` does not (`.` stops at a line
566
504
  // terminator), so it is not listed.
567
505
  if matches!(pattern, "" | ".*" | "^.*" | ".*$" | "(.*)" | "^(.*)" | "[\\s\\S]*" | "^[\\s\\S]*" | "^[\\s\\S]*$") {
568
506
  return Some(Matcher::Everything);
569
507
  }
570
508
  // Without the `u` flag, a pattern matches UTF-16 code units rather than characters: only the engine has that.
571
- if !facts.unicode {
509
+ if !unicode {
572
510
  return None;
573
511
  }
574
512
  match pattern {
@@ -582,9 +520,8 @@ fn choose(pattern: &str, facts: &ecmaregex::Facts) -> Option<Matcher> {
582
520
  && rest.len() > 1
583
521
  && !ends_with_escape(rest)
584
522
  && !rest.ends_with(['*', '+', '?', '}', '|', '(', '^'])
585
- && let Ok(rest_facts) = ecmaregex::check(rest)
586
- && rest_facts.unicode
587
- && let Some(m) = choose(rest, &rest_facts)
523
+ && grammar.is_valid_unicode(rest)
524
+ && let Some(m) = choose(rest, true, grammar)
588
525
  {
589
526
  return Some(m);
590
527
  }
@@ -592,24 +529,28 @@ fn choose(pattern: &str, facts: &ecmaregex::Facts) -> Option<Matcher> {
592
529
  return Some(Matcher::Line { min, max });
593
530
  }
594
531
  if let Some(texts) = whole_alternatives(pattern) {
595
- return Some(Matcher::Literals(Literals::new(texts)));
532
+ return (texts.len() <= MOST_LITERALS).then(|| Matcher::Literals(Literals::new(texts)));
596
533
  }
597
534
  if let Some(alts) = alternatives(pattern) {
598
- return Some(Matcher::Alternatives(alts));
599
- }
600
- if let Some(list) = SeparatedList::parse(pattern) {
601
- return Some(Matcher::SeparatedList(list));
535
+ let whole = |a: &Alternative| match a {
536
+ Alternative::Literal { start, end, .. } => *start && *end,
537
+ Alternative::Start(seq) => seq.to_end,
538
+ Alternative::End { .. } | Alternative::Anywhere { .. } => false,
539
+ };
540
+ return (alts.len() <= MOST_ALTERNATIVES && !alts.iter().any(whole)).then_some(Matcher::Alternatives(alts));
602
541
  }
603
542
  if let Some((set, bangs)) = excluded_class_with_word(pattern) {
604
543
  return Some(Matcher::ExcludedClassWithWord { set, bangs });
605
544
  }
606
545
  if let Some(seq) = Sequence::parse(pattern) {
607
546
  return Some(match seq.literal() {
547
+ // `^[a-z]*` with nothing it requires: every string.
548
+ Some(text) if text.is_empty() && !seq.to_end => Matcher::Everything,
608
549
  Some(text) => Matcher::Literal { text: text.into(), whole: seq.to_end },
609
550
  None => Matcher::Sequence(seq),
610
551
  });
611
552
  }
612
- linear(pattern, facts).map(Matcher::Regex)
553
+ None
613
554
  }
614
555
 
615
556
  /// `^.{m,n}$` (and `^.*$`, `^.+$`, `^.{m}$`, `^.{m,}$`, each also as `^(.…)$`): the bounds on the length of a line.
@@ -867,10 +808,13 @@ fn excluded_class_with_word(p: &str) -> Option<(CharSet, bool)> {
867
808
  .then_some((excluded, bangs))
868
809
  }
869
810
 
870
- /// The `regex` format: a valid ECMA-262 regular expression (with the `u` flag). A string over a limit of size is
871
- /// not read to its end and is not valid.
872
- pub(crate) fn is_valid_ecma_regex(s: &str) -> bool {
873
- ecmaregex::valid(s)
811
+ /// The `regex` format: a valid ECMA-262 regular expression in the grammar of the `u` flag, of the edition and version
812
+ /// of Unicode the dialect reads patterns with. It asks the grammar alone, as the .NET library's check does: the
813
+ /// engine's limits of size belong to compiling a pattern, so a string too large to compile is valid when its grammar
814
+ /// is. The engine's grammar-only reader takes time in proportion to the string and allocates nothing for one of
815
+ /// ordinary size.
816
+ pub(crate) fn is_valid_ecma_regex(s: &str, grammar: Grammar) -> bool {
817
+ Regex::is_valid_pattern_with(s, Grammar::flags(true), grammar.edition, grammar.unicode_version)
874
818
  }
875
819
 
876
820
  // ---------------------------------------------------------------------------------------------------------------------
@@ -1028,6 +972,16 @@ impl Sequence {
1028
972
  }
1029
973
  }
1030
974
  }
975
+ if !to_end {
976
+ // Without `$` a match needs only the start of the string: items at the end that can match nothing are
977
+ // dropped, and the last of the rest needs only its least count (`^[a-z]+` is `^[a-z]`).
978
+ while items.last().is_some_and(|item| item.min == 0) {
979
+ items.pop();
980
+ }
981
+ if let Some(last) = items.last_mut() {
982
+ last.max = last.min;
983
+ }
984
+ }
1031
985
  Some(Sequence { items: items.into_boxed_slice(), to_end })
1032
986
  }
1033
987
 
@@ -1083,231 +1037,6 @@ impl Sequence {
1083
1037
  }
1084
1038
  !self.to_end || at == b.len()
1085
1039
  }
1086
-
1087
- /// Greedily matches the items at the start of `s` (ignoring `to_end`): the bytes taken.
1088
- fn consume(&self, s: &str) -> Option<usize> {
1089
- let b = s.as_bytes();
1090
- let mut at = 0;
1091
- for item in self.items.iter() {
1092
- let mut n = 0u32;
1093
- while n < item.max && at < b.len() {
1094
- let c = b[at];
1095
- if c < 0x80 {
1096
- if !item.set.has_ascii(c) {
1097
- break;
1098
- }
1099
- at += 1;
1100
- } else {
1101
- let ch = s[at..].chars().next()?;
1102
- if !item.set.contains(ch) {
1103
- break;
1104
- }
1105
- at += ch.len_utf8();
1106
- }
1107
- n += 1;
1108
- }
1109
- if n < item.min {
1110
- return None;
1111
- }
1112
- }
1113
- Some(at)
1114
- }
1115
-
1116
- /// Whether greedy matching is exact when `next` can follow the items: every variable item is disjoint from the
1117
- /// items that can directly follow it (up to the first that cannot match nothing), `next` included.
1118
- fn greedy_before(items: &[Item], next: CharSet) -> bool {
1119
- items.iter().enumerate().all(|(i, item)| {
1120
- if item.min == item.max {
1121
- return true;
1122
- }
1123
- for following in &items[i + 1..] {
1124
- if !item.set.disjoint(following.set) {
1125
- return false;
1126
- }
1127
- if following.min > 0 {
1128
- return true;
1129
- }
1130
- }
1131
- item.set.disjoint(next)
1132
- })
1133
- }
1134
- }
1135
-
1136
- /// A list of items between separators: `^I(SR)*$` or `^I(SR)+$` (`final_` is `None`), or `^(RS)*F$` and
1137
- /// `^(RS)+F$`. The separator is a fixed sequence and no variable class can run into what follows it, so greedy
1138
- /// matching splits the string exactly where the pattern does.
1139
- #[derive(Debug)]
1140
- struct SeparatedList {
1141
- /// `I` of the first form.
1142
- first: Option<Sequence>,
1143
- repeated: Sequence,
1144
- separator: Sequence,
1145
- /// `F` of the second form, matched against the whole remainder.
1146
- final_: Option<Sequence>,
1147
- min_repeats: u32,
1148
- }
1149
-
1150
- impl SeparatedList {
1151
- fn is_match(&self, s: &str) -> bool {
1152
- let mut rest = s;
1153
- let mut repeats = 0;
1154
- if let Some(first) = &self.first {
1155
- let Some(n) = first.consume(rest) else { return false };
1156
- rest = &rest[n..];
1157
- loop {
1158
- if rest.is_empty() {
1159
- return repeats >= self.min_repeats;
1160
- }
1161
- let Some(n) = self.separator.consume(rest) else { return false };
1162
- rest = &rest[n..];
1163
- let Some(n) = self.repeated.consume(rest) else { return false };
1164
- rest = &rest[n..];
1165
- repeats += 1;
1166
- }
1167
- }
1168
- let final_ = self.final_.as_ref().expect("a separated list has a first or a final item");
1169
- // Whether the text is ASCII is worked out once, for the whole of it. The final sequence is tried after
1170
- // every item, and reading what is left of the text each time to see whether it is ASCII made the match take
1171
- // time quadratic in the number of items.
1172
- let ascii = s.is_ascii();
1173
- loop {
1174
- if repeats >= self.min_repeats
1175
- && (if ascii { final_.is_match_ascii(rest.as_bytes()) } else { final_.is_match_chars(rest) })
1176
- {
1177
- return true;
1178
- }
1179
- let Some(n) = self.repeated.consume(rest) else { return false };
1180
- let Some(m) = self.separator.consume(&rest[n..]) else { return false };
1181
- rest = &rest[n + m..];
1182
- repeats += 1;
1183
- }
1184
- }
1185
-
1186
- fn parse(p: &str) -> Option<SeparatedList> {
1187
- let body = p.strip_prefix('^')?.strip_suffix('$')?;
1188
- if ends_with_escape(body) {
1189
- return None;
1190
- }
1191
- let b = body.as_bytes();
1192
- // Top-level groups: (start, end) of each, where end is the index of its `)`.
1193
- let mut groups = Vec::new();
1194
- let (mut depth, mut in_class, mut i, mut open) = (0usize, false, 0, 0);
1195
- while i < b.len() {
1196
- match b[i] {
1197
- b'\\' => i += 1,
1198
- b'[' if !in_class => in_class = true,
1199
- b']' if in_class => in_class = false,
1200
- b'(' if !in_class => {
1201
- if depth == 0 {
1202
- open = i;
1203
- }
1204
- depth += 1;
1205
- }
1206
- b')' if !in_class => {
1207
- depth = depth.checked_sub(1)?;
1208
- if depth == 0 {
1209
- groups.push((open, i));
1210
- }
1211
- }
1212
- _ => {}
1213
- }
1214
- i += 1;
1215
- }
1216
- let quantified: Vec<_> = groups.iter().filter(|&&(_, e)| matches!(b.get(e + 1), Some(b'*' | b'+'))).collect();
1217
- let &&(open, close) = quantified.first().filter(|_| quantified.len() == 1)?;
1218
- let min_repeats = u32::from(b[close + 1] == b'+');
1219
- let group = strip_group(&body[open..=close])?;
1220
- let before = &body[..open];
1221
- let after = &body[close + 2..];
1222
- // The items of a text that is a class sequence. A `$` at the end of the text is not an item, and the list has
1223
- // no place for it: `^x(,a$)*$` is not `^x(,a)*$`. Such a text is not a part of a list.
1224
- let items = |t: &str| Sequence::parse(&format!("^{t}")).filter(|s| !s.to_end).map(|s| s.items.into_vec());
1225
- let seq = |items: &[Item], to_end| Sequence { items: items.into(), to_end };
1226
- let fixed = |items: &[Item]| !items.is_empty() && items.iter().all(|i| i.min == i.max && i.min > 0);
1227
- let g = items(group)?;
1228
- match (before.is_empty(), after.is_empty()) {
1229
- // ^I(SR)*$
1230
- (false, true) => {
1231
- let first = items(unwrap_group(before)?)?;
1232
- (1..g.len()).find_map(|k| {
1233
- let (separator, repeated) = g.split_at(k);
1234
- let next = separator[0].set;
1235
- (fixed(separator)
1236
- && Sequence::greedy_before(&first, next)
1237
- && Sequence::greedy_before(repeated, next))
1238
- .then(|| SeparatedList {
1239
- first: Some(seq(&first, false)),
1240
- repeated: seq(repeated, false),
1241
- separator: seq(separator, false),
1242
- final_: None,
1243
- min_repeats,
1244
- })
1245
- })
1246
- }
1247
- // ^(RS)*F$
1248
- (true, false) => {
1249
- let final_ = Sequence::parse(&format!("^{}$", unwrap_group(after)?))?;
1250
- (1..g.len()).find_map(|k| {
1251
- let (repeated, separator) = g.split_at(k);
1252
- (fixed(separator)
1253
- && Sequence::greedy_before(repeated, separator[0].set)
1254
- && final_stops_at(&final_, separator[0].set))
1255
- .then(|| SeparatedList {
1256
- first: None,
1257
- repeated: seq(repeated, false),
1258
- separator: seq(separator, false),
1259
- final_: Some(Sequence { items: final_.items.clone(), to_end: true }),
1260
- min_repeats,
1261
- })
1262
- })
1263
- }
1264
- _ => None,
1265
- }
1266
- }
1267
- }
1268
-
1269
- /// Whether the final sequence of a list cannot read past the separator that follows an item: no item of it holds a
1270
- /// character the first item of the separator holds. Tried at the start of an item, it then fails at that separator
1271
- /// at the latest, so trying it after every item reads the text a bounded number of times. A final sequence that can
1272
- /// read through separators would read to the end of the text from every item.
1273
- fn final_stops_at(final_: &Sequence, separator: CharSet) -> bool {
1274
- final_.items.iter().all(|item| item.set.disjoint(separator))
1275
- }
1276
-
1277
- /// The inside of `(...)` or `(?:...)`; `None` for other groups.
1278
- fn strip_group(g: &str) -> Option<&str> {
1279
- let inner = g.strip_prefix('(')?.strip_suffix(')')?;
1280
- match inner.strip_prefix("?:") {
1281
- Some(rest) => Some(rest),
1282
- None if inner.starts_with('?') => None,
1283
- None => Some(inner),
1284
- }
1285
- }
1286
-
1287
- /// A text that is one group wrapping everything, unwrapped; otherwise the text itself.
1288
- fn unwrap_group(t: &str) -> Option<&str> {
1289
- if t.starts_with('(') && t.ends_with(')') {
1290
- let inner = strip_group(t)?;
1291
- // Only when the parentheses enclose the whole text (`(a)(b)` is two groups).
1292
- let mut depth = 0i32;
1293
- let mut escaped = false;
1294
- for c in inner.chars() {
1295
- match c {
1296
- _ if escaped => escaped = false,
1297
- '\\' => escaped = true,
1298
- '(' => depth += 1,
1299
- ')' => {
1300
- depth -= 1;
1301
- if depth < 0 {
1302
- return None;
1303
- }
1304
- }
1305
- _ => {}
1306
- }
1307
- }
1308
- return Some(inner);
1309
- }
1310
- Some(t)
1311
1040
  }
1312
1041
 
1313
1042
  /// The set of a class escape (`\d`, `\w`, their negations, or an escaped punctuation character); `None` for `\s`
@@ -1427,338 +1156,26 @@ fn parse_quantifier(b: &[u8], mut i: usize) -> Option<(u32, u32, usize)> {
1427
1156
  Some((min, max, i))
1428
1157
  }
1429
1158
 
1430
- // ---------------------------------------------------------------------------------------------------------------------
1431
- // Translation to the regex crate
1432
-
1433
- /// ECMA-262 `\s`: WhiteSpace and LineTerminator.
1434
- const ECMA_SPACE: &str =
1435
- r"\t\n\x{B}\x{C}\r \x{A0}\x{1680}\x{2000}-\x{200A}\x{2028}\x{2029}\x{202F}\x{205F}\x{3000}\x{FEFF}";
1436
-
1437
- /// Translates an ECMA-262 pattern to the `regex` crate's syntax with the same meaning, or `None` when it uses
1438
- /// something outside the common subset. With `unicode` false, the pattern is read without the `u` flag (identity
1439
- /// escapes of any non-alphanumeric character), and the translation has its meaning on strings within the BMP.
1440
- fn translate(p: &str, unicode: bool) -> Option<String> {
1441
- let c: Vec<char> = p.chars().collect();
1442
- // Without the flag a character beyond the BMP is two code units, and what is around it (a quantifier, the ends
1443
- // of a range) applies to one of them. That has no translation.
1444
- if !unicode && c.iter().any(|&ch| ch as u32 > 0xFFFF) {
1445
- return None;
1446
- }
1447
- let mut out = String::with_capacity(p.len() * 2);
1448
- let mut i = 0;
1449
- while i < c.len() {
1450
- match c[i] {
1451
- '\\' => {
1452
- let (text, next) = escape(&c, i + 1, false, unicode)?;
1453
- out.push_str(&text);
1454
- i = next;
1455
- }
1456
- '[' => {
1457
- let (text, next) = class(&c, i + 1, unicode)?;
1458
- out.push_str(&text);
1459
- i = next;
1460
- }
1461
- '(' => {
1462
- if c.get(i + 1) == Some(&'?') {
1463
- match (c.get(i + 2), c.get(i + 3)) {
1464
- (Some(':'), _) => i += 3,
1465
- // A named group; lookbehinds (`(?<=`, `(?<!`) are not supported.
1466
- (Some('<'), Some(n)) if *n != '=' && *n != '!' => {
1467
- i += 3 + c[i + 3..].iter().position(|&x| x == '>')? + 1;
1468
- }
1469
- _ => return None,
1470
- }
1471
- } else {
1472
- i += 1;
1473
- }
1474
- out.push_str("(?:");
1475
- }
1476
- '{' => {
1477
- let end = i + c[i..].iter().position(|&x| x == '}')?;
1478
- let body: String = c[i + 1..end].iter().collect();
1479
- if body.is_empty() || !body.chars().all(|x| x.is_ascii_digit() || x == ',') {
1480
- return None;
1481
- }
1482
- out.push('{');
1483
- out.push_str(&body);
1484
- out.push('}');
1485
- i = end + 1;
1486
- }
1487
- '.' => {
1488
- out.push_str(r"[^\n\r\x{2028}\x{2029}]");
1489
- i += 1;
1490
- }
1491
- ch @ (')' | '|' | '^' | '$' | '*' | '+' | '?') => {
1492
- out.push(ch);
1493
- i += 1;
1494
- }
1495
- ch => {
1496
- push_literal(&mut out, ch);
1497
- i += 1;
1498
- }
1499
- }
1500
- }
1501
- Some(out)
1502
- }
1503
-
1504
- fn push_literal(out: &mut String, ch: char) {
1505
- use std::fmt::Write;
1506
- let _ = write!(out, "\\x{{{:X}}}", ch as u32);
1507
- }
1508
-
1509
- /// An escape after `\` at `i`: its translation and the index after it.
1510
- fn escape(c: &[char], i: usize, in_class: bool, unicode: bool) -> Option<(String, usize)> {
1511
- let e = *c.get(i)?;
1512
- let mut out = String::new();
1513
- let next = match e {
1514
- 'd' => {
1515
- out.push_str(if in_class { "0-9" } else { "[0-9]" });
1516
- i + 1
1517
- }
1518
- 'D' => {
1519
- out.push_str("[^0-9]");
1520
- i + 1
1521
- }
1522
- 'w' => {
1523
- out.push_str(if in_class { "0-9A-Za-z_" } else { "[0-9A-Za-z_]" });
1524
- i + 1
1525
- }
1526
- 'W' => {
1527
- out.push_str("[^0-9A-Za-z_]");
1528
- i + 1
1529
- }
1530
- 's' => {
1531
- if in_class {
1532
- out.push_str(ECMA_SPACE);
1533
- } else {
1534
- out.push('[');
1535
- out.push_str(ECMA_SPACE);
1536
- out.push(']');
1537
- }
1538
- i + 1
1539
- }
1540
- 'S' => {
1541
- out.push_str("[^");
1542
- out.push_str(ECMA_SPACE);
1543
- out.push(']');
1544
- i + 1
1545
- }
1546
- 'n' | 'r' | 't' | 'f' | 'v' => {
1547
- let ch = match e {
1548
- 'n' => '\n',
1549
- 'r' => '\r',
1550
- 't' => '\t',
1551
- 'f' => '\x0C',
1552
- _ => '\x0B',
1553
- };
1554
- push_literal(&mut out, ch);
1555
- i + 1
1556
- }
1557
- '0' if !c.get(i + 1).is_some_and(char::is_ascii_digit) => {
1558
- push_literal(&mut out, '\0');
1559
- i + 1
1560
- }
1561
- 'b' if in_class => {
1562
- push_literal(&mut out, '\x08');
1563
- i + 1
1564
- }
1565
- // ECMA's word boundaries are between ASCII word characters and the rest.
1566
- 'b' => {
1567
- out.push_str("(?-u:\\b)");
1568
- i + 1
1569
- }
1570
- // `\B` has no translation. `(?-u:\B)` also holds between the bytes of a character that takes several, where
1571
- // no match may start or end, and `regex` 1.13.1 then says there is no match where it finds one: for
1572
- // `s*?(?-u:\B)|..$` on `k\u{17f}k`, `is_match` is false and `find` is not. The engine has it.
1573
- 'B' => return None,
1574
- // A Unicode property is written as the ranges of the engine's own tables, and never by its name: the
1575
- // `regex` crate has tables of its own, of the version of Unicode its release carries, and a name would
1576
- // match what that version says. Without the `u` flag `\p` is the letter.
1577
- 'p' | 'P' if unicode && c.get(i + 1) == Some(&'{') => {
1578
- let end = i + 1 + c[i + 1..].iter().position(|&x| x == '}')?;
1579
- let body: String = c[i + 2..end].iter().collect();
1580
- let set = ecmaregex::property_set(&body)?;
1581
- let negated = e == 'P';
1582
- if set.ranges.is_empty() {
1583
- // No character has the property: nothing, or with `\P` everything.
1584
- out.push_str(if negated { r"[\x{0}-\x{10FFFF}]" } else { r"[^\x{0}-\x{10FFFF}]" });
1585
- } else if in_class && !negated {
1586
- push_ranges(&mut out, &set);
1587
- } else {
1588
- // A class of its own, which is also how a class holds the complement of a property.
1589
- out.push_str(if negated { "[^" } else { "[" });
1590
- push_ranges(&mut out, &set);
1591
- out.push(']');
1592
- }
1593
- end + 1
1594
- }
1595
- 'x' => {
1596
- let hex: String = c.get(i + 1..i + 3)?.iter().collect();
1597
- push_literal(&mut out, char::from_u32(u32::from_str_radix(&hex, 16).ok()?)?);
1598
- i + 3
1599
- }
1600
- // Without the `u` flag, `\u{...}` is not a code point escape.
1601
- 'u' if unicode || c.get(i + 1) != Some(&'{') => {
1602
- let (code, next) = unicode_escape(c, i + 1)?;
1603
- // Without the flag, two escapes that are a surrogate pair are two code units.
1604
- if !unicode && code > 0xFFFF {
1605
- return None;
1606
- }
1607
- push_literal(&mut out, char::from_u32(code)?);
1608
- next
1609
- }
1610
- // Identity escapes of syntax characters and `/` (and `-` in a class).
1611
- '^' | '$' | '\\' | '.' | '*' | '+' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '/' | '-' => {
1612
- push_literal(&mut out, e);
1613
- i + 1
1614
- }
1615
- // Without the `u` flag, any other non-alphanumeric character escapes to itself.
1616
- e if !unicode && !e.is_ascii_alphanumeric() => {
1617
- push_literal(&mut out, e);
1618
- i + 1
1619
- }
1620
- // Backreferences, \c, \k and the rest.
1621
- _ => return None,
1622
- };
1623
- Some((out, next))
1624
- }
1625
-
1626
- /// The ranges of a set, each end spelled as an escape. A surrogate is no character to the `regex` crate, and no
1627
- /// string holds one, so a range is written without the surrogates it spans. Two ranges that then touch, one ending
1628
- /// at U+D7FF and the next starting at U+E000, are written as one: see [`class`] for what `regex` makes of the two.
1629
- fn push_ranges(out: &mut String, set: &ecmaregex::CharSet) {
1630
- // The range that has been read and not yet written.
1631
- let mut pending: Option<(u32, u32)> = None;
1632
- let write = |out: &mut String, (lo, hi): (u32, u32)| {
1633
- if let (Some(lo), Some(hi)) = (char::from_u32(lo), char::from_u32(hi)) {
1634
- push_literal(out, lo);
1635
- if lo != hi {
1636
- out.push('-');
1637
- push_literal(out, hi);
1638
- }
1639
- }
1640
- };
1641
- for &[lo, hi] in set.ranges.as_chunks::<2>().0 {
1642
- // The ends that are surrogates move out of the surrogates. A range of surrogates alone is nothing.
1643
- let lo = if (0xD800..=0xDFFF).contains(&lo) { 0xE000 } else { lo as u32 };
1644
- let hi = if (0xD800..=0xDFFF).contains(&hi) { 0xD7FF } else { hi as u32 };
1645
- if lo > hi {
1646
- continue;
1647
- }
1648
- match pending {
1649
- Some((first, 0xD7FF)) if lo == 0xE000 => pending = Some((first, hi)),
1650
- Some(range) => {
1651
- write(out, range);
1652
- pending = Some((lo, hi));
1653
- }
1654
- None => pending = Some((lo, hi)),
1655
- }
1656
- }
1657
- if let Some(range) = pending {
1658
- write(out, range);
1659
- }
1660
- }
1661
-
1662
- /// `\u` escapes after the `u`: `XXXX` (joining a surrogate pair) or `{X...}`.
1663
- fn unicode_escape(c: &[char], i: usize) -> Option<(u32, usize)> {
1664
- if c.get(i) == Some(&'{') {
1665
- let end = i + c[i..].iter().position(|&x| x == '}')?;
1666
- let hex: String = c[i + 1..end].iter().collect();
1667
- return Some((u32::from_str_radix(&hex, 16).ok()?, end + 1));
1668
- }
1669
- let four = |at: usize| -> Option<u32> {
1670
- let hex: String = c.get(at..at + 4)?.iter().collect();
1671
- if hex.len() != 4 {
1672
- return None;
1673
- }
1674
- u32::from_str_radix(&hex, 16).ok()
1675
- };
1676
- let hi = four(i)?;
1677
- if (0xD800..0xDC00).contains(&hi) {
1678
- if c.get(i + 4) == Some(&'\\') && c.get(i + 5) == Some(&'u') {
1679
- let lo = four(i + 6)?;
1680
- if (0xDC00..0xE000).contains(&lo) {
1681
- return Some((0x10000 + ((hi - 0xD800) << 10) + (lo - 0xDC00), i + 10));
1682
- }
1683
- }
1684
- return None;
1685
- }
1686
- Some((hi, i + 4))
1687
- }
1688
-
1689
- /// A class after `[` at `i`: its translation (every member spelled as an escape, so nothing in it is special to the
1690
- /// `regex` crate) and the index after `]`.
1691
- fn class(c: &[char], mut i: usize, unicode: bool) -> Option<(String, usize)> {
1692
- let mut out = String::from("[");
1693
- if c.get(i) == Some(&'^') {
1694
- out.push('^');
1695
- i += 1;
1696
- }
1697
- let mut members = 0;
1698
- loop {
1699
- let ch = *c.get(i)?;
1700
- if ch == ']' {
1701
- i += 1;
1702
- break;
1703
- }
1704
- // One atom: a character (which can start a range) or a class escape.
1705
- let (atom, single, next) = if ch == '\\' {
1706
- let e = *c.get(i + 1)?;
1707
- let (text, next) = escape(c, i + 1, true, unicode)?;
1708
- let single = match e {
1709
- 'd' | 'D' | 'w' | 'W' | 's' | 'S' | 'p' | 'P' => None,
1710
- _ => Some(text.clone()),
1711
- };
1712
- (text, single, next)
1713
- } else {
1714
- let mut t = String::new();
1715
- push_literal(&mut t, ch);
1716
- (t.clone(), Some(t), i + 1)
1717
- };
1718
- i = next;
1719
- members += 1;
1720
- if c.get(i) == Some(&'-') && c.get(i + 1).is_some_and(|&n| n != ']') {
1721
- // A range: both ends single characters.
1722
- let lo = single?;
1723
- let (hi, next) = if c[i + 1] == '\\' {
1724
- let e = *c.get(i + 2)?;
1725
- if matches!(e, 'd' | 'D' | 'w' | 'W' | 's' | 'S' | 'p' | 'P') {
1726
- return None;
1727
- }
1728
- escape(c, i + 2, true, unicode)?
1729
- } else {
1730
- let mut t = String::new();
1731
- push_literal(&mut t, c[i + 1]);
1732
- (t, i + 2)
1733
- };
1734
- out.push_str(&lo);
1735
- out.push('-');
1736
- out.push_str(&hi);
1737
- i = next;
1738
- } else {
1739
- out.push_str(&atom);
1740
- }
1741
- }
1742
- if members == 0 {
1743
- return None; // `[]` and `[^]` have no counterpart.
1744
- }
1745
- // `regex` 1.13.1 (regex-syntax 0.8.11) negates wrongly a class with a range that ends at U+D7FF and another
1746
- // that starts at U+E000, the characters on either side of the surrogates: the complement takes both in, so
1747
- // `[^\x{D7FF}\x{E000}]` matches every character. A negated class that names both (every literal here is
1748
- // spelled `\x{...}`, so a `^` is a negation) is left to the engine.
1749
- if out.contains('^') && out.contains(r"\x{D7FF}") && out.contains(r"\x{E000}") {
1750
- return None;
1751
- }
1752
- out.push(']');
1753
- Some((out, i))
1754
- }
1755
-
1756
1159
  #[cfg(test)]
1757
1160
  mod tests {
1758
1161
  use super::*;
1759
1162
 
1163
+ const G: Grammar = Grammar { edition: Edition::Current, unicode_version: UnicodeVersion::Default };
1164
+
1165
+ /// A pattern compiled in today's grammar.
1166
+ fn compile(p: &str) -> Result<Arc<Pattern>, PatternError> {
1167
+ super::compile(p, G)
1168
+ }
1169
+
1760
1170
  const PATTERNS: &[&str] = &[
1761
1171
  "",
1172
+ // Sequences without `$`, which need only the start of the string.
1173
+ "^[a-z]+",
1174
+ "^[a-z]*",
1175
+ "^[a-zA-Z0-9_\\.\\-]*",
1176
+ "^a[b-c]{2,5}",
1177
+ "^[a-z]+[0-9]*",
1178
+ "^[a-z]{0,3}[0-9]{2,4}",
1762
1179
  ".*",
1763
1180
  "^.*",
1764
1181
  ".*$",
@@ -1919,18 +1336,17 @@ mod tests {
1919
1336
  seed
1920
1337
  };
1921
1338
  for &p in PATTERNS {
1922
- let reference = ecmaregex::compile(p).unwrap();
1339
+ let reference = G.build(p, true).or_else(|_| G.build(p, false)).unwrap();
1923
1340
  let compiled = compile(p).unwrap();
1924
1341
  for _ in 0..4000 {
1925
1342
  let len = next() % 9;
1926
1343
  let s: String = (0..len).map(|_| alphabet[(next() % alphabet.len() as u64) as usize]).collect();
1927
1344
  // Without the `u` flag the engine matches the UTF-16 code units of the string.
1928
- let found = reference.match_backtrack(&s, None).unwrap();
1345
+ let found = reference.is_match(&s).unwrap();
1929
1346
  for limit in [None, Some(Duration::from_secs(60)), Some(Duration::from_nanos(1))] {
1930
1347
  let mut budget = PatternBudget::new(PatternLimit::new(limit));
1931
1348
  let got = compiled.is_match(&s, &mut budget);
1932
- // A limit of a nanosecond gives `regex` no string, so this is the engine for every pattern on
1933
- // it. A match that reached its first check has no answer.
1349
+ // A match on the engine that reached its first check of the clock has no answer.
1934
1350
  assert!(got == found || budget.failed(), "{p:?} on {s:?} with a limit of {limit:?}");
1935
1351
  assert!(limit.is_some_and(|l| l.as_nanos() == 1) || !budget.failed(), "{p:?} on {s:?}");
1936
1352
  }
@@ -1946,71 +1362,55 @@ mod tests {
1946
1362
  for p in ["^x-", "^\\/", "^abc$"] {
1947
1363
  assert!(matches!(compile(p).unwrap().matcher, Matcher::Literal { .. }), "{p}");
1948
1364
  }
1949
- for p in ["^[a-z]*a$", "(base64key|awskms)://(.*)"] {
1950
- assert!(matches!(compile(p).unwrap().matcher, Matcher::Regex(_)), "{p}");
1951
- }
1365
+ assert!(matches!(compile("^[a-z]*a$").unwrap().matcher, Matcher::Engine(_)));
1952
1366
  for p in [".+", "^.+", "(.+)"] {
1953
1367
  assert!(matches!(compile(p).unwrap().matcher, Matcher::HasContent { .. }), "{p}");
1954
1368
  }
1955
1369
  for p in ["^.{1,256}$", "^.+$", "^.*$", "^(.*)$", "^(.+)$"] {
1956
1370
  assert!(matches!(compile(p).unwrap().matcher, Matcher::Line { .. }), "{p}");
1957
1371
  }
1958
- for p in ["^(ab|cd)$", "^(?:es|ES|x-)$"] {
1372
+ for p in ["^(ab|cd)$", "^(?:es|ES|x-)$", "^(CRLF|LF|crlf|lf)$"] {
1959
1373
  assert!(matches!(compile(p).unwrap().matcher, Matcher::Literals(_)), "{p}");
1960
1374
  }
1375
+ // Without `$`, a sequence is read only as far as it must be.
1376
+ for p in ["^[a-z]+", "^a[b-c]{2,5}"] {
1377
+ assert!(
1378
+ matches!(&compile(p).unwrap().matcher, Matcher::Sequence(seq) if seq.items.last().unwrap().max == seq.items.last().unwrap().min),
1379
+ "{p}"
1380
+ );
1381
+ }
1382
+ assert!(matches!(&compile("^[a-zA-Z0-9_\\.\\-]*").unwrap().matcher, Matcher::Everything));
1383
+ // At most two alternatives, neither anchored at both ends.
1384
+ for p in ["^ab|cd$", "a|b", "^([a|A][u|U][t|T][o|O])|([n|N][o|O][n|N][e|E])$", "(base64key|awskms)://(.*)"] {
1385
+ assert!(matches!(compile(p).unwrap().matcher, Matcher::Alternatives(_)), "{p}");
1386
+ }
1387
+ // The engine is faster for more literals, more alternatives, an alternative anchored at both ends, and a
1388
+ // separated list.
1961
1389
  for p in [
1962
- "^ab|cd$",
1963
- "a|b",
1964
- "^([a|A][u|U][t|T][o|O])|([n|N][o|O][n|N][e|E])$",
1390
+ "^(schemas|responses|parameters|examples|requestBodies)$",
1965
1391
  "^[Ee][Ss]2015(\\.([Cc][Oo][Rr][Ee]|[Pp][Rr][Oo][Xx][Yy]))?$",
1966
1392
  "^[1-5](?:[0-9]{2}|XX)$",
1967
1393
  "(^([0-9]+)\\.([0-9]+)$)|(^\\{[A-F0-9]{8}(-[A-F0-9]{4}){3}-[A-F0-9]{12}\\}$)",
1968
1394
  "^([t|T][o|O][p|P])|([c|C][e|E][n|N][t|T][e|E][r|R])|([b|B][o|O][t|T][t|T][o|O][m|M])$",
1969
1395
  "^([A-Za-z_][-A-Za-z0-9_.:]*)$",
1970
- ] {
1971
- assert!(matches!(compile(p).unwrap().matcher, Matcher::Alternatives(_)), "{p}");
1972
- }
1973
- for p in [
1396
+ "default|^[0-9]+$",
1974
1397
  "^([a-zA-Z0-9]{2,3})(-[a-zA-Z0-9]{1,6})*$",
1975
1398
  "^([a-zA-Z_$][a-zA-Z0-9_$]{0,39}\\.)*([a-zA-Z_$][a-zA-Z0-9_$]{0,39})$",
1976
- "^([a-z_$][a-z0-9_$]{0,39}\\.)*([a-zA-Z_$][a-zA-Z0-9_$]{0,39})$",
1977
- // An escaped `$` or `^` is a literal character of the list.
1978
1399
  "^x(,a\\$)*$",
1979
- "^x\\$(,a)*$",
1980
- "^(a\\$,)*b$",
1981
- "^x(,\\^a)*$",
1982
- "^(a,)*b\\$$",
1983
- "^x(,[$^]a)*$",
1984
- ] {
1985
- assert!(matches!(compile(p).unwrap().matcher, Matcher::SeparatedList(_)), "{p}");
1986
- }
1987
- // A list with an anchor or an assertion inside it has no shape.
1988
- for p in [
1989
1400
  "^x(,a$)*$",
1990
- "^x(?:,a$)+$",
1991
- "^x$(,a)*$",
1992
- "^(x$)(,a)*$",
1993
- "^$(,a)*$",
1994
- "^(a,$)*b$",
1995
- "^x(^,a)*$",
1996
- "^(^a,)*b$",
1997
- "^x(,(a$))*$",
1998
- "^x(,a\\b)*$",
1999
- "^x\\B(,a)*$",
2000
- "^x(,a(?=,|$))*$",
2001
1401
  "^x(?=,)(,a)*$",
2002
1402
  ] {
2003
- assert!(matches!(compile(p).unwrap().matcher, Matcher::Regex(_) | Matcher::Engine(_)), "{p}");
1403
+ assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
2004
1404
  }
2005
1405
  assert!(matches!(
2006
1406
  compile("^(?=[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$").unwrap().matcher,
2007
1407
  Matcher::ExcludedClassWithWord { .. }
2008
1408
  ));
2009
1409
  for p in ["\\bfoo", "^\\p{L}+$"] {
2010
- assert!(matches!(compile(p).unwrap().matcher, Matcher::Regex(_)), "{p}");
1410
+ assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
2011
1411
  }
2012
1412
  for p in ["^\\/[^\\*\\?\\&\\%]*(\\/\\*)?$", "a\\&.b", "^\\-a"] {
2013
- assert!(matches!(compile(p).unwrap().matcher, Matcher::RegexBmp(..)), "{p}");
1413
+ assert!(matches!(&compile(p).unwrap().matcher, Matcher::Engine(engine) if !engine.flags().unicode), "{p}");
2014
1414
  }
2015
1415
  for p in ["^((\\.(?!\\.)\\/)?\\w+\\/?)+$", "^\\1(a)", "\\Bs", "(?i:a)", "^[]", "\\cA"] {
2016
1416
  assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
@@ -2018,7 +1418,7 @@ mod tests {
2018
1418
  // Valid only without the `u` flag, and not in the syntax `regex` takes: the engine, by code unit.
2019
1419
  for p in ["^\\p{L}\\&", "^\\-?\u{10400}*$", "^\\-(?=a)"] {
2020
1420
  let compiled = compile(p).unwrap();
2021
- assert!(matches!(&compiled.matcher, Matcher::Engine(engine) if !engine.unicode()), "{p}");
1421
+ assert!(matches!(&compiled.matcher, Matcher::Engine(engine) if !engine.flags().unicode), "{p}");
2022
1422
  }
2023
1423
  }
2024
1424
  }
@@ -2027,6 +1427,13 @@ mod tests {
2027
1427
  mod limits {
2028
1428
  use super::*;
2029
1429
 
1430
+ const G: Grammar = Grammar { edition: Edition::Current, unicode_version: UnicodeVersion::Default };
1431
+
1432
+ /// A pattern compiled in today's grammar.
1433
+ fn compile(p: &str) -> Result<Arc<Pattern>, PatternError> {
1434
+ super::compile(p, G)
1435
+ }
1436
+
2030
1437
  /// Which matcher a pattern gets (the Go module's `TestAWideSearchIsLeftToTheEngine`). A sequence or a literal
2031
1438
  /// that would be searched for at every position of the text keeps its matcher up to 64 characters, and beyond
2032
1439
  /// that is left to a matcher that takes time proportional to the text or has the limit of a match.
@@ -2047,55 +1454,10 @@ mod limits {
2047
1454
  ] {
2048
1455
  let found = kind(&compile(&pattern).unwrap());
2049
1456
  assert_eq!(found == "alternatives", alternatives, "{pattern:.30} has the matcher {found}");
2050
- assert!(alternatives || matches!(found, "regex" | "engine"), "{pattern:.30} has the matcher {found}");
2051
- }
2052
- // The final sequence of a list must stop at the separator.
2053
- for (pattern, list) in [
2054
- (r"^(?:[a-z]+,)*[a-z]{2}[0-9]$", true),
2055
- (r"^(?:[a-z]+,)*[a-z,]+[0-9]$", false),
2056
- (r"^(?:[a-z]+,)*[0-9]{1,3}$", true),
2057
- ] {
2058
- let found = kind(&compile(pattern).unwrap());
2059
- assert_eq!(found == "list", list, "{pattern} has the matcher {found}");
2060
- }
2061
- }
2062
-
2063
- /// The `regex` crate is given no large program (the Go module's `TestTheRegexpPackageIsGivenNoLargeProgram`):
2064
- /// it writes a counted repetition out, so the pattern goes to the engine when the copies are many.
2065
- #[test]
2066
- fn regex_is_given_no_large_program() {
2067
- for (pattern, want) in [
2068
- (r"a{1000}".to_string(), "regex"),
2069
- (r"(?:a|b|ab)*c".to_string(), "regex"),
2070
- (r"\p{L}{1000}".repeat(90), "engine"),
2071
- (r"(?:[ab]{1000}){50}c".to_string(), "engine"),
2072
- (r"(?:a{100}){101}".to_string(), "engine"),
2073
- (r"(?:(?:a?){100}){100}b".to_string(), "engine"),
2074
- ] {
2075
- assert_eq!(kind(&compile(&pattern).unwrap()), want, "{pattern:.40}");
1457
+ assert!(alternatives || found == "engine", "{pattern:.30} has the matcher {found}");
2076
1458
  }
2077
1459
  }
2078
1460
 
2079
- /// A string too long for `regex` to answer within the limit of a match goes to the engine, which can be
2080
- /// stopped (the Go module's `TestALongTextIsNotGivenToTheRegexpPackage`).
2081
- #[test]
2082
- fn a_long_string_is_not_given_to_regex() {
2083
- let compiled = compile(r"^(?:[a-c\u00e9]{0,500}x)*$").unwrap();
2084
- let Matcher::Regex(linear) = &compiled.matcher else { panic!("the pattern is on {}", kind(&compiled)) };
2085
- assert!(linear.cost >= 500, "a program of {} instructions", linear.cost);
2086
- let short = "\u{e9}abcx".to_string();
2087
- let long = "\u{e9}abcx".repeat(100000);
2088
- let second = PatternLimit::new(Some(Duration::from_secs(1)));
2089
- let mut budget = PatternBudget::new(second);
2090
- assert!(linear.fits(&short, &budget) && !linear.fits(&long, &budget), "the strings are not on each side");
2091
- assert!(compiled.is_match(&short, &mut budget) && linear.engine.get().is_none(), "a short string");
2092
- assert!(compiled.is_match(&long, &mut budget) && !budget.failed(), "a long string");
2093
- assert!(matches!(linear.engine.get(), Some(Some(_))), "the engine was not made for the long string");
2094
- // With no limit every string is `regex`'s.
2095
- let mut unlimited = PatternBudget::new(PatternLimit::new(None));
2096
- assert!(linear.fits(&long, &unlimited) && !compiled.is_match(&format!("{long}y"), &mut unlimited));
2097
- }
2098
-
2099
1461
  /// The cache of compiled patterns is bounded in number and in size, and a pattern that has left it is still
2100
1462
  /// matched by whoever holds it.
2101
1463
  #[test]
@@ -2109,7 +1471,7 @@ mod limits {
2109
1471
  assert!(cache.patterns.len() <= MAX_CACHED_PATTERNS && cache.bytes <= MAX_CACHED_BYTES);
2110
1472
  assert_eq!(cache.patterns.len(), cache.order.len());
2111
1473
  assert_eq!(cache.bytes, cache.patterns.values().map(|p| p.retained).sum::<usize>());
2112
- assert!(cache.get("^cache-bound-0+$").is_none(), "the oldest pattern is still in the cache");
1474
+ assert!(cache.get("^cache-bound-0+$", G).is_none(), "the oldest pattern is still in the cache");
2113
1475
  }
2114
1476
  assert!(first.is_match("cache-bound-000", &mut budget));
2115
1477
  assert!(held[0].is_match("cache-bound-0x", &mut budget) && !held[7].is_match("cache-bound-0x", &mut budget));
@@ -2122,154 +1484,162 @@ mod limits {
2122
1484
  assert!(large[0].retained > 256 << 10, "a large pattern is counted as {} bytes", large[0].retained);
2123
1485
  assert!(cache().bytes <= MAX_CACHED_BYTES);
2124
1486
  // A pattern too large for the cache is not kept at all.
2125
- let huge = compile(&format!("^(?=huge){}$", "a".repeat(15000))).unwrap();
2126
- assert!(huge.retained > MAX_CACHED_PATTERN_BYTES && cache().get(&huge.source).is_none());
1487
+ let huge = compile(&format!("^(?=huge){}$", class.repeat(4))).unwrap();
1488
+ assert!(huge.retained > MAX_CACHED_PATTERN_BYTES && cache().get(&huge.source, G).is_none());
2127
1489
  }
1490
+ }
2128
1491
 
2129
- /// The `regex` crate compiles no program over its limit of size: a class of a Unicode property repeated often
2130
- /// is the engine's, which holds the class once.
2131
- #[test]
2132
- fn regex_compiles_no_program_over_its_limit() {
2133
- for (pattern, want) in [
2134
- (r"^\p{L}+$", "regex"),
2135
- (r"^[\p{L}\p{N}_-]{1,8}$", "regex"),
2136
- (r"^[\p{L}\p{N}_-]{1,64}$", "engine"),
2137
- (r"^[\p{Alphabetic}\p{Nd}]{1,255}$", "engine"),
2138
- ] {
2139
- let compiled = compile(pattern).unwrap();
2140
- assert_eq!(kind(&compiled), want, "{pattern}");
2141
- let mut budget = PatternBudget::new(PatternLimit::new(Some(Duration::from_secs(1))));
2142
- assert!(compiled.is_match("caf\u{e9}", &mut budget) && !compiled.is_match("caf\u{e9}!", &mut budget));
2143
- }
1492
+ #[cfg(test)]
1493
+ mod corpus_patterns {
1494
+ use std::sync::Arc;
1495
+
1496
+ use super::{Edition, Grammar, Pattern, PatternError, UnicodeVersion};
1497
+
1498
+ const G: Grammar = Grammar { edition: Edition::Current, unicode_version: UnicodeVersion::Default };
1499
+
1500
+ /// A pattern compiled in today's grammar.
1501
+ fn compile(p: &str) -> Result<Arc<Pattern>, PatternError> {
1502
+ super::compile(p, G)
2144
1503
  }
2145
1504
 
2146
- /// Alternatives of one character each are one class to a linear matcher, so a long alternation is a small
2147
- /// program, and `regex` keeps a long string for it. And a pattern that nests more deeply than `regex` reads by
2148
- /// default is still compiled by it, on a thread with a stack for the nesting.
1505
+ /// Lists which matcher each pattern in `CORVUS_PATTERNS` (corpus<TAB>pattern lines) gets, and what the cache
1506
+ /// counts for it.
2149
1507
  #[test]
2150
- fn regex_keeps_a_long_alternation_and_a_deep_nest() {
2151
- let alternatives: Vec<String> = (0..495u32).map(|k| char::from_u32(0x4E00 + k).unwrap().to_string()).collect();
2152
- let compiled = compile(&format!("(?:{})*z", alternatives.join("|"))).unwrap();
2153
- let Matcher::Regex(linear) = &compiled.matcher else { panic!("the alternation is on {}", kind(&compiled)) };
2154
- assert!(linear.cost <= 4, "a program of {} instructions", linear.cost);
2155
- let text = alternatives.concat().repeat(70);
2156
- assert!(text.len() >= 100_000);
2157
- let mut budget = PatternBudget::new(PatternLimit::new(Some(Duration::from_secs(1))));
2158
- assert!(linear.fits(&text, &budget) && !compiled.is_match(&text, &mut budget) && !budget.failed());
2159
- assert!(compiled.is_match(&format!("{text}z"), &mut budget) && linear.engine.get().is_none());
2160
-
2161
- for depth in [300, 999] {
2162
- let nested = format!("^{}a{}b$", "(".repeat(depth), "*)".repeat(depth));
2163
- let compiled = compile(&nested).unwrap();
2164
- assert_eq!(kind(&compiled), "regex", "{depth} nested groups");
2165
- let start = std::time::Instant::now();
2166
- assert!(
2167
- compiled.is_match("aaaab", &mut budget)
2168
- && !compiled.is_match("\u{e9}aaaaaaaaaaaaaaaaaaaaaaaaa", &mut budget)
2169
- );
2170
- assert!(!budget.failed() && start.elapsed() < Duration::from_secs(20), "{depth} nested groups");
1508
+ #[ignore]
1509
+ fn classify_corpus_patterns() {
1510
+ let Ok(path) = std::env::var("CORVUS_PATTERNS") else { return };
1511
+ for line in std::fs::read_to_string(path).unwrap().lines() {
1512
+ let (corpus, p) = line.split_once('\t').unwrap();
1513
+ let kind = compile(p).map(|c| super::kind(&c)).unwrap_or("invalid");
1514
+ let retained = compile(p).map_or(0, |c| c.retained);
1515
+ println!("{kind}\t{corpus}\t{p}\t{retained}");
2171
1516
  }
2172
1517
  }
2173
1518
 
2174
- /// The translation for `regex` names no Unicode class: every property is written as the ranges of the engine's
2175
- /// own tables, so the answer does not depend on the tables of the `regex` crate.
1519
+ /// Times each dedicated matcher against the engine over the corpora of jsonschema-benchmark in `CORVUS_CORPORA`
1520
+ /// (a directory of `<name>-schema.json` and `<name>-instances.jsonl`), so that a matcher the engine beats can be
1521
+ /// removed. The inputs of a pattern are the distinct strings of the corpus's instances (for `pattern`) or their
1522
+ /// property names (for `patternProperties`), at most 1,000. Each figure is the least of 7 rounds of at least 20 ms,
1523
+ /// after a warm-up of 200 ms, through `Pattern::is_match` with the default limit of a match. Run it with
1524
+ /// `cargo test --release --lib time_corpus_patterns -- --ignored --nocapture --test-threads 1`.
2176
1525
  #[test]
2177
- fn the_translation_carries_no_unicode_class() {
2178
- for p in [
2179
- r"^\p{L}+$",
2180
- r"\P{Lu}",
2181
- r"[\p{Script=Greek}\p{Nd}]",
2182
- r"[^\p{scx=Hira}]",
2183
- r"\d\w\s.",
2184
- r"\p{Any}\p{Assigned}\p{ASCII}",
2185
- ] {
2186
- let compiled = compile(p).unwrap();
2187
- assert_eq!(kind(&compiled), "regex", "{p}");
2188
- let translated = translate(p, true).unwrap();
2189
- for bad in [r"\p", r"\P", "(?i", "[[:", r"\d", r"\w", r"\s"] {
2190
- assert!(!translated.contains(bad), "the translation of {p} contains {bad}");
1526
+ #[ignore]
1527
+ fn time_corpus_patterns() {
1528
+ use std::collections::BTreeSet;
1529
+ use std::time::{Duration, Instant};
1530
+
1531
+ use super::{Literals, Matcher, PatternBudget, PatternLimit};
1532
+
1533
+ fn collect(v: &serde_json::Value, strings: &mut BTreeSet<String>, keys: &mut BTreeSet<String>) {
1534
+ match v {
1535
+ serde_json::Value::String(s) => {
1536
+ strings.insert(s.clone());
1537
+ }
1538
+ serde_json::Value::Array(a) => a.iter().for_each(|v| collect(v, strings, keys)),
1539
+ serde_json::Value::Object(o) => o.iter().for_each(|(k, v)| {
1540
+ keys.insert(k.clone());
1541
+ collect(v, strings, keys);
1542
+ }),
1543
+ _ => {}
2191
1544
  }
2192
1545
  }
2193
- // The four scripts Unicode 17 added are in the tables, whatever the `regex` crate's own tables hold.
2194
- let mut budget = PatternBudget::new(PatternLimit::new(None));
2195
- for (p, text) in [
2196
- (r"^\p{sc=Sidt}$", "\u{10940}"),
2197
- (r"^\p{sc=Tols}$", "\u{11DB0}"),
2198
- (r"^\p{sc=Berf}$", "\u{16EA0}"),
2199
- (r"^\p{sc=Tayo}$", "\u{1E6C0}"),
2200
- (r"^[\p{sc=Sidt}a]$", "\u{10940}"),
2201
- (r"^[^\P{sc=Sidt}]$", "\u{10940}"),
2202
- ] {
2203
- let compiled = compile(p).unwrap();
2204
- assert_eq!(kind(&compiled), "regex", "{p}");
2205
- assert!(compiled.is_match(text, &mut budget) && !compiled.is_match("a!", &mut budget), "{p}");
2206
- }
2207
- // The complement of a set whose ranges meet at the surrogates, which `regex` gets wrong when it is given
2208
- // the two ranges: Unknown holds U+D7FC to U+D7FF and U+E000 to U+F8FF. And the same written by hand, which
2209
- // is the engine's.
2210
- for (p, want) in [
2211
- (r"^\P{sc=Unknown}$", "regex"),
2212
- (r"^[^\p{scx=Unknown}]$", "regex"),
2213
- (r"^[^\p{Script=Unknown}a]$", "regex"),
2214
- (r"^[^\ud7ff\ue000]$", "engine"),
2215
- (r"^[^\ud7fc-\ud7ff\ue000-\uf8ff]$", "engine"),
2216
- (r"^[^\p{Co}\ud7ff]$", "engine"),
2217
- (r"^[^\p{Cn}\ue000]$", "engine"),
2218
- ] {
2219
- let compiled = compile(p).unwrap();
2220
- assert_eq!(kind(&compiled), want, "{p}");
2221
- let engine = ecmaregex::compile(p).unwrap();
2222
- for text in ["\u{D7FB}", "\u{D7FC}", "\u{D7FF}", "\u{E000}", "\u{E001}", "\u{F8FF}", "\u{F900}", "a", "b"] {
2223
- assert_eq!(
2224
- compiled.is_match(text, &mut budget),
2225
- engine.match_backtrack(text, None).unwrap(),
2226
- "{p} on {text:?}"
2227
- );
1546
+ fn patterns(v: &serde_json::Value, out: &mut BTreeSet<(&'static str, String)>) {
1547
+ match v {
1548
+ serde_json::Value::Array(a) => a.iter().for_each(|v| patterns(v, out)),
1549
+ serde_json::Value::Object(o) => {
1550
+ if let Some(serde_json::Value::String(p)) = o.get("pattern") {
1551
+ out.insert(("pattern", p.clone()));
1552
+ }
1553
+ if let Some(serde_json::Value::Object(pp)) = o.get("patternProperties") {
1554
+ pp.keys().for_each(|p| {
1555
+ out.insert(("patternProperties", p.clone()));
1556
+ });
1557
+ }
1558
+ o.values().for_each(|v| patterns(v, out));
1559
+ }
1560
+ _ => {}
2228
1561
  }
2229
- assert!(!compiled.is_match("\u{E000}", &mut budget), "{p} matches U+E000");
2230
- }
2231
- // A set with a range on each side of the surrogates and none of them is written as one range, and a range
2232
- // of surrogates alone as nothing.
2233
- let mut written = String::new();
2234
- push_ranges(&mut written, &ecmaregex::CharSet::new(&[0x61, 0x7A, 0xD700, 0xD7FF, 0xE000, 0xE0FF], false));
2235
- assert_eq!(written, r"\x{61}-\x{7A}\x{D700}-\x{E0FF}");
2236
- written.clear();
2237
- push_ranges(&mut written, &ecmaregex::CharSet::new(&[0xD7FF, 0xDBFF, 0xDC00, 0xDFFF, 0xDFFF, 0xE000], false));
2238
- assert_eq!(written, r"\x{D7FF}-\x{E000}");
2239
- written.clear();
2240
- push_ranges(&mut written, &ecmaregex::CharSet::new(&[0xD800, 0xDFFF], false));
2241
- assert_eq!(written, "");
2242
- // A property no character has, and its complement.
2243
- for (p, yes, no) in
2244
- [(r"^\P{sc=Hrkt}$", "a", "aa"), (r"^[\p{sc=Hrkt}a]$", "a", "b"), (r"^[^\P{sc=Hrkt}]?$", "", "a")]
2245
- {
2246
- let compiled = compile(p).unwrap();
2247
- assert_eq!(kind(&compiled), "regex", "{p}");
2248
- assert!(compiled.is_match(yes, &mut budget) && !compiled.is_match(no, &mut budget), "{p}");
2249
1562
  }
2250
- }
2251
- }
2252
-
2253
- #[cfg(test)]
2254
- mod corpus_patterns {
2255
- /// Lists which matcher each pattern in `CORVUS_PATTERNS` (corpus<TAB>pattern lines) gets.
2256
- #[test]
2257
- #[ignore]
2258
- fn classify_corpus_patterns() {
2259
- let Ok(path) = std::env::var("CORVUS_PATTERNS") else { return };
2260
- for line in std::fs::read_to_string(path).unwrap().lines() {
2261
- let (corpus, p) = line.split_once('\t').unwrap();
2262
- let kind = super::compile(p).map(|c| super::kind(&c)).unwrap_or("invalid");
2263
- // What the `regex` crate is given: the translation.
2264
- let unicode = crate::ecmaregex::valid(p);
2265
- let given = match kind {
2266
- "regex" | "regex-bmp" => super::translate(p, unicode).unwrap_or_default(),
2267
- _ => String::new(),
1563
+ fn time(p: &Pattern, inputs: &[String]) -> f64 {
1564
+ let limit = PatternLimit::new(Some(crate::DEFAULT_PATTERN_MATCH_LIMIT));
1565
+ let pass = || {
1566
+ let mut budget = PatternBudget::new(limit);
1567
+ inputs.iter().filter(|s| p.is_match(s, &mut budget)).count()
2268
1568
  };
2269
- // And what the cache counts for the pattern, which for a pattern on `regex` holds the estimate of
2270
- // that crate's program.
2271
- let retained = super::compile(p).map_or(0, |c| c.retained);
2272
- println!("{kind}\t{corpus}\t{p}\t{given}\t{retained}");
1569
+ let warm = Instant::now();
1570
+ while warm.elapsed() < Duration::from_millis(200) {
1571
+ std::hint::black_box(pass());
1572
+ }
1573
+ let mut best = f64::MAX;
1574
+ for _ in 0..7 {
1575
+ let (start, mut calls) = (Instant::now(), 0usize);
1576
+ while start.elapsed() < Duration::from_millis(20) {
1577
+ std::hint::black_box(pass());
1578
+ calls += inputs.len();
1579
+ }
1580
+ best = best.min(start.elapsed().as_nanos() as f64 / calls as f64);
1581
+ }
1582
+ best
1583
+ }
1584
+
1585
+ let Ok(dir) = std::env::var("CORVUS_CORPORA") else { return };
1586
+ let mut names: Vec<String> = std::fs::read_dir(&dir)
1587
+ .unwrap()
1588
+ .filter_map(|e| e.unwrap().file_name().to_str()?.strip_suffix("-schema.json").map(str::to_string))
1589
+ .collect();
1590
+ names.sort();
1591
+ // The alternatives a matcher tries one after another.
1592
+ fn alternatives(m: &Matcher) -> usize {
1593
+ match m {
1594
+ Matcher::Alternatives(a) => a.len(),
1595
+ Matcher::Literals(Literals(t)) => t.len(),
1596
+ _ => 1,
1597
+ }
1598
+ }
1599
+ println!("corpus\trole\tkind\talternatives\tshape ns\tengine ns\tshape/engine\tinputs\tpattern");
1600
+ for name in names {
1601
+ let schema: serde_json::Value =
1602
+ serde_json::from_str(&std::fs::read_to_string(format!("{dir}/{name}-schema.json")).unwrap()).unwrap();
1603
+ let (mut strings, mut keys) = (BTreeSet::new(), BTreeSet::new());
1604
+ for line in std::fs::read_to_string(format!("{dir}/{name}-instances.jsonl")).unwrap().lines() {
1605
+ if !line.trim().is_empty() {
1606
+ collect(&serde_json::from_str(line).unwrap(), &mut strings, &mut keys);
1607
+ }
1608
+ }
1609
+ let strings: Vec<String> = strings.into_iter().take(1000).collect();
1610
+ let keys: Vec<String> = keys.into_iter().take(1000).collect();
1611
+ let mut found = BTreeSet::new();
1612
+ patterns(&schema, &mut found);
1613
+ for (role, source) in found {
1614
+ let Ok(shape) = compile(&source) else { continue };
1615
+ if matches!(shape.matcher, Matcher::Engine(_)) {
1616
+ continue;
1617
+ }
1618
+ let inputs = if role == "pattern" { &strings } else { &keys };
1619
+ if inputs.is_empty() {
1620
+ continue;
1621
+ }
1622
+ let regex = G.build(&source, true).or_else(|_| G.build(&source, false)).unwrap();
1623
+ let engine = Pattern {
1624
+ source: source.clone(),
1625
+ grammar: G,
1626
+ matcher: Matcher::Engine(Box::new(regex)),
1627
+ retained: 0,
1628
+ };
1629
+ // The two must agree before their times mean anything.
1630
+ let mut budget = PatternBudget::new(PatternLimit::new(None));
1631
+ for s in inputs {
1632
+ assert_eq!(shape.is_match(s, &mut budget), engine.is_match(s, &mut budget), "{source:?} on {s:?}");
1633
+ }
1634
+ let (a, b) = (time(&shape, inputs), time(&engine, inputs));
1635
+ println!(
1636
+ "{name}\t{role}\t{}\t{}\t{a:.1}\t{b:.1}\t{:.3}\t{}\t{source}",
1637
+ super::kind(&shape),
1638
+ alternatives(&shape.matcher),
1639
+ a / b,
1640
+ inputs.len()
1641
+ );
1642
+ }
2273
1643
  }
2274
1644
  }
2275
1645
  }