corvus_json_schema 0.1.3 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,6 @@
1
- //! `pattern`/`patternProperties` matching with ECMA-262 semantics (the `u` flag), as JSON Schema specifies.
1
+ //! `pattern`/`patternProperties` matching with ECMA-262 semantics (the `u` flag), as JSON Schema specifies. A
2
+ //! pattern that is valid only without the flag is accepted too, and matched as ECMA-262 says for it: by UTF-16
3
+ //! code unit.
2
4
  //!
3
5
  //! A pattern gets the cheapest matcher that decides it exactly:
4
6
  //!
@@ -6,24 +8,78 @@
6
8
  //! - anchored sequences of quantified ASCII character classes and literals (`^[a-z][a-z0-9_]{0,29}$`, `^x-`,
7
9
  //! `^[@$_#]`), matched in one pass over the string (the TypeScript evaluator's class-sequence fast path);
8
10
  //! - alternatives of such sequences once groups are multiplied out (`^([a|A]uto)|([n|N]one)$`,
9
- //! `^[Ee][Ss]2015(\.([Cc]ore|[Pp]roxy))?$`), sets of literals, and separated lists (`^([a-z]+)(\.[a-z]+)*$`),
10
- //! as the C# evaluator's pattern matchers do;
11
- //! - patterns within the common subset of ECMA-262 and the `regex` crate's syntax, translated with ECMA semantics
12
- //! (`\d`/`\w` ASCII-only, ECMA's `\s` and `.`) and run by `regex`, which searches in linear time without
13
- //! allocating per match;
14
- //! - anything else (lookarounds, backreferences, word boundaries, Unicode properties) by `regress`, a backtracking
15
- //! ECMA-262 engine.
11
+ //! `^[Ee][Ss]2015(\.([Cc]ore|[Pp]roxy))?$`) and sets of literals, as the C# evaluator's pattern matchers do,
12
+ //! where they were measured faster than the engine (see [`MOST_LITERALS`]);
13
+ //! - anything else by [Corvus.Regex.Ecma262](https://docs.rs/corvus-regex-ecma262), an ECMA-262 engine that answers
14
+ //! every pattern as V8 does, as the pattern is written, and matches a pattern with no backreference in time
15
+ //! proportional to the pattern's size times the string.
16
16
  //!
17
- //! Patterns compile once per process (a pattern is immutable, so identical patterns share one matcher).
17
+ //! Whether a pattern is valid is the engine's answer, whichever matcher runs it, in the grammar of the edition the
18
+ //! dialect reads patterns with (ECMAScript 2020 for draft 2019-09 and draft 2020-12, today's for the others; see
19
+ //! [`Grammar`]), and so are the limits of size: a pattern over one is refused before any matcher reads it.
20
+ //!
21
+ //! Patterns compile once per process for each grammar (a pattern is immutable, so identical patterns share one
22
+ //! matcher).
18
23
 
19
- use std::collections::HashMap;
24
+ use std::collections::{HashMap, VecDeque};
20
25
  use std::fmt;
21
26
  use std::sync::{Arc, LazyLock, Mutex};
27
+ use std::time::Duration;
28
+
29
+ use corvus_regex_ecma262::{Edition, Flags, MatchError, Regex, RegexBuilder, UnicodeVersion};
30
+
31
+ use crate::dialect::Dialect;
32
+ use crate::options::{
33
+ CompileOptions, PatternMatchLimitError, PatternTooLargeError, SchemaCompilationError, SchemaCompilationErrorKind,
34
+ };
35
+
36
+ /// The grammar a pattern is read in: the edition of ECMA-262 and the version of Unicode.
37
+ #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
38
+ pub(crate) struct Grammar {
39
+ pub edition: Edition,
40
+ pub unicode_version: UnicodeVersion,
41
+ }
42
+
43
+ impl Grammar {
44
+ /// The grammar of a dialect's patterns: the edition the options give, or by default ECMAScript 2020 for draft
45
+ /// 2019-09 and draft 2020-12, which cite it, and today's for the earlier drafts; and the options' version of
46
+ /// Unicode.
47
+ pub(crate) fn of(dialect: Dialect, options: &CompileOptions) -> Grammar {
48
+ let edition = options.regex_edition.unwrap_or(if dialect >= Dialect::Draft201909 {
49
+ Edition::Es2020
50
+ } else {
51
+ Edition::Current
52
+ });
53
+ Grammar { edition, unicode_version: options.regex_unicode_version }
54
+ }
55
+
56
+ /// The flags of the grammar of the `u` flag, or of no flag.
57
+ fn flags(unicode: bool) -> Flags {
58
+ Flags { unicode, ..Flags::default() }
59
+ }
60
+
61
+ /// The pattern compiled in this grammar, with the `u` flag or with none.
62
+ fn build(self, pattern: &str, unicode: bool) -> Result<Regex, corvus_regex_ecma262::Error> {
63
+ RegexBuilder::new(pattern)
64
+ .with_flags(Grammar::flags(unicode))
65
+ .edition(self.edition)
66
+ .unicode_version(self.unicode_version)
67
+ .build()
68
+ }
69
+
70
+ /// Whether the pattern is valid in this grammar with the `u` flag.
71
+ fn is_valid_unicode(self, pattern: &str) -> bool {
72
+ Regex::is_valid_with(pattern, Grammar::flags(true), self.edition, self.unicode_version)
73
+ }
74
+ }
22
75
 
23
76
  /// A compiled `pattern`.
24
77
  pub(crate) struct Pattern {
25
78
  pub source: String,
79
+ grammar: Grammar,
26
80
  matcher: Matcher,
81
+ /// What the pattern is counted as in the cache of compiled patterns, in bytes (see [`Cache`]).
82
+ retained: usize,
27
83
  }
28
84
 
29
85
  enum Matcher {
@@ -35,7 +91,6 @@ enum Matcher {
35
91
  whole: bool,
36
92
  },
37
93
  Sequence(Sequence),
38
- SeparatedList(SeparatedList),
39
94
  /// `.+` (`^.+` with `start`): some (the first) character is not a line terminator.
40
95
  HasContent {
41
96
  start: bool,
@@ -55,12 +110,9 @@ enum Matcher {
55
110
  set: CharSet,
56
111
  bangs: bool,
57
112
  },
58
- Regex(regex::Regex),
59
- /// A pattern that is valid only without the `u` flag (an identity escape such as `\&`), so matching UTF-16 code
60
- /// units: translated for `regex`, which decides it exactly on strings with no character beyond the Basic
61
- /// Multilingual Plane (where code units are characters); regress decides the rest.
62
- RegexBmp(regex::Regex, regress::Regex),
63
- Regress(regress::Regex),
113
+ /// The `corvus-regex-ecma262` engine: by character for a pattern with the `u` flag, and by UTF-16 code unit for
114
+ /// one that is valid only without it.
115
+ Engine(Box<Regex>),
64
116
  }
65
117
 
66
118
  impl fmt::Debug for Pattern {
@@ -69,9 +121,71 @@ impl fmt::Debug for Pattern {
69
121
  }
70
122
  }
71
123
 
124
+ /// The limit of a match, as a validator holds it.
125
+ #[derive(Clone, Copy, Debug)]
126
+ pub(crate) struct PatternLimit {
127
+ /// The time one match may take, or `None` for no limit.
128
+ limit: Option<Duration>,
129
+ }
130
+
131
+ impl PatternLimit {
132
+ /// A limit of `limit` for each match. `None` and zero are no limit.
133
+ pub(crate) fn new(limit: Option<Duration>) -> PatternLimit {
134
+ PatternLimit { limit: limit.filter(|l| !l.is_zero()) }
135
+ }
136
+ }
137
+
138
+ /// What one evaluation, or the compilation of one schema, gives every match of a pattern: the time each may take.
139
+ /// It notes the first match that reached a limit, after which every match is abandoned at once.
140
+ pub(crate) struct PatternBudget {
141
+ limit: PatternLimit,
142
+ /// The match that reached a limit. It is boxed so that an evaluation, which holds a budget, stays small.
143
+ failure: Option<Box<PatternMatchLimitError>>,
144
+ }
145
+
146
+ impl PatternBudget {
147
+ #[inline(always)]
148
+ pub(crate) fn new(limit: PatternLimit) -> PatternBudget {
149
+ PatternBudget { limit, failure: None }
150
+ }
151
+
152
+ /// Whether a match has reached a limit, so that what the evaluation came to is not an answer.
153
+ #[inline(always)]
154
+ pub(crate) fn failed(&self) -> bool {
155
+ self.failure.is_some()
156
+ }
157
+
158
+ /// The error of the match that reached a limit, if one did.
159
+ pub(crate) fn take_failure(&mut self) -> Option<PatternMatchLimitError> {
160
+ self.failure.take().map(|e| *e)
161
+ }
162
+
163
+ /// Matches the string on the engine within the budget. When a limit is reached the answer is false, and it is
164
+ /// not an answer: the caller of the evaluation gets the error.
165
+ fn engine_match(&mut self, engine: &Regex, source: &str, s: &str) -> bool {
166
+ if self.failure.is_some() {
167
+ return false;
168
+ }
169
+ let result = match self.limit.limit {
170
+ Some(limit) => engine.is_match_within(s, limit),
171
+ None => engine.is_match(s),
172
+ };
173
+ result.unwrap_or_else(|e| {
174
+ self.failure = Some(Box::new(PatternMatchLimitError::new(
175
+ source,
176
+ self.limit.limit.unwrap_or_default(),
177
+ e == MatchError::Memory,
178
+ )));
179
+ false
180
+ })
181
+ }
182
+ }
183
+
72
184
  impl Pattern {
185
+ /// Whether the pattern matches somewhere in the string. The engine matches within the budget. Every other
186
+ /// matcher reads the string once, or a bounded number of times, and has no limit.
73
187
  #[inline]
74
- pub fn is_match(&self, s: &str) -> bool {
188
+ pub fn is_match(&self, s: &str, budget: &mut PatternBudget) -> bool {
75
189
  match &self.matcher {
76
190
  Matcher::Everything => true,
77
191
  Matcher::Literal { text, whole } => {
@@ -82,7 +196,6 @@ impl Pattern {
82
196
  }
83
197
  }
84
198
  Matcher::Sequence(seq) => seq.is_match(s),
85
- Matcher::SeparatedList(list) => list.is_match(s),
86
199
  Matcher::HasContent { start } => {
87
200
  if *start {
88
201
  s.chars().next().is_some_and(|c| !is_line_terminator(c))
@@ -115,20 +228,28 @@ impl Pattern {
115
228
  }
116
229
  word
117
230
  }
118
- Matcher::Regex(re) => re.is_match(s),
119
- // A UTF-8 lead byte of 0xF0 or above starts a character beyond the BMP.
120
- Matcher::RegexBmp(re, fallback) => {
121
- if s.bytes().any(|b| b >= 0xF0) {
122
- fallback.find(s).is_some()
123
- } else {
124
- re.is_match(s)
125
- }
126
- }
127
- Matcher::Regress(re) => re.find(s).is_some(),
231
+ Matcher::Engine(engine) => budget.engine_match(engine, &self.source, s),
128
232
  }
129
233
  }
130
234
  }
131
235
 
236
+ /// The kind of matcher a pattern has, for the tables of which pattern goes where.
237
+ #[cfg(test)]
238
+ pub(crate) fn kind(p: &Pattern) -> &'static str {
239
+ match &p.matcher {
240
+ Matcher::Engine(engine) if engine.flags().unicode => "engine",
241
+ Matcher::Engine(_) => "engine-units",
242
+ Matcher::Everything => "everything",
243
+ Matcher::Literal { .. } => "literal",
244
+ Matcher::Sequence(_) => "sequence",
245
+ Matcher::HasContent { .. } => "has-content",
246
+ Matcher::Line { .. } => "line",
247
+ Matcher::Literals(_) => "literals",
248
+ Matcher::Alternatives(_) => "alternatives",
249
+ Matcher::ExcludedClassWithWord { .. } => "excluded-class",
250
+ }
251
+ }
252
+
132
253
  /// ECMA-262 `LineTerminator`, which `.` does not match.
133
254
  #[inline]
134
255
  fn is_line_terminator(c: char) -> bool {
@@ -151,30 +272,34 @@ fn line_length(s: &str) -> Option<usize> {
151
272
  Some(n)
152
273
  }
153
274
 
154
- /// A set of strings: compared in turn when there are few, hashed otherwise.
155
- enum Literals {
156
- Few(Box<[Box<str>]>),
157
- Many(std::collections::HashSet<Box<str>>),
158
- }
275
+ /// A set of at most [`MOST_LITERALS`] strings, compared in turn.
276
+ struct Literals(Box<[Box<str>]>);
159
277
 
160
278
  impl Literals {
161
279
  fn new(texts: Vec<String>) -> Literals {
162
- if texts.len() <= 8 {
163
- Literals::Few(texts.into_iter().map(String::into_boxed_str).collect())
164
- } else {
165
- Literals::Many(texts.into_iter().map(String::into_boxed_str).collect())
166
- }
280
+ Literals(texts.into_iter().map(String::into_boxed_str).collect())
167
281
  }
168
282
 
169
283
  #[inline]
170
284
  fn contains(&self, s: &str) -> bool {
171
- match self {
172
- Literals::Few(texts) => texts.iter().any(|t| **t == *s),
173
- Literals::Many(set) => set.contains(s),
174
- }
285
+ self.0.iter().any(|t| **t == *s)
175
286
  }
176
287
  }
177
288
 
289
+ // What the dedicated matchers are kept for: each was timed against the engine, pattern by pattern, over the strings of
290
+ // the jsonschema-benchmark corpora (`corpus_patterns::time_corpus_patterns`), and a form is kept only where its matcher
291
+ // was the faster. Four literals compared in turn took 0.78 of the engine's time, and ten 3.2 times it. Two alternatives
292
+ // of which neither is anchored at both ends (`^([a|A]uto)|([n|N]one)$`) took 0.21 of it, since the engine searches for
293
+ // the unanchored one; three such took 1.4 to 1.6 times it, and every alternative anchored at both ends was faster on
294
+ // the engine (by 1.1 to 2.7 times), which reads the whole string once whatever the alternatives. A separated list was
295
+ // no faster than the engine (the same total time over the seven in the corpora), so it has no matcher of its own.
296
+
297
+ /// The most literals of [`Matcher::Literals`].
298
+ const MOST_LITERALS: usize = 4;
299
+
300
+ /// The most alternatives of [`Matcher::Alternatives`].
301
+ const MOST_ALTERNATIVES: usize = 2;
302
+
178
303
  /// One alternative of [`Matcher::Alternatives`]: a literal anchored at either end (or neither), a class sequence
179
304
  /// anchored at the start, or a fixed-width class sequence anchored at the end.
180
305
  enum Alternative {
@@ -237,43 +362,150 @@ impl Alternative {
237
362
  }
238
363
  }
239
364
 
240
- /// Compiles a pattern with the `u` flag, or (as many validators accept them) without it; the flag is returned.
241
- fn compile_regress(pattern: &str) -> Option<(regress::Regex, bool)> {
242
- match regress::Regex::with_flags(pattern, "u") {
243
- Ok(re) => Some((re, true)),
244
- Err(_) => regress::Regex::new(pattern).ok().map(|re| (re, false)),
365
+ /// The most patterns the process-wide cache holds.
366
+ const MAX_CACHED_PATTERNS: usize = 1024;
367
+
368
+ /// The most the patterns of the cache come to, in the bytes counted for each: 64 MiB. A pattern is counted as its
369
+ /// source and an estimate of what its matcher holds (see [`compile`]).
370
+ const MAX_CACHED_BYTES: usize = 64 << 20;
371
+
372
+ /// A pattern counted as more than this is compiled for the schema that has it and not kept in the cache.
373
+ const MAX_CACHED_PATTERN_BYTES: usize = MAX_CACHED_BYTES / 32;
374
+
375
+ /// The compiled patterns of the process, so that identical patterns share one matcher. It is bounded in number and
376
+ /// in size, since the patterns of a schema may come from outside: when it is over either bound the patterns that
377
+ /// were compiled first go, oldest first. A validator holds the patterns of its schema itself, so a pattern that has
378
+ /// left the cache is still matched by the validators that have it, and is compiled again only for a schema compiled
379
+ /// later.
380
+ #[derive(Default)]
381
+ struct Cache {
382
+ patterns: HashMap<(Arc<str>, Grammar), Arc<Pattern>>,
383
+ /// The sources and their grammars in the order they were compiled.
384
+ order: VecDeque<(Arc<str>, Grammar)>,
385
+ /// The sum of what the patterns are counted as.
386
+ bytes: usize,
387
+ }
388
+
389
+ impl Cache {
390
+ fn get(&self, source: &str, grammar: Grammar) -> Option<Arc<Pattern>> {
391
+ self.patterns.get(&(Arc::from(source), grammar)).cloned()
392
+ }
393
+
394
+ /// Keeps a pattern, and returns the one to use: the pattern itself, or the one another thread kept for the same
395
+ /// source meanwhile.
396
+ fn insert(&mut self, pattern: Arc<Pattern>) -> Arc<Pattern> {
397
+ if pattern.retained > MAX_CACHED_PATTERN_BYTES {
398
+ return pattern;
399
+ }
400
+ let key: (Arc<str>, Grammar) = (Arc::from(pattern.source.as_str()), pattern.grammar);
401
+ if let Some(kept) = self.patterns.get(&key) {
402
+ return kept.clone();
403
+ }
404
+ self.bytes += pattern.retained;
405
+ self.patterns.insert(key.clone(), pattern.clone());
406
+ self.order.push_back(key);
407
+ while self.patterns.len() > MAX_CACHED_PATTERNS || self.bytes > MAX_CACHED_BYTES {
408
+ let Some(oldest) = self.order.pop_front() else { break };
409
+ if let Some(evicted) = self.patterns.remove(&oldest) {
410
+ self.bytes -= evicted.retained;
411
+ }
412
+ }
413
+ pattern
414
+ }
415
+ }
416
+
417
+ static CACHE: LazyLock<Mutex<Cache>> = LazyLock::new(|| Mutex::new(Cache::default()));
418
+
419
+ fn cache() -> std::sync::MutexGuard<'static, Cache> {
420
+ CACHE.lock().unwrap_or_else(|e| e.into_inner())
421
+ }
422
+
423
+ /// Why a pattern does not compile.
424
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
425
+ pub(crate) enum PatternError {
426
+ /// It is not a valid ECMA-262 regular expression.
427
+ Invalid,
428
+ /// It is over a limit of size. It was not read to its end, so nothing is known of whether it is valid.
429
+ TooLarge(corvus_regex_ecma262::TooLargeError),
430
+ }
431
+
432
+ impl PatternError {
433
+ /// The error of a schema whose keyword `keyword` holds the pattern `pattern`.
434
+ pub(crate) fn of(self, pattern: &str, keyword: &'static str) -> SchemaCompilationError {
435
+ match self {
436
+ PatternError::Invalid => SchemaCompilationError::of_kind(
437
+ format!("Invalid regular expression '{pattern}' in {keyword}."),
438
+ SchemaCompilationErrorKind::InvalidPattern,
439
+ ),
440
+ PatternError::TooLarge(large) => {
441
+ let error =
442
+ PatternTooLargeError::new(pattern, keyword, large.limit.name(), large.value, large.to_string());
443
+ SchemaCompilationError::of_kind(error.to_string(), SchemaCompilationErrorKind::PatternTooLarge(error))
444
+ }
445
+ }
245
446
  }
246
447
  }
247
448
 
248
- static CACHE: LazyLock<Mutex<HashMap<String, Arc<Pattern>>>> = LazyLock::new(|| Mutex::new(HashMap::new()));
449
+ impl From<corvus_regex_ecma262::Error> for PatternError {
450
+ fn from(e: corvus_regex_ecma262::Error) -> PatternError {
451
+ match e {
452
+ corvus_regex_ecma262::Error::TooLarge(large) => PatternError::TooLarge(large),
453
+ // Not valid in the grammar, or (not given here) flags that are not.
454
+ _ => PatternError::Invalid,
455
+ }
456
+ }
457
+ }
249
458
 
250
- /// Compiles (or fetches from the process-wide cache) a pattern; `None` when it is not a valid ECMA-262 regex.
251
- pub(crate) fn compile(pattern: &str) -> Option<Arc<Pattern>> {
252
- if let Some(p) = CACHE.lock().unwrap().get(pattern) {
253
- return Some(p.clone());
459
+ /// A pattern in quotes for a message, or its first 120 bytes and its length when it is longer.
460
+ pub(crate) fn shown_pattern(pattern: &str) -> String {
461
+ const SHOWN: usize = 120;
462
+ if pattern.len() <= SHOWN {
463
+ return format!("'{pattern}'");
464
+ }
465
+ let mut cut = SHOWN;
466
+ while !pattern.is_char_boundary(cut) {
467
+ cut -= 1;
254
468
  }
255
- // Validity is ECMA-262's: a pattern regress rejects is an error, whichever matcher would run it.
256
- let (regress, unicode) = compile_regress(pattern)?;
257
- let matcher = match choose(pattern, unicode) {
258
- Some(m) => m,
259
- None if !unicode => match translate(pattern, false).and_then(|t| regex::Regex::new(&t).ok()) {
260
- Some(re) => Matcher::RegexBmp(re, regress),
261
- None => Matcher::Regress(regress),
469
+ format!("'{}...' ({} bytes)", &pattern[..cut], pattern.len())
470
+ }
471
+
472
+ /// Compiles (or fetches from the process-wide cache) a pattern, in the grammar of its dialect.
473
+ pub(crate) fn compile(pattern: &str, grammar: Grammar) -> Result<Arc<Pattern>, PatternError> {
474
+ if let Some(p) = cache().get(pattern, grammar) {
475
+ return Ok(p);
476
+ }
477
+ // Validity is ECMA-262's: a pattern the engine refuses is an error, whichever matcher would run it. A pattern is
478
+ // read with the u flag, and failing that with no flag, whose grammar (Annex B) many schemas need. A pattern valid
479
+ // with the flag that a dedicated matcher takes is read and never built: building the engine's program and
480
+ // automata costs far more than reading. The engine is built only for the patterns it matches, and for those over
481
+ // a limit of size, which it refuses (the reading refuses them too, without saying why).
482
+ let matcher = match grammar.is_valid_unicode(pattern).then(|| choose(pattern, true, grammar)).flatten() {
483
+ Some(shape) => shape,
484
+ None => match grammar.build(pattern, true) {
485
+ Ok(engine) => Matcher::Engine(Box::new(engine)),
486
+ Err(corvus_regex_ecma262::Error::TooLarge(large)) => return Err(PatternError::TooLarge(large)),
487
+ Err(_) => {
488
+ let engine = grammar.build(pattern, false)?;
489
+ choose(pattern, false, grammar).unwrap_or_else(|| Matcher::Engine(Box::new(engine)))
490
+ }
262
491
  },
263
- None => Matcher::Regress(regress),
264
492
  };
265
- let p = Arc::new(Pattern { source: pattern.to_string(), matcher });
266
- CACHE.lock().unwrap().insert(pattern.to_string(), p.clone());
267
- Some(p)
493
+ // What the cache counts for the pattern (see `MAX_CACHED_BYTES`): its source, and for the engine an estimate of
494
+ // its program and automata, which grow with the pattern.
495
+ let mut retained = pattern.len() + 256;
496
+ if matches!(matcher, Matcher::Engine(_)) {
497
+ retained += 16 * pattern.len();
498
+ }
499
+ Ok(cache().insert(Arc::new(Pattern { source: pattern.to_string(), grammar, matcher, retained })))
268
500
  }
269
501
 
270
- fn choose(pattern: &str, unicode: bool) -> Option<Matcher> {
502
+ fn choose(pattern: &str, unicode: bool, grammar: Grammar) -> Option<Matcher> {
271
503
  // Unanchored (or start-anchored) `.*` finds an empty match in any string; `^.*$` does not (`.` stops at a line
272
504
  // terminator), so it is not listed.
273
505
  if matches!(pattern, "" | ".*" | "^.*" | ".*$" | "(.*)" | "^(.*)" | "[\\s\\S]*" | "^[\\s\\S]*" | "^[\\s\\S]*$") {
274
506
  return Some(Matcher::Everything);
275
507
  }
276
- // Without the `u` flag, a pattern matches UTF-16 code units rather than characters: only regress has that.
508
+ // Without the `u` flag, a pattern matches UTF-16 code units rather than characters: only the engine has that.
277
509
  if !unicode {
278
510
  return None;
279
511
  }
@@ -288,7 +520,8 @@ fn choose(pattern: &str, unicode: bool) -> Option<Matcher> {
288
520
  && rest.len() > 1
289
521
  && !ends_with_escape(rest)
290
522
  && !rest.ends_with(['*', '+', '?', '}', '|', '(', '^'])
291
- && let Some(m) = choose(rest, unicode)
523
+ && grammar.is_valid_unicode(rest)
524
+ && let Some(m) = choose(rest, true, grammar)
292
525
  {
293
526
  return Some(m);
294
527
  }
@@ -296,25 +529,28 @@ fn choose(pattern: &str, unicode: bool) -> Option<Matcher> {
296
529
  return Some(Matcher::Line { min, max });
297
530
  }
298
531
  if let Some(texts) = whole_alternatives(pattern) {
299
- return Some(Matcher::Literals(Literals::new(texts)));
532
+ return (texts.len() <= MOST_LITERALS).then(|| Matcher::Literals(Literals::new(texts)));
300
533
  }
301
534
  if let Some(alts) = alternatives(pattern) {
302
- return Some(Matcher::Alternatives(alts));
303
- }
304
- if let Some(list) = SeparatedList::parse(pattern) {
305
- return Some(Matcher::SeparatedList(list));
535
+ let whole = |a: &Alternative| match a {
536
+ Alternative::Literal { start, end, .. } => *start && *end,
537
+ Alternative::Start(seq) => seq.to_end,
538
+ Alternative::End { .. } | Alternative::Anywhere { .. } => false,
539
+ };
540
+ return (alts.len() <= MOST_ALTERNATIVES && !alts.iter().any(whole)).then_some(Matcher::Alternatives(alts));
306
541
  }
307
542
  if let Some((set, bangs)) = excluded_class_with_word(pattern) {
308
543
  return Some(Matcher::ExcludedClassWithWord { set, bangs });
309
544
  }
310
545
  if let Some(seq) = Sequence::parse(pattern) {
311
546
  return Some(match seq.literal() {
547
+ // `^[a-z]*` with nothing it requires: every string.
548
+ Some(text) if text.is_empty() && !seq.to_end => Matcher::Everything,
312
549
  Some(text) => Matcher::Literal { text: text.into(), whole: seq.to_end },
313
550
  None => Matcher::Sequence(seq),
314
551
  });
315
552
  }
316
- let translated = translate(pattern, true)?;
317
- regex::Regex::new(&translated).ok().map(Matcher::Regex)
553
+ None
318
554
  }
319
555
 
320
556
  /// `^.{m,n}$` (and `^.*$`, `^.+$`, `^.{m}$`, `^.{m,}$`, each also as `^(.…)$`): the bounds on the length of a line.
@@ -393,12 +629,18 @@ const MAX_ALTERNATIVES: usize = 64;
393
629
  /// `regex` crate's single automaton pass (measured on jsconfig's case-folded `lib` names).
394
630
  const MAX_SEQUENCE_ALTERNATIVES: usize = 4;
395
631
 
632
+ /// The widest class sequence, in characters, and the longest literal, in bytes, that is searched for at every
633
+ /// position of a text. The search for a sequence takes the text times this, and these matchers have no limit, so a
634
+ /// wider one is left to the engine, which has the limit of a match. (The rule is the Go module's, so that the ports
635
+ /// give a pattern the same kind of matcher.)
636
+ const MAX_ANYWHERE_WIDTH: usize = 64;
637
+
396
638
  /// Two or more alternatives once groups of alternatives (and optional groups) are expanded into whole alternatives
397
639
  /// (`^a|b|c$`, `^([a|A]uto)|([n|N]one)$`, `^[Ee][Ss]2015(\.([Cc]ore|[Pp]roxy))?$`), each a literal or a class
398
640
  /// sequence anchored where its own `^` and `$` say.
399
641
  fn alternatives(p: &str) -> Option<Box<[Alternative]>> {
400
642
  let c: Vec<char> = p.chars().collect();
401
- let (parts, end) = expand(&c, 0)?;
643
+ let (parts, end) = expand(&c, 0, 0)?;
402
644
  // A single alternative is only new here when a group was expanded (Sequence::parse takes the rest).
403
645
  if end != c.len() || parts.is_empty() || (parts.len() == 1 && parts[0] == p) {
404
646
  return None;
@@ -412,6 +654,10 @@ fn alternatives(p: &str) -> Option<Box<[Alternative]>> {
412
654
  _ => (false, part),
413
655
  };
414
656
  if let Some(text) = literal_text(body) {
657
+ // A literal that is anchored at neither end is searched for at every position of the text.
658
+ if !start && !end && text.len() > MAX_ANYWHERE_WIDTH {
659
+ return None;
660
+ }
415
661
  return Some(Alternative::Literal { text: text.into(), start, end });
416
662
  }
417
663
  let seq = Sequence::parse(&format!("^{body}{}", if end { "$" } else { "" }))?;
@@ -420,7 +666,11 @@ fn alternatives(p: &str) -> Option<Box<[Alternative]>> {
420
666
  }
421
667
  let width =
422
668
  seq.items.iter().all(|i| i.min == i.max).then(|| seq.items.iter().map(|i| i.min as usize).sum())?;
423
- Some(if end { Alternative::End { seq, width } } else { Alternative::Anywhere { seq, width } })
669
+ if end {
670
+ return Some(Alternative::End { seq, width });
671
+ }
672
+ // A sequence that is not anchored is tried at every position of the text, and reads its width at each.
673
+ (width <= MAX_ANYWHERE_WIDTH).then_some(Alternative::Anywhere { seq, width })
424
674
  })
425
675
  .collect::<Option<_>>()?;
426
676
  let sequences = alternatives.iter().any(|a| !matches!(a, Alternative::Literal { .. }));
@@ -432,10 +682,18 @@ fn ends_with_escape(t: &str) -> bool {
432
682
  t.bytes().rev().take_while(|&b| b == b'\\').count() % 2 == 1
433
683
  }
434
684
 
685
+ /// The deepest nesting of groups that is multiplied out. `expand` goes a level deeper for each, and a pattern may
686
+ /// nest a thousand deep, which the stack of a thread cannot be relied on to hold. A pattern that nests deeper than
687
+ /// this is left to the matchers after this one.
688
+ const MAX_EXPANDED_NESTING: usize = 32;
689
+
435
690
  /// Expands the alternatives from `i` up to an unmatched `)` or the end: groups of alternatives multiply out, a group
436
691
  /// quantified by `?` also contributes the empty alternative, and everything else is copied. `None` for lookarounds,
437
- /// other quantified groups, or too many alternatives.
438
- fn expand(c: &[char], mut i: usize) -> Option<(Vec<String>, usize)> {
692
+ /// other quantified groups, too many alternatives, or groups nested deeper than `MAX_EXPANDED_NESTING`.
693
+ fn expand(c: &[char], mut i: usize, depth: usize) -> Option<(Vec<String>, usize)> {
694
+ if depth > MAX_EXPANDED_NESTING {
695
+ return None;
696
+ }
439
697
  let mut all = Vec::new();
440
698
  let mut branch = vec![String::new()];
441
699
  while i < c.len() {
@@ -454,7 +712,7 @@ fn expand(c: &[char], mut i: usize) -> Option<(Vec<String>, usize)> {
454
712
  }
455
713
  i += 2;
456
714
  }
457
- let (mut inner, next) = expand(c, i)?;
715
+ let (mut inner, next) = expand(c, i, depth + 1)?;
458
716
  if c.get(next) != Some(&')') {
459
717
  return None;
460
718
  }
@@ -550,9 +808,13 @@ fn excluded_class_with_word(p: &str) -> Option<(CharSet, bool)> {
550
808
  .then_some((excluded, bangs))
551
809
  }
552
810
 
553
- /// The `regex` format: a valid ECMA-262 regular expression (with the `u` flag).
554
- pub(crate) fn is_valid_ecma_regex(s: &str) -> bool {
555
- regress::Regex::with_flags(s, "u").is_ok()
811
+ /// The `regex` format: a valid ECMA-262 regular expression in the grammar of the `u` flag, of the edition and version
812
+ /// of Unicode the dialect reads patterns with. It asks the grammar alone, as the .NET library's check does: the
813
+ /// engine's limits of size belong to compiling a pattern, so a string too large to compile is valid when its grammar
814
+ /// is. The engine's grammar-only reader takes time in proportion to the string and allocates nothing for one of
815
+ /// ordinary size.
816
+ pub(crate) fn is_valid_ecma_regex(s: &str, grammar: Grammar) -> bool {
817
+ Regex::is_valid_pattern_with(s, Grammar::flags(true), grammar.edition, grammar.unicode_version)
556
818
  }
557
819
 
558
820
  // ---------------------------------------------------------------------------------------------------------------------
@@ -710,6 +972,16 @@ impl Sequence {
710
972
  }
711
973
  }
712
974
  }
975
+ if !to_end {
976
+ // Without `$` a match needs only the start of the string: items at the end that can match nothing are
977
+ // dropped, and the last of the rest needs only its least count (`^[a-z]+` is `^[a-z]`).
978
+ while items.last().is_some_and(|item| item.min == 0) {
979
+ items.pop();
980
+ }
981
+ if let Some(last) = items.last_mut() {
982
+ last.max = last.min;
983
+ }
984
+ }
713
985
  Some(Sequence { items: items.into_boxed_slice(), to_end })
714
986
  }
715
987
 
@@ -765,212 +1037,6 @@ impl Sequence {
765
1037
  }
766
1038
  !self.to_end || at == b.len()
767
1039
  }
768
-
769
- /// Greedily matches the items at the start of `s` (ignoring `to_end`): the bytes taken.
770
- fn consume(&self, s: &str) -> Option<usize> {
771
- let b = s.as_bytes();
772
- let mut at = 0;
773
- for item in self.items.iter() {
774
- let mut n = 0u32;
775
- while n < item.max && at < b.len() {
776
- let c = b[at];
777
- if c < 0x80 {
778
- if !item.set.has_ascii(c) {
779
- break;
780
- }
781
- at += 1;
782
- } else {
783
- let ch = s[at..].chars().next()?;
784
- if !item.set.contains(ch) {
785
- break;
786
- }
787
- at += ch.len_utf8();
788
- }
789
- n += 1;
790
- }
791
- if n < item.min {
792
- return None;
793
- }
794
- }
795
- Some(at)
796
- }
797
-
798
- /// Whether greedy matching is exact when `next` can follow the items: every variable item is disjoint from the
799
- /// items that can directly follow it (up to the first that cannot match nothing), `next` included.
800
- fn greedy_before(items: &[Item], next: CharSet) -> bool {
801
- items.iter().enumerate().all(|(i, item)| {
802
- if item.min == item.max {
803
- return true;
804
- }
805
- for following in &items[i + 1..] {
806
- if !item.set.disjoint(following.set) {
807
- return false;
808
- }
809
- if following.min > 0 {
810
- return true;
811
- }
812
- }
813
- item.set.disjoint(next)
814
- })
815
- }
816
- }
817
-
818
- /// A list of items between separators: `^I(SR)*$` or `^I(SR)+$` (`final_` is `None`), or `^(RS)*F$` and
819
- /// `^(RS)+F$`. The separator is a fixed sequence and no variable class can run into what follows it, so greedy
820
- /// matching splits the string exactly where the pattern does.
821
- #[derive(Debug)]
822
- struct SeparatedList {
823
- /// `I` of the first form.
824
- first: Option<Sequence>,
825
- repeated: Sequence,
826
- separator: Sequence,
827
- /// `F` of the second form, matched against the whole remainder.
828
- final_: Option<Sequence>,
829
- min_repeats: u32,
830
- }
831
-
832
- impl SeparatedList {
833
- fn is_match(&self, s: &str) -> bool {
834
- let mut rest = s;
835
- let mut repeats = 0;
836
- if let Some(first) = &self.first {
837
- let Some(n) = first.consume(rest) else { return false };
838
- rest = &rest[n..];
839
- loop {
840
- if rest.is_empty() {
841
- return repeats >= self.min_repeats;
842
- }
843
- let Some(n) = self.separator.consume(rest) else { return false };
844
- rest = &rest[n..];
845
- let Some(n) = self.repeated.consume(rest) else { return false };
846
- rest = &rest[n..];
847
- repeats += 1;
848
- }
849
- }
850
- let final_ = self.final_.as_ref().expect("a separated list has a first or a final item");
851
- loop {
852
- if repeats >= self.min_repeats && final_.is_match(rest) {
853
- return true;
854
- }
855
- let Some(n) = self.repeated.consume(rest) else { return false };
856
- let Some(m) = self.separator.consume(&rest[n..]) else { return false };
857
- rest = &rest[n + m..];
858
- repeats += 1;
859
- }
860
- }
861
-
862
- fn parse(p: &str) -> Option<SeparatedList> {
863
- let body = p.strip_prefix('^')?.strip_suffix('$')?;
864
- if ends_with_escape(body) {
865
- return None;
866
- }
867
- let b = body.as_bytes();
868
- // Top-level groups: (start, end) of each, where end is the index of its `)`.
869
- let mut groups = Vec::new();
870
- let (mut depth, mut in_class, mut i, mut open) = (0usize, false, 0, 0);
871
- while i < b.len() {
872
- match b[i] {
873
- b'\\' => i += 1,
874
- b'[' if !in_class => in_class = true,
875
- b']' if in_class => in_class = false,
876
- b'(' if !in_class => {
877
- if depth == 0 {
878
- open = i;
879
- }
880
- depth += 1;
881
- }
882
- b')' if !in_class => {
883
- depth = depth.checked_sub(1)?;
884
- if depth == 0 {
885
- groups.push((open, i));
886
- }
887
- }
888
- _ => {}
889
- }
890
- i += 1;
891
- }
892
- let quantified: Vec<_> = groups.iter().filter(|&&(_, e)| matches!(b.get(e + 1), Some(b'*' | b'+'))).collect();
893
- let &&(open, close) = quantified.first().filter(|_| quantified.len() == 1)?;
894
- let min_repeats = u32::from(b[close + 1] == b'+');
895
- let group = strip_group(&body[open..=close])?;
896
- let before = &body[..open];
897
- let after = &body[close + 2..];
898
- let items = |t: &str| Sequence::parse(&format!("^{t}")).map(|s| s.items.into_vec());
899
- let seq = |items: &[Item], to_end| Sequence { items: items.into(), to_end };
900
- let fixed = |items: &[Item]| !items.is_empty() && items.iter().all(|i| i.min == i.max && i.min > 0);
901
- let g = items(group)?;
902
- match (before.is_empty(), after.is_empty()) {
903
- // ^I(SR)*$
904
- (false, true) => {
905
- let first = items(unwrap_group(before)?)?;
906
- (1..g.len()).find_map(|k| {
907
- let (separator, repeated) = g.split_at(k);
908
- let next = separator[0].set;
909
- (fixed(separator)
910
- && Sequence::greedy_before(&first, next)
911
- && Sequence::greedy_before(repeated, next))
912
- .then(|| SeparatedList {
913
- first: Some(seq(&first, false)),
914
- repeated: seq(repeated, false),
915
- separator: seq(separator, false),
916
- final_: None,
917
- min_repeats,
918
- })
919
- })
920
- }
921
- // ^(RS)*F$
922
- (true, false) => {
923
- let final_ = Sequence::parse(&format!("^{}$", unwrap_group(after)?))?;
924
- (1..g.len()).find_map(|k| {
925
- let (repeated, separator) = g.split_at(k);
926
- (fixed(separator) && Sequence::greedy_before(repeated, separator[0].set)).then(|| SeparatedList {
927
- first: None,
928
- repeated: seq(repeated, false),
929
- separator: seq(separator, false),
930
- final_: Some(Sequence { items: final_.items.clone(), to_end: true }),
931
- min_repeats,
932
- })
933
- })
934
- }
935
- _ => None,
936
- }
937
- }
938
- }
939
-
940
- /// The inside of `(...)` or `(?:...)`; `None` for other groups.
941
- fn strip_group(g: &str) -> Option<&str> {
942
- let inner = g.strip_prefix('(')?.strip_suffix(')')?;
943
- match inner.strip_prefix("?:") {
944
- Some(rest) => Some(rest),
945
- None if inner.starts_with('?') => None,
946
- None => Some(inner),
947
- }
948
- }
949
-
950
- /// A text that is one group wrapping everything, unwrapped; otherwise the text itself.
951
- fn unwrap_group(t: &str) -> Option<&str> {
952
- if t.starts_with('(') && t.ends_with(')') {
953
- let inner = strip_group(t)?;
954
- // Only when the parentheses enclose the whole text (`(a)(b)` is two groups).
955
- let mut depth = 0i32;
956
- let mut escaped = false;
957
- for c in inner.chars() {
958
- match c {
959
- _ if escaped => escaped = false,
960
- '\\' => escaped = true,
961
- '(' => depth += 1,
962
- ')' => {
963
- depth -= 1;
964
- if depth < 0 {
965
- return None;
966
- }
967
- }
968
- _ => {}
969
- }
970
- }
971
- return Some(inner);
972
- }
973
- Some(t)
974
1040
  }
975
1041
 
976
1042
  /// The set of a class escape (`\d`, `\w`, their negations, or an escaped punctuation character); `None` for `\s`
@@ -1090,280 +1156,26 @@ fn parse_quantifier(b: &[u8], mut i: usize) -> Option<(u32, u32, usize)> {
1090
1156
  Some((min, max, i))
1091
1157
  }
1092
1158
 
1093
- // ---------------------------------------------------------------------------------------------------------------------
1094
- // Translation to the regex crate
1095
-
1096
- /// ECMA-262 `\s`: WhiteSpace and LineTerminator.
1097
- const ECMA_SPACE: &str =
1098
- r"\t\n\x{B}\x{C}\r \x{A0}\x{1680}\x{2000}-\x{200A}\x{2028}\x{2029}\x{202F}\x{205F}\x{3000}\x{FEFF}";
1099
-
1100
- /// Translates an ECMA-262 pattern to the `regex` crate's syntax with the same meaning, or `None` when it uses
1101
- /// something outside the common subset. With `unicode` false, the pattern is read without the `u` flag (identity
1102
- /// escapes of any non-alphanumeric character), and the translation has its meaning on strings within the BMP.
1103
- fn translate(p: &str, unicode: bool) -> Option<String> {
1104
- let c: Vec<char> = p.chars().collect();
1105
- let mut out = String::with_capacity(p.len() * 2);
1106
- let mut i = 0;
1107
- while i < c.len() {
1108
- match c[i] {
1109
- '\\' => {
1110
- let (text, next) = escape(&c, i + 1, false, unicode)?;
1111
- out.push_str(&text);
1112
- i = next;
1113
- }
1114
- '[' => {
1115
- let (text, next) = class(&c, i + 1, unicode)?;
1116
- out.push_str(&text);
1117
- i = next;
1118
- }
1119
- '(' => {
1120
- if c.get(i + 1) == Some(&'?') {
1121
- match (c.get(i + 2), c.get(i + 3)) {
1122
- (Some(':'), _) => i += 3,
1123
- // A named group; lookbehinds (`(?<=`, `(?<!`) are not supported.
1124
- (Some('<'), Some(n)) if *n != '=' && *n != '!' => {
1125
- i += 3 + c[i + 3..].iter().position(|&x| x == '>')? + 1;
1126
- }
1127
- _ => return None,
1128
- }
1129
- } else {
1130
- i += 1;
1131
- }
1132
- out.push_str("(?:");
1133
- }
1134
- '{' => {
1135
- let end = i + c[i..].iter().position(|&x| x == '}')?;
1136
- let body: String = c[i + 1..end].iter().collect();
1137
- if body.is_empty() || !body.chars().all(|x| x.is_ascii_digit() || x == ',') {
1138
- return None;
1139
- }
1140
- out.push('{');
1141
- out.push_str(&body);
1142
- out.push('}');
1143
- i = end + 1;
1144
- }
1145
- '.' => {
1146
- out.push_str(r"[^\n\r\x{2028}\x{2029}]");
1147
- i += 1;
1148
- }
1149
- ch @ (')' | '|' | '^' | '$' | '*' | '+' | '?') => {
1150
- out.push(ch);
1151
- i += 1;
1152
- }
1153
- ch => {
1154
- push_literal(&mut out, ch);
1155
- i += 1;
1156
- }
1157
- }
1158
- }
1159
- Some(out)
1160
- }
1161
-
1162
- fn push_literal(out: &mut String, ch: char) {
1163
- use std::fmt::Write;
1164
- let _ = write!(out, "\\x{{{:X}}}", ch as u32);
1165
- }
1166
-
1167
- /// An escape after `\` at `i`: its translation and the index after it.
1168
- fn escape(c: &[char], i: usize, in_class: bool, unicode: bool) -> Option<(String, usize)> {
1169
- let e = *c.get(i)?;
1170
- let mut out = String::new();
1171
- let next = match e {
1172
- 'd' => {
1173
- out.push_str(if in_class { "0-9" } else { "[0-9]" });
1174
- i + 1
1175
- }
1176
- 'D' => {
1177
- out.push_str("[^0-9]");
1178
- i + 1
1179
- }
1180
- 'w' => {
1181
- out.push_str(if in_class { "0-9A-Za-z_" } else { "[0-9A-Za-z_]" });
1182
- i + 1
1183
- }
1184
- 'W' => {
1185
- out.push_str("[^0-9A-Za-z_]");
1186
- i + 1
1187
- }
1188
- 's' => {
1189
- if in_class {
1190
- out.push_str(ECMA_SPACE);
1191
- } else {
1192
- out.push('[');
1193
- out.push_str(ECMA_SPACE);
1194
- out.push(']');
1195
- }
1196
- i + 1
1197
- }
1198
- 'S' => {
1199
- out.push_str("[^");
1200
- out.push_str(ECMA_SPACE);
1201
- out.push(']');
1202
- i + 1
1203
- }
1204
- 'n' | 'r' | 't' | 'f' | 'v' => {
1205
- let ch = match e {
1206
- 'n' => '\n',
1207
- 'r' => '\r',
1208
- 't' => '\t',
1209
- 'f' => '\x0C',
1210
- _ => '\x0B',
1211
- };
1212
- push_literal(&mut out, ch);
1213
- i + 1
1214
- }
1215
- '0' if !c.get(i + 1).is_some_and(char::is_ascii_digit) => {
1216
- push_literal(&mut out, '\0');
1217
- i + 1
1218
- }
1219
- 'b' if in_class => {
1220
- push_literal(&mut out, '\x08');
1221
- i + 1
1222
- }
1223
- // ECMA's word boundaries are between ASCII word characters and the rest.
1224
- 'b' => {
1225
- out.push_str("(?-u:\\b)");
1226
- i + 1
1227
- }
1228
- 'B' => {
1229
- out.push_str("(?-u:\\B)");
1230
- i + 1
1231
- }
1232
- // Unicode properties (validated as ECMA-262 names by regress) have the same names in the `regex` crate.
1233
- // Without the `u` flag `\p` is the letter.
1234
- 'p' | 'P' if unicode && c.get(i + 1) == Some(&'{') => {
1235
- let end = i + 1 + c[i + 1..].iter().position(|&x| x == '}')?;
1236
- let body: String = c[i + 2..end].iter().collect();
1237
- if body.is_empty() || !body.chars().all(|x| x.is_ascii_alphanumeric() || x == '_' || x == '=') {
1238
- return None;
1239
- }
1240
- out.push('\\');
1241
- out.push(e);
1242
- out.push('{');
1243
- out.push_str(&body);
1244
- out.push('}');
1245
- end + 1
1246
- }
1247
- 'x' => {
1248
- let hex: String = c.get(i + 1..i + 3)?.iter().collect();
1249
- push_literal(&mut out, char::from_u32(u32::from_str_radix(&hex, 16).ok()?)?);
1250
- i + 3
1251
- }
1252
- // Without the `u` flag, `\u{...}` is not a code point escape.
1253
- 'u' if unicode || c.get(i + 1) != Some(&'{') => {
1254
- let (code, next) = unicode_escape(c, i + 1)?;
1255
- push_literal(&mut out, char::from_u32(code)?);
1256
- next
1257
- }
1258
- // Identity escapes of syntax characters and `/` (and `-` in a class).
1259
- '^' | '$' | '\\' | '.' | '*' | '+' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '/' | '-' => {
1260
- push_literal(&mut out, e);
1261
- i + 1
1262
- }
1263
- // Without the `u` flag, any other non-alphanumeric character escapes to itself.
1264
- e if !unicode && !e.is_ascii_alphanumeric() => {
1265
- push_literal(&mut out, e);
1266
- i + 1
1267
- }
1268
- // Backreferences, \c, \k and the rest.
1269
- _ => return None,
1270
- };
1271
- Some((out, next))
1272
- }
1273
-
1274
- /// `\u` escapes after the `u`: `XXXX` (joining a surrogate pair) or `{X...}`.
1275
- fn unicode_escape(c: &[char], i: usize) -> Option<(u32, usize)> {
1276
- if c.get(i) == Some(&'{') {
1277
- let end = i + c[i..].iter().position(|&x| x == '}')?;
1278
- let hex: String = c[i + 1..end].iter().collect();
1279
- return Some((u32::from_str_radix(&hex, 16).ok()?, end + 1));
1280
- }
1281
- let four = |at: usize| -> Option<u32> {
1282
- let hex: String = c.get(at..at + 4)?.iter().collect();
1283
- if hex.len() != 4 {
1284
- return None;
1285
- }
1286
- u32::from_str_radix(&hex, 16).ok()
1287
- };
1288
- let hi = four(i)?;
1289
- if (0xD800..0xDC00).contains(&hi) {
1290
- if c.get(i + 4) == Some(&'\\') && c.get(i + 5) == Some(&'u') {
1291
- let lo = four(i + 6)?;
1292
- if (0xDC00..0xE000).contains(&lo) {
1293
- return Some((0x10000 + ((hi - 0xD800) << 10) + (lo - 0xDC00), i + 10));
1294
- }
1295
- }
1296
- return None;
1297
- }
1298
- Some((hi, i + 4))
1299
- }
1300
-
1301
- /// A class after `[` at `i`: its translation (every member spelled as an escape, so nothing in it is special to the
1302
- /// `regex` crate) and the index after `]`.
1303
- fn class(c: &[char], mut i: usize, unicode: bool) -> Option<(String, usize)> {
1304
- let mut out = String::from("[");
1305
- if c.get(i) == Some(&'^') {
1306
- out.push('^');
1307
- i += 1;
1308
- }
1309
- let mut members = 0;
1310
- loop {
1311
- let ch = *c.get(i)?;
1312
- if ch == ']' {
1313
- i += 1;
1314
- break;
1315
- }
1316
- // One atom: a character (which can start a range) or a class escape.
1317
- let (atom, single, next) = if ch == '\\' {
1318
- let e = *c.get(i + 1)?;
1319
- let (text, next) = escape(c, i + 1, true, unicode)?;
1320
- let single = match e {
1321
- 'd' | 'D' | 'w' | 'W' | 's' | 'S' | 'p' | 'P' => None,
1322
- _ => Some(text.clone()),
1323
- };
1324
- (text, single, next)
1325
- } else {
1326
- let mut t = String::new();
1327
- push_literal(&mut t, ch);
1328
- (t.clone(), Some(t), i + 1)
1329
- };
1330
- i = next;
1331
- members += 1;
1332
- if c.get(i) == Some(&'-') && c.get(i + 1).is_some_and(|&n| n != ']') {
1333
- // A range: both ends single characters.
1334
- let lo = single?;
1335
- let (hi, next) = if c[i + 1] == '\\' {
1336
- let e = *c.get(i + 2)?;
1337
- if matches!(e, 'd' | 'D' | 'w' | 'W' | 's' | 'S' | 'p' | 'P') {
1338
- return None;
1339
- }
1340
- escape(c, i + 2, true, unicode)?
1341
- } else {
1342
- let mut t = String::new();
1343
- push_literal(&mut t, c[i + 1]);
1344
- (t, i + 2)
1345
- };
1346
- out.push_str(&lo);
1347
- out.push('-');
1348
- out.push_str(&hi);
1349
- i = next;
1350
- } else {
1351
- out.push_str(&atom);
1352
- }
1353
- }
1354
- if members == 0 {
1355
- return None; // `[]` and `[^]` have no counterpart.
1356
- }
1357
- out.push(']');
1358
- Some((out, i))
1359
- }
1360
-
1361
1159
  #[cfg(test)]
1362
1160
  mod tests {
1363
1161
  use super::*;
1364
1162
 
1163
+ const G: Grammar = Grammar { edition: Edition::Current, unicode_version: UnicodeVersion::Default };
1164
+
1165
+ /// A pattern compiled in today's grammar.
1166
+ fn compile(p: &str) -> Result<Arc<Pattern>, PatternError> {
1167
+ super::compile(p, G)
1168
+ }
1169
+
1365
1170
  const PATTERNS: &[&str] = &[
1366
1171
  "",
1172
+ // Sequences without `$`, which need only the start of the string.
1173
+ "^[a-z]+",
1174
+ "^[a-z]*",
1175
+ "^[a-zA-Z0-9_\\.\\-]*",
1176
+ "^a[b-c]{2,5}",
1177
+ "^[a-z]+[0-9]*",
1178
+ "^[a-z]{0,3}[0-9]{2,4}",
1367
1179
  ".*",
1368
1180
  "^.*",
1369
1181
  ".*$",
@@ -1484,6 +1296,20 @@ mod tests {
1484
1296
  "^(é,)*a$",
1485
1297
  "^(?=!+[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$",
1486
1298
  "^(?=!+[^a]+$)(?=(.*\\w)).+$",
1299
+ // Lists with an anchor or an assertion inside them, and with an escaped anchor.
1300
+ "^x(,a$)*$",
1301
+ "^a(,a$)+$",
1302
+ "^x$(,a)*$",
1303
+ "^(a$)(,a)*$",
1304
+ "^$(,a)*$",
1305
+ "^(a,$)*b$",
1306
+ "^([a-z]+,$)+a$",
1307
+ "^a(^,a)*$",
1308
+ "^(^a,)*b$",
1309
+ "^a(,a\\b)*$",
1310
+ "^a(,a\\$)*$",
1311
+ "^a\\$(,a)*$",
1312
+ "^(a\\$,)*b$",
1487
1313
  // Valid only without the `u` flag (identity escapes), so matching code units.
1488
1314
  "^\\/[^\\*\\?\\&\\%]*(\\/\\*)?$",
1489
1315
  "^[\\&\\@\\_]+$",
@@ -1493,9 +1319,10 @@ mod tests {
1493
1319
  "\\uD83D\\uDE00|\\&",
1494
1320
  ];
1495
1321
 
1496
- /// Every matcher agrees with regress on strings over an alphabet that exercises classes, anchors and non-ASCII.
1322
+ /// Every matcher agrees with the engine's backtracking matcher on strings over an alphabet that exercises
1323
+ /// classes, anchors and non-ASCII.
1497
1324
  #[test]
1498
- fn matchers_agree_with_regress() {
1325
+ fn matchers_agree_with_the_engine() {
1499
1326
  let alphabet = [
1500
1327
  "a", "b", "z", "A", "X", "Z", "0", "1", "5", "9", "_", "-", ".", ":", "/", "@", "#", "$", "*", "!", "{",
1501
1328
  "}", "|", " ", "\n", "\u{a0}", "é", "µ", "😀", "\u{2028}", "x-", "es", "ES", "ms", "txt", "Au", "to", "No",
@@ -1509,12 +1336,20 @@ mod tests {
1509
1336
  seed
1510
1337
  };
1511
1338
  for &p in PATTERNS {
1512
- let (reference, _) = compile_regress(p).unwrap();
1339
+ let reference = G.build(p, true).or_else(|_| G.build(p, false)).unwrap();
1513
1340
  let compiled = compile(p).unwrap();
1514
1341
  for _ in 0..4000 {
1515
1342
  let len = next() % 9;
1516
1343
  let s: String = (0..len).map(|_| alphabet[(next() % alphabet.len() as u64) as usize]).collect();
1517
- assert_eq!(compiled.is_match(&s), reference.find(&s).is_some(), "{p:?} on {s:?}");
1344
+ // Without the `u` flag the engine matches the UTF-16 code units of the string.
1345
+ let found = reference.is_match(&s).unwrap();
1346
+ for limit in [None, Some(Duration::from_secs(60)), Some(Duration::from_nanos(1))] {
1347
+ let mut budget = PatternBudget::new(PatternLimit::new(limit));
1348
+ let got = compiled.is_match(&s, &mut budget);
1349
+ // A match on the engine that reached its first check of the clock has no answer.
1350
+ assert!(got == found || budget.failed(), "{p:?} on {s:?} with a limit of {limit:?}");
1351
+ assert!(limit.is_some_and(|l| l.as_nanos() == 1) || !budget.failed(), "{p:?} on {s:?}");
1352
+ }
1518
1353
  }
1519
1354
  }
1520
1355
  }
@@ -1527,70 +1362,284 @@ mod tests {
1527
1362
  for p in ["^x-", "^\\/", "^abc$"] {
1528
1363
  assert!(matches!(compile(p).unwrap().matcher, Matcher::Literal { .. }), "{p}");
1529
1364
  }
1530
- for p in ["^[a-z]*a$", "(base64key|awskms)://(.*)"] {
1531
- assert!(matches!(compile(p).unwrap().matcher, Matcher::Regex(_)), "{p}");
1532
- }
1365
+ assert!(matches!(compile("^[a-z]*a$").unwrap().matcher, Matcher::Engine(_)));
1533
1366
  for p in [".+", "^.+", "(.+)"] {
1534
1367
  assert!(matches!(compile(p).unwrap().matcher, Matcher::HasContent { .. }), "{p}");
1535
1368
  }
1536
1369
  for p in ["^.{1,256}$", "^.+$", "^.*$", "^(.*)$", "^(.+)$"] {
1537
1370
  assert!(matches!(compile(p).unwrap().matcher, Matcher::Line { .. }), "{p}");
1538
1371
  }
1539
- for p in ["^(ab|cd)$", "^(?:es|ES|x-)$"] {
1372
+ for p in ["^(ab|cd)$", "^(?:es|ES|x-)$", "^(CRLF|LF|crlf|lf)$"] {
1540
1373
  assert!(matches!(compile(p).unwrap().matcher, Matcher::Literals(_)), "{p}");
1541
1374
  }
1375
+ // Without `$`, a sequence is read only as far as it must be.
1376
+ for p in ["^[a-z]+", "^a[b-c]{2,5}"] {
1377
+ assert!(
1378
+ matches!(&compile(p).unwrap().matcher, Matcher::Sequence(seq) if seq.items.last().unwrap().max == seq.items.last().unwrap().min),
1379
+ "{p}"
1380
+ );
1381
+ }
1382
+ assert!(matches!(&compile("^[a-zA-Z0-9_\\.\\-]*").unwrap().matcher, Matcher::Everything));
1383
+ // At most two alternatives, neither anchored at both ends.
1384
+ for p in ["^ab|cd$", "a|b", "^([a|A][u|U][t|T][o|O])|([n|N][o|O][n|N][e|E])$", "(base64key|awskms)://(.*)"] {
1385
+ assert!(matches!(compile(p).unwrap().matcher, Matcher::Alternatives(_)), "{p}");
1386
+ }
1387
+ // The engine is faster for more literals, more alternatives, an alternative anchored at both ends, and a
1388
+ // separated list.
1542
1389
  for p in [
1543
- "^ab|cd$",
1544
- "a|b",
1545
- "^([a|A][u|U][t|T][o|O])|([n|N][o|O][n|N][e|E])$",
1390
+ "^(schemas|responses|parameters|examples|requestBodies)$",
1546
1391
  "^[Ee][Ss]2015(\\.([Cc][Oo][Rr][Ee]|[Pp][Rr][Oo][Xx][Yy]))?$",
1547
1392
  "^[1-5](?:[0-9]{2}|XX)$",
1548
1393
  "(^([0-9]+)\\.([0-9]+)$)|(^\\{[A-F0-9]{8}(-[A-F0-9]{4}){3}-[A-F0-9]{12}\\}$)",
1549
1394
  "^([t|T][o|O][p|P])|([c|C][e|E][n|N][t|T][e|E][r|R])|([b|B][o|O][t|T][t|T][o|O][m|M])$",
1550
1395
  "^([A-Za-z_][-A-Za-z0-9_.:]*)$",
1551
- ] {
1552
- assert!(matches!(compile(p).unwrap().matcher, Matcher::Alternatives(_)), "{p}");
1553
- }
1554
- for p in [
1396
+ "default|^[0-9]+$",
1555
1397
  "^([a-zA-Z0-9]{2,3})(-[a-zA-Z0-9]{1,6})*$",
1556
1398
  "^([a-zA-Z_$][a-zA-Z0-9_$]{0,39}\\.)*([a-zA-Z_$][a-zA-Z0-9_$]{0,39})$",
1557
- "^([a-z_$][a-z0-9_$]{0,39}\\.)*([a-zA-Z_$][a-zA-Z0-9_$]{0,39})$",
1399
+ "^x(,a\\$)*$",
1400
+ "^x(,a$)*$",
1401
+ "^x(?=,)(,a)*$",
1558
1402
  ] {
1559
- assert!(matches!(compile(p).unwrap().matcher, Matcher::SeparatedList(_)), "{p}");
1403
+ assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
1560
1404
  }
1561
1405
  assert!(matches!(
1562
1406
  compile("^(?=[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$").unwrap().matcher,
1563
1407
  Matcher::ExcludedClassWithWord { .. }
1564
1408
  ));
1565
1409
  for p in ["\\bfoo", "^\\p{L}+$"] {
1566
- assert!(matches!(compile(p).unwrap().matcher, Matcher::Regex(_)), "{p}");
1410
+ assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
1567
1411
  }
1568
1412
  for p in ["^\\/[^\\*\\?\\&\\%]*(\\/\\*)?$", "a\\&.b", "^\\-a"] {
1569
- assert!(matches!(compile(p).unwrap().matcher, Matcher::RegexBmp(..)), "{p}");
1413
+ assert!(matches!(&compile(p).unwrap().matcher, Matcher::Engine(engine) if !engine.flags().unicode), "{p}");
1570
1414
  }
1571
- for p in ["^((\\.(?!\\.)\\/)?\\w+\\/?)+$", "^\\1(a)", "^\\p{L}\\&"] {
1572
- assert!(matches!(compile(p).unwrap().matcher, Matcher::Regress(_)), "{p}");
1415
+ for p in ["^((\\.(?!\\.)\\/)?\\w+\\/?)+$", "^\\1(a)", "\\Bs", "(?i:a)", "^[]", "\\cA"] {
1416
+ assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
1417
+ }
1418
+ // Valid only without the `u` flag, and not in the syntax `regex` takes: the engine, by code unit.
1419
+ for p in ["^\\p{L}\\&", "^\\-?\u{10400}*$", "^\\-(?=a)"] {
1420
+ let compiled = compile(p).unwrap();
1421
+ assert!(matches!(&compiled.matcher, Matcher::Engine(engine) if !engine.flags().unicode), "{p}");
1422
+ }
1423
+ }
1424
+ }
1425
+
1426
+ #[cfg(test)]
1427
+ mod limits {
1428
+ use super::*;
1429
+
1430
+ const G: Grammar = Grammar { edition: Edition::Current, unicode_version: UnicodeVersion::Default };
1431
+
1432
+ /// A pattern compiled in today's grammar.
1433
+ fn compile(p: &str) -> Result<Arc<Pattern>, PatternError> {
1434
+ super::compile(p, G)
1435
+ }
1436
+
1437
+ /// Which matcher a pattern gets (the Go module's `TestAWideSearchIsLeftToTheEngine`). A sequence or a literal
1438
+ /// that would be searched for at every position of the text keeps its matcher up to 64 characters, and beyond
1439
+ /// that is left to a matcher that takes time proportional to the text or has the limit of a match.
1440
+ #[test]
1441
+ fn a_wide_search_is_left_to_the_engine() {
1442
+ let a = |n: usize| "a".repeat(n);
1443
+ for (pattern, alternatives) in [
1444
+ (r"[a-z]{64}[0-9]{0}|x".to_string(), true),
1445
+ (r"[a-z]{63}[0-9]|x".to_string(), true),
1446
+ (r"[a-z]{64}[0-9]|x".to_string(), false),
1447
+ (r"[a-z]{50000}[0-9]|x".to_string(), false),
1448
+ (r"^[a-z]{50000}[0-9]|x".to_string(), true),
1449
+ (r"[a-z]{50000}[0-9]$|^x".to_string(), true),
1450
+ (format!("{}|^x", a(64)), true),
1451
+ (format!("{}|^x", a(65)), false),
1452
+ (format!("^{}|^x", a(65)), true),
1453
+ (format!("{}$|^x", a(65)), true),
1454
+ ] {
1455
+ let found = kind(&compile(&pattern).unwrap());
1456
+ assert_eq!(found == "alternatives", alternatives, "{pattern:.30} has the matcher {found}");
1457
+ assert!(alternatives || found == "engine", "{pattern:.30} has the matcher {found}");
1573
1458
  }
1574
1459
  }
1460
+
1461
+ /// The cache of compiled patterns is bounded in number and in size, and a pattern that has left it is still
1462
+ /// matched by whoever holds it.
1463
+ #[test]
1464
+ fn the_cache_of_patterns_is_bounded() {
1465
+ let mut budget = PatternBudget::new(PatternLimit::new(None));
1466
+ let first = compile("^cache-bound-0+$").unwrap();
1467
+ let held: Vec<Arc<Pattern>> =
1468
+ (0..3 * MAX_CACHED_PATTERNS).map(|n| compile(&format!("^cache-bound-{n}(?=x)x$")).unwrap()).collect();
1469
+ {
1470
+ let cache = cache();
1471
+ assert!(cache.patterns.len() <= MAX_CACHED_PATTERNS && cache.bytes <= MAX_CACHED_BYTES);
1472
+ assert_eq!(cache.patterns.len(), cache.order.len());
1473
+ assert_eq!(cache.bytes, cache.patterns.values().map(|p| p.retained).sum::<usize>());
1474
+ assert!(cache.get("^cache-bound-0+$", G).is_none(), "the oldest pattern is still in the cache");
1475
+ }
1476
+ assert!(first.is_match("cache-bound-000", &mut budget));
1477
+ assert!(held[0].is_match("cache-bound-0x", &mut budget) && !held[7].is_match("cache-bound-0x", &mut budget));
1478
+ // The same source compiles again, to a matcher of its own.
1479
+ let again = compile("^cache-bound-0+$").unwrap();
1480
+ assert!(!Arc::ptr_eq(&first, &again) && again.is_match("cache-bound-0", &mut budget));
1481
+ // Large patterns leave the cache by size, well before their number is the bound.
1482
+ let class = r"[\u0100\u0102\u0104\u0106]".repeat(1500);
1483
+ let large: Vec<Arc<Pattern>> = (0..200).map(|n| compile(&format!("^(?=large-{n}){class}$")).unwrap()).collect();
1484
+ assert!(large[0].retained > 256 << 10, "a large pattern is counted as {} bytes", large[0].retained);
1485
+ assert!(cache().bytes <= MAX_CACHED_BYTES);
1486
+ // A pattern too large for the cache is not kept at all.
1487
+ let huge = compile(&format!("^(?=huge){}$", class.repeat(4))).unwrap();
1488
+ assert!(huge.retained > MAX_CACHED_PATTERN_BYTES && cache().get(&huge.source, G).is_none());
1489
+ }
1575
1490
  }
1576
1491
 
1577
1492
  #[cfg(test)]
1578
1493
  mod corpus_patterns {
1579
- /// Lists which matcher each pattern in `CORVUS_PATTERNS` (corpus<TAB>pattern lines) gets.
1494
+ use std::sync::Arc;
1495
+
1496
+ use super::{Edition, Grammar, Pattern, PatternError, UnicodeVersion};
1497
+
1498
+ const G: Grammar = Grammar { edition: Edition::Current, unicode_version: UnicodeVersion::Default };
1499
+
1500
+ /// A pattern compiled in today's grammar.
1501
+ fn compile(p: &str) -> Result<Arc<Pattern>, PatternError> {
1502
+ super::compile(p, G)
1503
+ }
1504
+
1505
+ /// Lists which matcher each pattern in `CORVUS_PATTERNS` (corpus<TAB>pattern lines) gets, and what the cache
1506
+ /// counts for it.
1580
1507
  #[test]
1581
1508
  #[ignore]
1582
1509
  fn classify_corpus_patterns() {
1583
1510
  let Ok(path) = std::env::var("CORVUS_PATTERNS") else { return };
1584
1511
  for line in std::fs::read_to_string(path).unwrap().lines() {
1585
1512
  let (corpus, p) = line.split_once('\t').unwrap();
1586
- let kind = super::compile(p)
1587
- .map(|c| match &c.matcher {
1588
- super::Matcher::Regex(_) => "regex",
1589
- super::Matcher::Regress(_) => "regress",
1590
- _ => "fast",
1591
- })
1592
- .unwrap_or("invalid");
1593
- println!("{kind}\t{corpus}\t{p}");
1513
+ let kind = compile(p).map(|c| super::kind(&c)).unwrap_or("invalid");
1514
+ let retained = compile(p).map_or(0, |c| c.retained);
1515
+ println!("{kind}\t{corpus}\t{p}\t{retained}");
1516
+ }
1517
+ }
1518
+
1519
+ /// Times each dedicated matcher against the engine over the corpora of jsonschema-benchmark in `CORVUS_CORPORA`
1520
+ /// (a directory of `<name>-schema.json` and `<name>-instances.jsonl`), so that a matcher the engine beats can be
1521
+ /// removed. The inputs of a pattern are the distinct strings of the corpus's instances (for `pattern`) or their
1522
+ /// property names (for `patternProperties`), at most 1,000. Each figure is the least of 7 rounds of at least 20 ms,
1523
+ /// after a warm-up of 200 ms, through `Pattern::is_match` with the default limit of a match. Run it with
1524
+ /// `cargo test --release --lib time_corpus_patterns -- --ignored --nocapture --test-threads 1`.
1525
+ #[test]
1526
+ #[ignore]
1527
+ fn time_corpus_patterns() {
1528
+ use std::collections::BTreeSet;
1529
+ use std::time::{Duration, Instant};
1530
+
1531
+ use super::{Literals, Matcher, PatternBudget, PatternLimit};
1532
+
1533
+ fn collect(v: &serde_json::Value, strings: &mut BTreeSet<String>, keys: &mut BTreeSet<String>) {
1534
+ match v {
1535
+ serde_json::Value::String(s) => {
1536
+ strings.insert(s.clone());
1537
+ }
1538
+ serde_json::Value::Array(a) => a.iter().for_each(|v| collect(v, strings, keys)),
1539
+ serde_json::Value::Object(o) => o.iter().for_each(|(k, v)| {
1540
+ keys.insert(k.clone());
1541
+ collect(v, strings, keys);
1542
+ }),
1543
+ _ => {}
1544
+ }
1545
+ }
1546
+ fn patterns(v: &serde_json::Value, out: &mut BTreeSet<(&'static str, String)>) {
1547
+ match v {
1548
+ serde_json::Value::Array(a) => a.iter().for_each(|v| patterns(v, out)),
1549
+ serde_json::Value::Object(o) => {
1550
+ if let Some(serde_json::Value::String(p)) = o.get("pattern") {
1551
+ out.insert(("pattern", p.clone()));
1552
+ }
1553
+ if let Some(serde_json::Value::Object(pp)) = o.get("patternProperties") {
1554
+ pp.keys().for_each(|p| {
1555
+ out.insert(("patternProperties", p.clone()));
1556
+ });
1557
+ }
1558
+ o.values().for_each(|v| patterns(v, out));
1559
+ }
1560
+ _ => {}
1561
+ }
1562
+ }
1563
+ fn time(p: &Pattern, inputs: &[String]) -> f64 {
1564
+ let limit = PatternLimit::new(Some(crate::DEFAULT_PATTERN_MATCH_LIMIT));
1565
+ let pass = || {
1566
+ let mut budget = PatternBudget::new(limit);
1567
+ inputs.iter().filter(|s| p.is_match(s, &mut budget)).count()
1568
+ };
1569
+ let warm = Instant::now();
1570
+ while warm.elapsed() < Duration::from_millis(200) {
1571
+ std::hint::black_box(pass());
1572
+ }
1573
+ let mut best = f64::MAX;
1574
+ for _ in 0..7 {
1575
+ let (start, mut calls) = (Instant::now(), 0usize);
1576
+ while start.elapsed() < Duration::from_millis(20) {
1577
+ std::hint::black_box(pass());
1578
+ calls += inputs.len();
1579
+ }
1580
+ best = best.min(start.elapsed().as_nanos() as f64 / calls as f64);
1581
+ }
1582
+ best
1583
+ }
1584
+
1585
+ let Ok(dir) = std::env::var("CORVUS_CORPORA") else { return };
1586
+ let mut names: Vec<String> = std::fs::read_dir(&dir)
1587
+ .unwrap()
1588
+ .filter_map(|e| e.unwrap().file_name().to_str()?.strip_suffix("-schema.json").map(str::to_string))
1589
+ .collect();
1590
+ names.sort();
1591
+ // The alternatives a matcher tries one after another.
1592
+ fn alternatives(m: &Matcher) -> usize {
1593
+ match m {
1594
+ Matcher::Alternatives(a) => a.len(),
1595
+ Matcher::Literals(Literals(t)) => t.len(),
1596
+ _ => 1,
1597
+ }
1598
+ }
1599
+ println!("corpus\trole\tkind\talternatives\tshape ns\tengine ns\tshape/engine\tinputs\tpattern");
1600
+ for name in names {
1601
+ let schema: serde_json::Value =
1602
+ serde_json::from_str(&std::fs::read_to_string(format!("{dir}/{name}-schema.json")).unwrap()).unwrap();
1603
+ let (mut strings, mut keys) = (BTreeSet::new(), BTreeSet::new());
1604
+ for line in std::fs::read_to_string(format!("{dir}/{name}-instances.jsonl")).unwrap().lines() {
1605
+ if !line.trim().is_empty() {
1606
+ collect(&serde_json::from_str(line).unwrap(), &mut strings, &mut keys);
1607
+ }
1608
+ }
1609
+ let strings: Vec<String> = strings.into_iter().take(1000).collect();
1610
+ let keys: Vec<String> = keys.into_iter().take(1000).collect();
1611
+ let mut found = BTreeSet::new();
1612
+ patterns(&schema, &mut found);
1613
+ for (role, source) in found {
1614
+ let Ok(shape) = compile(&source) else { continue };
1615
+ if matches!(shape.matcher, Matcher::Engine(_)) {
1616
+ continue;
1617
+ }
1618
+ let inputs = if role == "pattern" { &strings } else { &keys };
1619
+ if inputs.is_empty() {
1620
+ continue;
1621
+ }
1622
+ let regex = G.build(&source, true).or_else(|_| G.build(&source, false)).unwrap();
1623
+ let engine = Pattern {
1624
+ source: source.clone(),
1625
+ grammar: G,
1626
+ matcher: Matcher::Engine(Box::new(regex)),
1627
+ retained: 0,
1628
+ };
1629
+ // The two must agree before their times mean anything.
1630
+ let mut budget = PatternBudget::new(PatternLimit::new(None));
1631
+ for s in inputs {
1632
+ assert_eq!(shape.is_match(s, &mut budget), engine.is_match(s, &mut budget), "{source:?} on {s:?}");
1633
+ }
1634
+ let (a, b) = (time(&shape, inputs), time(&engine, inputs));
1635
+ println!(
1636
+ "{name}\t{role}\t{}\t{}\t{a:.1}\t{b:.1}\t{:.3}\t{}\t{source}",
1637
+ super::kind(&shape),
1638
+ alternatives(&shape.matcher),
1639
+ a / b,
1640
+ inputs.len()
1641
+ );
1642
+ }
1594
1643
  }
1595
1644
  }
1596
1645
  }