corvus_json_schema 0.1.3 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +13 -12
- data/README.md +12 -1
- data/THIRD-PARTY-NOTICES.txt +96 -0
- data/VERSIONHISTORY.md +33 -0
- data/ext/corvus_json_schema/Cargo.toml +2 -2
- data/ext/corvus_json_schema/clippy.toml +3 -4
- data/ext/corvus_json_schema/src/lib.rs +70 -20
- data/ext/corvus_json_schema/vendor/corvus-json-schema/Cargo.toml +2 -2
- data/ext/corvus_json_schema/vendor/corvus-json-schema/README.md +68 -4
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/compiler.rs +91 -97
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/document.rs +2 -2
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/eval/plan/fused.rs +13 -9
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/eval/plan.rs +12 -11
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/eval.rs +105 -73
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/formats.rs +102 -7
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/lib.rs +165 -30
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/loader.rs +19 -21
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/numbers.rs +1 -1
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/options.rs +211 -4
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/pattern.rs +635 -586
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/uri.rs +4 -4
- data/lib/corvus_json_schema/version.rb +1 -1
- data/lib/corvus_json_schema.rb +18 -3
- metadata +3 -2
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
//! `pattern`/`patternProperties` matching with ECMA-262 semantics (the `u` flag), as JSON Schema specifies.
|
|
1
|
+
//! `pattern`/`patternProperties` matching with ECMA-262 semantics (the `u` flag), as JSON Schema specifies. A
|
|
2
|
+
//! pattern that is valid only without the flag is accepted too, and matched as ECMA-262 says for it: by UTF-16
|
|
3
|
+
//! code unit.
|
|
2
4
|
//!
|
|
3
5
|
//! A pattern gets the cheapest matcher that decides it exactly:
|
|
4
6
|
//!
|
|
@@ -6,24 +8,78 @@
|
|
|
6
8
|
//! - anchored sequences of quantified ASCII character classes and literals (`^[a-z][a-z0-9_]{0,29}$`, `^x-`,
|
|
7
9
|
//! `^[@$_#]`), matched in one pass over the string (the TypeScript evaluator's class-sequence fast path);
|
|
8
10
|
//! - alternatives of such sequences once groups are multiplied out (`^([a|A]uto)|([n|N]one)$`,
|
|
9
|
-
//! `^[Ee][Ss]2015(\.([Cc]ore|[Pp]roxy))?$`)
|
|
10
|
-
//!
|
|
11
|
-
//! -
|
|
12
|
-
//!
|
|
13
|
-
//!
|
|
14
|
-
//! - anything else (lookarounds, backreferences, word boundaries, Unicode properties) by `regress`, a backtracking
|
|
15
|
-
//! ECMA-262 engine.
|
|
11
|
+
//! `^[Ee][Ss]2015(\.([Cc]ore|[Pp]roxy))?$`) and sets of literals, as the C# evaluator's pattern matchers do,
|
|
12
|
+
//! where they were measured faster than the engine (see [`MOST_LITERALS`]);
|
|
13
|
+
//! - anything else by [Corvus.Regex.Ecma262](https://docs.rs/corvus-regex-ecma262), an ECMA-262 engine that answers
|
|
14
|
+
//! every pattern as V8 does, as the pattern is written, and matches a pattern with no backreference in time
|
|
15
|
+
//! proportional to the pattern's size times the string.
|
|
16
16
|
//!
|
|
17
|
-
//!
|
|
17
|
+
//! Whether a pattern is valid is the engine's answer, whichever matcher runs it, in the grammar of the edition the
|
|
18
|
+
//! dialect reads patterns with (ECMAScript 2020 for draft 2019-09 and draft 2020-12, today's for the others; see
|
|
19
|
+
//! [`Grammar`]), and so are the limits of size: a pattern over one is refused before any matcher reads it.
|
|
20
|
+
//!
|
|
21
|
+
//! Patterns compile once per process for each grammar (a pattern is immutable, so identical patterns share one
|
|
22
|
+
//! matcher).
|
|
18
23
|
|
|
19
|
-
use std::collections::HashMap;
|
|
24
|
+
use std::collections::{HashMap, VecDeque};
|
|
20
25
|
use std::fmt;
|
|
21
26
|
use std::sync::{Arc, LazyLock, Mutex};
|
|
27
|
+
use std::time::Duration;
|
|
28
|
+
|
|
29
|
+
use corvus_regex_ecma262::{Edition, Flags, MatchError, Regex, RegexBuilder, UnicodeVersion};
|
|
30
|
+
|
|
31
|
+
use crate::dialect::Dialect;
|
|
32
|
+
use crate::options::{
|
|
33
|
+
CompileOptions, PatternMatchLimitError, PatternTooLargeError, SchemaCompilationError, SchemaCompilationErrorKind,
|
|
34
|
+
};
|
|
35
|
+
|
|
36
|
+
/// The grammar a pattern is read in: the edition of ECMA-262 and the version of Unicode.
|
|
37
|
+
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
|
38
|
+
pub(crate) struct Grammar {
|
|
39
|
+
pub edition: Edition,
|
|
40
|
+
pub unicode_version: UnicodeVersion,
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
impl Grammar {
|
|
44
|
+
/// The grammar of a dialect's patterns: the edition the options give, or by default ECMAScript 2020 for draft
|
|
45
|
+
/// 2019-09 and draft 2020-12, which cite it, and today's for the earlier drafts; and the options' version of
|
|
46
|
+
/// Unicode.
|
|
47
|
+
pub(crate) fn of(dialect: Dialect, options: &CompileOptions) -> Grammar {
|
|
48
|
+
let edition = options.regex_edition.unwrap_or(if dialect >= Dialect::Draft201909 {
|
|
49
|
+
Edition::Es2020
|
|
50
|
+
} else {
|
|
51
|
+
Edition::Current
|
|
52
|
+
});
|
|
53
|
+
Grammar { edition, unicode_version: options.regex_unicode_version }
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/// The flags of the grammar of the `u` flag, or of no flag.
|
|
57
|
+
fn flags(unicode: bool) -> Flags {
|
|
58
|
+
Flags { unicode, ..Flags::default() }
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/// The pattern compiled in this grammar, with the `u` flag or with none.
|
|
62
|
+
fn build(self, pattern: &str, unicode: bool) -> Result<Regex, corvus_regex_ecma262::Error> {
|
|
63
|
+
RegexBuilder::new(pattern)
|
|
64
|
+
.with_flags(Grammar::flags(unicode))
|
|
65
|
+
.edition(self.edition)
|
|
66
|
+
.unicode_version(self.unicode_version)
|
|
67
|
+
.build()
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/// Whether the pattern is valid in this grammar with the `u` flag.
|
|
71
|
+
fn is_valid_unicode(self, pattern: &str) -> bool {
|
|
72
|
+
Regex::is_valid_with(pattern, Grammar::flags(true), self.edition, self.unicode_version)
|
|
73
|
+
}
|
|
74
|
+
}
|
|
22
75
|
|
|
23
76
|
/// A compiled `pattern`.
|
|
24
77
|
pub(crate) struct Pattern {
|
|
25
78
|
pub source: String,
|
|
79
|
+
grammar: Grammar,
|
|
26
80
|
matcher: Matcher,
|
|
81
|
+
/// What the pattern is counted as in the cache of compiled patterns, in bytes (see [`Cache`]).
|
|
82
|
+
retained: usize,
|
|
27
83
|
}
|
|
28
84
|
|
|
29
85
|
enum Matcher {
|
|
@@ -35,7 +91,6 @@ enum Matcher {
|
|
|
35
91
|
whole: bool,
|
|
36
92
|
},
|
|
37
93
|
Sequence(Sequence),
|
|
38
|
-
SeparatedList(SeparatedList),
|
|
39
94
|
/// `.+` (`^.+` with `start`): some (the first) character is not a line terminator.
|
|
40
95
|
HasContent {
|
|
41
96
|
start: bool,
|
|
@@ -55,12 +110,9 @@ enum Matcher {
|
|
|
55
110
|
set: CharSet,
|
|
56
111
|
bangs: bool,
|
|
57
112
|
},
|
|
58
|
-
|
|
59
|
-
///
|
|
60
|
-
|
|
61
|
-
/// Multilingual Plane (where code units are characters); regress decides the rest.
|
|
62
|
-
RegexBmp(regex::Regex, regress::Regex),
|
|
63
|
-
Regress(regress::Regex),
|
|
113
|
+
/// The `corvus-regex-ecma262` engine: by character for a pattern with the `u` flag, and by UTF-16 code unit for
|
|
114
|
+
/// one that is valid only without it.
|
|
115
|
+
Engine(Box<Regex>),
|
|
64
116
|
}
|
|
65
117
|
|
|
66
118
|
impl fmt::Debug for Pattern {
|
|
@@ -69,9 +121,71 @@ impl fmt::Debug for Pattern {
|
|
|
69
121
|
}
|
|
70
122
|
}
|
|
71
123
|
|
|
124
|
+
/// The limit of a match, as a validator holds it.
|
|
125
|
+
#[derive(Clone, Copy, Debug)]
|
|
126
|
+
pub(crate) struct PatternLimit {
|
|
127
|
+
/// The time one match may take, or `None` for no limit.
|
|
128
|
+
limit: Option<Duration>,
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
impl PatternLimit {
|
|
132
|
+
/// A limit of `limit` for each match. `None` and zero are no limit.
|
|
133
|
+
pub(crate) fn new(limit: Option<Duration>) -> PatternLimit {
|
|
134
|
+
PatternLimit { limit: limit.filter(|l| !l.is_zero()) }
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
|
|
138
|
+
/// What one evaluation, or the compilation of one schema, gives every match of a pattern: the time each may take.
|
|
139
|
+
/// It notes the first match that reached a limit, after which every match is abandoned at once.
|
|
140
|
+
pub(crate) struct PatternBudget {
|
|
141
|
+
limit: PatternLimit,
|
|
142
|
+
/// The match that reached a limit. It is boxed so that an evaluation, which holds a budget, stays small.
|
|
143
|
+
failure: Option<Box<PatternMatchLimitError>>,
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
impl PatternBudget {
|
|
147
|
+
#[inline(always)]
|
|
148
|
+
pub(crate) fn new(limit: PatternLimit) -> PatternBudget {
|
|
149
|
+
PatternBudget { limit, failure: None }
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
/// Whether a match has reached a limit, so that what the evaluation came to is not an answer.
|
|
153
|
+
#[inline(always)]
|
|
154
|
+
pub(crate) fn failed(&self) -> bool {
|
|
155
|
+
self.failure.is_some()
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
/// The error of the match that reached a limit, if one did.
|
|
159
|
+
pub(crate) fn take_failure(&mut self) -> Option<PatternMatchLimitError> {
|
|
160
|
+
self.failure.take().map(|e| *e)
|
|
161
|
+
}
|
|
162
|
+
|
|
163
|
+
/// Matches the string on the engine within the budget. When a limit is reached the answer is false, and it is
|
|
164
|
+
/// not an answer: the caller of the evaluation gets the error.
|
|
165
|
+
fn engine_match(&mut self, engine: &Regex, source: &str, s: &str) -> bool {
|
|
166
|
+
if self.failure.is_some() {
|
|
167
|
+
return false;
|
|
168
|
+
}
|
|
169
|
+
let result = match self.limit.limit {
|
|
170
|
+
Some(limit) => engine.is_match_within(s, limit),
|
|
171
|
+
None => engine.is_match(s),
|
|
172
|
+
};
|
|
173
|
+
result.unwrap_or_else(|e| {
|
|
174
|
+
self.failure = Some(Box::new(PatternMatchLimitError::new(
|
|
175
|
+
source,
|
|
176
|
+
self.limit.limit.unwrap_or_default(),
|
|
177
|
+
e == MatchError::Memory,
|
|
178
|
+
)));
|
|
179
|
+
false
|
|
180
|
+
})
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
72
184
|
impl Pattern {
|
|
185
|
+
/// Whether the pattern matches somewhere in the string. The engine matches within the budget. Every other
|
|
186
|
+
/// matcher reads the string once, or a bounded number of times, and has no limit.
|
|
73
187
|
#[inline]
|
|
74
|
-
pub fn is_match(&self, s: &str) -> bool {
|
|
188
|
+
pub fn is_match(&self, s: &str, budget: &mut PatternBudget) -> bool {
|
|
75
189
|
match &self.matcher {
|
|
76
190
|
Matcher::Everything => true,
|
|
77
191
|
Matcher::Literal { text, whole } => {
|
|
@@ -82,7 +196,6 @@ impl Pattern {
|
|
|
82
196
|
}
|
|
83
197
|
}
|
|
84
198
|
Matcher::Sequence(seq) => seq.is_match(s),
|
|
85
|
-
Matcher::SeparatedList(list) => list.is_match(s),
|
|
86
199
|
Matcher::HasContent { start } => {
|
|
87
200
|
if *start {
|
|
88
201
|
s.chars().next().is_some_and(|c| !is_line_terminator(c))
|
|
@@ -115,20 +228,28 @@ impl Pattern {
|
|
|
115
228
|
}
|
|
116
229
|
word
|
|
117
230
|
}
|
|
118
|
-
Matcher::
|
|
119
|
-
// A UTF-8 lead byte of 0xF0 or above starts a character beyond the BMP.
|
|
120
|
-
Matcher::RegexBmp(re, fallback) => {
|
|
121
|
-
if s.bytes().any(|b| b >= 0xF0) {
|
|
122
|
-
fallback.find(s).is_some()
|
|
123
|
-
} else {
|
|
124
|
-
re.is_match(s)
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
Matcher::Regress(re) => re.find(s).is_some(),
|
|
231
|
+
Matcher::Engine(engine) => budget.engine_match(engine, &self.source, s),
|
|
128
232
|
}
|
|
129
233
|
}
|
|
130
234
|
}
|
|
131
235
|
|
|
236
|
+
/// The kind of matcher a pattern has, for the tables of which pattern goes where.
|
|
237
|
+
#[cfg(test)]
|
|
238
|
+
pub(crate) fn kind(p: &Pattern) -> &'static str {
|
|
239
|
+
match &p.matcher {
|
|
240
|
+
Matcher::Engine(engine) if engine.flags().unicode => "engine",
|
|
241
|
+
Matcher::Engine(_) => "engine-units",
|
|
242
|
+
Matcher::Everything => "everything",
|
|
243
|
+
Matcher::Literal { .. } => "literal",
|
|
244
|
+
Matcher::Sequence(_) => "sequence",
|
|
245
|
+
Matcher::HasContent { .. } => "has-content",
|
|
246
|
+
Matcher::Line { .. } => "line",
|
|
247
|
+
Matcher::Literals(_) => "literals",
|
|
248
|
+
Matcher::Alternatives(_) => "alternatives",
|
|
249
|
+
Matcher::ExcludedClassWithWord { .. } => "excluded-class",
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
132
253
|
/// ECMA-262 `LineTerminator`, which `.` does not match.
|
|
133
254
|
#[inline]
|
|
134
255
|
fn is_line_terminator(c: char) -> bool {
|
|
@@ -151,30 +272,34 @@ fn line_length(s: &str) -> Option<usize> {
|
|
|
151
272
|
Some(n)
|
|
152
273
|
}
|
|
153
274
|
|
|
154
|
-
/// A set of strings
|
|
155
|
-
|
|
156
|
-
Few(Box<[Box<str>]>),
|
|
157
|
-
Many(std::collections::HashSet<Box<str>>),
|
|
158
|
-
}
|
|
275
|
+
/// A set of at most [`MOST_LITERALS`] strings, compared in turn.
|
|
276
|
+
struct Literals(Box<[Box<str>]>);
|
|
159
277
|
|
|
160
278
|
impl Literals {
|
|
161
279
|
fn new(texts: Vec<String>) -> Literals {
|
|
162
|
-
|
|
163
|
-
Literals::Few(texts.into_iter().map(String::into_boxed_str).collect())
|
|
164
|
-
} else {
|
|
165
|
-
Literals::Many(texts.into_iter().map(String::into_boxed_str).collect())
|
|
166
|
-
}
|
|
280
|
+
Literals(texts.into_iter().map(String::into_boxed_str).collect())
|
|
167
281
|
}
|
|
168
282
|
|
|
169
283
|
#[inline]
|
|
170
284
|
fn contains(&self, s: &str) -> bool {
|
|
171
|
-
|
|
172
|
-
Literals::Few(texts) => texts.iter().any(|t| **t == *s),
|
|
173
|
-
Literals::Many(set) => set.contains(s),
|
|
174
|
-
}
|
|
285
|
+
self.0.iter().any(|t| **t == *s)
|
|
175
286
|
}
|
|
176
287
|
}
|
|
177
288
|
|
|
289
|
+
// What the dedicated matchers are kept for: each was timed against the engine, pattern by pattern, over the strings of
|
|
290
|
+
// the jsonschema-benchmark corpora (`corpus_patterns::time_corpus_patterns`), and a form is kept only where its matcher
|
|
291
|
+
// was the faster. Four literals compared in turn took 0.78 of the engine's time, and ten 3.2 times it. Two alternatives
|
|
292
|
+
// of which neither is anchored at both ends (`^([a|A]uto)|([n|N]one)$`) took 0.21 of it, since the engine searches for
|
|
293
|
+
// the unanchored one; three such took 1.4 to 1.6 times it, and every alternative anchored at both ends was faster on
|
|
294
|
+
// the engine (by 1.1 to 2.7 times), which reads the whole string once whatever the alternatives. A separated list was
|
|
295
|
+
// no faster than the engine (the same total time over the seven in the corpora), so it has no matcher of its own.
|
|
296
|
+
|
|
297
|
+
/// The most literals of [`Matcher::Literals`].
|
|
298
|
+
const MOST_LITERALS: usize = 4;
|
|
299
|
+
|
|
300
|
+
/// The most alternatives of [`Matcher::Alternatives`].
|
|
301
|
+
const MOST_ALTERNATIVES: usize = 2;
|
|
302
|
+
|
|
178
303
|
/// One alternative of [`Matcher::Alternatives`]: a literal anchored at either end (or neither), a class sequence
|
|
179
304
|
/// anchored at the start, or a fixed-width class sequence anchored at the end.
|
|
180
305
|
enum Alternative {
|
|
@@ -237,43 +362,150 @@ impl Alternative {
|
|
|
237
362
|
}
|
|
238
363
|
}
|
|
239
364
|
|
|
240
|
-
///
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
365
|
+
/// The most patterns the process-wide cache holds.
|
|
366
|
+
const MAX_CACHED_PATTERNS: usize = 1024;
|
|
367
|
+
|
|
368
|
+
/// The most the patterns of the cache come to, in the bytes counted for each: 64 MiB. A pattern is counted as its
|
|
369
|
+
/// source and an estimate of what its matcher holds (see [`compile`]).
|
|
370
|
+
const MAX_CACHED_BYTES: usize = 64 << 20;
|
|
371
|
+
|
|
372
|
+
/// A pattern counted as more than this is compiled for the schema that has it and not kept in the cache.
|
|
373
|
+
const MAX_CACHED_PATTERN_BYTES: usize = MAX_CACHED_BYTES / 32;
|
|
374
|
+
|
|
375
|
+
/// The compiled patterns of the process, so that identical patterns share one matcher. It is bounded in number and
|
|
376
|
+
/// in size, since the patterns of a schema may come from outside: when it is over either bound the patterns that
|
|
377
|
+
/// were compiled first go, oldest first. A validator holds the patterns of its schema itself, so a pattern that has
|
|
378
|
+
/// left the cache is still matched by the validators that have it, and is compiled again only for a schema compiled
|
|
379
|
+
/// later.
|
|
380
|
+
#[derive(Default)]
|
|
381
|
+
struct Cache {
|
|
382
|
+
patterns: HashMap<(Arc<str>, Grammar), Arc<Pattern>>,
|
|
383
|
+
/// The sources and their grammars in the order they were compiled.
|
|
384
|
+
order: VecDeque<(Arc<str>, Grammar)>,
|
|
385
|
+
/// The sum of what the patterns are counted as.
|
|
386
|
+
bytes: usize,
|
|
387
|
+
}
|
|
388
|
+
|
|
389
|
+
impl Cache {
|
|
390
|
+
fn get(&self, source: &str, grammar: Grammar) -> Option<Arc<Pattern>> {
|
|
391
|
+
self.patterns.get(&(Arc::from(source), grammar)).cloned()
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
/// Keeps a pattern, and returns the one to use: the pattern itself, or the one another thread kept for the same
|
|
395
|
+
/// source meanwhile.
|
|
396
|
+
fn insert(&mut self, pattern: Arc<Pattern>) -> Arc<Pattern> {
|
|
397
|
+
if pattern.retained > MAX_CACHED_PATTERN_BYTES {
|
|
398
|
+
return pattern;
|
|
399
|
+
}
|
|
400
|
+
let key: (Arc<str>, Grammar) = (Arc::from(pattern.source.as_str()), pattern.grammar);
|
|
401
|
+
if let Some(kept) = self.patterns.get(&key) {
|
|
402
|
+
return kept.clone();
|
|
403
|
+
}
|
|
404
|
+
self.bytes += pattern.retained;
|
|
405
|
+
self.patterns.insert(key.clone(), pattern.clone());
|
|
406
|
+
self.order.push_back(key);
|
|
407
|
+
while self.patterns.len() > MAX_CACHED_PATTERNS || self.bytes > MAX_CACHED_BYTES {
|
|
408
|
+
let Some(oldest) = self.order.pop_front() else { break };
|
|
409
|
+
if let Some(evicted) = self.patterns.remove(&oldest) {
|
|
410
|
+
self.bytes -= evicted.retained;
|
|
411
|
+
}
|
|
412
|
+
}
|
|
413
|
+
pattern
|
|
414
|
+
}
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
static CACHE: LazyLock<Mutex<Cache>> = LazyLock::new(|| Mutex::new(Cache::default()));
|
|
418
|
+
|
|
419
|
+
fn cache() -> std::sync::MutexGuard<'static, Cache> {
|
|
420
|
+
CACHE.lock().unwrap_or_else(|e| e.into_inner())
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
/// Why a pattern does not compile.
|
|
424
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
425
|
+
pub(crate) enum PatternError {
|
|
426
|
+
/// It is not a valid ECMA-262 regular expression.
|
|
427
|
+
Invalid,
|
|
428
|
+
/// It is over a limit of size. It was not read to its end, so nothing is known of whether it is valid.
|
|
429
|
+
TooLarge(corvus_regex_ecma262::TooLargeError),
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
impl PatternError {
|
|
433
|
+
/// The error of a schema whose keyword `keyword` holds the pattern `pattern`.
|
|
434
|
+
pub(crate) fn of(self, pattern: &str, keyword: &'static str) -> SchemaCompilationError {
|
|
435
|
+
match self {
|
|
436
|
+
PatternError::Invalid => SchemaCompilationError::of_kind(
|
|
437
|
+
format!("Invalid regular expression '{pattern}' in {keyword}."),
|
|
438
|
+
SchemaCompilationErrorKind::InvalidPattern,
|
|
439
|
+
),
|
|
440
|
+
PatternError::TooLarge(large) => {
|
|
441
|
+
let error =
|
|
442
|
+
PatternTooLargeError::new(pattern, keyword, large.limit.name(), large.value, large.to_string());
|
|
443
|
+
SchemaCompilationError::of_kind(error.to_string(), SchemaCompilationErrorKind::PatternTooLarge(error))
|
|
444
|
+
}
|
|
445
|
+
}
|
|
245
446
|
}
|
|
246
447
|
}
|
|
247
448
|
|
|
248
|
-
|
|
449
|
+
impl From<corvus_regex_ecma262::Error> for PatternError {
|
|
450
|
+
fn from(e: corvus_regex_ecma262::Error) -> PatternError {
|
|
451
|
+
match e {
|
|
452
|
+
corvus_regex_ecma262::Error::TooLarge(large) => PatternError::TooLarge(large),
|
|
453
|
+
// Not valid in the grammar, or (not given here) flags that are not.
|
|
454
|
+
_ => PatternError::Invalid,
|
|
455
|
+
}
|
|
456
|
+
}
|
|
457
|
+
}
|
|
249
458
|
|
|
250
|
-
///
|
|
251
|
-
pub(crate) fn
|
|
252
|
-
|
|
253
|
-
|
|
459
|
+
/// A pattern in quotes for a message, or its first 120 bytes and its length when it is longer.
|
|
460
|
+
pub(crate) fn shown_pattern(pattern: &str) -> String {
|
|
461
|
+
const SHOWN: usize = 120;
|
|
462
|
+
if pattern.len() <= SHOWN {
|
|
463
|
+
return format!("'{pattern}'");
|
|
464
|
+
}
|
|
465
|
+
let mut cut = SHOWN;
|
|
466
|
+
while !pattern.is_char_boundary(cut) {
|
|
467
|
+
cut -= 1;
|
|
254
468
|
}
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
469
|
+
format!("'{}...' ({} bytes)", &pattern[..cut], pattern.len())
|
|
470
|
+
}
|
|
471
|
+
|
|
472
|
+
/// Compiles (or fetches from the process-wide cache) a pattern, in the grammar of its dialect.
|
|
473
|
+
pub(crate) fn compile(pattern: &str, grammar: Grammar) -> Result<Arc<Pattern>, PatternError> {
|
|
474
|
+
if let Some(p) = cache().get(pattern, grammar) {
|
|
475
|
+
return Ok(p);
|
|
476
|
+
}
|
|
477
|
+
// Validity is ECMA-262's: a pattern the engine refuses is an error, whichever matcher would run it. A pattern is
|
|
478
|
+
// read with the u flag, and failing that with no flag, whose grammar (Annex B) many schemas need. A pattern valid
|
|
479
|
+
// with the flag that a dedicated matcher takes is read and never built: building the engine's program and
|
|
480
|
+
// automata costs far more than reading. The engine is built only for the patterns it matches, and for those over
|
|
481
|
+
// a limit of size, which it refuses (the reading refuses them too, without saying why).
|
|
482
|
+
let matcher = match grammar.is_valid_unicode(pattern).then(|| choose(pattern, true, grammar)).flatten() {
|
|
483
|
+
Some(shape) => shape,
|
|
484
|
+
None => match grammar.build(pattern, true) {
|
|
485
|
+
Ok(engine) => Matcher::Engine(Box::new(engine)),
|
|
486
|
+
Err(corvus_regex_ecma262::Error::TooLarge(large)) => return Err(PatternError::TooLarge(large)),
|
|
487
|
+
Err(_) => {
|
|
488
|
+
let engine = grammar.build(pattern, false)?;
|
|
489
|
+
choose(pattern, false, grammar).unwrap_or_else(|| Matcher::Engine(Box::new(engine)))
|
|
490
|
+
}
|
|
262
491
|
},
|
|
263
|
-
None => Matcher::Regress(regress),
|
|
264
492
|
};
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
493
|
+
// What the cache counts for the pattern (see `MAX_CACHED_BYTES`): its source, and for the engine an estimate of
|
|
494
|
+
// its program and automata, which grow with the pattern.
|
|
495
|
+
let mut retained = pattern.len() + 256;
|
|
496
|
+
if matches!(matcher, Matcher::Engine(_)) {
|
|
497
|
+
retained += 16 * pattern.len();
|
|
498
|
+
}
|
|
499
|
+
Ok(cache().insert(Arc::new(Pattern { source: pattern.to_string(), grammar, matcher, retained })))
|
|
268
500
|
}
|
|
269
501
|
|
|
270
|
-
fn choose(pattern: &str, unicode: bool) -> Option<Matcher> {
|
|
502
|
+
fn choose(pattern: &str, unicode: bool, grammar: Grammar) -> Option<Matcher> {
|
|
271
503
|
// Unanchored (or start-anchored) `.*` finds an empty match in any string; `^.*$` does not (`.` stops at a line
|
|
272
504
|
// terminator), so it is not listed.
|
|
273
505
|
if matches!(pattern, "" | ".*" | "^.*" | ".*$" | "(.*)" | "^(.*)" | "[\\s\\S]*" | "^[\\s\\S]*" | "^[\\s\\S]*$") {
|
|
274
506
|
return Some(Matcher::Everything);
|
|
275
507
|
}
|
|
276
|
-
// Without the `u` flag, a pattern matches UTF-16 code units rather than characters: only
|
|
508
|
+
// Without the `u` flag, a pattern matches UTF-16 code units rather than characters: only the engine has that.
|
|
277
509
|
if !unicode {
|
|
278
510
|
return None;
|
|
279
511
|
}
|
|
@@ -288,7 +520,8 @@ fn choose(pattern: &str, unicode: bool) -> Option<Matcher> {
|
|
|
288
520
|
&& rest.len() > 1
|
|
289
521
|
&& !ends_with_escape(rest)
|
|
290
522
|
&& !rest.ends_with(['*', '+', '?', '}', '|', '(', '^'])
|
|
291
|
-
&&
|
|
523
|
+
&& grammar.is_valid_unicode(rest)
|
|
524
|
+
&& let Some(m) = choose(rest, true, grammar)
|
|
292
525
|
{
|
|
293
526
|
return Some(m);
|
|
294
527
|
}
|
|
@@ -296,25 +529,28 @@ fn choose(pattern: &str, unicode: bool) -> Option<Matcher> {
|
|
|
296
529
|
return Some(Matcher::Line { min, max });
|
|
297
530
|
}
|
|
298
531
|
if let Some(texts) = whole_alternatives(pattern) {
|
|
299
|
-
return
|
|
532
|
+
return (texts.len() <= MOST_LITERALS).then(|| Matcher::Literals(Literals::new(texts)));
|
|
300
533
|
}
|
|
301
534
|
if let Some(alts) = alternatives(pattern) {
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
535
|
+
let whole = |a: &Alternative| match a {
|
|
536
|
+
Alternative::Literal { start, end, .. } => *start && *end,
|
|
537
|
+
Alternative::Start(seq) => seq.to_end,
|
|
538
|
+
Alternative::End { .. } | Alternative::Anywhere { .. } => false,
|
|
539
|
+
};
|
|
540
|
+
return (alts.len() <= MOST_ALTERNATIVES && !alts.iter().any(whole)).then_some(Matcher::Alternatives(alts));
|
|
306
541
|
}
|
|
307
542
|
if let Some((set, bangs)) = excluded_class_with_word(pattern) {
|
|
308
543
|
return Some(Matcher::ExcludedClassWithWord { set, bangs });
|
|
309
544
|
}
|
|
310
545
|
if let Some(seq) = Sequence::parse(pattern) {
|
|
311
546
|
return Some(match seq.literal() {
|
|
547
|
+
// `^[a-z]*` with nothing it requires: every string.
|
|
548
|
+
Some(text) if text.is_empty() && !seq.to_end => Matcher::Everything,
|
|
312
549
|
Some(text) => Matcher::Literal { text: text.into(), whole: seq.to_end },
|
|
313
550
|
None => Matcher::Sequence(seq),
|
|
314
551
|
});
|
|
315
552
|
}
|
|
316
|
-
|
|
317
|
-
regex::Regex::new(&translated).ok().map(Matcher::Regex)
|
|
553
|
+
None
|
|
318
554
|
}
|
|
319
555
|
|
|
320
556
|
/// `^.{m,n}$` (and `^.*$`, `^.+$`, `^.{m}$`, `^.{m,}$`, each also as `^(.…)$`): the bounds on the length of a line.
|
|
@@ -393,12 +629,18 @@ const MAX_ALTERNATIVES: usize = 64;
|
|
|
393
629
|
/// `regex` crate's single automaton pass (measured on jsconfig's case-folded `lib` names).
|
|
394
630
|
const MAX_SEQUENCE_ALTERNATIVES: usize = 4;
|
|
395
631
|
|
|
632
|
+
/// The widest class sequence, in characters, and the longest literal, in bytes, that is searched for at every
|
|
633
|
+
/// position of a text. The search for a sequence takes the text times this, and these matchers have no limit, so a
|
|
634
|
+
/// wider one is left to the engine, which has the limit of a match. (The rule is the Go module's, so that the ports
|
|
635
|
+
/// give a pattern the same kind of matcher.)
|
|
636
|
+
const MAX_ANYWHERE_WIDTH: usize = 64;
|
|
637
|
+
|
|
396
638
|
/// Two or more alternatives once groups of alternatives (and optional groups) are expanded into whole alternatives
|
|
397
639
|
/// (`^a|b|c$`, `^([a|A]uto)|([n|N]one)$`, `^[Ee][Ss]2015(\.([Cc]ore|[Pp]roxy))?$`), each a literal or a class
|
|
398
640
|
/// sequence anchored where its own `^` and `$` say.
|
|
399
641
|
fn alternatives(p: &str) -> Option<Box<[Alternative]>> {
|
|
400
642
|
let c: Vec<char> = p.chars().collect();
|
|
401
|
-
let (parts, end) = expand(&c, 0)?;
|
|
643
|
+
let (parts, end) = expand(&c, 0, 0)?;
|
|
402
644
|
// A single alternative is only new here when a group was expanded (Sequence::parse takes the rest).
|
|
403
645
|
if end != c.len() || parts.is_empty() || (parts.len() == 1 && parts[0] == p) {
|
|
404
646
|
return None;
|
|
@@ -412,6 +654,10 @@ fn alternatives(p: &str) -> Option<Box<[Alternative]>> {
|
|
|
412
654
|
_ => (false, part),
|
|
413
655
|
};
|
|
414
656
|
if let Some(text) = literal_text(body) {
|
|
657
|
+
// A literal that is anchored at neither end is searched for at every position of the text.
|
|
658
|
+
if !start && !end && text.len() > MAX_ANYWHERE_WIDTH {
|
|
659
|
+
return None;
|
|
660
|
+
}
|
|
415
661
|
return Some(Alternative::Literal { text: text.into(), start, end });
|
|
416
662
|
}
|
|
417
663
|
let seq = Sequence::parse(&format!("^{body}{}", if end { "$" } else { "" }))?;
|
|
@@ -420,7 +666,11 @@ fn alternatives(p: &str) -> Option<Box<[Alternative]>> {
|
|
|
420
666
|
}
|
|
421
667
|
let width =
|
|
422
668
|
seq.items.iter().all(|i| i.min == i.max).then(|| seq.items.iter().map(|i| i.min as usize).sum())?;
|
|
423
|
-
|
|
669
|
+
if end {
|
|
670
|
+
return Some(Alternative::End { seq, width });
|
|
671
|
+
}
|
|
672
|
+
// A sequence that is not anchored is tried at every position of the text, and reads its width at each.
|
|
673
|
+
(width <= MAX_ANYWHERE_WIDTH).then_some(Alternative::Anywhere { seq, width })
|
|
424
674
|
})
|
|
425
675
|
.collect::<Option<_>>()?;
|
|
426
676
|
let sequences = alternatives.iter().any(|a| !matches!(a, Alternative::Literal { .. }));
|
|
@@ -432,10 +682,18 @@ fn ends_with_escape(t: &str) -> bool {
|
|
|
432
682
|
t.bytes().rev().take_while(|&b| b == b'\\').count() % 2 == 1
|
|
433
683
|
}
|
|
434
684
|
|
|
685
|
+
/// The deepest nesting of groups that is multiplied out. `expand` goes a level deeper for each, and a pattern may
|
|
686
|
+
/// nest a thousand deep, which the stack of a thread cannot be relied on to hold. A pattern that nests deeper than
|
|
687
|
+
/// this is left to the matchers after this one.
|
|
688
|
+
const MAX_EXPANDED_NESTING: usize = 32;
|
|
689
|
+
|
|
435
690
|
/// Expands the alternatives from `i` up to an unmatched `)` or the end: groups of alternatives multiply out, a group
|
|
436
691
|
/// quantified by `?` also contributes the empty alternative, and everything else is copied. `None` for lookarounds,
|
|
437
|
-
/// other quantified groups,
|
|
438
|
-
fn expand(c: &[char], mut i: usize) -> Option<(Vec<String>, usize)> {
|
|
692
|
+
/// other quantified groups, too many alternatives, or groups nested deeper than `MAX_EXPANDED_NESTING`.
|
|
693
|
+
fn expand(c: &[char], mut i: usize, depth: usize) -> Option<(Vec<String>, usize)> {
|
|
694
|
+
if depth > MAX_EXPANDED_NESTING {
|
|
695
|
+
return None;
|
|
696
|
+
}
|
|
439
697
|
let mut all = Vec::new();
|
|
440
698
|
let mut branch = vec![String::new()];
|
|
441
699
|
while i < c.len() {
|
|
@@ -454,7 +712,7 @@ fn expand(c: &[char], mut i: usize) -> Option<(Vec<String>, usize)> {
|
|
|
454
712
|
}
|
|
455
713
|
i += 2;
|
|
456
714
|
}
|
|
457
|
-
let (mut inner, next) = expand(c, i)?;
|
|
715
|
+
let (mut inner, next) = expand(c, i, depth + 1)?;
|
|
458
716
|
if c.get(next) != Some(&')') {
|
|
459
717
|
return None;
|
|
460
718
|
}
|
|
@@ -550,9 +808,13 @@ fn excluded_class_with_word(p: &str) -> Option<(CharSet, bool)> {
|
|
|
550
808
|
.then_some((excluded, bangs))
|
|
551
809
|
}
|
|
552
810
|
|
|
553
|
-
/// The `regex` format: a valid ECMA-262 regular expression
|
|
554
|
-
|
|
555
|
-
|
|
811
|
+
/// The `regex` format: a valid ECMA-262 regular expression in the grammar of the `u` flag, of the edition and version
|
|
812
|
+
/// of Unicode the dialect reads patterns with. It asks the grammar alone, as the .NET library's check does: the
|
|
813
|
+
/// engine's limits of size belong to compiling a pattern, so a string too large to compile is valid when its grammar
|
|
814
|
+
/// is. The engine's grammar-only reader takes time in proportion to the string and allocates nothing for one of
|
|
815
|
+
/// ordinary size.
|
|
816
|
+
pub(crate) fn is_valid_ecma_regex(s: &str, grammar: Grammar) -> bool {
|
|
817
|
+
Regex::is_valid_pattern_with(s, Grammar::flags(true), grammar.edition, grammar.unicode_version)
|
|
556
818
|
}
|
|
557
819
|
|
|
558
820
|
// ---------------------------------------------------------------------------------------------------------------------
|
|
@@ -710,6 +972,16 @@ impl Sequence {
|
|
|
710
972
|
}
|
|
711
973
|
}
|
|
712
974
|
}
|
|
975
|
+
if !to_end {
|
|
976
|
+
// Without `$` a match needs only the start of the string: items at the end that can match nothing are
|
|
977
|
+
// dropped, and the last of the rest needs only its least count (`^[a-z]+` is `^[a-z]`).
|
|
978
|
+
while items.last().is_some_and(|item| item.min == 0) {
|
|
979
|
+
items.pop();
|
|
980
|
+
}
|
|
981
|
+
if let Some(last) = items.last_mut() {
|
|
982
|
+
last.max = last.min;
|
|
983
|
+
}
|
|
984
|
+
}
|
|
713
985
|
Some(Sequence { items: items.into_boxed_slice(), to_end })
|
|
714
986
|
}
|
|
715
987
|
|
|
@@ -765,212 +1037,6 @@ impl Sequence {
|
|
|
765
1037
|
}
|
|
766
1038
|
!self.to_end || at == b.len()
|
|
767
1039
|
}
|
|
768
|
-
|
|
769
|
-
/// Greedily matches the items at the start of `s` (ignoring `to_end`): the bytes taken.
|
|
770
|
-
fn consume(&self, s: &str) -> Option<usize> {
|
|
771
|
-
let b = s.as_bytes();
|
|
772
|
-
let mut at = 0;
|
|
773
|
-
for item in self.items.iter() {
|
|
774
|
-
let mut n = 0u32;
|
|
775
|
-
while n < item.max && at < b.len() {
|
|
776
|
-
let c = b[at];
|
|
777
|
-
if c < 0x80 {
|
|
778
|
-
if !item.set.has_ascii(c) {
|
|
779
|
-
break;
|
|
780
|
-
}
|
|
781
|
-
at += 1;
|
|
782
|
-
} else {
|
|
783
|
-
let ch = s[at..].chars().next()?;
|
|
784
|
-
if !item.set.contains(ch) {
|
|
785
|
-
break;
|
|
786
|
-
}
|
|
787
|
-
at += ch.len_utf8();
|
|
788
|
-
}
|
|
789
|
-
n += 1;
|
|
790
|
-
}
|
|
791
|
-
if n < item.min {
|
|
792
|
-
return None;
|
|
793
|
-
}
|
|
794
|
-
}
|
|
795
|
-
Some(at)
|
|
796
|
-
}
|
|
797
|
-
|
|
798
|
-
/// Whether greedy matching is exact when `next` can follow the items: every variable item is disjoint from the
|
|
799
|
-
/// items that can directly follow it (up to the first that cannot match nothing), `next` included.
|
|
800
|
-
fn greedy_before(items: &[Item], next: CharSet) -> bool {
|
|
801
|
-
items.iter().enumerate().all(|(i, item)| {
|
|
802
|
-
if item.min == item.max {
|
|
803
|
-
return true;
|
|
804
|
-
}
|
|
805
|
-
for following in &items[i + 1..] {
|
|
806
|
-
if !item.set.disjoint(following.set) {
|
|
807
|
-
return false;
|
|
808
|
-
}
|
|
809
|
-
if following.min > 0 {
|
|
810
|
-
return true;
|
|
811
|
-
}
|
|
812
|
-
}
|
|
813
|
-
item.set.disjoint(next)
|
|
814
|
-
})
|
|
815
|
-
}
|
|
816
|
-
}
|
|
817
|
-
|
|
818
|
-
/// A list of items between separators: `^I(SR)*$` or `^I(SR)+$` (`final_` is `None`), or `^(RS)*F$` and
|
|
819
|
-
/// `^(RS)+F$`. The separator is a fixed sequence and no variable class can run into what follows it, so greedy
|
|
820
|
-
/// matching splits the string exactly where the pattern does.
|
|
821
|
-
#[derive(Debug)]
|
|
822
|
-
struct SeparatedList {
|
|
823
|
-
/// `I` of the first form.
|
|
824
|
-
first: Option<Sequence>,
|
|
825
|
-
repeated: Sequence,
|
|
826
|
-
separator: Sequence,
|
|
827
|
-
/// `F` of the second form, matched against the whole remainder.
|
|
828
|
-
final_: Option<Sequence>,
|
|
829
|
-
min_repeats: u32,
|
|
830
|
-
}
|
|
831
|
-
|
|
832
|
-
impl SeparatedList {
|
|
833
|
-
fn is_match(&self, s: &str) -> bool {
|
|
834
|
-
let mut rest = s;
|
|
835
|
-
let mut repeats = 0;
|
|
836
|
-
if let Some(first) = &self.first {
|
|
837
|
-
let Some(n) = first.consume(rest) else { return false };
|
|
838
|
-
rest = &rest[n..];
|
|
839
|
-
loop {
|
|
840
|
-
if rest.is_empty() {
|
|
841
|
-
return repeats >= self.min_repeats;
|
|
842
|
-
}
|
|
843
|
-
let Some(n) = self.separator.consume(rest) else { return false };
|
|
844
|
-
rest = &rest[n..];
|
|
845
|
-
let Some(n) = self.repeated.consume(rest) else { return false };
|
|
846
|
-
rest = &rest[n..];
|
|
847
|
-
repeats += 1;
|
|
848
|
-
}
|
|
849
|
-
}
|
|
850
|
-
let final_ = self.final_.as_ref().expect("a separated list has a first or a final item");
|
|
851
|
-
loop {
|
|
852
|
-
if repeats >= self.min_repeats && final_.is_match(rest) {
|
|
853
|
-
return true;
|
|
854
|
-
}
|
|
855
|
-
let Some(n) = self.repeated.consume(rest) else { return false };
|
|
856
|
-
let Some(m) = self.separator.consume(&rest[n..]) else { return false };
|
|
857
|
-
rest = &rest[n + m..];
|
|
858
|
-
repeats += 1;
|
|
859
|
-
}
|
|
860
|
-
}
|
|
861
|
-
|
|
862
|
-
fn parse(p: &str) -> Option<SeparatedList> {
|
|
863
|
-
let body = p.strip_prefix('^')?.strip_suffix('$')?;
|
|
864
|
-
if ends_with_escape(body) {
|
|
865
|
-
return None;
|
|
866
|
-
}
|
|
867
|
-
let b = body.as_bytes();
|
|
868
|
-
// Top-level groups: (start, end) of each, where end is the index of its `)`.
|
|
869
|
-
let mut groups = Vec::new();
|
|
870
|
-
let (mut depth, mut in_class, mut i, mut open) = (0usize, false, 0, 0);
|
|
871
|
-
while i < b.len() {
|
|
872
|
-
match b[i] {
|
|
873
|
-
b'\\' => i += 1,
|
|
874
|
-
b'[' if !in_class => in_class = true,
|
|
875
|
-
b']' if in_class => in_class = false,
|
|
876
|
-
b'(' if !in_class => {
|
|
877
|
-
if depth == 0 {
|
|
878
|
-
open = i;
|
|
879
|
-
}
|
|
880
|
-
depth += 1;
|
|
881
|
-
}
|
|
882
|
-
b')' if !in_class => {
|
|
883
|
-
depth = depth.checked_sub(1)?;
|
|
884
|
-
if depth == 0 {
|
|
885
|
-
groups.push((open, i));
|
|
886
|
-
}
|
|
887
|
-
}
|
|
888
|
-
_ => {}
|
|
889
|
-
}
|
|
890
|
-
i += 1;
|
|
891
|
-
}
|
|
892
|
-
let quantified: Vec<_> = groups.iter().filter(|&&(_, e)| matches!(b.get(e + 1), Some(b'*' | b'+'))).collect();
|
|
893
|
-
let &&(open, close) = quantified.first().filter(|_| quantified.len() == 1)?;
|
|
894
|
-
let min_repeats = u32::from(b[close + 1] == b'+');
|
|
895
|
-
let group = strip_group(&body[open..=close])?;
|
|
896
|
-
let before = &body[..open];
|
|
897
|
-
let after = &body[close + 2..];
|
|
898
|
-
let items = |t: &str| Sequence::parse(&format!("^{t}")).map(|s| s.items.into_vec());
|
|
899
|
-
let seq = |items: &[Item], to_end| Sequence { items: items.into(), to_end };
|
|
900
|
-
let fixed = |items: &[Item]| !items.is_empty() && items.iter().all(|i| i.min == i.max && i.min > 0);
|
|
901
|
-
let g = items(group)?;
|
|
902
|
-
match (before.is_empty(), after.is_empty()) {
|
|
903
|
-
// ^I(SR)*$
|
|
904
|
-
(false, true) => {
|
|
905
|
-
let first = items(unwrap_group(before)?)?;
|
|
906
|
-
(1..g.len()).find_map(|k| {
|
|
907
|
-
let (separator, repeated) = g.split_at(k);
|
|
908
|
-
let next = separator[0].set;
|
|
909
|
-
(fixed(separator)
|
|
910
|
-
&& Sequence::greedy_before(&first, next)
|
|
911
|
-
&& Sequence::greedy_before(repeated, next))
|
|
912
|
-
.then(|| SeparatedList {
|
|
913
|
-
first: Some(seq(&first, false)),
|
|
914
|
-
repeated: seq(repeated, false),
|
|
915
|
-
separator: seq(separator, false),
|
|
916
|
-
final_: None,
|
|
917
|
-
min_repeats,
|
|
918
|
-
})
|
|
919
|
-
})
|
|
920
|
-
}
|
|
921
|
-
// ^(RS)*F$
|
|
922
|
-
(true, false) => {
|
|
923
|
-
let final_ = Sequence::parse(&format!("^{}$", unwrap_group(after)?))?;
|
|
924
|
-
(1..g.len()).find_map(|k| {
|
|
925
|
-
let (repeated, separator) = g.split_at(k);
|
|
926
|
-
(fixed(separator) && Sequence::greedy_before(repeated, separator[0].set)).then(|| SeparatedList {
|
|
927
|
-
first: None,
|
|
928
|
-
repeated: seq(repeated, false),
|
|
929
|
-
separator: seq(separator, false),
|
|
930
|
-
final_: Some(Sequence { items: final_.items.clone(), to_end: true }),
|
|
931
|
-
min_repeats,
|
|
932
|
-
})
|
|
933
|
-
})
|
|
934
|
-
}
|
|
935
|
-
_ => None,
|
|
936
|
-
}
|
|
937
|
-
}
|
|
938
|
-
}
|
|
939
|
-
|
|
940
|
-
/// The inside of `(...)` or `(?:...)`; `None` for other groups.
|
|
941
|
-
fn strip_group(g: &str) -> Option<&str> {
|
|
942
|
-
let inner = g.strip_prefix('(')?.strip_suffix(')')?;
|
|
943
|
-
match inner.strip_prefix("?:") {
|
|
944
|
-
Some(rest) => Some(rest),
|
|
945
|
-
None if inner.starts_with('?') => None,
|
|
946
|
-
None => Some(inner),
|
|
947
|
-
}
|
|
948
|
-
}
|
|
949
|
-
|
|
950
|
-
/// A text that is one group wrapping everything, unwrapped; otherwise the text itself.
|
|
951
|
-
fn unwrap_group(t: &str) -> Option<&str> {
|
|
952
|
-
if t.starts_with('(') && t.ends_with(')') {
|
|
953
|
-
let inner = strip_group(t)?;
|
|
954
|
-
// Only when the parentheses enclose the whole text (`(a)(b)` is two groups).
|
|
955
|
-
let mut depth = 0i32;
|
|
956
|
-
let mut escaped = false;
|
|
957
|
-
for c in inner.chars() {
|
|
958
|
-
match c {
|
|
959
|
-
_ if escaped => escaped = false,
|
|
960
|
-
'\\' => escaped = true,
|
|
961
|
-
'(' => depth += 1,
|
|
962
|
-
')' => {
|
|
963
|
-
depth -= 1;
|
|
964
|
-
if depth < 0 {
|
|
965
|
-
return None;
|
|
966
|
-
}
|
|
967
|
-
}
|
|
968
|
-
_ => {}
|
|
969
|
-
}
|
|
970
|
-
}
|
|
971
|
-
return Some(inner);
|
|
972
|
-
}
|
|
973
|
-
Some(t)
|
|
974
1040
|
}
|
|
975
1041
|
|
|
976
1042
|
/// The set of a class escape (`\d`, `\w`, their negations, or an escaped punctuation character); `None` for `\s`
|
|
@@ -1090,280 +1156,26 @@ fn parse_quantifier(b: &[u8], mut i: usize) -> Option<(u32, u32, usize)> {
|
|
|
1090
1156
|
Some((min, max, i))
|
|
1091
1157
|
}
|
|
1092
1158
|
|
|
1093
|
-
// ---------------------------------------------------------------------------------------------------------------------
|
|
1094
|
-
// Translation to the regex crate
|
|
1095
|
-
|
|
1096
|
-
/// ECMA-262 `\s`: WhiteSpace and LineTerminator.
|
|
1097
|
-
const ECMA_SPACE: &str =
|
|
1098
|
-
r"\t\n\x{B}\x{C}\r \x{A0}\x{1680}\x{2000}-\x{200A}\x{2028}\x{2029}\x{202F}\x{205F}\x{3000}\x{FEFF}";
|
|
1099
|
-
|
|
1100
|
-
/// Translates an ECMA-262 pattern to the `regex` crate's syntax with the same meaning, or `None` when it uses
|
|
1101
|
-
/// something outside the common subset. With `unicode` false, the pattern is read without the `u` flag (identity
|
|
1102
|
-
/// escapes of any non-alphanumeric character), and the translation has its meaning on strings within the BMP.
|
|
1103
|
-
fn translate(p: &str, unicode: bool) -> Option<String> {
|
|
1104
|
-
let c: Vec<char> = p.chars().collect();
|
|
1105
|
-
let mut out = String::with_capacity(p.len() * 2);
|
|
1106
|
-
let mut i = 0;
|
|
1107
|
-
while i < c.len() {
|
|
1108
|
-
match c[i] {
|
|
1109
|
-
'\\' => {
|
|
1110
|
-
let (text, next) = escape(&c, i + 1, false, unicode)?;
|
|
1111
|
-
out.push_str(&text);
|
|
1112
|
-
i = next;
|
|
1113
|
-
}
|
|
1114
|
-
'[' => {
|
|
1115
|
-
let (text, next) = class(&c, i + 1, unicode)?;
|
|
1116
|
-
out.push_str(&text);
|
|
1117
|
-
i = next;
|
|
1118
|
-
}
|
|
1119
|
-
'(' => {
|
|
1120
|
-
if c.get(i + 1) == Some(&'?') {
|
|
1121
|
-
match (c.get(i + 2), c.get(i + 3)) {
|
|
1122
|
-
(Some(':'), _) => i += 3,
|
|
1123
|
-
// A named group; lookbehinds (`(?<=`, `(?<!`) are not supported.
|
|
1124
|
-
(Some('<'), Some(n)) if *n != '=' && *n != '!' => {
|
|
1125
|
-
i += 3 + c[i + 3..].iter().position(|&x| x == '>')? + 1;
|
|
1126
|
-
}
|
|
1127
|
-
_ => return None,
|
|
1128
|
-
}
|
|
1129
|
-
} else {
|
|
1130
|
-
i += 1;
|
|
1131
|
-
}
|
|
1132
|
-
out.push_str("(?:");
|
|
1133
|
-
}
|
|
1134
|
-
'{' => {
|
|
1135
|
-
let end = i + c[i..].iter().position(|&x| x == '}')?;
|
|
1136
|
-
let body: String = c[i + 1..end].iter().collect();
|
|
1137
|
-
if body.is_empty() || !body.chars().all(|x| x.is_ascii_digit() || x == ',') {
|
|
1138
|
-
return None;
|
|
1139
|
-
}
|
|
1140
|
-
out.push('{');
|
|
1141
|
-
out.push_str(&body);
|
|
1142
|
-
out.push('}');
|
|
1143
|
-
i = end + 1;
|
|
1144
|
-
}
|
|
1145
|
-
'.' => {
|
|
1146
|
-
out.push_str(r"[^\n\r\x{2028}\x{2029}]");
|
|
1147
|
-
i += 1;
|
|
1148
|
-
}
|
|
1149
|
-
ch @ (')' | '|' | '^' | '$' | '*' | '+' | '?') => {
|
|
1150
|
-
out.push(ch);
|
|
1151
|
-
i += 1;
|
|
1152
|
-
}
|
|
1153
|
-
ch => {
|
|
1154
|
-
push_literal(&mut out, ch);
|
|
1155
|
-
i += 1;
|
|
1156
|
-
}
|
|
1157
|
-
}
|
|
1158
|
-
}
|
|
1159
|
-
Some(out)
|
|
1160
|
-
}
|
|
1161
|
-
|
|
1162
|
-
fn push_literal(out: &mut String, ch: char) {
|
|
1163
|
-
use std::fmt::Write;
|
|
1164
|
-
let _ = write!(out, "\\x{{{:X}}}", ch as u32);
|
|
1165
|
-
}
|
|
1166
|
-
|
|
1167
|
-
/// An escape after `\` at `i`: its translation and the index after it.
|
|
1168
|
-
fn escape(c: &[char], i: usize, in_class: bool, unicode: bool) -> Option<(String, usize)> {
|
|
1169
|
-
let e = *c.get(i)?;
|
|
1170
|
-
let mut out = String::new();
|
|
1171
|
-
let next = match e {
|
|
1172
|
-
'd' => {
|
|
1173
|
-
out.push_str(if in_class { "0-9" } else { "[0-9]" });
|
|
1174
|
-
i + 1
|
|
1175
|
-
}
|
|
1176
|
-
'D' => {
|
|
1177
|
-
out.push_str("[^0-9]");
|
|
1178
|
-
i + 1
|
|
1179
|
-
}
|
|
1180
|
-
'w' => {
|
|
1181
|
-
out.push_str(if in_class { "0-9A-Za-z_" } else { "[0-9A-Za-z_]" });
|
|
1182
|
-
i + 1
|
|
1183
|
-
}
|
|
1184
|
-
'W' => {
|
|
1185
|
-
out.push_str("[^0-9A-Za-z_]");
|
|
1186
|
-
i + 1
|
|
1187
|
-
}
|
|
1188
|
-
's' => {
|
|
1189
|
-
if in_class {
|
|
1190
|
-
out.push_str(ECMA_SPACE);
|
|
1191
|
-
} else {
|
|
1192
|
-
out.push('[');
|
|
1193
|
-
out.push_str(ECMA_SPACE);
|
|
1194
|
-
out.push(']');
|
|
1195
|
-
}
|
|
1196
|
-
i + 1
|
|
1197
|
-
}
|
|
1198
|
-
'S' => {
|
|
1199
|
-
out.push_str("[^");
|
|
1200
|
-
out.push_str(ECMA_SPACE);
|
|
1201
|
-
out.push(']');
|
|
1202
|
-
i + 1
|
|
1203
|
-
}
|
|
1204
|
-
'n' | 'r' | 't' | 'f' | 'v' => {
|
|
1205
|
-
let ch = match e {
|
|
1206
|
-
'n' => '\n',
|
|
1207
|
-
'r' => '\r',
|
|
1208
|
-
't' => '\t',
|
|
1209
|
-
'f' => '\x0C',
|
|
1210
|
-
_ => '\x0B',
|
|
1211
|
-
};
|
|
1212
|
-
push_literal(&mut out, ch);
|
|
1213
|
-
i + 1
|
|
1214
|
-
}
|
|
1215
|
-
'0' if !c.get(i + 1).is_some_and(char::is_ascii_digit) => {
|
|
1216
|
-
push_literal(&mut out, '\0');
|
|
1217
|
-
i + 1
|
|
1218
|
-
}
|
|
1219
|
-
'b' if in_class => {
|
|
1220
|
-
push_literal(&mut out, '\x08');
|
|
1221
|
-
i + 1
|
|
1222
|
-
}
|
|
1223
|
-
// ECMA's word boundaries are between ASCII word characters and the rest.
|
|
1224
|
-
'b' => {
|
|
1225
|
-
out.push_str("(?-u:\\b)");
|
|
1226
|
-
i + 1
|
|
1227
|
-
}
|
|
1228
|
-
'B' => {
|
|
1229
|
-
out.push_str("(?-u:\\B)");
|
|
1230
|
-
i + 1
|
|
1231
|
-
}
|
|
1232
|
-
// Unicode properties (validated as ECMA-262 names by regress) have the same names in the `regex` crate.
|
|
1233
|
-
// Without the `u` flag `\p` is the letter.
|
|
1234
|
-
'p' | 'P' if unicode && c.get(i + 1) == Some(&'{') => {
|
|
1235
|
-
let end = i + 1 + c[i + 1..].iter().position(|&x| x == '}')?;
|
|
1236
|
-
let body: String = c[i + 2..end].iter().collect();
|
|
1237
|
-
if body.is_empty() || !body.chars().all(|x| x.is_ascii_alphanumeric() || x == '_' || x == '=') {
|
|
1238
|
-
return None;
|
|
1239
|
-
}
|
|
1240
|
-
out.push('\\');
|
|
1241
|
-
out.push(e);
|
|
1242
|
-
out.push('{');
|
|
1243
|
-
out.push_str(&body);
|
|
1244
|
-
out.push('}');
|
|
1245
|
-
end + 1
|
|
1246
|
-
}
|
|
1247
|
-
'x' => {
|
|
1248
|
-
let hex: String = c.get(i + 1..i + 3)?.iter().collect();
|
|
1249
|
-
push_literal(&mut out, char::from_u32(u32::from_str_radix(&hex, 16).ok()?)?);
|
|
1250
|
-
i + 3
|
|
1251
|
-
}
|
|
1252
|
-
// Without the `u` flag, `\u{...}` is not a code point escape.
|
|
1253
|
-
'u' if unicode || c.get(i + 1) != Some(&'{') => {
|
|
1254
|
-
let (code, next) = unicode_escape(c, i + 1)?;
|
|
1255
|
-
push_literal(&mut out, char::from_u32(code)?);
|
|
1256
|
-
next
|
|
1257
|
-
}
|
|
1258
|
-
// Identity escapes of syntax characters and `/` (and `-` in a class).
|
|
1259
|
-
'^' | '$' | '\\' | '.' | '*' | '+' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '/' | '-' => {
|
|
1260
|
-
push_literal(&mut out, e);
|
|
1261
|
-
i + 1
|
|
1262
|
-
}
|
|
1263
|
-
// Without the `u` flag, any other non-alphanumeric character escapes to itself.
|
|
1264
|
-
e if !unicode && !e.is_ascii_alphanumeric() => {
|
|
1265
|
-
push_literal(&mut out, e);
|
|
1266
|
-
i + 1
|
|
1267
|
-
}
|
|
1268
|
-
// Backreferences, \c, \k and the rest.
|
|
1269
|
-
_ => return None,
|
|
1270
|
-
};
|
|
1271
|
-
Some((out, next))
|
|
1272
|
-
}
|
|
1273
|
-
|
|
1274
|
-
/// `\u` escapes after the `u`: `XXXX` (joining a surrogate pair) or `{X...}`.
|
|
1275
|
-
fn unicode_escape(c: &[char], i: usize) -> Option<(u32, usize)> {
|
|
1276
|
-
if c.get(i) == Some(&'{') {
|
|
1277
|
-
let end = i + c[i..].iter().position(|&x| x == '}')?;
|
|
1278
|
-
let hex: String = c[i + 1..end].iter().collect();
|
|
1279
|
-
return Some((u32::from_str_radix(&hex, 16).ok()?, end + 1));
|
|
1280
|
-
}
|
|
1281
|
-
let four = |at: usize| -> Option<u32> {
|
|
1282
|
-
let hex: String = c.get(at..at + 4)?.iter().collect();
|
|
1283
|
-
if hex.len() != 4 {
|
|
1284
|
-
return None;
|
|
1285
|
-
}
|
|
1286
|
-
u32::from_str_radix(&hex, 16).ok()
|
|
1287
|
-
};
|
|
1288
|
-
let hi = four(i)?;
|
|
1289
|
-
if (0xD800..0xDC00).contains(&hi) {
|
|
1290
|
-
if c.get(i + 4) == Some(&'\\') && c.get(i + 5) == Some(&'u') {
|
|
1291
|
-
let lo = four(i + 6)?;
|
|
1292
|
-
if (0xDC00..0xE000).contains(&lo) {
|
|
1293
|
-
return Some((0x10000 + ((hi - 0xD800) << 10) + (lo - 0xDC00), i + 10));
|
|
1294
|
-
}
|
|
1295
|
-
}
|
|
1296
|
-
return None;
|
|
1297
|
-
}
|
|
1298
|
-
Some((hi, i + 4))
|
|
1299
|
-
}
|
|
1300
|
-
|
|
1301
|
-
/// A class after `[` at `i`: its translation (every member spelled as an escape, so nothing in it is special to the
|
|
1302
|
-
/// `regex` crate) and the index after `]`.
|
|
1303
|
-
fn class(c: &[char], mut i: usize, unicode: bool) -> Option<(String, usize)> {
|
|
1304
|
-
let mut out = String::from("[");
|
|
1305
|
-
if c.get(i) == Some(&'^') {
|
|
1306
|
-
out.push('^');
|
|
1307
|
-
i += 1;
|
|
1308
|
-
}
|
|
1309
|
-
let mut members = 0;
|
|
1310
|
-
loop {
|
|
1311
|
-
let ch = *c.get(i)?;
|
|
1312
|
-
if ch == ']' {
|
|
1313
|
-
i += 1;
|
|
1314
|
-
break;
|
|
1315
|
-
}
|
|
1316
|
-
// One atom: a character (which can start a range) or a class escape.
|
|
1317
|
-
let (atom, single, next) = if ch == '\\' {
|
|
1318
|
-
let e = *c.get(i + 1)?;
|
|
1319
|
-
let (text, next) = escape(c, i + 1, true, unicode)?;
|
|
1320
|
-
let single = match e {
|
|
1321
|
-
'd' | 'D' | 'w' | 'W' | 's' | 'S' | 'p' | 'P' => None,
|
|
1322
|
-
_ => Some(text.clone()),
|
|
1323
|
-
};
|
|
1324
|
-
(text, single, next)
|
|
1325
|
-
} else {
|
|
1326
|
-
let mut t = String::new();
|
|
1327
|
-
push_literal(&mut t, ch);
|
|
1328
|
-
(t.clone(), Some(t), i + 1)
|
|
1329
|
-
};
|
|
1330
|
-
i = next;
|
|
1331
|
-
members += 1;
|
|
1332
|
-
if c.get(i) == Some(&'-') && c.get(i + 1).is_some_and(|&n| n != ']') {
|
|
1333
|
-
// A range: both ends single characters.
|
|
1334
|
-
let lo = single?;
|
|
1335
|
-
let (hi, next) = if c[i + 1] == '\\' {
|
|
1336
|
-
let e = *c.get(i + 2)?;
|
|
1337
|
-
if matches!(e, 'd' | 'D' | 'w' | 'W' | 's' | 'S' | 'p' | 'P') {
|
|
1338
|
-
return None;
|
|
1339
|
-
}
|
|
1340
|
-
escape(c, i + 2, true, unicode)?
|
|
1341
|
-
} else {
|
|
1342
|
-
let mut t = String::new();
|
|
1343
|
-
push_literal(&mut t, c[i + 1]);
|
|
1344
|
-
(t, i + 2)
|
|
1345
|
-
};
|
|
1346
|
-
out.push_str(&lo);
|
|
1347
|
-
out.push('-');
|
|
1348
|
-
out.push_str(&hi);
|
|
1349
|
-
i = next;
|
|
1350
|
-
} else {
|
|
1351
|
-
out.push_str(&atom);
|
|
1352
|
-
}
|
|
1353
|
-
}
|
|
1354
|
-
if members == 0 {
|
|
1355
|
-
return None; // `[]` and `[^]` have no counterpart.
|
|
1356
|
-
}
|
|
1357
|
-
out.push(']');
|
|
1358
|
-
Some((out, i))
|
|
1359
|
-
}
|
|
1360
|
-
|
|
1361
1159
|
#[cfg(test)]
|
|
1362
1160
|
mod tests {
|
|
1363
1161
|
use super::*;
|
|
1364
1162
|
|
|
1163
|
+
const G: Grammar = Grammar { edition: Edition::Current, unicode_version: UnicodeVersion::Default };
|
|
1164
|
+
|
|
1165
|
+
/// A pattern compiled in today's grammar.
|
|
1166
|
+
fn compile(p: &str) -> Result<Arc<Pattern>, PatternError> {
|
|
1167
|
+
super::compile(p, G)
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1365
1170
|
const PATTERNS: &[&str] = &[
|
|
1366
1171
|
"",
|
|
1172
|
+
// Sequences without `$`, which need only the start of the string.
|
|
1173
|
+
"^[a-z]+",
|
|
1174
|
+
"^[a-z]*",
|
|
1175
|
+
"^[a-zA-Z0-9_\\.\\-]*",
|
|
1176
|
+
"^a[b-c]{2,5}",
|
|
1177
|
+
"^[a-z]+[0-9]*",
|
|
1178
|
+
"^[a-z]{0,3}[0-9]{2,4}",
|
|
1367
1179
|
".*",
|
|
1368
1180
|
"^.*",
|
|
1369
1181
|
".*$",
|
|
@@ -1484,6 +1296,20 @@ mod tests {
|
|
|
1484
1296
|
"^(é,)*a$",
|
|
1485
1297
|
"^(?=!+[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$",
|
|
1486
1298
|
"^(?=!+[^a]+$)(?=(.*\\w)).+$",
|
|
1299
|
+
// Lists with an anchor or an assertion inside them, and with an escaped anchor.
|
|
1300
|
+
"^x(,a$)*$",
|
|
1301
|
+
"^a(,a$)+$",
|
|
1302
|
+
"^x$(,a)*$",
|
|
1303
|
+
"^(a$)(,a)*$",
|
|
1304
|
+
"^$(,a)*$",
|
|
1305
|
+
"^(a,$)*b$",
|
|
1306
|
+
"^([a-z]+,$)+a$",
|
|
1307
|
+
"^a(^,a)*$",
|
|
1308
|
+
"^(^a,)*b$",
|
|
1309
|
+
"^a(,a\\b)*$",
|
|
1310
|
+
"^a(,a\\$)*$",
|
|
1311
|
+
"^a\\$(,a)*$",
|
|
1312
|
+
"^(a\\$,)*b$",
|
|
1487
1313
|
// Valid only without the `u` flag (identity escapes), so matching code units.
|
|
1488
1314
|
"^\\/[^\\*\\?\\&\\%]*(\\/\\*)?$",
|
|
1489
1315
|
"^[\\&\\@\\_]+$",
|
|
@@ -1493,9 +1319,10 @@ mod tests {
|
|
|
1493
1319
|
"\\uD83D\\uDE00|\\&",
|
|
1494
1320
|
];
|
|
1495
1321
|
|
|
1496
|
-
/// Every matcher agrees with
|
|
1322
|
+
/// Every matcher agrees with the engine's backtracking matcher on strings over an alphabet that exercises
|
|
1323
|
+
/// classes, anchors and non-ASCII.
|
|
1497
1324
|
#[test]
|
|
1498
|
-
fn
|
|
1325
|
+
fn matchers_agree_with_the_engine() {
|
|
1499
1326
|
let alphabet = [
|
|
1500
1327
|
"a", "b", "z", "A", "X", "Z", "0", "1", "5", "9", "_", "-", ".", ":", "/", "@", "#", "$", "*", "!", "{",
|
|
1501
1328
|
"}", "|", " ", "\n", "\u{a0}", "é", "µ", "😀", "\u{2028}", "x-", "es", "ES", "ms", "txt", "Au", "to", "No",
|
|
@@ -1509,12 +1336,20 @@ mod tests {
|
|
|
1509
1336
|
seed
|
|
1510
1337
|
};
|
|
1511
1338
|
for &p in PATTERNS {
|
|
1512
|
-
let (
|
|
1339
|
+
let reference = G.build(p, true).or_else(|_| G.build(p, false)).unwrap();
|
|
1513
1340
|
let compiled = compile(p).unwrap();
|
|
1514
1341
|
for _ in 0..4000 {
|
|
1515
1342
|
let len = next() % 9;
|
|
1516
1343
|
let s: String = (0..len).map(|_| alphabet[(next() % alphabet.len() as u64) as usize]).collect();
|
|
1517
|
-
|
|
1344
|
+
// Without the `u` flag the engine matches the UTF-16 code units of the string.
|
|
1345
|
+
let found = reference.is_match(&s).unwrap();
|
|
1346
|
+
for limit in [None, Some(Duration::from_secs(60)), Some(Duration::from_nanos(1))] {
|
|
1347
|
+
let mut budget = PatternBudget::new(PatternLimit::new(limit));
|
|
1348
|
+
let got = compiled.is_match(&s, &mut budget);
|
|
1349
|
+
// A match on the engine that reached its first check of the clock has no answer.
|
|
1350
|
+
assert!(got == found || budget.failed(), "{p:?} on {s:?} with a limit of {limit:?}");
|
|
1351
|
+
assert!(limit.is_some_and(|l| l.as_nanos() == 1) || !budget.failed(), "{p:?} on {s:?}");
|
|
1352
|
+
}
|
|
1518
1353
|
}
|
|
1519
1354
|
}
|
|
1520
1355
|
}
|
|
@@ -1527,70 +1362,284 @@ mod tests {
|
|
|
1527
1362
|
for p in ["^x-", "^\\/", "^abc$"] {
|
|
1528
1363
|
assert!(matches!(compile(p).unwrap().matcher, Matcher::Literal { .. }), "{p}");
|
|
1529
1364
|
}
|
|
1530
|
-
|
|
1531
|
-
assert!(matches!(compile(p).unwrap().matcher, Matcher::Regex(_)), "{p}");
|
|
1532
|
-
}
|
|
1365
|
+
assert!(matches!(compile("^[a-z]*a$").unwrap().matcher, Matcher::Engine(_)));
|
|
1533
1366
|
for p in [".+", "^.+", "(.+)"] {
|
|
1534
1367
|
assert!(matches!(compile(p).unwrap().matcher, Matcher::HasContent { .. }), "{p}");
|
|
1535
1368
|
}
|
|
1536
1369
|
for p in ["^.{1,256}$", "^.+$", "^.*$", "^(.*)$", "^(.+)$"] {
|
|
1537
1370
|
assert!(matches!(compile(p).unwrap().matcher, Matcher::Line { .. }), "{p}");
|
|
1538
1371
|
}
|
|
1539
|
-
for p in ["^(ab|cd)$", "^(?:es|ES|x-)$"] {
|
|
1372
|
+
for p in ["^(ab|cd)$", "^(?:es|ES|x-)$", "^(CRLF|LF|crlf|lf)$"] {
|
|
1540
1373
|
assert!(matches!(compile(p).unwrap().matcher, Matcher::Literals(_)), "{p}");
|
|
1541
1374
|
}
|
|
1375
|
+
// Without `$`, a sequence is read only as far as it must be.
|
|
1376
|
+
for p in ["^[a-z]+", "^a[b-c]{2,5}"] {
|
|
1377
|
+
assert!(
|
|
1378
|
+
matches!(&compile(p).unwrap().matcher, Matcher::Sequence(seq) if seq.items.last().unwrap().max == seq.items.last().unwrap().min),
|
|
1379
|
+
"{p}"
|
|
1380
|
+
);
|
|
1381
|
+
}
|
|
1382
|
+
assert!(matches!(&compile("^[a-zA-Z0-9_\\.\\-]*").unwrap().matcher, Matcher::Everything));
|
|
1383
|
+
// At most two alternatives, neither anchored at both ends.
|
|
1384
|
+
for p in ["^ab|cd$", "a|b", "^([a|A][u|U][t|T][o|O])|([n|N][o|O][n|N][e|E])$", "(base64key|awskms)://(.*)"] {
|
|
1385
|
+
assert!(matches!(compile(p).unwrap().matcher, Matcher::Alternatives(_)), "{p}");
|
|
1386
|
+
}
|
|
1387
|
+
// The engine is faster for more literals, more alternatives, an alternative anchored at both ends, and a
|
|
1388
|
+
// separated list.
|
|
1542
1389
|
for p in [
|
|
1543
|
-
"^
|
|
1544
|
-
"a|b",
|
|
1545
|
-
"^([a|A][u|U][t|T][o|O])|([n|N][o|O][n|N][e|E])$",
|
|
1390
|
+
"^(schemas|responses|parameters|examples|requestBodies)$",
|
|
1546
1391
|
"^[Ee][Ss]2015(\\.([Cc][Oo][Rr][Ee]|[Pp][Rr][Oo][Xx][Yy]))?$",
|
|
1547
1392
|
"^[1-5](?:[0-9]{2}|XX)$",
|
|
1548
1393
|
"(^([0-9]+)\\.([0-9]+)$)|(^\\{[A-F0-9]{8}(-[A-F0-9]{4}){3}-[A-F0-9]{12}\\}$)",
|
|
1549
1394
|
"^([t|T][o|O][p|P])|([c|C][e|E][n|N][t|T][e|E][r|R])|([b|B][o|O][t|T][t|T][o|O][m|M])$",
|
|
1550
1395
|
"^([A-Za-z_][-A-Za-z0-9_.:]*)$",
|
|
1551
|
-
|
|
1552
|
-
assert!(matches!(compile(p).unwrap().matcher, Matcher::Alternatives(_)), "{p}");
|
|
1553
|
-
}
|
|
1554
|
-
for p in [
|
|
1396
|
+
"default|^[0-9]+$",
|
|
1555
1397
|
"^([a-zA-Z0-9]{2,3})(-[a-zA-Z0-9]{1,6})*$",
|
|
1556
1398
|
"^([a-zA-Z_$][a-zA-Z0-9_$]{0,39}\\.)*([a-zA-Z_$][a-zA-Z0-9_$]{0,39})$",
|
|
1557
|
-
"^(
|
|
1399
|
+
"^x(,a\\$)*$",
|
|
1400
|
+
"^x(,a$)*$",
|
|
1401
|
+
"^x(?=,)(,a)*$",
|
|
1558
1402
|
] {
|
|
1559
|
-
assert!(matches!(compile(p).unwrap().matcher, Matcher::
|
|
1403
|
+
assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
|
|
1560
1404
|
}
|
|
1561
1405
|
assert!(matches!(
|
|
1562
1406
|
compile("^(?=[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$").unwrap().matcher,
|
|
1563
1407
|
Matcher::ExcludedClassWithWord { .. }
|
|
1564
1408
|
));
|
|
1565
1409
|
for p in ["\\bfoo", "^\\p{L}+$"] {
|
|
1566
|
-
assert!(matches!(compile(p).unwrap().matcher, Matcher::
|
|
1410
|
+
assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
|
|
1567
1411
|
}
|
|
1568
1412
|
for p in ["^\\/[^\\*\\?\\&\\%]*(\\/\\*)?$", "a\\&.b", "^\\-a"] {
|
|
1569
|
-
assert!(matches!(compile(p).unwrap().matcher, Matcher::
|
|
1413
|
+
assert!(matches!(&compile(p).unwrap().matcher, Matcher::Engine(engine) if !engine.flags().unicode), "{p}");
|
|
1570
1414
|
}
|
|
1571
|
-
for p in ["^((\\.(?!\\.)\\/)?\\w+\\/?)+$", "^\\1(a)", "
|
|
1572
|
-
assert!(matches!(compile(p).unwrap().matcher, Matcher::
|
|
1415
|
+
for p in ["^((\\.(?!\\.)\\/)?\\w+\\/?)+$", "^\\1(a)", "\\Bs", "(?i:a)", "^[]", "\\cA"] {
|
|
1416
|
+
assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
|
|
1417
|
+
}
|
|
1418
|
+
// Valid only without the `u` flag, and not in the syntax `regex` takes: the engine, by code unit.
|
|
1419
|
+
for p in ["^\\p{L}\\&", "^\\-?\u{10400}*$", "^\\-(?=a)"] {
|
|
1420
|
+
let compiled = compile(p).unwrap();
|
|
1421
|
+
assert!(matches!(&compiled.matcher, Matcher::Engine(engine) if !engine.flags().unicode), "{p}");
|
|
1422
|
+
}
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
|
|
1426
|
+
#[cfg(test)]
|
|
1427
|
+
mod limits {
|
|
1428
|
+
use super::*;
|
|
1429
|
+
|
|
1430
|
+
const G: Grammar = Grammar { edition: Edition::Current, unicode_version: UnicodeVersion::Default };
|
|
1431
|
+
|
|
1432
|
+
/// A pattern compiled in today's grammar.
|
|
1433
|
+
fn compile(p: &str) -> Result<Arc<Pattern>, PatternError> {
|
|
1434
|
+
super::compile(p, G)
|
|
1435
|
+
}
|
|
1436
|
+
|
|
1437
|
+
/// Which matcher a pattern gets (the Go module's `TestAWideSearchIsLeftToTheEngine`). A sequence or a literal
|
|
1438
|
+
/// that would be searched for at every position of the text keeps its matcher up to 64 characters, and beyond
|
|
1439
|
+
/// that is left to a matcher that takes time proportional to the text or has the limit of a match.
|
|
1440
|
+
#[test]
|
|
1441
|
+
fn a_wide_search_is_left_to_the_engine() {
|
|
1442
|
+
let a = |n: usize| "a".repeat(n);
|
|
1443
|
+
for (pattern, alternatives) in [
|
|
1444
|
+
(r"[a-z]{64}[0-9]{0}|x".to_string(), true),
|
|
1445
|
+
(r"[a-z]{63}[0-9]|x".to_string(), true),
|
|
1446
|
+
(r"[a-z]{64}[0-9]|x".to_string(), false),
|
|
1447
|
+
(r"[a-z]{50000}[0-9]|x".to_string(), false),
|
|
1448
|
+
(r"^[a-z]{50000}[0-9]|x".to_string(), true),
|
|
1449
|
+
(r"[a-z]{50000}[0-9]$|^x".to_string(), true),
|
|
1450
|
+
(format!("{}|^x", a(64)), true),
|
|
1451
|
+
(format!("{}|^x", a(65)), false),
|
|
1452
|
+
(format!("^{}|^x", a(65)), true),
|
|
1453
|
+
(format!("{}$|^x", a(65)), true),
|
|
1454
|
+
] {
|
|
1455
|
+
let found = kind(&compile(&pattern).unwrap());
|
|
1456
|
+
assert_eq!(found == "alternatives", alternatives, "{pattern:.30} has the matcher {found}");
|
|
1457
|
+
assert!(alternatives || found == "engine", "{pattern:.30} has the matcher {found}");
|
|
1573
1458
|
}
|
|
1574
1459
|
}
|
|
1460
|
+
|
|
1461
|
+
/// The cache of compiled patterns is bounded in number and in size, and a pattern that has left it is still
|
|
1462
|
+
/// matched by whoever holds it.
|
|
1463
|
+
#[test]
|
|
1464
|
+
fn the_cache_of_patterns_is_bounded() {
|
|
1465
|
+
let mut budget = PatternBudget::new(PatternLimit::new(None));
|
|
1466
|
+
let first = compile("^cache-bound-0+$").unwrap();
|
|
1467
|
+
let held: Vec<Arc<Pattern>> =
|
|
1468
|
+
(0..3 * MAX_CACHED_PATTERNS).map(|n| compile(&format!("^cache-bound-{n}(?=x)x$")).unwrap()).collect();
|
|
1469
|
+
{
|
|
1470
|
+
let cache = cache();
|
|
1471
|
+
assert!(cache.patterns.len() <= MAX_CACHED_PATTERNS && cache.bytes <= MAX_CACHED_BYTES);
|
|
1472
|
+
assert_eq!(cache.patterns.len(), cache.order.len());
|
|
1473
|
+
assert_eq!(cache.bytes, cache.patterns.values().map(|p| p.retained).sum::<usize>());
|
|
1474
|
+
assert!(cache.get("^cache-bound-0+$", G).is_none(), "the oldest pattern is still in the cache");
|
|
1475
|
+
}
|
|
1476
|
+
assert!(first.is_match("cache-bound-000", &mut budget));
|
|
1477
|
+
assert!(held[0].is_match("cache-bound-0x", &mut budget) && !held[7].is_match("cache-bound-0x", &mut budget));
|
|
1478
|
+
// The same source compiles again, to a matcher of its own.
|
|
1479
|
+
let again = compile("^cache-bound-0+$").unwrap();
|
|
1480
|
+
assert!(!Arc::ptr_eq(&first, &again) && again.is_match("cache-bound-0", &mut budget));
|
|
1481
|
+
// Large patterns leave the cache by size, well before their number is the bound.
|
|
1482
|
+
let class = r"[\u0100\u0102\u0104\u0106]".repeat(1500);
|
|
1483
|
+
let large: Vec<Arc<Pattern>> = (0..200).map(|n| compile(&format!("^(?=large-{n}){class}$")).unwrap()).collect();
|
|
1484
|
+
assert!(large[0].retained > 256 << 10, "a large pattern is counted as {} bytes", large[0].retained);
|
|
1485
|
+
assert!(cache().bytes <= MAX_CACHED_BYTES);
|
|
1486
|
+
// A pattern too large for the cache is not kept at all.
|
|
1487
|
+
let huge = compile(&format!("^(?=huge){}$", class.repeat(4))).unwrap();
|
|
1488
|
+
assert!(huge.retained > MAX_CACHED_PATTERN_BYTES && cache().get(&huge.source, G).is_none());
|
|
1489
|
+
}
|
|
1575
1490
|
}
|
|
1576
1491
|
|
|
1577
1492
|
#[cfg(test)]
|
|
1578
1493
|
mod corpus_patterns {
|
|
1579
|
-
|
|
1494
|
+
use std::sync::Arc;
|
|
1495
|
+
|
|
1496
|
+
use super::{Edition, Grammar, Pattern, PatternError, UnicodeVersion};
|
|
1497
|
+
|
|
1498
|
+
const G: Grammar = Grammar { edition: Edition::Current, unicode_version: UnicodeVersion::Default };
|
|
1499
|
+
|
|
1500
|
+
/// A pattern compiled in today's grammar.
|
|
1501
|
+
fn compile(p: &str) -> Result<Arc<Pattern>, PatternError> {
|
|
1502
|
+
super::compile(p, G)
|
|
1503
|
+
}
|
|
1504
|
+
|
|
1505
|
+
/// Lists which matcher each pattern in `CORVUS_PATTERNS` (corpus<TAB>pattern lines) gets, and what the cache
|
|
1506
|
+
/// counts for it.
|
|
1580
1507
|
#[test]
|
|
1581
1508
|
#[ignore]
|
|
1582
1509
|
fn classify_corpus_patterns() {
|
|
1583
1510
|
let Ok(path) = std::env::var("CORVUS_PATTERNS") else { return };
|
|
1584
1511
|
for line in std::fs::read_to_string(path).unwrap().lines() {
|
|
1585
1512
|
let (corpus, p) = line.split_once('\t').unwrap();
|
|
1586
|
-
let kind =
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
1513
|
+
let kind = compile(p).map(|c| super::kind(&c)).unwrap_or("invalid");
|
|
1514
|
+
let retained = compile(p).map_or(0, |c| c.retained);
|
|
1515
|
+
println!("{kind}\t{corpus}\t{p}\t{retained}");
|
|
1516
|
+
}
|
|
1517
|
+
}
|
|
1518
|
+
|
|
1519
|
+
/// Times each dedicated matcher against the engine over the corpora of jsonschema-benchmark in `CORVUS_CORPORA`
|
|
1520
|
+
/// (a directory of `<name>-schema.json` and `<name>-instances.jsonl`), so that a matcher the engine beats can be
|
|
1521
|
+
/// removed. The inputs of a pattern are the distinct strings of the corpus's instances (for `pattern`) or their
|
|
1522
|
+
/// property names (for `patternProperties`), at most 1,000. Each figure is the least of 7 rounds of at least 20 ms,
|
|
1523
|
+
/// after a warm-up of 200 ms, through `Pattern::is_match` with the default limit of a match. Run it with
|
|
1524
|
+
/// `cargo test --release --lib time_corpus_patterns -- --ignored --nocapture --test-threads 1`.
|
|
1525
|
+
#[test]
|
|
1526
|
+
#[ignore]
|
|
1527
|
+
fn time_corpus_patterns() {
|
|
1528
|
+
use std::collections::BTreeSet;
|
|
1529
|
+
use std::time::{Duration, Instant};
|
|
1530
|
+
|
|
1531
|
+
use super::{Literals, Matcher, PatternBudget, PatternLimit};
|
|
1532
|
+
|
|
1533
|
+
fn collect(v: &serde_json::Value, strings: &mut BTreeSet<String>, keys: &mut BTreeSet<String>) {
|
|
1534
|
+
match v {
|
|
1535
|
+
serde_json::Value::String(s) => {
|
|
1536
|
+
strings.insert(s.clone());
|
|
1537
|
+
}
|
|
1538
|
+
serde_json::Value::Array(a) => a.iter().for_each(|v| collect(v, strings, keys)),
|
|
1539
|
+
serde_json::Value::Object(o) => o.iter().for_each(|(k, v)| {
|
|
1540
|
+
keys.insert(k.clone());
|
|
1541
|
+
collect(v, strings, keys);
|
|
1542
|
+
}),
|
|
1543
|
+
_ => {}
|
|
1544
|
+
}
|
|
1545
|
+
}
|
|
1546
|
+
fn patterns(v: &serde_json::Value, out: &mut BTreeSet<(&'static str, String)>) {
|
|
1547
|
+
match v {
|
|
1548
|
+
serde_json::Value::Array(a) => a.iter().for_each(|v| patterns(v, out)),
|
|
1549
|
+
serde_json::Value::Object(o) => {
|
|
1550
|
+
if let Some(serde_json::Value::String(p)) = o.get("pattern") {
|
|
1551
|
+
out.insert(("pattern", p.clone()));
|
|
1552
|
+
}
|
|
1553
|
+
if let Some(serde_json::Value::Object(pp)) = o.get("patternProperties") {
|
|
1554
|
+
pp.keys().for_each(|p| {
|
|
1555
|
+
out.insert(("patternProperties", p.clone()));
|
|
1556
|
+
});
|
|
1557
|
+
}
|
|
1558
|
+
o.values().for_each(|v| patterns(v, out));
|
|
1559
|
+
}
|
|
1560
|
+
_ => {}
|
|
1561
|
+
}
|
|
1562
|
+
}
|
|
1563
|
+
fn time(p: &Pattern, inputs: &[String]) -> f64 {
|
|
1564
|
+
let limit = PatternLimit::new(Some(crate::DEFAULT_PATTERN_MATCH_LIMIT));
|
|
1565
|
+
let pass = || {
|
|
1566
|
+
let mut budget = PatternBudget::new(limit);
|
|
1567
|
+
inputs.iter().filter(|s| p.is_match(s, &mut budget)).count()
|
|
1568
|
+
};
|
|
1569
|
+
let warm = Instant::now();
|
|
1570
|
+
while warm.elapsed() < Duration::from_millis(200) {
|
|
1571
|
+
std::hint::black_box(pass());
|
|
1572
|
+
}
|
|
1573
|
+
let mut best = f64::MAX;
|
|
1574
|
+
for _ in 0..7 {
|
|
1575
|
+
let (start, mut calls) = (Instant::now(), 0usize);
|
|
1576
|
+
while start.elapsed() < Duration::from_millis(20) {
|
|
1577
|
+
std::hint::black_box(pass());
|
|
1578
|
+
calls += inputs.len();
|
|
1579
|
+
}
|
|
1580
|
+
best = best.min(start.elapsed().as_nanos() as f64 / calls as f64);
|
|
1581
|
+
}
|
|
1582
|
+
best
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
let Ok(dir) = std::env::var("CORVUS_CORPORA") else { return };
|
|
1586
|
+
let mut names: Vec<String> = std::fs::read_dir(&dir)
|
|
1587
|
+
.unwrap()
|
|
1588
|
+
.filter_map(|e| e.unwrap().file_name().to_str()?.strip_suffix("-schema.json").map(str::to_string))
|
|
1589
|
+
.collect();
|
|
1590
|
+
names.sort();
|
|
1591
|
+
// The alternatives a matcher tries one after another.
|
|
1592
|
+
fn alternatives(m: &Matcher) -> usize {
|
|
1593
|
+
match m {
|
|
1594
|
+
Matcher::Alternatives(a) => a.len(),
|
|
1595
|
+
Matcher::Literals(Literals(t)) => t.len(),
|
|
1596
|
+
_ => 1,
|
|
1597
|
+
}
|
|
1598
|
+
}
|
|
1599
|
+
println!("corpus\trole\tkind\talternatives\tshape ns\tengine ns\tshape/engine\tinputs\tpattern");
|
|
1600
|
+
for name in names {
|
|
1601
|
+
let schema: serde_json::Value =
|
|
1602
|
+
serde_json::from_str(&std::fs::read_to_string(format!("{dir}/{name}-schema.json")).unwrap()).unwrap();
|
|
1603
|
+
let (mut strings, mut keys) = (BTreeSet::new(), BTreeSet::new());
|
|
1604
|
+
for line in std::fs::read_to_string(format!("{dir}/{name}-instances.jsonl")).unwrap().lines() {
|
|
1605
|
+
if !line.trim().is_empty() {
|
|
1606
|
+
collect(&serde_json::from_str(line).unwrap(), &mut strings, &mut keys);
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
let strings: Vec<String> = strings.into_iter().take(1000).collect();
|
|
1610
|
+
let keys: Vec<String> = keys.into_iter().take(1000).collect();
|
|
1611
|
+
let mut found = BTreeSet::new();
|
|
1612
|
+
patterns(&schema, &mut found);
|
|
1613
|
+
for (role, source) in found {
|
|
1614
|
+
let Ok(shape) = compile(&source) else { continue };
|
|
1615
|
+
if matches!(shape.matcher, Matcher::Engine(_)) {
|
|
1616
|
+
continue;
|
|
1617
|
+
}
|
|
1618
|
+
let inputs = if role == "pattern" { &strings } else { &keys };
|
|
1619
|
+
if inputs.is_empty() {
|
|
1620
|
+
continue;
|
|
1621
|
+
}
|
|
1622
|
+
let regex = G.build(&source, true).or_else(|_| G.build(&source, false)).unwrap();
|
|
1623
|
+
let engine = Pattern {
|
|
1624
|
+
source: source.clone(),
|
|
1625
|
+
grammar: G,
|
|
1626
|
+
matcher: Matcher::Engine(Box::new(regex)),
|
|
1627
|
+
retained: 0,
|
|
1628
|
+
};
|
|
1629
|
+
// The two must agree before their times mean anything.
|
|
1630
|
+
let mut budget = PatternBudget::new(PatternLimit::new(None));
|
|
1631
|
+
for s in inputs {
|
|
1632
|
+
assert_eq!(shape.is_match(s, &mut budget), engine.is_match(s, &mut budget), "{source:?} on {s:?}");
|
|
1633
|
+
}
|
|
1634
|
+
let (a, b) = (time(&shape, inputs), time(&engine, inputs));
|
|
1635
|
+
println!(
|
|
1636
|
+
"{name}\t{role}\t{}\t{}\t{a:.1}\t{b:.1}\t{:.3}\t{}\t{source}",
|
|
1637
|
+
super::kind(&shape),
|
|
1638
|
+
alternatives(&shape.matcher),
|
|
1639
|
+
a / b,
|
|
1640
|
+
inputs.len()
|
|
1641
|
+
);
|
|
1642
|
+
}
|
|
1594
1643
|
}
|
|
1595
1644
|
}
|
|
1596
1645
|
}
|