corvus_json_schema 0.1.4 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +13 -2
- data/README.md +6 -4
- data/THIRD-PARTY-NOTICES.txt +15 -14
- data/VERSIONHISTORY.md +9 -0
- data/ext/corvus_json_schema/Cargo.toml +2 -2
- data/ext/corvus_json_schema/src/lib.rs +29 -2
- data/ext/corvus_json_schema/vendor/corvus-json-schema/Cargo.toml +4 -9
- data/ext/corvus_json_schema/vendor/corvus-json-schema/README.md +30 -49
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/compiler.rs +5 -3
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/formats.rs +7 -6
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/lib.rs +4 -4
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/options.rs +12 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/pattern.rs +344 -974
- data/lib/corvus_json_schema/version.rb +1 -1
- data/lib/corvus_json_schema.rb +8 -3
- metadata +2 -15
- data/ext/corvus_json_schema/vendor/corvus-json-schema/THIRD-PARTY-NOTICES.txt +0 -93
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/charset.rs +0 -271
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/compile.rs +0 -941
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/dfa.rs +0 -366
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/fold.rs +0 -138
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/mod.rs +0 -329
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/parse.rs +0 -1128
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/re2.rs +0 -129
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/tables_gen.rs +0 -6685
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/tests.rs +0 -1672
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/ucd.rs +0 -309
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/units.rs +0 -165
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/vm.rs +0 -981
|
@@ -8,33 +8,75 @@
|
|
|
8
8
|
//! - anchored sequences of quantified ASCII character classes and literals (`^[a-z][a-z0-9_]{0,29}$`, `^x-`,
|
|
9
9
|
//! `^[@$_#]`), matched in one pass over the string (the TypeScript evaluator's class-sequence fast path);
|
|
10
10
|
//! - alternatives of such sequences once groups are multiplied out (`^([a|A]uto)|([n|N]one)$`,
|
|
11
|
-
//! `^[Ee][Ss]2015(\.([Cc]ore|[Pp]roxy))?$`)
|
|
12
|
-
//!
|
|
13
|
-
//! -
|
|
14
|
-
//!
|
|
15
|
-
//!
|
|
16
|
-
//! - anything else (lookarounds, backreferences, `\B`, modifier groups, and every pattern `regex` is not given) by
|
|
17
|
-
//! the crate's own ECMA-262 engine (`ecmaregex`), a port of the engine of the Go module. It answers every pattern
|
|
18
|
-
//! as ECMA-262 does, as the pattern is written.
|
|
11
|
+
//! `^[Ee][Ss]2015(\.([Cc]ore|[Pp]roxy))?$`) and sets of literals, as the C# evaluator's pattern matchers do,
|
|
12
|
+
//! where they were measured faster than the engine (see [`MOST_LITERALS`]);
|
|
13
|
+
//! - anything else by [Corvus.Regex.Ecma262](https://docs.rs/corvus-regex-ecma262), an ECMA-262 engine that answers
|
|
14
|
+
//! every pattern as V8 does, as the pattern is written, and matches a pattern with no backreference in time
|
|
15
|
+
//! proportional to the pattern's size times the string.
|
|
19
16
|
//!
|
|
20
|
-
//! Whether a pattern is valid is the engine's
|
|
21
|
-
//!
|
|
17
|
+
//! Whether a pattern is valid is the engine's answer, whichever matcher runs it, in the grammar of the edition the
|
|
18
|
+
//! dialect reads patterns with (ECMAScript 2020 for draft 2019-09 and draft 2020-12, today's for the others; see
|
|
19
|
+
//! [`Grammar`]), and so are the limits of size: a pattern over one is refused before any matcher reads it.
|
|
22
20
|
//!
|
|
23
|
-
//! Patterns compile once per process (a pattern is immutable, so identical patterns share one
|
|
21
|
+
//! Patterns compile once per process for each grammar (a pattern is immutable, so identical patterns share one
|
|
22
|
+
//! matcher).
|
|
24
23
|
|
|
25
24
|
use std::collections::{HashMap, VecDeque};
|
|
26
25
|
use std::fmt;
|
|
27
|
-
use std::sync::{Arc, LazyLock, Mutex
|
|
26
|
+
use std::sync::{Arc, LazyLock, Mutex};
|
|
28
27
|
use std::time::Duration;
|
|
29
28
|
|
|
30
|
-
use
|
|
29
|
+
use corvus_regex_ecma262::{Edition, Flags, MatchError, Regex, RegexBuilder, UnicodeVersion};
|
|
30
|
+
|
|
31
|
+
use crate::dialect::Dialect;
|
|
31
32
|
use crate::options::{
|
|
32
|
-
PatternMatchLimitError, PatternTooLargeError, SchemaCompilationError, SchemaCompilationErrorKind,
|
|
33
|
+
CompileOptions, PatternMatchLimitError, PatternTooLargeError, SchemaCompilationError, SchemaCompilationErrorKind,
|
|
33
34
|
};
|
|
34
35
|
|
|
36
|
+
/// The grammar a pattern is read in: the edition of ECMA-262 and the version of Unicode.
|
|
37
|
+
#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
|
|
38
|
+
pub(crate) struct Grammar {
|
|
39
|
+
pub edition: Edition,
|
|
40
|
+
pub unicode_version: UnicodeVersion,
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
impl Grammar {
|
|
44
|
+
/// The grammar of a dialect's patterns: the edition the options give, or by default ECMAScript 2020 for draft
|
|
45
|
+
/// 2019-09 and draft 2020-12, which cite it, and today's for the earlier drafts; and the options' version of
|
|
46
|
+
/// Unicode.
|
|
47
|
+
pub(crate) fn of(dialect: Dialect, options: &CompileOptions) -> Grammar {
|
|
48
|
+
let edition = options.regex_edition.unwrap_or(if dialect >= Dialect::Draft201909 {
|
|
49
|
+
Edition::Es2020
|
|
50
|
+
} else {
|
|
51
|
+
Edition::Current
|
|
52
|
+
});
|
|
53
|
+
Grammar { edition, unicode_version: options.regex_unicode_version }
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/// The flags of the grammar of the `u` flag, or of no flag.
|
|
57
|
+
fn flags(unicode: bool) -> Flags {
|
|
58
|
+
Flags { unicode, ..Flags::default() }
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
/// The pattern compiled in this grammar, with the `u` flag or with none.
|
|
62
|
+
fn build(self, pattern: &str, unicode: bool) -> Result<Regex, corvus_regex_ecma262::Error> {
|
|
63
|
+
RegexBuilder::new(pattern)
|
|
64
|
+
.with_flags(Grammar::flags(unicode))
|
|
65
|
+
.edition(self.edition)
|
|
66
|
+
.unicode_version(self.unicode_version)
|
|
67
|
+
.build()
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/// Whether the pattern is valid in this grammar with the `u` flag.
|
|
71
|
+
fn is_valid_unicode(self, pattern: &str) -> bool {
|
|
72
|
+
Regex::is_valid_with(pattern, Grammar::flags(true), self.edition, self.unicode_version)
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
35
76
|
/// A compiled `pattern`.
|
|
36
77
|
pub(crate) struct Pattern {
|
|
37
78
|
pub source: String,
|
|
79
|
+
grammar: Grammar,
|
|
38
80
|
matcher: Matcher,
|
|
39
81
|
/// What the pattern is counted as in the cache of compiled patterns, in bytes (see [`Cache`]).
|
|
40
82
|
retained: usize,
|
|
@@ -49,7 +91,6 @@ enum Matcher {
|
|
|
49
91
|
whole: bool,
|
|
50
92
|
},
|
|
51
93
|
Sequence(Sequence),
|
|
52
|
-
SeparatedList(SeparatedList),
|
|
53
94
|
/// `.+` (`^.+` with `start`): some (the first) character is not a line terminator.
|
|
54
95
|
HasContent {
|
|
55
96
|
start: bool,
|
|
@@ -69,14 +110,9 @@ enum Matcher {
|
|
|
69
110
|
set: CharSet,
|
|
70
111
|
bangs: bool,
|
|
71
112
|
},
|
|
72
|
-
|
|
73
|
-
///
|
|
74
|
-
|
|
75
|
-
/// Multilingual Plane (where code units are characters). The engine decides the rest, by code unit.
|
|
76
|
-
RegexBmp(Linear),
|
|
77
|
-
/// The crate's ECMA-262 engine: by character for a pattern with the `u` flag, and by UTF-16 code unit for one
|
|
78
|
-
/// that is valid only without it.
|
|
79
|
-
Engine(ecmaregex::Regexp),
|
|
113
|
+
/// The `corvus-regex-ecma262` engine: by character for a pattern with the `u` flag, and by UTF-16 code unit for
|
|
114
|
+
/// one that is valid only without it.
|
|
115
|
+
Engine(Box<Regex>),
|
|
80
116
|
}
|
|
81
117
|
|
|
82
118
|
impl fmt::Debug for Pattern {
|
|
@@ -85,55 +121,17 @@ impl fmt::Debug for Pattern {
|
|
|
85
121
|
}
|
|
86
122
|
}
|
|
87
123
|
|
|
88
|
-
/// A pattern compiled by the `regex` crate, which matches in time proportional to its program times the string and
|
|
89
|
-
/// cannot be stopped, with the engine for the strings it is not given.
|
|
90
|
-
struct Linear {
|
|
91
|
-
re: regex::Regex,
|
|
92
|
-
/// The number of instructions estimated for the program (`ecmaregex::Facts::linear_cost`), and one at least.
|
|
93
|
-
cost: u64,
|
|
94
|
-
/// The engine for the pattern: compiled with the pattern when it is valid only without the `u` flag, and
|
|
95
|
-
/// otherwise for the first string that is too long for `regex` to answer within the limit of a match.
|
|
96
|
-
engine: OnceLock<Option<ecmaregex::Regexp>>,
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
impl Linear {
|
|
100
|
-
/// Whether `regex` answers the string within the limit of a match: its program times the string, at
|
|
101
|
-
/// [`LINEAR_STEP`] for each, is within the limit. With no limit every string is.
|
|
102
|
-
#[inline(always)]
|
|
103
|
-
fn fits(&self, s: &str, budget: &PatternBudget) -> bool {
|
|
104
|
-
(s.len() as u64).saturating_mul(self.cost) <= budget.limit.linear
|
|
105
|
-
}
|
|
106
|
-
|
|
107
|
-
#[cold]
|
|
108
|
-
fn engine(&self, source: &str) -> Option<&ecmaregex::Regexp> {
|
|
109
|
-
self.engine.get_or_init(|| ecmaregex::compile(source).ok()).as_ref()
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
|
|
113
|
-
/// The time the `regex` crate can take for one instruction of its program and one byte of the string, when every
|
|
114
|
-
/// instruction has a thread on it (it then runs its simulation of the automaton, which is its slowest matcher). A
|
|
115
|
-
/// match on `regex` cannot be stopped, so a match with a limit goes there only when the size of the program times
|
|
116
|
-
/// the length of the string times this is within the limit. Any other goes to the engine, which reads the clock.
|
|
117
|
-
/// The figure is the Go module's for the regexp package of Go. For the `regex` crate the most that was measured,
|
|
118
|
-
/// with patterns that defeat its lazy automaton on a text of 100,000 bytes, was 3.6 ns.
|
|
119
|
-
const LINEAR_STEP: Duration = Duration::from_nanos(16);
|
|
120
|
-
|
|
121
124
|
/// The limit of a match, as a validator holds it.
|
|
122
125
|
#[derive(Clone, Copy, Debug)]
|
|
123
126
|
pub(crate) struct PatternLimit {
|
|
124
127
|
/// The time one match may take, or `None` for no limit.
|
|
125
128
|
limit: Option<Duration>,
|
|
126
|
-
/// The program of a `regex` matcher times the bytes of a string that the limit allows (see [`LINEAR_STEP`]).
|
|
127
|
-
linear: u64,
|
|
128
129
|
}
|
|
129
130
|
|
|
130
131
|
impl PatternLimit {
|
|
131
132
|
/// A limit of `limit` for each match. `None` and zero are no limit.
|
|
132
133
|
pub(crate) fn new(limit: Option<Duration>) -> PatternLimit {
|
|
133
|
-
|
|
134
|
-
let linear =
|
|
135
|
-
limit.map_or(u64::MAX, |l| u64::try_from(l.as_nanos() / LINEAR_STEP.as_nanos()).unwrap_or(u64::MAX));
|
|
136
|
-
PatternLimit { limit, linear }
|
|
134
|
+
PatternLimit { limit: limit.filter(|l| !l.is_zero()) }
|
|
137
135
|
}
|
|
138
136
|
}
|
|
139
137
|
|
|
@@ -164,20 +162,19 @@ impl PatternBudget {
|
|
|
164
162
|
|
|
165
163
|
/// Matches the string on the engine within the budget. When a limit is reached the answer is false, and it is
|
|
166
164
|
/// not an answer: the caller of the evaluation gets the error.
|
|
167
|
-
fn engine_match(&mut self, engine:
|
|
165
|
+
fn engine_match(&mut self, engine: &Regex, source: &str, s: &str) -> bool {
|
|
168
166
|
if self.failure.is_some() {
|
|
169
167
|
return false;
|
|
170
168
|
}
|
|
171
|
-
let result = match
|
|
172
|
-
Some(
|
|
173
|
-
|
|
174
|
-
None => Err(ecmaregex::MatchError::Memory),
|
|
169
|
+
let result = match self.limit.limit {
|
|
170
|
+
Some(limit) => engine.is_match_within(s, limit),
|
|
171
|
+
None => engine.is_match(s),
|
|
175
172
|
};
|
|
176
173
|
result.unwrap_or_else(|e| {
|
|
177
174
|
self.failure = Some(Box::new(PatternMatchLimitError::new(
|
|
178
175
|
source,
|
|
179
176
|
self.limit.limit.unwrap_or_default(),
|
|
180
|
-
e ==
|
|
177
|
+
e == MatchError::Memory,
|
|
181
178
|
)));
|
|
182
179
|
false
|
|
183
180
|
})
|
|
@@ -199,7 +196,6 @@ impl Pattern {
|
|
|
199
196
|
}
|
|
200
197
|
}
|
|
201
198
|
Matcher::Sequence(seq) => seq.is_match(s),
|
|
202
|
-
Matcher::SeparatedList(list) => list.is_match(s),
|
|
203
199
|
Matcher::HasContent { start } => {
|
|
204
200
|
if *start {
|
|
205
201
|
s.chars().next().is_some_and(|c| !is_line_terminator(c))
|
|
@@ -232,22 +228,7 @@ impl Pattern {
|
|
|
232
228
|
}
|
|
233
229
|
word
|
|
234
230
|
}
|
|
235
|
-
Matcher::
|
|
236
|
-
if linear.fits(s, budget) {
|
|
237
|
-
linear.re.is_match(s)
|
|
238
|
-
} else {
|
|
239
|
-
budget.engine_match(linear.engine(&self.source), &self.source, s)
|
|
240
|
-
}
|
|
241
|
-
}
|
|
242
|
-
// A UTF-8 lead byte of 0xF0 or above starts a character beyond the BMP.
|
|
243
|
-
Matcher::RegexBmp(linear) => {
|
|
244
|
-
if linear.fits(s, budget) && !ecmaregex::has_supplementary(s.as_bytes()) {
|
|
245
|
-
linear.re.is_match(s)
|
|
246
|
-
} else {
|
|
247
|
-
budget.engine_match(linear.engine(&self.source), &self.source, s)
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
Matcher::Engine(engine) => budget.engine_match(Some(engine), &self.source, s),
|
|
231
|
+
Matcher::Engine(engine) => budget.engine_match(engine, &self.source, s),
|
|
251
232
|
}
|
|
252
233
|
}
|
|
253
234
|
}
|
|
@@ -256,14 +237,11 @@ impl Pattern {
|
|
|
256
237
|
#[cfg(test)]
|
|
257
238
|
pub(crate) fn kind(p: &Pattern) -> &'static str {
|
|
258
239
|
match &p.matcher {
|
|
259
|
-
Matcher::
|
|
260
|
-
Matcher::RegexBmp(..) => "regex-bmp",
|
|
261
|
-
Matcher::Engine(engine) if engine.unicode() => "engine",
|
|
240
|
+
Matcher::Engine(engine) if engine.flags().unicode => "engine",
|
|
262
241
|
Matcher::Engine(_) => "engine-units",
|
|
263
242
|
Matcher::Everything => "everything",
|
|
264
243
|
Matcher::Literal { .. } => "literal",
|
|
265
244
|
Matcher::Sequence(_) => "sequence",
|
|
266
|
-
Matcher::SeparatedList(_) => "list",
|
|
267
245
|
Matcher::HasContent { .. } => "has-content",
|
|
268
246
|
Matcher::Line { .. } => "line",
|
|
269
247
|
Matcher::Literals(_) => "literals",
|
|
@@ -294,30 +272,34 @@ fn line_length(s: &str) -> Option<usize> {
|
|
|
294
272
|
Some(n)
|
|
295
273
|
}
|
|
296
274
|
|
|
297
|
-
/// A set of strings
|
|
298
|
-
|
|
299
|
-
Few(Box<[Box<str>]>),
|
|
300
|
-
Many(std::collections::HashSet<Box<str>>),
|
|
301
|
-
}
|
|
275
|
+
/// A set of at most [`MOST_LITERALS`] strings, compared in turn.
|
|
276
|
+
struct Literals(Box<[Box<str>]>);
|
|
302
277
|
|
|
303
278
|
impl Literals {
|
|
304
279
|
fn new(texts: Vec<String>) -> Literals {
|
|
305
|
-
|
|
306
|
-
Literals::Few(texts.into_iter().map(String::into_boxed_str).collect())
|
|
307
|
-
} else {
|
|
308
|
-
Literals::Many(texts.into_iter().map(String::into_boxed_str).collect())
|
|
309
|
-
}
|
|
280
|
+
Literals(texts.into_iter().map(String::into_boxed_str).collect())
|
|
310
281
|
}
|
|
311
282
|
|
|
312
283
|
#[inline]
|
|
313
284
|
fn contains(&self, s: &str) -> bool {
|
|
314
|
-
|
|
315
|
-
Literals::Few(texts) => texts.iter().any(|t| **t == *s),
|
|
316
|
-
Literals::Many(set) => set.contains(s),
|
|
317
|
-
}
|
|
285
|
+
self.0.iter().any(|t| **t == *s)
|
|
318
286
|
}
|
|
319
287
|
}
|
|
320
288
|
|
|
289
|
+
// What the dedicated matchers are kept for: each was timed against the engine, pattern by pattern, over the strings of
|
|
290
|
+
// the jsonschema-benchmark corpora (`corpus_patterns::time_corpus_patterns`), and a form is kept only where its matcher
|
|
291
|
+
// was the faster. Four literals compared in turn took 0.78 of the engine's time, and ten 3.2 times it. Two alternatives
|
|
292
|
+
// of which neither is anchored at both ends (`^([a|A]uto)|([n|N]one)$`) took 0.21 of it, since the engine searches for
|
|
293
|
+
// the unanchored one; three such took 1.4 to 1.6 times it, and every alternative anchored at both ends was faster on
|
|
294
|
+
// the engine (by 1.1 to 2.7 times), which reads the whole string once whatever the alternatives. A separated list was
|
|
295
|
+
// no faster than the engine (the same total time over the seven in the corpora), so it has no matcher of its own.
|
|
296
|
+
|
|
297
|
+
/// The most literals of [`Matcher::Literals`].
|
|
298
|
+
const MOST_LITERALS: usize = 4;
|
|
299
|
+
|
|
300
|
+
/// The most alternatives of [`Matcher::Alternatives`].
|
|
301
|
+
const MOST_ALTERNATIVES: usize = 2;
|
|
302
|
+
|
|
321
303
|
/// One alternative of [`Matcher::Alternatives`]: a literal anchored at either end (or neither), a class sequence
|
|
322
304
|
/// anchored at the start, or a fixed-width class sequence anchored at the end.
|
|
323
305
|
enum Alternative {
|
|
@@ -384,22 +366,12 @@ impl Alternative {
|
|
|
384
366
|
const MAX_CACHED_PATTERNS: usize = 1024;
|
|
385
367
|
|
|
386
368
|
/// The most the patterns of the cache come to, in the bytes counted for each: 64 MiB. A pattern is counted as its
|
|
387
|
-
/// source
|
|
388
|
-
/// for a pattern on the `regex` crate the memory estimated for that crate's program (`ecmaregex::Facts::linear_size`,
|
|
389
|
-
/// which is an estimate: the crate does not say what it holds), up to three times [`REGEX_SIZE_LIMIT`].
|
|
369
|
+
/// source and an estimate of what its matcher holds (see [`compile`]).
|
|
390
370
|
const MAX_CACHED_BYTES: usize = 64 << 20;
|
|
391
371
|
|
|
392
372
|
/// A pattern counted as more than this is compiled for the schema that has it and not kept in the cache.
|
|
393
373
|
const MAX_CACHED_PATTERN_BYTES: usize = MAX_CACHED_BYTES / 32;
|
|
394
374
|
|
|
395
|
-
/// The largest program the `regex` crate compiles for a pattern, in bytes: 2 MiB. (Its own default is 10 MiB.) A
|
|
396
|
-
/// pattern that needs more is the engine's, which holds a class once however often the pattern repeats it.
|
|
397
|
-
const REGEX_SIZE_LIMIT: usize = 2 << 20;
|
|
398
|
-
|
|
399
|
-
/// The most the `regex` crate's lazy automaton caches for one pattern and one thread, in bytes: 2 MiB, which is
|
|
400
|
-
/// that crate's default, said here so that it stays so.
|
|
401
|
-
const REGEX_DFA_SIZE_LIMIT: usize = 2 << 20;
|
|
402
|
-
|
|
403
375
|
/// The compiled patterns of the process, so that identical patterns share one matcher. It is bounded in number and
|
|
404
376
|
/// in size, since the patterns of a schema may come from outside: when it is over either bound the patterns that
|
|
405
377
|
/// were compiled first go, oldest first. A validator holds the patterns of its schema itself, so a pattern that has
|
|
@@ -407,16 +379,16 @@ const REGEX_DFA_SIZE_LIMIT: usize = 2 << 20;
|
|
|
407
379
|
/// later.
|
|
408
380
|
#[derive(Default)]
|
|
409
381
|
struct Cache {
|
|
410
|
-
patterns: HashMap<Arc<str>, Arc<Pattern>>,
|
|
411
|
-
/// The sources in the order they were compiled.
|
|
412
|
-
order: VecDeque<Arc<str
|
|
382
|
+
patterns: HashMap<(Arc<str>, Grammar), Arc<Pattern>>,
|
|
383
|
+
/// The sources and their grammars in the order they were compiled.
|
|
384
|
+
order: VecDeque<(Arc<str>, Grammar)>,
|
|
413
385
|
/// The sum of what the patterns are counted as.
|
|
414
386
|
bytes: usize,
|
|
415
387
|
}
|
|
416
388
|
|
|
417
389
|
impl Cache {
|
|
418
|
-
fn get(&self, source: &str) -> Option<Arc<Pattern>> {
|
|
419
|
-
self.patterns.get(source).cloned()
|
|
390
|
+
fn get(&self, source: &str, grammar: Grammar) -> Option<Arc<Pattern>> {
|
|
391
|
+
self.patterns.get(&(Arc::from(source), grammar)).cloned()
|
|
420
392
|
}
|
|
421
393
|
|
|
422
394
|
/// Keeps a pattern, and returns the one to use: the pattern itself, or the one another thread kept for the same
|
|
@@ -425,13 +397,13 @@ impl Cache {
|
|
|
425
397
|
if pattern.retained > MAX_CACHED_PATTERN_BYTES {
|
|
426
398
|
return pattern;
|
|
427
399
|
}
|
|
428
|
-
|
|
400
|
+
let key: (Arc<str>, Grammar) = (Arc::from(pattern.source.as_str()), pattern.grammar);
|
|
401
|
+
if let Some(kept) = self.patterns.get(&key) {
|
|
429
402
|
return kept.clone();
|
|
430
403
|
}
|
|
431
|
-
let source: Arc<str> = Arc::from(pattern.source.as_str());
|
|
432
404
|
self.bytes += pattern.retained;
|
|
433
|
-
self.patterns.insert(
|
|
434
|
-
self.order.push_back(
|
|
405
|
+
self.patterns.insert(key.clone(), pattern.clone());
|
|
406
|
+
self.order.push_back(key);
|
|
435
407
|
while self.patterns.len() > MAX_CACHED_PATTERNS || self.bytes > MAX_CACHED_BYTES {
|
|
436
408
|
let Some(oldest) = self.order.pop_front() else { break };
|
|
437
409
|
if let Some(evicted) = self.patterns.remove(&oldest) {
|
|
@@ -454,7 +426,7 @@ pub(crate) enum PatternError {
|
|
|
454
426
|
/// It is not a valid ECMA-262 regular expression.
|
|
455
427
|
Invalid,
|
|
456
428
|
/// It is over a limit of size. It was not read to its end, so nothing is known of whether it is valid.
|
|
457
|
-
TooLarge(
|
|
429
|
+
TooLarge(corvus_regex_ecma262::TooLargeError),
|
|
458
430
|
}
|
|
459
431
|
|
|
460
432
|
impl PatternError {
|
|
@@ -474,11 +446,12 @@ impl PatternError {
|
|
|
474
446
|
}
|
|
475
447
|
}
|
|
476
448
|
|
|
477
|
-
impl From<
|
|
478
|
-
fn from(e:
|
|
449
|
+
impl From<corvus_regex_ecma262::Error> for PatternError {
|
|
450
|
+
fn from(e: corvus_regex_ecma262::Error) -> PatternError {
|
|
479
451
|
match e {
|
|
480
|
-
|
|
481
|
-
|
|
452
|
+
corvus_regex_ecma262::Error::TooLarge(large) => PatternError::TooLarge(large),
|
|
453
|
+
// Not valid in the grammar, or (not given here) flags that are not.
|
|
454
|
+
_ => PatternError::Invalid,
|
|
482
455
|
}
|
|
483
456
|
}
|
|
484
457
|
}
|
|
@@ -496,79 +469,44 @@ pub(crate) fn shown_pattern(pattern: &str) -> String {
|
|
|
496
469
|
format!("'{}...' ({} bytes)", &pattern[..cut], pattern.len())
|
|
497
470
|
}
|
|
498
471
|
|
|
499
|
-
/// Compiles (or fetches from the process-wide cache) a pattern.
|
|
500
|
-
pub(crate) fn compile(pattern: &str) -> Result<Arc<Pattern>, PatternError> {
|
|
501
|
-
if let Some(p) = cache().get(pattern) {
|
|
472
|
+
/// Compiles (or fetches from the process-wide cache) a pattern, in the grammar of its dialect.
|
|
473
|
+
pub(crate) fn compile(pattern: &str, grammar: Grammar) -> Result<Arc<Pattern>, PatternError> {
|
|
474
|
+
if let Some(p) = cache().get(pattern, grammar) {
|
|
502
475
|
return Ok(p);
|
|
503
476
|
}
|
|
504
|
-
//
|
|
505
|
-
//
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
None => match
|
|
512
|
-
|
|
513
|
-
|
|
477
|
+
// Validity is ECMA-262's: a pattern the engine refuses is an error, whichever matcher would run it. A pattern is
|
|
478
|
+
// read with the u flag, and failing that with no flag, whose grammar (Annex B) many schemas need. A pattern valid
|
|
479
|
+
// with the flag that a dedicated matcher takes is read and never built: building the engine's program and
|
|
480
|
+
// automata costs far more than reading. The engine is built only for the patterns it matches, and for those over
|
|
481
|
+
// a limit of size, which it refuses (the reading refuses them too, without saying why).
|
|
482
|
+
let matcher = match grammar.is_valid_unicode(pattern).then(|| choose(pattern, true, grammar)).flatten() {
|
|
483
|
+
Some(shape) => shape,
|
|
484
|
+
None => match grammar.build(pattern, true) {
|
|
485
|
+
Ok(engine) => Matcher::Engine(Box::new(engine)),
|
|
486
|
+
Err(corvus_regex_ecma262::Error::TooLarge(large)) => return Err(PatternError::TooLarge(large)),
|
|
487
|
+
Err(_) => {
|
|
488
|
+
let engine = grammar.build(pattern, false)?;
|
|
489
|
+
choose(pattern, false, grammar).unwrap_or_else(|| Matcher::Engine(Box::new(engine)))
|
|
490
|
+
}
|
|
514
491
|
},
|
|
515
492
|
};
|
|
516
|
-
// What the cache counts for the pattern (see `MAX_CACHED_BYTES`)
|
|
493
|
+
// What the cache counts for the pattern (see `MAX_CACHED_BYTES`): its source, and for the engine an estimate of
|
|
494
|
+
// its program and automata, which grow with the pattern.
|
|
517
495
|
let mut retained = pattern.len() + 256;
|
|
518
|
-
if matches!(matcher, Matcher::
|
|
519
|
-
retained +=
|
|
520
|
-
}
|
|
521
|
-
if matches!(matcher, Matcher::Regex(_) | Matcher::RegexBmp(_)) {
|
|
522
|
-
retained += (facts.linear_size as usize).min(3 * REGEX_SIZE_LIMIT);
|
|
523
|
-
}
|
|
524
|
-
Ok(cache().insert(Arc::new(Pattern { source: pattern.to_string(), matcher, retained })))
|
|
525
|
-
}
|
|
526
|
-
|
|
527
|
-
/// The pattern translated for the `regex` crate and compiled by it, when it is in the syntax both have and its
|
|
528
|
-
/// program is not large (see `ecmaregex::MAX_RE2_COST`: `regex` writes a counted repetition out, and takes time
|
|
529
|
-
/// proportional to its program to compile and, for every character of a text, to match).
|
|
530
|
-
fn linear(pattern: &str, facts: &ecmaregex::Facts) -> Option<Linear> {
|
|
531
|
-
if facts.re2_cost > ecmaregex::MAX_RE2_COST {
|
|
532
|
-
return None;
|
|
496
|
+
if matches!(matcher, Matcher::Engine(_)) {
|
|
497
|
+
retained += 16 * pattern.len();
|
|
533
498
|
}
|
|
534
|
-
|
|
535
|
-
let re = if ecmaregex::shallow(pattern) {
|
|
536
|
-
regex::RegexBuilder::new(&translation)
|
|
537
|
-
.size_limit(REGEX_SIZE_LIMIT)
|
|
538
|
-
.dfa_size_limit(REGEX_DFA_SIZE_LIMIT)
|
|
539
|
-
.build()
|
|
540
|
-
.ok()?
|
|
541
|
-
} else {
|
|
542
|
-
// `regex` reads 250 levels of nesting unless it is told otherwise, because what it does with a pattern goes
|
|
543
|
-
// a level deeper for each. A pattern that opens many groups is compiled on a thread with a stack for the
|
|
544
|
-
// 1,000 levels the engine's own limit allows (a group, its quantifier and a class inside it are three
|
|
545
|
-
// levels to `regex`), so that it is not the engine's for its nesting alone.
|
|
546
|
-
ecmaregex::deep(|| {
|
|
547
|
-
regex::RegexBuilder::new(&translation)
|
|
548
|
-
.size_limit(REGEX_SIZE_LIMIT)
|
|
549
|
-
.dfa_size_limit(REGEX_DFA_SIZE_LIMIT)
|
|
550
|
-
.nest_limit(4 * ecmaregex::MAX_NESTING as u32)
|
|
551
|
-
.build()
|
|
552
|
-
.ok()
|
|
553
|
-
})??
|
|
554
|
-
};
|
|
555
|
-
let engine = OnceLock::new();
|
|
556
|
-
if !facts.unicode || !ecmaregex::shallow(pattern) {
|
|
557
|
-
// The engine has the strings with a character beyond the Basic Multilingual Plane, so it is compiled now.
|
|
558
|
-
// So is that of a pattern that nests deeply, which is compiled on a stack of its own.
|
|
559
|
-
let _ = engine.set(Some(ecmaregex::compile(pattern).ok()?));
|
|
560
|
-
}
|
|
561
|
-
Some(Linear { re, cost: facts.linear_cost.max(1) as u64, engine })
|
|
499
|
+
Ok(cache().insert(Arc::new(Pattern { source: pattern.to_string(), grammar, matcher, retained })))
|
|
562
500
|
}
|
|
563
501
|
|
|
564
|
-
fn choose(pattern: &str,
|
|
502
|
+
fn choose(pattern: &str, unicode: bool, grammar: Grammar) -> Option<Matcher> {
|
|
565
503
|
// Unanchored (or start-anchored) `.*` finds an empty match in any string; `^.*$` does not (`.` stops at a line
|
|
566
504
|
// terminator), so it is not listed.
|
|
567
505
|
if matches!(pattern, "" | ".*" | "^.*" | ".*$" | "(.*)" | "^(.*)" | "[\\s\\S]*" | "^[\\s\\S]*" | "^[\\s\\S]*$") {
|
|
568
506
|
return Some(Matcher::Everything);
|
|
569
507
|
}
|
|
570
508
|
// Without the `u` flag, a pattern matches UTF-16 code units rather than characters: only the engine has that.
|
|
571
|
-
if !
|
|
509
|
+
if !unicode {
|
|
572
510
|
return None;
|
|
573
511
|
}
|
|
574
512
|
match pattern {
|
|
@@ -582,9 +520,8 @@ fn choose(pattern: &str, facts: &ecmaregex::Facts) -> Option<Matcher> {
|
|
|
582
520
|
&& rest.len() > 1
|
|
583
521
|
&& !ends_with_escape(rest)
|
|
584
522
|
&& !rest.ends_with(['*', '+', '?', '}', '|', '(', '^'])
|
|
585
|
-
&&
|
|
586
|
-
&&
|
|
587
|
-
&& let Some(m) = choose(rest, &rest_facts)
|
|
523
|
+
&& grammar.is_valid_unicode(rest)
|
|
524
|
+
&& let Some(m) = choose(rest, true, grammar)
|
|
588
525
|
{
|
|
589
526
|
return Some(m);
|
|
590
527
|
}
|
|
@@ -592,24 +529,28 @@ fn choose(pattern: &str, facts: &ecmaregex::Facts) -> Option<Matcher> {
|
|
|
592
529
|
return Some(Matcher::Line { min, max });
|
|
593
530
|
}
|
|
594
531
|
if let Some(texts) = whole_alternatives(pattern) {
|
|
595
|
-
return
|
|
532
|
+
return (texts.len() <= MOST_LITERALS).then(|| Matcher::Literals(Literals::new(texts)));
|
|
596
533
|
}
|
|
597
534
|
if let Some(alts) = alternatives(pattern) {
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
535
|
+
let whole = |a: &Alternative| match a {
|
|
536
|
+
Alternative::Literal { start, end, .. } => *start && *end,
|
|
537
|
+
Alternative::Start(seq) => seq.to_end,
|
|
538
|
+
Alternative::End { .. } | Alternative::Anywhere { .. } => false,
|
|
539
|
+
};
|
|
540
|
+
return (alts.len() <= MOST_ALTERNATIVES && !alts.iter().any(whole)).then_some(Matcher::Alternatives(alts));
|
|
602
541
|
}
|
|
603
542
|
if let Some((set, bangs)) = excluded_class_with_word(pattern) {
|
|
604
543
|
return Some(Matcher::ExcludedClassWithWord { set, bangs });
|
|
605
544
|
}
|
|
606
545
|
if let Some(seq) = Sequence::parse(pattern) {
|
|
607
546
|
return Some(match seq.literal() {
|
|
547
|
+
// `^[a-z]*` with nothing it requires: every string.
|
|
548
|
+
Some(text) if text.is_empty() && !seq.to_end => Matcher::Everything,
|
|
608
549
|
Some(text) => Matcher::Literal { text: text.into(), whole: seq.to_end },
|
|
609
550
|
None => Matcher::Sequence(seq),
|
|
610
551
|
});
|
|
611
552
|
}
|
|
612
|
-
|
|
553
|
+
None
|
|
613
554
|
}
|
|
614
555
|
|
|
615
556
|
/// `^.{m,n}$` (and `^.*$`, `^.+$`, `^.{m}$`, `^.{m,}$`, each also as `^(.…)$`): the bounds on the length of a line.
|
|
@@ -867,10 +808,13 @@ fn excluded_class_with_word(p: &str) -> Option<(CharSet, bool)> {
|
|
|
867
808
|
.then_some((excluded, bangs))
|
|
868
809
|
}
|
|
869
810
|
|
|
870
|
-
/// The `regex` format: a valid ECMA-262 regular expression
|
|
871
|
-
///
|
|
872
|
-
|
|
873
|
-
|
|
811
|
+
/// The `regex` format: a valid ECMA-262 regular expression in the grammar of the `u` flag, of the edition and version
|
|
812
|
+
/// of Unicode the dialect reads patterns with. It asks the grammar alone, as the .NET library's check does: the
|
|
813
|
+
/// engine's limits of size belong to compiling a pattern, so a string too large to compile is valid when its grammar
|
|
814
|
+
/// is. The engine's grammar-only reader takes time in proportion to the string and allocates nothing for one of
|
|
815
|
+
/// ordinary size.
|
|
816
|
+
pub(crate) fn is_valid_ecma_regex(s: &str, grammar: Grammar) -> bool {
|
|
817
|
+
Regex::is_valid_pattern_with(s, Grammar::flags(true), grammar.edition, grammar.unicode_version)
|
|
874
818
|
}
|
|
875
819
|
|
|
876
820
|
// ---------------------------------------------------------------------------------------------------------------------
|
|
@@ -1028,6 +972,16 @@ impl Sequence {
|
|
|
1028
972
|
}
|
|
1029
973
|
}
|
|
1030
974
|
}
|
|
975
|
+
if !to_end {
|
|
976
|
+
// Without `$` a match needs only the start of the string: items at the end that can match nothing are
|
|
977
|
+
// dropped, and the last of the rest needs only its least count (`^[a-z]+` is `^[a-z]`).
|
|
978
|
+
while items.last().is_some_and(|item| item.min == 0) {
|
|
979
|
+
items.pop();
|
|
980
|
+
}
|
|
981
|
+
if let Some(last) = items.last_mut() {
|
|
982
|
+
last.max = last.min;
|
|
983
|
+
}
|
|
984
|
+
}
|
|
1031
985
|
Some(Sequence { items: items.into_boxed_slice(), to_end })
|
|
1032
986
|
}
|
|
1033
987
|
|
|
@@ -1083,231 +1037,6 @@ impl Sequence {
|
|
|
1083
1037
|
}
|
|
1084
1038
|
!self.to_end || at == b.len()
|
|
1085
1039
|
}
|
|
1086
|
-
|
|
1087
|
-
/// Greedily matches the items at the start of `s` (ignoring `to_end`): the bytes taken.
|
|
1088
|
-
fn consume(&self, s: &str) -> Option<usize> {
|
|
1089
|
-
let b = s.as_bytes();
|
|
1090
|
-
let mut at = 0;
|
|
1091
|
-
for item in self.items.iter() {
|
|
1092
|
-
let mut n = 0u32;
|
|
1093
|
-
while n < item.max && at < b.len() {
|
|
1094
|
-
let c = b[at];
|
|
1095
|
-
if c < 0x80 {
|
|
1096
|
-
if !item.set.has_ascii(c) {
|
|
1097
|
-
break;
|
|
1098
|
-
}
|
|
1099
|
-
at += 1;
|
|
1100
|
-
} else {
|
|
1101
|
-
let ch = s[at..].chars().next()?;
|
|
1102
|
-
if !item.set.contains(ch) {
|
|
1103
|
-
break;
|
|
1104
|
-
}
|
|
1105
|
-
at += ch.len_utf8();
|
|
1106
|
-
}
|
|
1107
|
-
n += 1;
|
|
1108
|
-
}
|
|
1109
|
-
if n < item.min {
|
|
1110
|
-
return None;
|
|
1111
|
-
}
|
|
1112
|
-
}
|
|
1113
|
-
Some(at)
|
|
1114
|
-
}
|
|
1115
|
-
|
|
1116
|
-
/// Whether greedy matching is exact when `next` can follow the items: every variable item is disjoint from the
|
|
1117
|
-
/// items that can directly follow it (up to the first that cannot match nothing), `next` included.
|
|
1118
|
-
fn greedy_before(items: &[Item], next: CharSet) -> bool {
|
|
1119
|
-
items.iter().enumerate().all(|(i, item)| {
|
|
1120
|
-
if item.min == item.max {
|
|
1121
|
-
return true;
|
|
1122
|
-
}
|
|
1123
|
-
for following in &items[i + 1..] {
|
|
1124
|
-
if !item.set.disjoint(following.set) {
|
|
1125
|
-
return false;
|
|
1126
|
-
}
|
|
1127
|
-
if following.min > 0 {
|
|
1128
|
-
return true;
|
|
1129
|
-
}
|
|
1130
|
-
}
|
|
1131
|
-
item.set.disjoint(next)
|
|
1132
|
-
})
|
|
1133
|
-
}
|
|
1134
|
-
}
|
|
1135
|
-
|
|
1136
|
-
/// A list of items between separators: `^I(SR)*$` or `^I(SR)+$` (`final_` is `None`), or `^(RS)*F$` and
|
|
1137
|
-
/// `^(RS)+F$`. The separator is a fixed sequence and no variable class can run into what follows it, so greedy
|
|
1138
|
-
/// matching splits the string exactly where the pattern does.
|
|
1139
|
-
#[derive(Debug)]
|
|
1140
|
-
struct SeparatedList {
|
|
1141
|
-
/// `I` of the first form.
|
|
1142
|
-
first: Option<Sequence>,
|
|
1143
|
-
repeated: Sequence,
|
|
1144
|
-
separator: Sequence,
|
|
1145
|
-
/// `F` of the second form, matched against the whole remainder.
|
|
1146
|
-
final_: Option<Sequence>,
|
|
1147
|
-
min_repeats: u32,
|
|
1148
|
-
}
|
|
1149
|
-
|
|
1150
|
-
impl SeparatedList {
|
|
1151
|
-
fn is_match(&self, s: &str) -> bool {
|
|
1152
|
-
let mut rest = s;
|
|
1153
|
-
let mut repeats = 0;
|
|
1154
|
-
if let Some(first) = &self.first {
|
|
1155
|
-
let Some(n) = first.consume(rest) else { return false };
|
|
1156
|
-
rest = &rest[n..];
|
|
1157
|
-
loop {
|
|
1158
|
-
if rest.is_empty() {
|
|
1159
|
-
return repeats >= self.min_repeats;
|
|
1160
|
-
}
|
|
1161
|
-
let Some(n) = self.separator.consume(rest) else { return false };
|
|
1162
|
-
rest = &rest[n..];
|
|
1163
|
-
let Some(n) = self.repeated.consume(rest) else { return false };
|
|
1164
|
-
rest = &rest[n..];
|
|
1165
|
-
repeats += 1;
|
|
1166
|
-
}
|
|
1167
|
-
}
|
|
1168
|
-
let final_ = self.final_.as_ref().expect("a separated list has a first or a final item");
|
|
1169
|
-
// Whether the text is ASCII is worked out once, for the whole of it. The final sequence is tried after
|
|
1170
|
-
// every item, and reading what is left of the text each time to see whether it is ASCII made the match take
|
|
1171
|
-
// time quadratic in the number of items.
|
|
1172
|
-
let ascii = s.is_ascii();
|
|
1173
|
-
loop {
|
|
1174
|
-
if repeats >= self.min_repeats
|
|
1175
|
-
&& (if ascii { final_.is_match_ascii(rest.as_bytes()) } else { final_.is_match_chars(rest) })
|
|
1176
|
-
{
|
|
1177
|
-
return true;
|
|
1178
|
-
}
|
|
1179
|
-
let Some(n) = self.repeated.consume(rest) else { return false };
|
|
1180
|
-
let Some(m) = self.separator.consume(&rest[n..]) else { return false };
|
|
1181
|
-
rest = &rest[n + m..];
|
|
1182
|
-
repeats += 1;
|
|
1183
|
-
}
|
|
1184
|
-
}
|
|
1185
|
-
|
|
1186
|
-
fn parse(p: &str) -> Option<SeparatedList> {
|
|
1187
|
-
let body = p.strip_prefix('^')?.strip_suffix('$')?;
|
|
1188
|
-
if ends_with_escape(body) {
|
|
1189
|
-
return None;
|
|
1190
|
-
}
|
|
1191
|
-
let b = body.as_bytes();
|
|
1192
|
-
// Top-level groups: (start, end) of each, where end is the index of its `)`.
|
|
1193
|
-
let mut groups = Vec::new();
|
|
1194
|
-
let (mut depth, mut in_class, mut i, mut open) = (0usize, false, 0, 0);
|
|
1195
|
-
while i < b.len() {
|
|
1196
|
-
match b[i] {
|
|
1197
|
-
b'\\' => i += 1,
|
|
1198
|
-
b'[' if !in_class => in_class = true,
|
|
1199
|
-
b']' if in_class => in_class = false,
|
|
1200
|
-
b'(' if !in_class => {
|
|
1201
|
-
if depth == 0 {
|
|
1202
|
-
open = i;
|
|
1203
|
-
}
|
|
1204
|
-
depth += 1;
|
|
1205
|
-
}
|
|
1206
|
-
b')' if !in_class => {
|
|
1207
|
-
depth = depth.checked_sub(1)?;
|
|
1208
|
-
if depth == 0 {
|
|
1209
|
-
groups.push((open, i));
|
|
1210
|
-
}
|
|
1211
|
-
}
|
|
1212
|
-
_ => {}
|
|
1213
|
-
}
|
|
1214
|
-
i += 1;
|
|
1215
|
-
}
|
|
1216
|
-
let quantified: Vec<_> = groups.iter().filter(|&&(_, e)| matches!(b.get(e + 1), Some(b'*' | b'+'))).collect();
|
|
1217
|
-
let &&(open, close) = quantified.first().filter(|_| quantified.len() == 1)?;
|
|
1218
|
-
let min_repeats = u32::from(b[close + 1] == b'+');
|
|
1219
|
-
let group = strip_group(&body[open..=close])?;
|
|
1220
|
-
let before = &body[..open];
|
|
1221
|
-
let after = &body[close + 2..];
|
|
1222
|
-
// The items of a text that is a class sequence. A `$` at the end of the text is not an item, and the list has
|
|
1223
|
-
// no place for it: `^x(,a$)*$` is not `^x(,a)*$`. Such a text is not a part of a list.
|
|
1224
|
-
let items = |t: &str| Sequence::parse(&format!("^{t}")).filter(|s| !s.to_end).map(|s| s.items.into_vec());
|
|
1225
|
-
let seq = |items: &[Item], to_end| Sequence { items: items.into(), to_end };
|
|
1226
|
-
let fixed = |items: &[Item]| !items.is_empty() && items.iter().all(|i| i.min == i.max && i.min > 0);
|
|
1227
|
-
let g = items(group)?;
|
|
1228
|
-
match (before.is_empty(), after.is_empty()) {
|
|
1229
|
-
// ^I(SR)*$
|
|
1230
|
-
(false, true) => {
|
|
1231
|
-
let first = items(unwrap_group(before)?)?;
|
|
1232
|
-
(1..g.len()).find_map(|k| {
|
|
1233
|
-
let (separator, repeated) = g.split_at(k);
|
|
1234
|
-
let next = separator[0].set;
|
|
1235
|
-
(fixed(separator)
|
|
1236
|
-
&& Sequence::greedy_before(&first, next)
|
|
1237
|
-
&& Sequence::greedy_before(repeated, next))
|
|
1238
|
-
.then(|| SeparatedList {
|
|
1239
|
-
first: Some(seq(&first, false)),
|
|
1240
|
-
repeated: seq(repeated, false),
|
|
1241
|
-
separator: seq(separator, false),
|
|
1242
|
-
final_: None,
|
|
1243
|
-
min_repeats,
|
|
1244
|
-
})
|
|
1245
|
-
})
|
|
1246
|
-
}
|
|
1247
|
-
// ^(RS)*F$
|
|
1248
|
-
(true, false) => {
|
|
1249
|
-
let final_ = Sequence::parse(&format!("^{}$", unwrap_group(after)?))?;
|
|
1250
|
-
(1..g.len()).find_map(|k| {
|
|
1251
|
-
let (repeated, separator) = g.split_at(k);
|
|
1252
|
-
(fixed(separator)
|
|
1253
|
-
&& Sequence::greedy_before(repeated, separator[0].set)
|
|
1254
|
-
&& final_stops_at(&final_, separator[0].set))
|
|
1255
|
-
.then(|| SeparatedList {
|
|
1256
|
-
first: None,
|
|
1257
|
-
repeated: seq(repeated, false),
|
|
1258
|
-
separator: seq(separator, false),
|
|
1259
|
-
final_: Some(Sequence { items: final_.items.clone(), to_end: true }),
|
|
1260
|
-
min_repeats,
|
|
1261
|
-
})
|
|
1262
|
-
})
|
|
1263
|
-
}
|
|
1264
|
-
_ => None,
|
|
1265
|
-
}
|
|
1266
|
-
}
|
|
1267
|
-
}
|
|
1268
|
-
|
|
1269
|
-
/// Whether the final sequence of a list cannot read past the separator that follows an item: no item of it holds a
|
|
1270
|
-
/// character the first item of the separator holds. Tried at the start of an item, it then fails at that separator
|
|
1271
|
-
/// at the latest, so trying it after every item reads the text a bounded number of times. A final sequence that can
|
|
1272
|
-
/// read through separators would read to the end of the text from every item.
|
|
1273
|
-
fn final_stops_at(final_: &Sequence, separator: CharSet) -> bool {
|
|
1274
|
-
final_.items.iter().all(|item| item.set.disjoint(separator))
|
|
1275
|
-
}
|
|
1276
|
-
|
|
1277
|
-
/// The inside of `(...)` or `(?:...)`; `None` for other groups.
|
|
1278
|
-
fn strip_group(g: &str) -> Option<&str> {
|
|
1279
|
-
let inner = g.strip_prefix('(')?.strip_suffix(')')?;
|
|
1280
|
-
match inner.strip_prefix("?:") {
|
|
1281
|
-
Some(rest) => Some(rest),
|
|
1282
|
-
None if inner.starts_with('?') => None,
|
|
1283
|
-
None => Some(inner),
|
|
1284
|
-
}
|
|
1285
|
-
}
|
|
1286
|
-
|
|
1287
|
-
/// A text that is one group wrapping everything, unwrapped; otherwise the text itself.
|
|
1288
|
-
fn unwrap_group(t: &str) -> Option<&str> {
|
|
1289
|
-
if t.starts_with('(') && t.ends_with(')') {
|
|
1290
|
-
let inner = strip_group(t)?;
|
|
1291
|
-
// Only when the parentheses enclose the whole text (`(a)(b)` is two groups).
|
|
1292
|
-
let mut depth = 0i32;
|
|
1293
|
-
let mut escaped = false;
|
|
1294
|
-
for c in inner.chars() {
|
|
1295
|
-
match c {
|
|
1296
|
-
_ if escaped => escaped = false,
|
|
1297
|
-
'\\' => escaped = true,
|
|
1298
|
-
'(' => depth += 1,
|
|
1299
|
-
')' => {
|
|
1300
|
-
depth -= 1;
|
|
1301
|
-
if depth < 0 {
|
|
1302
|
-
return None;
|
|
1303
|
-
}
|
|
1304
|
-
}
|
|
1305
|
-
_ => {}
|
|
1306
|
-
}
|
|
1307
|
-
}
|
|
1308
|
-
return Some(inner);
|
|
1309
|
-
}
|
|
1310
|
-
Some(t)
|
|
1311
1040
|
}
|
|
1312
1041
|
|
|
1313
1042
|
/// The set of a class escape (`\d`, `\w`, their negations, or an escaped punctuation character); `None` for `\s`
|
|
@@ -1427,338 +1156,26 @@ fn parse_quantifier(b: &[u8], mut i: usize) -> Option<(u32, u32, usize)> {
|
|
|
1427
1156
|
Some((min, max, i))
|
|
1428
1157
|
}
|
|
1429
1158
|
|
|
1430
|
-
// ---------------------------------------------------------------------------------------------------------------------
|
|
1431
|
-
// Translation to the regex crate
|
|
1432
|
-
|
|
1433
|
-
/// ECMA-262 `\s`: WhiteSpace and LineTerminator.
|
|
1434
|
-
const ECMA_SPACE: &str =
|
|
1435
|
-
r"\t\n\x{B}\x{C}\r \x{A0}\x{1680}\x{2000}-\x{200A}\x{2028}\x{2029}\x{202F}\x{205F}\x{3000}\x{FEFF}";
|
|
1436
|
-
|
|
1437
|
-
/// Translates an ECMA-262 pattern to the `regex` crate's syntax with the same meaning, or `None` when it uses
|
|
1438
|
-
/// something outside the common subset. With `unicode` false, the pattern is read without the `u` flag (identity
|
|
1439
|
-
/// escapes of any non-alphanumeric character), and the translation has its meaning on strings within the BMP.
|
|
1440
|
-
fn translate(p: &str, unicode: bool) -> Option<String> {
|
|
1441
|
-
let c: Vec<char> = p.chars().collect();
|
|
1442
|
-
// Without the flag a character beyond the BMP is two code units, and what is around it (a quantifier, the ends
|
|
1443
|
-
// of a range) applies to one of them. That has no translation.
|
|
1444
|
-
if !unicode && c.iter().any(|&ch| ch as u32 > 0xFFFF) {
|
|
1445
|
-
return None;
|
|
1446
|
-
}
|
|
1447
|
-
let mut out = String::with_capacity(p.len() * 2);
|
|
1448
|
-
let mut i = 0;
|
|
1449
|
-
while i < c.len() {
|
|
1450
|
-
match c[i] {
|
|
1451
|
-
'\\' => {
|
|
1452
|
-
let (text, next) = escape(&c, i + 1, false, unicode)?;
|
|
1453
|
-
out.push_str(&text);
|
|
1454
|
-
i = next;
|
|
1455
|
-
}
|
|
1456
|
-
'[' => {
|
|
1457
|
-
let (text, next) = class(&c, i + 1, unicode)?;
|
|
1458
|
-
out.push_str(&text);
|
|
1459
|
-
i = next;
|
|
1460
|
-
}
|
|
1461
|
-
'(' => {
|
|
1462
|
-
if c.get(i + 1) == Some(&'?') {
|
|
1463
|
-
match (c.get(i + 2), c.get(i + 3)) {
|
|
1464
|
-
(Some(':'), _) => i += 3,
|
|
1465
|
-
// A named group; lookbehinds (`(?<=`, `(?<!`) are not supported.
|
|
1466
|
-
(Some('<'), Some(n)) if *n != '=' && *n != '!' => {
|
|
1467
|
-
i += 3 + c[i + 3..].iter().position(|&x| x == '>')? + 1;
|
|
1468
|
-
}
|
|
1469
|
-
_ => return None,
|
|
1470
|
-
}
|
|
1471
|
-
} else {
|
|
1472
|
-
i += 1;
|
|
1473
|
-
}
|
|
1474
|
-
out.push_str("(?:");
|
|
1475
|
-
}
|
|
1476
|
-
'{' => {
|
|
1477
|
-
let end = i + c[i..].iter().position(|&x| x == '}')?;
|
|
1478
|
-
let body: String = c[i + 1..end].iter().collect();
|
|
1479
|
-
if body.is_empty() || !body.chars().all(|x| x.is_ascii_digit() || x == ',') {
|
|
1480
|
-
return None;
|
|
1481
|
-
}
|
|
1482
|
-
out.push('{');
|
|
1483
|
-
out.push_str(&body);
|
|
1484
|
-
out.push('}');
|
|
1485
|
-
i = end + 1;
|
|
1486
|
-
}
|
|
1487
|
-
'.' => {
|
|
1488
|
-
out.push_str(r"[^\n\r\x{2028}\x{2029}]");
|
|
1489
|
-
i += 1;
|
|
1490
|
-
}
|
|
1491
|
-
ch @ (')' | '|' | '^' | '$' | '*' | '+' | '?') => {
|
|
1492
|
-
out.push(ch);
|
|
1493
|
-
i += 1;
|
|
1494
|
-
}
|
|
1495
|
-
ch => {
|
|
1496
|
-
push_literal(&mut out, ch);
|
|
1497
|
-
i += 1;
|
|
1498
|
-
}
|
|
1499
|
-
}
|
|
1500
|
-
}
|
|
1501
|
-
Some(out)
|
|
1502
|
-
}
|
|
1503
|
-
|
|
1504
|
-
fn push_literal(out: &mut String, ch: char) {
|
|
1505
|
-
use std::fmt::Write;
|
|
1506
|
-
let _ = write!(out, "\\x{{{:X}}}", ch as u32);
|
|
1507
|
-
}
|
|
1508
|
-
|
|
1509
|
-
/// An escape after `\` at `i`: its translation and the index after it.
|
|
1510
|
-
fn escape(c: &[char], i: usize, in_class: bool, unicode: bool) -> Option<(String, usize)> {
|
|
1511
|
-
let e = *c.get(i)?;
|
|
1512
|
-
let mut out = String::new();
|
|
1513
|
-
let next = match e {
|
|
1514
|
-
'd' => {
|
|
1515
|
-
out.push_str(if in_class { "0-9" } else { "[0-9]" });
|
|
1516
|
-
i + 1
|
|
1517
|
-
}
|
|
1518
|
-
'D' => {
|
|
1519
|
-
out.push_str("[^0-9]");
|
|
1520
|
-
i + 1
|
|
1521
|
-
}
|
|
1522
|
-
'w' => {
|
|
1523
|
-
out.push_str(if in_class { "0-9A-Za-z_" } else { "[0-9A-Za-z_]" });
|
|
1524
|
-
i + 1
|
|
1525
|
-
}
|
|
1526
|
-
'W' => {
|
|
1527
|
-
out.push_str("[^0-9A-Za-z_]");
|
|
1528
|
-
i + 1
|
|
1529
|
-
}
|
|
1530
|
-
's' => {
|
|
1531
|
-
if in_class {
|
|
1532
|
-
out.push_str(ECMA_SPACE);
|
|
1533
|
-
} else {
|
|
1534
|
-
out.push('[');
|
|
1535
|
-
out.push_str(ECMA_SPACE);
|
|
1536
|
-
out.push(']');
|
|
1537
|
-
}
|
|
1538
|
-
i + 1
|
|
1539
|
-
}
|
|
1540
|
-
'S' => {
|
|
1541
|
-
out.push_str("[^");
|
|
1542
|
-
out.push_str(ECMA_SPACE);
|
|
1543
|
-
out.push(']');
|
|
1544
|
-
i + 1
|
|
1545
|
-
}
|
|
1546
|
-
'n' | 'r' | 't' | 'f' | 'v' => {
|
|
1547
|
-
let ch = match e {
|
|
1548
|
-
'n' => '\n',
|
|
1549
|
-
'r' => '\r',
|
|
1550
|
-
't' => '\t',
|
|
1551
|
-
'f' => '\x0C',
|
|
1552
|
-
_ => '\x0B',
|
|
1553
|
-
};
|
|
1554
|
-
push_literal(&mut out, ch);
|
|
1555
|
-
i + 1
|
|
1556
|
-
}
|
|
1557
|
-
'0' if !c.get(i + 1).is_some_and(char::is_ascii_digit) => {
|
|
1558
|
-
push_literal(&mut out, '\0');
|
|
1559
|
-
i + 1
|
|
1560
|
-
}
|
|
1561
|
-
'b' if in_class => {
|
|
1562
|
-
push_literal(&mut out, '\x08');
|
|
1563
|
-
i + 1
|
|
1564
|
-
}
|
|
1565
|
-
// ECMA's word boundaries are between ASCII word characters and the rest.
|
|
1566
|
-
'b' => {
|
|
1567
|
-
out.push_str("(?-u:\\b)");
|
|
1568
|
-
i + 1
|
|
1569
|
-
}
|
|
1570
|
-
// `\B` has no translation. `(?-u:\B)` also holds between the bytes of a character that takes several, where
|
|
1571
|
-
// no match may start or end, and `regex` 1.13.1 then says there is no match where it finds one: for
|
|
1572
|
-
// `s*?(?-u:\B)|..$` on `k\u{17f}k`, `is_match` is false and `find` is not. The engine has it.
|
|
1573
|
-
'B' => return None,
|
|
1574
|
-
// A Unicode property is written as the ranges of the engine's own tables, and never by its name: the
|
|
1575
|
-
// `regex` crate has tables of its own, of the version of Unicode its release carries, and a name would
|
|
1576
|
-
// match what that version says. Without the `u` flag `\p` is the letter.
|
|
1577
|
-
'p' | 'P' if unicode && c.get(i + 1) == Some(&'{') => {
|
|
1578
|
-
let end = i + 1 + c[i + 1..].iter().position(|&x| x == '}')?;
|
|
1579
|
-
let body: String = c[i + 2..end].iter().collect();
|
|
1580
|
-
let set = ecmaregex::property_set(&body)?;
|
|
1581
|
-
let negated = e == 'P';
|
|
1582
|
-
if set.ranges.is_empty() {
|
|
1583
|
-
// No character has the property: nothing, or with `\P` everything.
|
|
1584
|
-
out.push_str(if negated { r"[\x{0}-\x{10FFFF}]" } else { r"[^\x{0}-\x{10FFFF}]" });
|
|
1585
|
-
} else if in_class && !negated {
|
|
1586
|
-
push_ranges(&mut out, &set);
|
|
1587
|
-
} else {
|
|
1588
|
-
// A class of its own, which is also how a class holds the complement of a property.
|
|
1589
|
-
out.push_str(if negated { "[^" } else { "[" });
|
|
1590
|
-
push_ranges(&mut out, &set);
|
|
1591
|
-
out.push(']');
|
|
1592
|
-
}
|
|
1593
|
-
end + 1
|
|
1594
|
-
}
|
|
1595
|
-
'x' => {
|
|
1596
|
-
let hex: String = c.get(i + 1..i + 3)?.iter().collect();
|
|
1597
|
-
push_literal(&mut out, char::from_u32(u32::from_str_radix(&hex, 16).ok()?)?);
|
|
1598
|
-
i + 3
|
|
1599
|
-
}
|
|
1600
|
-
// Without the `u` flag, `\u{...}` is not a code point escape.
|
|
1601
|
-
'u' if unicode || c.get(i + 1) != Some(&'{') => {
|
|
1602
|
-
let (code, next) = unicode_escape(c, i + 1)?;
|
|
1603
|
-
// Without the flag, two escapes that are a surrogate pair are two code units.
|
|
1604
|
-
if !unicode && code > 0xFFFF {
|
|
1605
|
-
return None;
|
|
1606
|
-
}
|
|
1607
|
-
push_literal(&mut out, char::from_u32(code)?);
|
|
1608
|
-
next
|
|
1609
|
-
}
|
|
1610
|
-
// Identity escapes of syntax characters and `/` (and `-` in a class).
|
|
1611
|
-
'^' | '$' | '\\' | '.' | '*' | '+' | '?' | '(' | ')' | '[' | ']' | '{' | '}' | '|' | '/' | '-' => {
|
|
1612
|
-
push_literal(&mut out, e);
|
|
1613
|
-
i + 1
|
|
1614
|
-
}
|
|
1615
|
-
// Without the `u` flag, any other non-alphanumeric character escapes to itself.
|
|
1616
|
-
e if !unicode && !e.is_ascii_alphanumeric() => {
|
|
1617
|
-
push_literal(&mut out, e);
|
|
1618
|
-
i + 1
|
|
1619
|
-
}
|
|
1620
|
-
// Backreferences, \c, \k and the rest.
|
|
1621
|
-
_ => return None,
|
|
1622
|
-
};
|
|
1623
|
-
Some((out, next))
|
|
1624
|
-
}
|
|
1625
|
-
|
|
1626
|
-
/// The ranges of a set, each end spelled as an escape. A surrogate is no character to the `regex` crate, and no
|
|
1627
|
-
/// string holds one, so a range is written without the surrogates it spans. Two ranges that then touch, one ending
|
|
1628
|
-
/// at U+D7FF and the next starting at U+E000, are written as one: see [`class`] for what `regex` makes of the two.
|
|
1629
|
-
fn push_ranges(out: &mut String, set: &ecmaregex::CharSet) {
|
|
1630
|
-
// The range that has been read and not yet written.
|
|
1631
|
-
let mut pending: Option<(u32, u32)> = None;
|
|
1632
|
-
let write = |out: &mut String, (lo, hi): (u32, u32)| {
|
|
1633
|
-
if let (Some(lo), Some(hi)) = (char::from_u32(lo), char::from_u32(hi)) {
|
|
1634
|
-
push_literal(out, lo);
|
|
1635
|
-
if lo != hi {
|
|
1636
|
-
out.push('-');
|
|
1637
|
-
push_literal(out, hi);
|
|
1638
|
-
}
|
|
1639
|
-
}
|
|
1640
|
-
};
|
|
1641
|
-
for &[lo, hi] in set.ranges.as_chunks::<2>().0 {
|
|
1642
|
-
// The ends that are surrogates move out of the surrogates. A range of surrogates alone is nothing.
|
|
1643
|
-
let lo = if (0xD800..=0xDFFF).contains(&lo) { 0xE000 } else { lo as u32 };
|
|
1644
|
-
let hi = if (0xD800..=0xDFFF).contains(&hi) { 0xD7FF } else { hi as u32 };
|
|
1645
|
-
if lo > hi {
|
|
1646
|
-
continue;
|
|
1647
|
-
}
|
|
1648
|
-
match pending {
|
|
1649
|
-
Some((first, 0xD7FF)) if lo == 0xE000 => pending = Some((first, hi)),
|
|
1650
|
-
Some(range) => {
|
|
1651
|
-
write(out, range);
|
|
1652
|
-
pending = Some((lo, hi));
|
|
1653
|
-
}
|
|
1654
|
-
None => pending = Some((lo, hi)),
|
|
1655
|
-
}
|
|
1656
|
-
}
|
|
1657
|
-
if let Some(range) = pending {
|
|
1658
|
-
write(out, range);
|
|
1659
|
-
}
|
|
1660
|
-
}
|
|
1661
|
-
|
|
1662
|
-
/// `\u` escapes after the `u`: `XXXX` (joining a surrogate pair) or `{X...}`.
|
|
1663
|
-
fn unicode_escape(c: &[char], i: usize) -> Option<(u32, usize)> {
|
|
1664
|
-
if c.get(i) == Some(&'{') {
|
|
1665
|
-
let end = i + c[i..].iter().position(|&x| x == '}')?;
|
|
1666
|
-
let hex: String = c[i + 1..end].iter().collect();
|
|
1667
|
-
return Some((u32::from_str_radix(&hex, 16).ok()?, end + 1));
|
|
1668
|
-
}
|
|
1669
|
-
let four = |at: usize| -> Option<u32> {
|
|
1670
|
-
let hex: String = c.get(at..at + 4)?.iter().collect();
|
|
1671
|
-
if hex.len() != 4 {
|
|
1672
|
-
return None;
|
|
1673
|
-
}
|
|
1674
|
-
u32::from_str_radix(&hex, 16).ok()
|
|
1675
|
-
};
|
|
1676
|
-
let hi = four(i)?;
|
|
1677
|
-
if (0xD800..0xDC00).contains(&hi) {
|
|
1678
|
-
if c.get(i + 4) == Some(&'\\') && c.get(i + 5) == Some(&'u') {
|
|
1679
|
-
let lo = four(i + 6)?;
|
|
1680
|
-
if (0xDC00..0xE000).contains(&lo) {
|
|
1681
|
-
return Some((0x10000 + ((hi - 0xD800) << 10) + (lo - 0xDC00), i + 10));
|
|
1682
|
-
}
|
|
1683
|
-
}
|
|
1684
|
-
return None;
|
|
1685
|
-
}
|
|
1686
|
-
Some((hi, i + 4))
|
|
1687
|
-
}
|
|
1688
|
-
|
|
1689
|
-
/// A class after `[` at `i`: its translation (every member spelled as an escape, so nothing in it is special to the
|
|
1690
|
-
/// `regex` crate) and the index after `]`.
|
|
1691
|
-
fn class(c: &[char], mut i: usize, unicode: bool) -> Option<(String, usize)> {
|
|
1692
|
-
let mut out = String::from("[");
|
|
1693
|
-
if c.get(i) == Some(&'^') {
|
|
1694
|
-
out.push('^');
|
|
1695
|
-
i += 1;
|
|
1696
|
-
}
|
|
1697
|
-
let mut members = 0;
|
|
1698
|
-
loop {
|
|
1699
|
-
let ch = *c.get(i)?;
|
|
1700
|
-
if ch == ']' {
|
|
1701
|
-
i += 1;
|
|
1702
|
-
break;
|
|
1703
|
-
}
|
|
1704
|
-
// One atom: a character (which can start a range) or a class escape.
|
|
1705
|
-
let (atom, single, next) = if ch == '\\' {
|
|
1706
|
-
let e = *c.get(i + 1)?;
|
|
1707
|
-
let (text, next) = escape(c, i + 1, true, unicode)?;
|
|
1708
|
-
let single = match e {
|
|
1709
|
-
'd' | 'D' | 'w' | 'W' | 's' | 'S' | 'p' | 'P' => None,
|
|
1710
|
-
_ => Some(text.clone()),
|
|
1711
|
-
};
|
|
1712
|
-
(text, single, next)
|
|
1713
|
-
} else {
|
|
1714
|
-
let mut t = String::new();
|
|
1715
|
-
push_literal(&mut t, ch);
|
|
1716
|
-
(t.clone(), Some(t), i + 1)
|
|
1717
|
-
};
|
|
1718
|
-
i = next;
|
|
1719
|
-
members += 1;
|
|
1720
|
-
if c.get(i) == Some(&'-') && c.get(i + 1).is_some_and(|&n| n != ']') {
|
|
1721
|
-
// A range: both ends single characters.
|
|
1722
|
-
let lo = single?;
|
|
1723
|
-
let (hi, next) = if c[i + 1] == '\\' {
|
|
1724
|
-
let e = *c.get(i + 2)?;
|
|
1725
|
-
if matches!(e, 'd' | 'D' | 'w' | 'W' | 's' | 'S' | 'p' | 'P') {
|
|
1726
|
-
return None;
|
|
1727
|
-
}
|
|
1728
|
-
escape(c, i + 2, true, unicode)?
|
|
1729
|
-
} else {
|
|
1730
|
-
let mut t = String::new();
|
|
1731
|
-
push_literal(&mut t, c[i + 1]);
|
|
1732
|
-
(t, i + 2)
|
|
1733
|
-
};
|
|
1734
|
-
out.push_str(&lo);
|
|
1735
|
-
out.push('-');
|
|
1736
|
-
out.push_str(&hi);
|
|
1737
|
-
i = next;
|
|
1738
|
-
} else {
|
|
1739
|
-
out.push_str(&atom);
|
|
1740
|
-
}
|
|
1741
|
-
}
|
|
1742
|
-
if members == 0 {
|
|
1743
|
-
return None; // `[]` and `[^]` have no counterpart.
|
|
1744
|
-
}
|
|
1745
|
-
// `regex` 1.13.1 (regex-syntax 0.8.11) negates wrongly a class with a range that ends at U+D7FF and another
|
|
1746
|
-
// that starts at U+E000, the characters on either side of the surrogates: the complement takes both in, so
|
|
1747
|
-
// `[^\x{D7FF}\x{E000}]` matches every character. A negated class that names both (every literal here is
|
|
1748
|
-
// spelled `\x{...}`, so a `^` is a negation) is left to the engine.
|
|
1749
|
-
if out.contains('^') && out.contains(r"\x{D7FF}") && out.contains(r"\x{E000}") {
|
|
1750
|
-
return None;
|
|
1751
|
-
}
|
|
1752
|
-
out.push(']');
|
|
1753
|
-
Some((out, i))
|
|
1754
|
-
}
|
|
1755
|
-
|
|
1756
1159
|
#[cfg(test)]
|
|
1757
1160
|
mod tests {
|
|
1758
1161
|
use super::*;
|
|
1759
1162
|
|
|
1163
|
+
const G: Grammar = Grammar { edition: Edition::Current, unicode_version: UnicodeVersion::Default };
|
|
1164
|
+
|
|
1165
|
+
/// A pattern compiled in today's grammar.
|
|
1166
|
+
fn compile(p: &str) -> Result<Arc<Pattern>, PatternError> {
|
|
1167
|
+
super::compile(p, G)
|
|
1168
|
+
}
|
|
1169
|
+
|
|
1760
1170
|
const PATTERNS: &[&str] = &[
|
|
1761
1171
|
"",
|
|
1172
|
+
// Sequences without `$`, which need only the start of the string.
|
|
1173
|
+
"^[a-z]+",
|
|
1174
|
+
"^[a-z]*",
|
|
1175
|
+
"^[a-zA-Z0-9_\\.\\-]*",
|
|
1176
|
+
"^a[b-c]{2,5}",
|
|
1177
|
+
"^[a-z]+[0-9]*",
|
|
1178
|
+
"^[a-z]{0,3}[0-9]{2,4}",
|
|
1762
1179
|
".*",
|
|
1763
1180
|
"^.*",
|
|
1764
1181
|
".*$",
|
|
@@ -1919,18 +1336,17 @@ mod tests {
|
|
|
1919
1336
|
seed
|
|
1920
1337
|
};
|
|
1921
1338
|
for &p in PATTERNS {
|
|
1922
|
-
let reference =
|
|
1339
|
+
let reference = G.build(p, true).or_else(|_| G.build(p, false)).unwrap();
|
|
1923
1340
|
let compiled = compile(p).unwrap();
|
|
1924
1341
|
for _ in 0..4000 {
|
|
1925
1342
|
let len = next() % 9;
|
|
1926
1343
|
let s: String = (0..len).map(|_| alphabet[(next() % alphabet.len() as u64) as usize]).collect();
|
|
1927
1344
|
// Without the `u` flag the engine matches the UTF-16 code units of the string.
|
|
1928
|
-
let found = reference.
|
|
1345
|
+
let found = reference.is_match(&s).unwrap();
|
|
1929
1346
|
for limit in [None, Some(Duration::from_secs(60)), Some(Duration::from_nanos(1))] {
|
|
1930
1347
|
let mut budget = PatternBudget::new(PatternLimit::new(limit));
|
|
1931
1348
|
let got = compiled.is_match(&s, &mut budget);
|
|
1932
|
-
// A
|
|
1933
|
-
// it. A match that reached its first check has no answer.
|
|
1349
|
+
// A match on the engine that reached its first check of the clock has no answer.
|
|
1934
1350
|
assert!(got == found || budget.failed(), "{p:?} on {s:?} with a limit of {limit:?}");
|
|
1935
1351
|
assert!(limit.is_some_and(|l| l.as_nanos() == 1) || !budget.failed(), "{p:?} on {s:?}");
|
|
1936
1352
|
}
|
|
@@ -1946,71 +1362,55 @@ mod tests {
|
|
|
1946
1362
|
for p in ["^x-", "^\\/", "^abc$"] {
|
|
1947
1363
|
assert!(matches!(compile(p).unwrap().matcher, Matcher::Literal { .. }), "{p}");
|
|
1948
1364
|
}
|
|
1949
|
-
|
|
1950
|
-
assert!(matches!(compile(p).unwrap().matcher, Matcher::Regex(_)), "{p}");
|
|
1951
|
-
}
|
|
1365
|
+
assert!(matches!(compile("^[a-z]*a$").unwrap().matcher, Matcher::Engine(_)));
|
|
1952
1366
|
for p in [".+", "^.+", "(.+)"] {
|
|
1953
1367
|
assert!(matches!(compile(p).unwrap().matcher, Matcher::HasContent { .. }), "{p}");
|
|
1954
1368
|
}
|
|
1955
1369
|
for p in ["^.{1,256}$", "^.+$", "^.*$", "^(.*)$", "^(.+)$"] {
|
|
1956
1370
|
assert!(matches!(compile(p).unwrap().matcher, Matcher::Line { .. }), "{p}");
|
|
1957
1371
|
}
|
|
1958
|
-
for p in ["^(ab|cd)$", "^(?:es|ES|x-)$"] {
|
|
1372
|
+
for p in ["^(ab|cd)$", "^(?:es|ES|x-)$", "^(CRLF|LF|crlf|lf)$"] {
|
|
1959
1373
|
assert!(matches!(compile(p).unwrap().matcher, Matcher::Literals(_)), "{p}");
|
|
1960
1374
|
}
|
|
1375
|
+
// Without `$`, a sequence is read only as far as it must be.
|
|
1376
|
+
for p in ["^[a-z]+", "^a[b-c]{2,5}"] {
|
|
1377
|
+
assert!(
|
|
1378
|
+
matches!(&compile(p).unwrap().matcher, Matcher::Sequence(seq) if seq.items.last().unwrap().max == seq.items.last().unwrap().min),
|
|
1379
|
+
"{p}"
|
|
1380
|
+
);
|
|
1381
|
+
}
|
|
1382
|
+
assert!(matches!(&compile("^[a-zA-Z0-9_\\.\\-]*").unwrap().matcher, Matcher::Everything));
|
|
1383
|
+
// At most two alternatives, neither anchored at both ends.
|
|
1384
|
+
for p in ["^ab|cd$", "a|b", "^([a|A][u|U][t|T][o|O])|([n|N][o|O][n|N][e|E])$", "(base64key|awskms)://(.*)"] {
|
|
1385
|
+
assert!(matches!(compile(p).unwrap().matcher, Matcher::Alternatives(_)), "{p}");
|
|
1386
|
+
}
|
|
1387
|
+
// The engine is faster for more literals, more alternatives, an alternative anchored at both ends, and a
|
|
1388
|
+
// separated list.
|
|
1961
1389
|
for p in [
|
|
1962
|
-
"^
|
|
1963
|
-
"a|b",
|
|
1964
|
-
"^([a|A][u|U][t|T][o|O])|([n|N][o|O][n|N][e|E])$",
|
|
1390
|
+
"^(schemas|responses|parameters|examples|requestBodies)$",
|
|
1965
1391
|
"^[Ee][Ss]2015(\\.([Cc][Oo][Rr][Ee]|[Pp][Rr][Oo][Xx][Yy]))?$",
|
|
1966
1392
|
"^[1-5](?:[0-9]{2}|XX)$",
|
|
1967
1393
|
"(^([0-9]+)\\.([0-9]+)$)|(^\\{[A-F0-9]{8}(-[A-F0-9]{4}){3}-[A-F0-9]{12}\\}$)",
|
|
1968
1394
|
"^([t|T][o|O][p|P])|([c|C][e|E][n|N][t|T][e|E][r|R])|([b|B][o|O][t|T][t|T][o|O][m|M])$",
|
|
1969
1395
|
"^([A-Za-z_][-A-Za-z0-9_.:]*)$",
|
|
1970
|
-
|
|
1971
|
-
assert!(matches!(compile(p).unwrap().matcher, Matcher::Alternatives(_)), "{p}");
|
|
1972
|
-
}
|
|
1973
|
-
for p in [
|
|
1396
|
+
"default|^[0-9]+$",
|
|
1974
1397
|
"^([a-zA-Z0-9]{2,3})(-[a-zA-Z0-9]{1,6})*$",
|
|
1975
1398
|
"^([a-zA-Z_$][a-zA-Z0-9_$]{0,39}\\.)*([a-zA-Z_$][a-zA-Z0-9_$]{0,39})$",
|
|
1976
|
-
"^([a-z_$][a-z0-9_$]{0,39}\\.)*([a-zA-Z_$][a-zA-Z0-9_$]{0,39})$",
|
|
1977
|
-
// An escaped `$` or `^` is a literal character of the list.
|
|
1978
1399
|
"^x(,a\\$)*$",
|
|
1979
|
-
"^x\\$(,a)*$",
|
|
1980
|
-
"^(a\\$,)*b$",
|
|
1981
|
-
"^x(,\\^a)*$",
|
|
1982
|
-
"^(a,)*b\\$$",
|
|
1983
|
-
"^x(,[$^]a)*$",
|
|
1984
|
-
] {
|
|
1985
|
-
assert!(matches!(compile(p).unwrap().matcher, Matcher::SeparatedList(_)), "{p}");
|
|
1986
|
-
}
|
|
1987
|
-
// A list with an anchor or an assertion inside it has no shape.
|
|
1988
|
-
for p in [
|
|
1989
1400
|
"^x(,a$)*$",
|
|
1990
|
-
"^x(?:,a$)+$",
|
|
1991
|
-
"^x$(,a)*$",
|
|
1992
|
-
"^(x$)(,a)*$",
|
|
1993
|
-
"^$(,a)*$",
|
|
1994
|
-
"^(a,$)*b$",
|
|
1995
|
-
"^x(^,a)*$",
|
|
1996
|
-
"^(^a,)*b$",
|
|
1997
|
-
"^x(,(a$))*$",
|
|
1998
|
-
"^x(,a\\b)*$",
|
|
1999
|
-
"^x\\B(,a)*$",
|
|
2000
|
-
"^x(,a(?=,|$))*$",
|
|
2001
1401
|
"^x(?=,)(,a)*$",
|
|
2002
1402
|
] {
|
|
2003
|
-
assert!(matches!(compile(p).unwrap().matcher, Matcher::
|
|
1403
|
+
assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
|
|
2004
1404
|
}
|
|
2005
1405
|
assert!(matches!(
|
|
2006
1406
|
compile("^(?=[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$").unwrap().matcher,
|
|
2007
1407
|
Matcher::ExcludedClassWithWord { .. }
|
|
2008
1408
|
));
|
|
2009
1409
|
for p in ["\\bfoo", "^\\p{L}+$"] {
|
|
2010
|
-
assert!(matches!(compile(p).unwrap().matcher, Matcher::
|
|
1410
|
+
assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
|
|
2011
1411
|
}
|
|
2012
1412
|
for p in ["^\\/[^\\*\\?\\&\\%]*(\\/\\*)?$", "a\\&.b", "^\\-a"] {
|
|
2013
|
-
assert!(matches!(compile(p).unwrap().matcher, Matcher::
|
|
1413
|
+
assert!(matches!(&compile(p).unwrap().matcher, Matcher::Engine(engine) if !engine.flags().unicode), "{p}");
|
|
2014
1414
|
}
|
|
2015
1415
|
for p in ["^((\\.(?!\\.)\\/)?\\w+\\/?)+$", "^\\1(a)", "\\Bs", "(?i:a)", "^[]", "\\cA"] {
|
|
2016
1416
|
assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
|
|
@@ -2018,7 +1418,7 @@ mod tests {
|
|
|
2018
1418
|
// Valid only without the `u` flag, and not in the syntax `regex` takes: the engine, by code unit.
|
|
2019
1419
|
for p in ["^\\p{L}\\&", "^\\-?\u{10400}*$", "^\\-(?=a)"] {
|
|
2020
1420
|
let compiled = compile(p).unwrap();
|
|
2021
|
-
assert!(matches!(&compiled.matcher, Matcher::Engine(engine) if !engine.
|
|
1421
|
+
assert!(matches!(&compiled.matcher, Matcher::Engine(engine) if !engine.flags().unicode), "{p}");
|
|
2022
1422
|
}
|
|
2023
1423
|
}
|
|
2024
1424
|
}
|
|
@@ -2027,6 +1427,13 @@ mod tests {
|
|
|
2027
1427
|
mod limits {
|
|
2028
1428
|
use super::*;
|
|
2029
1429
|
|
|
1430
|
+
const G: Grammar = Grammar { edition: Edition::Current, unicode_version: UnicodeVersion::Default };
|
|
1431
|
+
|
|
1432
|
+
/// A pattern compiled in today's grammar.
|
|
1433
|
+
fn compile(p: &str) -> Result<Arc<Pattern>, PatternError> {
|
|
1434
|
+
super::compile(p, G)
|
|
1435
|
+
}
|
|
1436
|
+
|
|
2030
1437
|
/// Which matcher a pattern gets (the Go module's `TestAWideSearchIsLeftToTheEngine`). A sequence or a literal
|
|
2031
1438
|
/// that would be searched for at every position of the text keeps its matcher up to 64 characters, and beyond
|
|
2032
1439
|
/// that is left to a matcher that takes time proportional to the text or has the limit of a match.
|
|
@@ -2047,55 +1454,10 @@ mod limits {
|
|
|
2047
1454
|
] {
|
|
2048
1455
|
let found = kind(&compile(&pattern).unwrap());
|
|
2049
1456
|
assert_eq!(found == "alternatives", alternatives, "{pattern:.30} has the matcher {found}");
|
|
2050
|
-
assert!(alternatives ||
|
|
2051
|
-
}
|
|
2052
|
-
// The final sequence of a list must stop at the separator.
|
|
2053
|
-
for (pattern, list) in [
|
|
2054
|
-
(r"^(?:[a-z]+,)*[a-z]{2}[0-9]$", true),
|
|
2055
|
-
(r"^(?:[a-z]+,)*[a-z,]+[0-9]$", false),
|
|
2056
|
-
(r"^(?:[a-z]+,)*[0-9]{1,3}$", true),
|
|
2057
|
-
] {
|
|
2058
|
-
let found = kind(&compile(pattern).unwrap());
|
|
2059
|
-
assert_eq!(found == "list", list, "{pattern} has the matcher {found}");
|
|
2060
|
-
}
|
|
2061
|
-
}
|
|
2062
|
-
|
|
2063
|
-
/// The `regex` crate is given no large program (the Go module's `TestTheRegexpPackageIsGivenNoLargeProgram`):
|
|
2064
|
-
/// it writes a counted repetition out, so the pattern goes to the engine when the copies are many.
|
|
2065
|
-
#[test]
|
|
2066
|
-
fn regex_is_given_no_large_program() {
|
|
2067
|
-
for (pattern, want) in [
|
|
2068
|
-
(r"a{1000}".to_string(), "regex"),
|
|
2069
|
-
(r"(?:a|b|ab)*c".to_string(), "regex"),
|
|
2070
|
-
(r"\p{L}{1000}".repeat(90), "engine"),
|
|
2071
|
-
(r"(?:[ab]{1000}){50}c".to_string(), "engine"),
|
|
2072
|
-
(r"(?:a{100}){101}".to_string(), "engine"),
|
|
2073
|
-
(r"(?:(?:a?){100}){100}b".to_string(), "engine"),
|
|
2074
|
-
] {
|
|
2075
|
-
assert_eq!(kind(&compile(&pattern).unwrap()), want, "{pattern:.40}");
|
|
1457
|
+
assert!(alternatives || found == "engine", "{pattern:.30} has the matcher {found}");
|
|
2076
1458
|
}
|
|
2077
1459
|
}
|
|
2078
1460
|
|
|
2079
|
-
/// A string too long for `regex` to answer within the limit of a match goes to the engine, which can be
|
|
2080
|
-
/// stopped (the Go module's `TestALongTextIsNotGivenToTheRegexpPackage`).
|
|
2081
|
-
#[test]
|
|
2082
|
-
fn a_long_string_is_not_given_to_regex() {
|
|
2083
|
-
let compiled = compile(r"^(?:[a-c\u00e9]{0,500}x)*$").unwrap();
|
|
2084
|
-
let Matcher::Regex(linear) = &compiled.matcher else { panic!("the pattern is on {}", kind(&compiled)) };
|
|
2085
|
-
assert!(linear.cost >= 500, "a program of {} instructions", linear.cost);
|
|
2086
|
-
let short = "\u{e9}abcx".to_string();
|
|
2087
|
-
let long = "\u{e9}abcx".repeat(100000);
|
|
2088
|
-
let second = PatternLimit::new(Some(Duration::from_secs(1)));
|
|
2089
|
-
let mut budget = PatternBudget::new(second);
|
|
2090
|
-
assert!(linear.fits(&short, &budget) && !linear.fits(&long, &budget), "the strings are not on each side");
|
|
2091
|
-
assert!(compiled.is_match(&short, &mut budget) && linear.engine.get().is_none(), "a short string");
|
|
2092
|
-
assert!(compiled.is_match(&long, &mut budget) && !budget.failed(), "a long string");
|
|
2093
|
-
assert!(matches!(linear.engine.get(), Some(Some(_))), "the engine was not made for the long string");
|
|
2094
|
-
// With no limit every string is `regex`'s.
|
|
2095
|
-
let mut unlimited = PatternBudget::new(PatternLimit::new(None));
|
|
2096
|
-
assert!(linear.fits(&long, &unlimited) && !compiled.is_match(&format!("{long}y"), &mut unlimited));
|
|
2097
|
-
}
|
|
2098
|
-
|
|
2099
1461
|
/// The cache of compiled patterns is bounded in number and in size, and a pattern that has left it is still
|
|
2100
1462
|
/// matched by whoever holds it.
|
|
2101
1463
|
#[test]
|
|
@@ -2109,7 +1471,7 @@ mod limits {
|
|
|
2109
1471
|
assert!(cache.patterns.len() <= MAX_CACHED_PATTERNS && cache.bytes <= MAX_CACHED_BYTES);
|
|
2110
1472
|
assert_eq!(cache.patterns.len(), cache.order.len());
|
|
2111
1473
|
assert_eq!(cache.bytes, cache.patterns.values().map(|p| p.retained).sum::<usize>());
|
|
2112
|
-
assert!(cache.get("^cache-bound-0+$").is_none(), "the oldest pattern is still in the cache");
|
|
1474
|
+
assert!(cache.get("^cache-bound-0+$", G).is_none(), "the oldest pattern is still in the cache");
|
|
2113
1475
|
}
|
|
2114
1476
|
assert!(first.is_match("cache-bound-000", &mut budget));
|
|
2115
1477
|
assert!(held[0].is_match("cache-bound-0x", &mut budget) && !held[7].is_match("cache-bound-0x", &mut budget));
|
|
@@ -2122,154 +1484,162 @@ mod limits {
|
|
|
2122
1484
|
assert!(large[0].retained > 256 << 10, "a large pattern is counted as {} bytes", large[0].retained);
|
|
2123
1485
|
assert!(cache().bytes <= MAX_CACHED_BYTES);
|
|
2124
1486
|
// A pattern too large for the cache is not kept at all.
|
|
2125
|
-
let huge = compile(&format!("^(?=huge){}$",
|
|
2126
|
-
assert!(huge.retained > MAX_CACHED_PATTERN_BYTES && cache().get(&huge.source).is_none());
|
|
1487
|
+
let huge = compile(&format!("^(?=huge){}$", class.repeat(4))).unwrap();
|
|
1488
|
+
assert!(huge.retained > MAX_CACHED_PATTERN_BYTES && cache().get(&huge.source, G).is_none());
|
|
2127
1489
|
}
|
|
1490
|
+
}
|
|
2128
1491
|
|
|
2129
|
-
|
|
2130
|
-
|
|
2131
|
-
|
|
2132
|
-
|
|
2133
|
-
|
|
2134
|
-
|
|
2135
|
-
|
|
2136
|
-
|
|
2137
|
-
|
|
2138
|
-
|
|
2139
|
-
|
|
2140
|
-
assert_eq!(kind(&compiled), want, "{pattern}");
|
|
2141
|
-
let mut budget = PatternBudget::new(PatternLimit::new(Some(Duration::from_secs(1))));
|
|
2142
|
-
assert!(compiled.is_match("caf\u{e9}", &mut budget) && !compiled.is_match("caf\u{e9}!", &mut budget));
|
|
2143
|
-
}
|
|
1492
|
+
#[cfg(test)]
|
|
1493
|
+
mod corpus_patterns {
|
|
1494
|
+
use std::sync::Arc;
|
|
1495
|
+
|
|
1496
|
+
use super::{Edition, Grammar, Pattern, PatternError, UnicodeVersion};
|
|
1497
|
+
|
|
1498
|
+
const G: Grammar = Grammar { edition: Edition::Current, unicode_version: UnicodeVersion::Default };
|
|
1499
|
+
|
|
1500
|
+
/// A pattern compiled in today's grammar.
|
|
1501
|
+
fn compile(p: &str) -> Result<Arc<Pattern>, PatternError> {
|
|
1502
|
+
super::compile(p, G)
|
|
2144
1503
|
}
|
|
2145
1504
|
|
|
2146
|
-
///
|
|
2147
|
-
///
|
|
2148
|
-
/// default is still compiled by it, on a thread with a stack for the nesting.
|
|
1505
|
+
/// Lists which matcher each pattern in `CORVUS_PATTERNS` (corpus<TAB>pattern lines) gets, and what the cache
|
|
1506
|
+
/// counts for it.
|
|
2149
1507
|
#[test]
|
|
2150
|
-
|
|
2151
|
-
|
|
2152
|
-
let
|
|
2153
|
-
|
|
2154
|
-
|
|
2155
|
-
|
|
2156
|
-
|
|
2157
|
-
|
|
2158
|
-
assert!(linear.fits(&text, &budget) && !compiled.is_match(&text, &mut budget) && !budget.failed());
|
|
2159
|
-
assert!(compiled.is_match(&format!("{text}z"), &mut budget) && linear.engine.get().is_none());
|
|
2160
|
-
|
|
2161
|
-
for depth in [300, 999] {
|
|
2162
|
-
let nested = format!("^{}a{}b$", "(".repeat(depth), "*)".repeat(depth));
|
|
2163
|
-
let compiled = compile(&nested).unwrap();
|
|
2164
|
-
assert_eq!(kind(&compiled), "regex", "{depth} nested groups");
|
|
2165
|
-
let start = std::time::Instant::now();
|
|
2166
|
-
assert!(
|
|
2167
|
-
compiled.is_match("aaaab", &mut budget)
|
|
2168
|
-
&& !compiled.is_match("\u{e9}aaaaaaaaaaaaaaaaaaaaaaaaa", &mut budget)
|
|
2169
|
-
);
|
|
2170
|
-
assert!(!budget.failed() && start.elapsed() < Duration::from_secs(20), "{depth} nested groups");
|
|
1508
|
+
#[ignore]
|
|
1509
|
+
fn classify_corpus_patterns() {
|
|
1510
|
+
let Ok(path) = std::env::var("CORVUS_PATTERNS") else { return };
|
|
1511
|
+
for line in std::fs::read_to_string(path).unwrap().lines() {
|
|
1512
|
+
let (corpus, p) = line.split_once('\t').unwrap();
|
|
1513
|
+
let kind = compile(p).map(|c| super::kind(&c)).unwrap_or("invalid");
|
|
1514
|
+
let retained = compile(p).map_or(0, |c| c.retained);
|
|
1515
|
+
println!("{kind}\t{corpus}\t{p}\t{retained}");
|
|
2171
1516
|
}
|
|
2172
1517
|
}
|
|
2173
1518
|
|
|
2174
|
-
///
|
|
2175
|
-
///
|
|
1519
|
+
/// Times each dedicated matcher against the engine over the corpora of jsonschema-benchmark in `CORVUS_CORPORA`
|
|
1520
|
+
/// (a directory of `<name>-schema.json` and `<name>-instances.jsonl`), so that a matcher the engine beats can be
|
|
1521
|
+
/// removed. The inputs of a pattern are the distinct strings of the corpus's instances (for `pattern`) or their
|
|
1522
|
+
/// property names (for `patternProperties`), at most 1,000. Each figure is the least of 7 rounds of at least 20 ms,
|
|
1523
|
+
/// after a warm-up of 200 ms, through `Pattern::is_match` with the default limit of a match. Run it with
|
|
1524
|
+
/// `cargo test --release --lib time_corpus_patterns -- --ignored --nocapture --test-threads 1`.
|
|
2176
1525
|
#[test]
|
|
2177
|
-
|
|
2178
|
-
|
|
2179
|
-
|
|
2180
|
-
|
|
2181
|
-
|
|
2182
|
-
|
|
2183
|
-
|
|
2184
|
-
|
|
2185
|
-
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
|
|
1526
|
+
#[ignore]
|
|
1527
|
+
fn time_corpus_patterns() {
|
|
1528
|
+
use std::collections::BTreeSet;
|
|
1529
|
+
use std::time::{Duration, Instant};
|
|
1530
|
+
|
|
1531
|
+
use super::{Literals, Matcher, PatternBudget, PatternLimit};
|
|
1532
|
+
|
|
1533
|
+
fn collect(v: &serde_json::Value, strings: &mut BTreeSet<String>, keys: &mut BTreeSet<String>) {
|
|
1534
|
+
match v {
|
|
1535
|
+
serde_json::Value::String(s) => {
|
|
1536
|
+
strings.insert(s.clone());
|
|
1537
|
+
}
|
|
1538
|
+
serde_json::Value::Array(a) => a.iter().for_each(|v| collect(v, strings, keys)),
|
|
1539
|
+
serde_json::Value::Object(o) => o.iter().for_each(|(k, v)| {
|
|
1540
|
+
keys.insert(k.clone());
|
|
1541
|
+
collect(v, strings, keys);
|
|
1542
|
+
}),
|
|
1543
|
+
_ => {}
|
|
2191
1544
|
}
|
|
2192
1545
|
}
|
|
2193
|
-
|
|
2194
|
-
|
|
2195
|
-
|
|
2196
|
-
|
|
2197
|
-
|
|
2198
|
-
|
|
2199
|
-
|
|
2200
|
-
|
|
2201
|
-
|
|
2202
|
-
|
|
2203
|
-
|
|
2204
|
-
|
|
2205
|
-
|
|
2206
|
-
|
|
2207
|
-
|
|
2208
|
-
// the two ranges: Unknown holds U+D7FC to U+D7FF and U+E000 to U+F8FF. And the same written by hand, which
|
|
2209
|
-
// is the engine's.
|
|
2210
|
-
for (p, want) in [
|
|
2211
|
-
(r"^\P{sc=Unknown}$", "regex"),
|
|
2212
|
-
(r"^[^\p{scx=Unknown}]$", "regex"),
|
|
2213
|
-
(r"^[^\p{Script=Unknown}a]$", "regex"),
|
|
2214
|
-
(r"^[^\ud7ff\ue000]$", "engine"),
|
|
2215
|
-
(r"^[^\ud7fc-\ud7ff\ue000-\uf8ff]$", "engine"),
|
|
2216
|
-
(r"^[^\p{Co}\ud7ff]$", "engine"),
|
|
2217
|
-
(r"^[^\p{Cn}\ue000]$", "engine"),
|
|
2218
|
-
] {
|
|
2219
|
-
let compiled = compile(p).unwrap();
|
|
2220
|
-
assert_eq!(kind(&compiled), want, "{p}");
|
|
2221
|
-
let engine = ecmaregex::compile(p).unwrap();
|
|
2222
|
-
for text in ["\u{D7FB}", "\u{D7FC}", "\u{D7FF}", "\u{E000}", "\u{E001}", "\u{F8FF}", "\u{F900}", "a", "b"] {
|
|
2223
|
-
assert_eq!(
|
|
2224
|
-
compiled.is_match(text, &mut budget),
|
|
2225
|
-
engine.match_backtrack(text, None).unwrap(),
|
|
2226
|
-
"{p} on {text:?}"
|
|
2227
|
-
);
|
|
1546
|
+
fn patterns(v: &serde_json::Value, out: &mut BTreeSet<(&'static str, String)>) {
|
|
1547
|
+
match v {
|
|
1548
|
+
serde_json::Value::Array(a) => a.iter().for_each(|v| patterns(v, out)),
|
|
1549
|
+
serde_json::Value::Object(o) => {
|
|
1550
|
+
if let Some(serde_json::Value::String(p)) = o.get("pattern") {
|
|
1551
|
+
out.insert(("pattern", p.clone()));
|
|
1552
|
+
}
|
|
1553
|
+
if let Some(serde_json::Value::Object(pp)) = o.get("patternProperties") {
|
|
1554
|
+
pp.keys().for_each(|p| {
|
|
1555
|
+
out.insert(("patternProperties", p.clone()));
|
|
1556
|
+
});
|
|
1557
|
+
}
|
|
1558
|
+
o.values().for_each(|v| patterns(v, out));
|
|
1559
|
+
}
|
|
1560
|
+
_ => {}
|
|
2228
1561
|
}
|
|
2229
|
-
assert!(!compiled.is_match("\u{E000}", &mut budget), "{p} matches U+E000");
|
|
2230
|
-
}
|
|
2231
|
-
// A set with a range on each side of the surrogates and none of them is written as one range, and a range
|
|
2232
|
-
// of surrogates alone as nothing.
|
|
2233
|
-
let mut written = String::new();
|
|
2234
|
-
push_ranges(&mut written, &ecmaregex::CharSet::new(&[0x61, 0x7A, 0xD700, 0xD7FF, 0xE000, 0xE0FF], false));
|
|
2235
|
-
assert_eq!(written, r"\x{61}-\x{7A}\x{D700}-\x{E0FF}");
|
|
2236
|
-
written.clear();
|
|
2237
|
-
push_ranges(&mut written, &ecmaregex::CharSet::new(&[0xD7FF, 0xDBFF, 0xDC00, 0xDFFF, 0xDFFF, 0xE000], false));
|
|
2238
|
-
assert_eq!(written, r"\x{D7FF}-\x{E000}");
|
|
2239
|
-
written.clear();
|
|
2240
|
-
push_ranges(&mut written, &ecmaregex::CharSet::new(&[0xD800, 0xDFFF], false));
|
|
2241
|
-
assert_eq!(written, "");
|
|
2242
|
-
// A property no character has, and its complement.
|
|
2243
|
-
for (p, yes, no) in
|
|
2244
|
-
[(r"^\P{sc=Hrkt}$", "a", "aa"), (r"^[\p{sc=Hrkt}a]$", "a", "b"), (r"^[^\P{sc=Hrkt}]?$", "", "a")]
|
|
2245
|
-
{
|
|
2246
|
-
let compiled = compile(p).unwrap();
|
|
2247
|
-
assert_eq!(kind(&compiled), "regex", "{p}");
|
|
2248
|
-
assert!(compiled.is_match(yes, &mut budget) && !compiled.is_match(no, &mut budget), "{p}");
|
|
2249
1562
|
}
|
|
2250
|
-
|
|
2251
|
-
|
|
2252
|
-
|
|
2253
|
-
|
|
2254
|
-
|
|
2255
|
-
/// Lists which matcher each pattern in `CORVUS_PATTERNS` (corpus<TAB>pattern lines) gets.
|
|
2256
|
-
#[test]
|
|
2257
|
-
#[ignore]
|
|
2258
|
-
fn classify_corpus_patterns() {
|
|
2259
|
-
let Ok(path) = std::env::var("CORVUS_PATTERNS") else { return };
|
|
2260
|
-
for line in std::fs::read_to_string(path).unwrap().lines() {
|
|
2261
|
-
let (corpus, p) = line.split_once('\t').unwrap();
|
|
2262
|
-
let kind = super::compile(p).map(|c| super::kind(&c)).unwrap_or("invalid");
|
|
2263
|
-
// What the `regex` crate is given: the translation.
|
|
2264
|
-
let unicode = crate::ecmaregex::valid(p);
|
|
2265
|
-
let given = match kind {
|
|
2266
|
-
"regex" | "regex-bmp" => super::translate(p, unicode).unwrap_or_default(),
|
|
2267
|
-
_ => String::new(),
|
|
1563
|
+
fn time(p: &Pattern, inputs: &[String]) -> f64 {
|
|
1564
|
+
let limit = PatternLimit::new(Some(crate::DEFAULT_PATTERN_MATCH_LIMIT));
|
|
1565
|
+
let pass = || {
|
|
1566
|
+
let mut budget = PatternBudget::new(limit);
|
|
1567
|
+
inputs.iter().filter(|s| p.is_match(s, &mut budget)).count()
|
|
2268
1568
|
};
|
|
2269
|
-
|
|
2270
|
-
|
|
2271
|
-
|
|
2272
|
-
|
|
1569
|
+
let warm = Instant::now();
|
|
1570
|
+
while warm.elapsed() < Duration::from_millis(200) {
|
|
1571
|
+
std::hint::black_box(pass());
|
|
1572
|
+
}
|
|
1573
|
+
let mut best = f64::MAX;
|
|
1574
|
+
for _ in 0..7 {
|
|
1575
|
+
let (start, mut calls) = (Instant::now(), 0usize);
|
|
1576
|
+
while start.elapsed() < Duration::from_millis(20) {
|
|
1577
|
+
std::hint::black_box(pass());
|
|
1578
|
+
calls += inputs.len();
|
|
1579
|
+
}
|
|
1580
|
+
best = best.min(start.elapsed().as_nanos() as f64 / calls as f64);
|
|
1581
|
+
}
|
|
1582
|
+
best
|
|
1583
|
+
}
|
|
1584
|
+
|
|
1585
|
+
let Ok(dir) = std::env::var("CORVUS_CORPORA") else { return };
|
|
1586
|
+
let mut names: Vec<String> = std::fs::read_dir(&dir)
|
|
1587
|
+
.unwrap()
|
|
1588
|
+
.filter_map(|e| e.unwrap().file_name().to_str()?.strip_suffix("-schema.json").map(str::to_string))
|
|
1589
|
+
.collect();
|
|
1590
|
+
names.sort();
|
|
1591
|
+
// The alternatives a matcher tries one after another.
|
|
1592
|
+
fn alternatives(m: &Matcher) -> usize {
|
|
1593
|
+
match m {
|
|
1594
|
+
Matcher::Alternatives(a) => a.len(),
|
|
1595
|
+
Matcher::Literals(Literals(t)) => t.len(),
|
|
1596
|
+
_ => 1,
|
|
1597
|
+
}
|
|
1598
|
+
}
|
|
1599
|
+
println!("corpus\trole\tkind\talternatives\tshape ns\tengine ns\tshape/engine\tinputs\tpattern");
|
|
1600
|
+
for name in names {
|
|
1601
|
+
let schema: serde_json::Value =
|
|
1602
|
+
serde_json::from_str(&std::fs::read_to_string(format!("{dir}/{name}-schema.json")).unwrap()).unwrap();
|
|
1603
|
+
let (mut strings, mut keys) = (BTreeSet::new(), BTreeSet::new());
|
|
1604
|
+
for line in std::fs::read_to_string(format!("{dir}/{name}-instances.jsonl")).unwrap().lines() {
|
|
1605
|
+
if !line.trim().is_empty() {
|
|
1606
|
+
collect(&serde_json::from_str(line).unwrap(), &mut strings, &mut keys);
|
|
1607
|
+
}
|
|
1608
|
+
}
|
|
1609
|
+
let strings: Vec<String> = strings.into_iter().take(1000).collect();
|
|
1610
|
+
let keys: Vec<String> = keys.into_iter().take(1000).collect();
|
|
1611
|
+
let mut found = BTreeSet::new();
|
|
1612
|
+
patterns(&schema, &mut found);
|
|
1613
|
+
for (role, source) in found {
|
|
1614
|
+
let Ok(shape) = compile(&source) else { continue };
|
|
1615
|
+
if matches!(shape.matcher, Matcher::Engine(_)) {
|
|
1616
|
+
continue;
|
|
1617
|
+
}
|
|
1618
|
+
let inputs = if role == "pattern" { &strings } else { &keys };
|
|
1619
|
+
if inputs.is_empty() {
|
|
1620
|
+
continue;
|
|
1621
|
+
}
|
|
1622
|
+
let regex = G.build(&source, true).or_else(|_| G.build(&source, false)).unwrap();
|
|
1623
|
+
let engine = Pattern {
|
|
1624
|
+
source: source.clone(),
|
|
1625
|
+
grammar: G,
|
|
1626
|
+
matcher: Matcher::Engine(Box::new(regex)),
|
|
1627
|
+
retained: 0,
|
|
1628
|
+
};
|
|
1629
|
+
// The two must agree before their times mean anything.
|
|
1630
|
+
let mut budget = PatternBudget::new(PatternLimit::new(None));
|
|
1631
|
+
for s in inputs {
|
|
1632
|
+
assert_eq!(shape.is_match(s, &mut budget), engine.is_match(s, &mut budget), "{source:?} on {s:?}");
|
|
1633
|
+
}
|
|
1634
|
+
let (a, b) = (time(&shape, inputs), time(&engine, inputs));
|
|
1635
|
+
println!(
|
|
1636
|
+
"{name}\t{role}\t{}\t{}\t{a:.1}\t{b:.1}\t{:.3}\t{}\t{source}",
|
|
1637
|
+
super::kind(&shape),
|
|
1638
|
+
alternatives(&shape.matcher),
|
|
1639
|
+
a / b,
|
|
1640
|
+
inputs.len()
|
|
1641
|
+
);
|
|
1642
|
+
}
|
|
2273
1643
|
}
|
|
2274
1644
|
}
|
|
2275
1645
|
}
|