corvus_json_schema 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Cargo.lock +2 -12
- data/README.md +10 -1
- data/THIRD-PARTY-NOTICES.txt +95 -0
- data/VERSIONHISTORY.md +24 -0
- data/ext/corvus_json_schema/Cargo.toml +2 -2
- data/ext/corvus_json_schema/clippy.toml +3 -4
- data/ext/corvus_json_schema/src/lib.rs +42 -19
- data/ext/corvus_json_schema/vendor/corvus-json-schema/Cargo.toml +9 -4
- data/ext/corvus_json_schema/vendor/corvus-json-schema/README.md +87 -4
- data/ext/corvus_json_schema/vendor/corvus-json-schema/THIRD-PARTY-NOTICES.txt +93 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/compiler.rs +88 -96
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/document.rs +2 -2
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/charset.rs +271 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/compile.rs +941 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/dfa.rs +366 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/fold.rs +138 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/mod.rs +329 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/parse.rs +1128 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/re2.rs +129 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/tables_gen.rs +6685 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/tests.rs +1672 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/ucd.rs +309 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/units.rs +165 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/vm.rs +981 -0
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/eval/plan/fused.rs +13 -9
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/eval/plan.rs +12 -11
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/eval.rs +105 -73
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/formats.rs +95 -1
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/lib.rs +165 -30
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/loader.rs +19 -21
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/numbers.rs +1 -1
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/options.rs +199 -4
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/pattern.rs +758 -79
- data/ext/corvus_json_schema/vendor/corvus-json-schema/src/uri.rs +4 -4
- data/lib/corvus_json_schema/version.rb +1 -1
- data/lib/corvus_json_schema.rb +13 -3
- metadata +16 -2
|
@@ -1,4 +1,6 @@
|
|
|
1
|
-
//! `pattern`/`patternProperties` matching with ECMA-262 semantics (the `u` flag), as JSON Schema specifies.
|
|
1
|
+
//! `pattern`/`patternProperties` matching with ECMA-262 semantics (the `u` flag), as JSON Schema specifies. A
|
|
2
|
+
//! pattern that is valid only without the flag is accepted too, and matched as ECMA-262 says for it: by UTF-16
|
|
3
|
+
//! code unit.
|
|
2
4
|
//!
|
|
3
5
|
//! A pattern gets the cheapest matcher that decides it exactly:
|
|
4
6
|
//!
|
|
@@ -11,19 +13,31 @@
|
|
|
11
13
|
//! - patterns within the common subset of ECMA-262 and the `regex` crate's syntax, translated with ECMA semantics
|
|
12
14
|
//! (`\d`/`\w` ASCII-only, ECMA's `\s` and `.`) and run by `regex`, which searches in linear time without
|
|
13
15
|
//! allocating per match;
|
|
14
|
-
//! - anything else (lookarounds, backreferences,
|
|
15
|
-
//! ECMA-262 engine.
|
|
16
|
+
//! - anything else (lookarounds, backreferences, `\B`, modifier groups, and every pattern `regex` is not given) by
|
|
17
|
+
//! the crate's own ECMA-262 engine (`ecmaregex`), a port of the engine of the Go module. It answers every pattern
|
|
18
|
+
//! as ECMA-262 does, as the pattern is written.
|
|
19
|
+
//!
|
|
20
|
+
//! Whether a pattern is valid is the engine's parser's answer, whichever matcher runs it, and so are the limits of
|
|
21
|
+
//! size: a pattern over one is refused before any matcher reads it.
|
|
16
22
|
//!
|
|
17
23
|
//! Patterns compile once per process (a pattern is immutable, so identical patterns share one matcher).
|
|
18
24
|
|
|
19
|
-
use std::collections::HashMap;
|
|
25
|
+
use std::collections::{HashMap, VecDeque};
|
|
20
26
|
use std::fmt;
|
|
21
|
-
use std::sync::{Arc, LazyLock, Mutex};
|
|
27
|
+
use std::sync::{Arc, LazyLock, Mutex, OnceLock};
|
|
28
|
+
use std::time::Duration;
|
|
29
|
+
|
|
30
|
+
use crate::ecmaregex;
|
|
31
|
+
use crate::options::{
|
|
32
|
+
PatternMatchLimitError, PatternTooLargeError, SchemaCompilationError, SchemaCompilationErrorKind,
|
|
33
|
+
};
|
|
22
34
|
|
|
23
35
|
/// A compiled `pattern`.
|
|
24
36
|
pub(crate) struct Pattern {
|
|
25
37
|
pub source: String,
|
|
26
38
|
matcher: Matcher,
|
|
39
|
+
/// What the pattern is counted as in the cache of compiled patterns, in bytes (see [`Cache`]).
|
|
40
|
+
retained: usize,
|
|
27
41
|
}
|
|
28
42
|
|
|
29
43
|
enum Matcher {
|
|
@@ -55,12 +69,14 @@ enum Matcher {
|
|
|
55
69
|
set: CharSet,
|
|
56
70
|
bangs: bool,
|
|
57
71
|
},
|
|
58
|
-
Regex(
|
|
72
|
+
Regex(Linear),
|
|
59
73
|
/// A pattern that is valid only without the `u` flag (an identity escape such as `\&`), so matching UTF-16 code
|
|
60
74
|
/// units: translated for `regex`, which decides it exactly on strings with no character beyond the Basic
|
|
61
|
-
/// Multilingual Plane (where code units are characters)
|
|
62
|
-
RegexBmp(
|
|
63
|
-
|
|
75
|
+
/// Multilingual Plane (where code units are characters). The engine decides the rest, by code unit.
|
|
76
|
+
RegexBmp(Linear),
|
|
77
|
+
/// The crate's ECMA-262 engine: by character for a pattern with the `u` flag, and by UTF-16 code unit for one
|
|
78
|
+
/// that is valid only without it.
|
|
79
|
+
Engine(ecmaregex::Regexp),
|
|
64
80
|
}
|
|
65
81
|
|
|
66
82
|
impl fmt::Debug for Pattern {
|
|
@@ -69,9 +85,110 @@ impl fmt::Debug for Pattern {
|
|
|
69
85
|
}
|
|
70
86
|
}
|
|
71
87
|
|
|
88
|
+
/// A pattern compiled by the `regex` crate, which matches in time proportional to its program times the string and
|
|
89
|
+
/// cannot be stopped, with the engine for the strings it is not given.
|
|
90
|
+
struct Linear {
|
|
91
|
+
re: regex::Regex,
|
|
92
|
+
/// The number of instructions estimated for the program (`ecmaregex::Facts::linear_cost`), and one at least.
|
|
93
|
+
cost: u64,
|
|
94
|
+
/// The engine for the pattern: compiled with the pattern when it is valid only without the `u` flag, and
|
|
95
|
+
/// otherwise for the first string that is too long for `regex` to answer within the limit of a match.
|
|
96
|
+
engine: OnceLock<Option<ecmaregex::Regexp>>,
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
impl Linear {
|
|
100
|
+
/// Whether `regex` answers the string within the limit of a match: its program times the string, at
|
|
101
|
+
/// [`LINEAR_STEP`] for each, is within the limit. With no limit every string is.
|
|
102
|
+
#[inline(always)]
|
|
103
|
+
fn fits(&self, s: &str, budget: &PatternBudget) -> bool {
|
|
104
|
+
(s.len() as u64).saturating_mul(self.cost) <= budget.limit.linear
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
#[cold]
|
|
108
|
+
fn engine(&self, source: &str) -> Option<&ecmaregex::Regexp> {
|
|
109
|
+
self.engine.get_or_init(|| ecmaregex::compile(source).ok()).as_ref()
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/// The time the `regex` crate can take for one instruction of its program and one byte of the string, when every
|
|
114
|
+
/// instruction has a thread on it (it then runs its simulation of the automaton, which is its slowest matcher). A
|
|
115
|
+
/// match on `regex` cannot be stopped, so a match with a limit goes there only when the size of the program times
|
|
116
|
+
/// the length of the string times this is within the limit. Any other goes to the engine, which reads the clock.
|
|
117
|
+
/// The figure is the Go module's for the regexp package of Go. For the `regex` crate the most that was measured,
|
|
118
|
+
/// with patterns that defeat its lazy automaton on a text of 100,000 bytes, was 3.6 ns.
|
|
119
|
+
const LINEAR_STEP: Duration = Duration::from_nanos(16);
|
|
120
|
+
|
|
121
|
+
/// The limit of a match, as a validator holds it.
|
|
122
|
+
#[derive(Clone, Copy, Debug)]
|
|
123
|
+
pub(crate) struct PatternLimit {
|
|
124
|
+
/// The time one match may take, or `None` for no limit.
|
|
125
|
+
limit: Option<Duration>,
|
|
126
|
+
/// The program of a `regex` matcher times the bytes of a string that the limit allows (see [`LINEAR_STEP`]).
|
|
127
|
+
linear: u64,
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
impl PatternLimit {
|
|
131
|
+
/// A limit of `limit` for each match. `None` and zero are no limit.
|
|
132
|
+
pub(crate) fn new(limit: Option<Duration>) -> PatternLimit {
|
|
133
|
+
let limit = limit.filter(|l| !l.is_zero());
|
|
134
|
+
let linear =
|
|
135
|
+
limit.map_or(u64::MAX, |l| u64::try_from(l.as_nanos() / LINEAR_STEP.as_nanos()).unwrap_or(u64::MAX));
|
|
136
|
+
PatternLimit { limit, linear }
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
/// What one evaluation, or the compilation of one schema, gives every match of a pattern: the time each may take.
|
|
141
|
+
/// It notes the first match that reached a limit, after which every match is abandoned at once.
|
|
142
|
+
pub(crate) struct PatternBudget {
|
|
143
|
+
limit: PatternLimit,
|
|
144
|
+
/// The match that reached a limit. It is boxed so that an evaluation, which holds a budget, stays small.
|
|
145
|
+
failure: Option<Box<PatternMatchLimitError>>,
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
impl PatternBudget {
|
|
149
|
+
#[inline(always)]
|
|
150
|
+
pub(crate) fn new(limit: PatternLimit) -> PatternBudget {
|
|
151
|
+
PatternBudget { limit, failure: None }
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
/// Whether a match has reached a limit, so that what the evaluation came to is not an answer.
|
|
155
|
+
#[inline(always)]
|
|
156
|
+
pub(crate) fn failed(&self) -> bool {
|
|
157
|
+
self.failure.is_some()
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/// The error of the match that reached a limit, if one did.
|
|
161
|
+
pub(crate) fn take_failure(&mut self) -> Option<PatternMatchLimitError> {
|
|
162
|
+
self.failure.take().map(|e| *e)
|
|
163
|
+
}
|
|
164
|
+
|
|
165
|
+
/// Matches the string on the engine within the budget. When a limit is reached the answer is false, and it is
|
|
166
|
+
/// not an answer: the caller of the evaluation gets the error.
|
|
167
|
+
fn engine_match(&mut self, engine: Option<&ecmaregex::Regexp>, source: &str, s: &str) -> bool {
|
|
168
|
+
if self.failure.is_some() {
|
|
169
|
+
return false;
|
|
170
|
+
}
|
|
171
|
+
let result = match engine {
|
|
172
|
+
Some(engine) => engine.match_limit(s, self.limit.limit),
|
|
173
|
+
// No engine could be made for the pattern (see `ecmaregex::compile`), so there is no answer.
|
|
174
|
+
None => Err(ecmaregex::MatchError::Memory),
|
|
175
|
+
};
|
|
176
|
+
result.unwrap_or_else(|e| {
|
|
177
|
+
self.failure = Some(Box::new(PatternMatchLimitError::new(
|
|
178
|
+
source,
|
|
179
|
+
self.limit.limit.unwrap_or_default(),
|
|
180
|
+
e == ecmaregex::MatchError::Memory,
|
|
181
|
+
)));
|
|
182
|
+
false
|
|
183
|
+
})
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
|
|
72
187
|
impl Pattern {
|
|
188
|
+
/// Whether the pattern matches somewhere in the string. The engine matches within the budget. Every other
|
|
189
|
+
/// matcher reads the string once, or a bounded number of times, and has no limit.
|
|
73
190
|
#[inline]
|
|
74
|
-
pub fn is_match(&self, s: &str) -> bool {
|
|
191
|
+
pub fn is_match(&self, s: &str, budget: &mut PatternBudget) -> bool {
|
|
75
192
|
match &self.matcher {
|
|
76
193
|
Matcher::Everything => true,
|
|
77
194
|
Matcher::Literal { text, whole } => {
|
|
@@ -115,20 +232,46 @@ impl Pattern {
|
|
|
115
232
|
}
|
|
116
233
|
word
|
|
117
234
|
}
|
|
118
|
-
Matcher::Regex(
|
|
235
|
+
Matcher::Regex(linear) => {
|
|
236
|
+
if linear.fits(s, budget) {
|
|
237
|
+
linear.re.is_match(s)
|
|
238
|
+
} else {
|
|
239
|
+
budget.engine_match(linear.engine(&self.source), &self.source, s)
|
|
240
|
+
}
|
|
241
|
+
}
|
|
119
242
|
// A UTF-8 lead byte of 0xF0 or above starts a character beyond the BMP.
|
|
120
|
-
Matcher::RegexBmp(
|
|
121
|
-
if
|
|
122
|
-
|
|
243
|
+
Matcher::RegexBmp(linear) => {
|
|
244
|
+
if linear.fits(s, budget) && !ecmaregex::has_supplementary(s.as_bytes()) {
|
|
245
|
+
linear.re.is_match(s)
|
|
123
246
|
} else {
|
|
124
|
-
|
|
247
|
+
budget.engine_match(linear.engine(&self.source), &self.source, s)
|
|
125
248
|
}
|
|
126
249
|
}
|
|
127
|
-
Matcher::
|
|
250
|
+
Matcher::Engine(engine) => budget.engine_match(Some(engine), &self.source, s),
|
|
128
251
|
}
|
|
129
252
|
}
|
|
130
253
|
}
|
|
131
254
|
|
|
255
|
+
/// The kind of matcher a pattern has, for the tables of which pattern goes where.
|
|
256
|
+
#[cfg(test)]
|
|
257
|
+
pub(crate) fn kind(p: &Pattern) -> &'static str {
|
|
258
|
+
match &p.matcher {
|
|
259
|
+
Matcher::Regex(_) => "regex",
|
|
260
|
+
Matcher::RegexBmp(..) => "regex-bmp",
|
|
261
|
+
Matcher::Engine(engine) if engine.unicode() => "engine",
|
|
262
|
+
Matcher::Engine(_) => "engine-units",
|
|
263
|
+
Matcher::Everything => "everything",
|
|
264
|
+
Matcher::Literal { .. } => "literal",
|
|
265
|
+
Matcher::Sequence(_) => "sequence",
|
|
266
|
+
Matcher::SeparatedList(_) => "list",
|
|
267
|
+
Matcher::HasContent { .. } => "has-content",
|
|
268
|
+
Matcher::Line { .. } => "line",
|
|
269
|
+
Matcher::Literals(_) => "literals",
|
|
270
|
+
Matcher::Alternatives(_) => "alternatives",
|
|
271
|
+
Matcher::ExcludedClassWithWord { .. } => "excluded-class",
|
|
272
|
+
}
|
|
273
|
+
}
|
|
274
|
+
|
|
132
275
|
/// ECMA-262 `LineTerminator`, which `.` does not match.
|
|
133
276
|
#[inline]
|
|
134
277
|
fn is_line_terminator(c: char) -> bool {
|
|
@@ -237,44 +380,195 @@ impl Alternative {
|
|
|
237
380
|
}
|
|
238
381
|
}
|
|
239
382
|
|
|
240
|
-
///
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
383
|
+
/// The most patterns the process-wide cache holds.
|
|
384
|
+
const MAX_CACHED_PATTERNS: usize = 1024;
|
|
385
|
+
|
|
386
|
+
/// The most the patterns of the cache come to, in the bytes counted for each: 64 MiB. A pattern is counted as its
|
|
387
|
+
/// source, the size the engine's parser counted for it four times over (its program is proportional to that), and
|
|
388
|
+
/// for a pattern on the `regex` crate the memory estimated for that crate's program (`ecmaregex::Facts::linear_size`,
|
|
389
|
+
/// which is an estimate: the crate does not say what it holds), up to three times [`REGEX_SIZE_LIMIT`].
|
|
390
|
+
const MAX_CACHED_BYTES: usize = 64 << 20;
|
|
391
|
+
|
|
392
|
+
/// A pattern counted as more than this is compiled for the schema that has it and not kept in the cache.
|
|
393
|
+
const MAX_CACHED_PATTERN_BYTES: usize = MAX_CACHED_BYTES / 32;
|
|
394
|
+
|
|
395
|
+
/// The largest program the `regex` crate compiles for a pattern, in bytes: 2 MiB. (Its own default is 10 MiB.) A
|
|
396
|
+
/// pattern that needs more is the engine's, which holds a class once however often the pattern repeats it.
|
|
397
|
+
const REGEX_SIZE_LIMIT: usize = 2 << 20;
|
|
398
|
+
|
|
399
|
+
/// The most the `regex` crate's lazy automaton caches for one pattern and one thread, in bytes: 2 MiB, which is
|
|
400
|
+
/// that crate's default, said here so that it stays so.
|
|
401
|
+
const REGEX_DFA_SIZE_LIMIT: usize = 2 << 20;
|
|
402
|
+
|
|
403
|
+
/// The compiled patterns of the process, so that identical patterns share one matcher. It is bounded in number and
|
|
404
|
+
/// in size, since the patterns of a schema may come from outside: when it is over either bound the patterns that
|
|
405
|
+
/// were compiled first go, oldest first. A validator holds the patterns of its schema itself, so a pattern that has
|
|
406
|
+
/// left the cache is still matched by the validators that have it, and is compiled again only for a schema compiled
|
|
407
|
+
/// later.
|
|
408
|
+
#[derive(Default)]
|
|
409
|
+
struct Cache {
|
|
410
|
+
patterns: HashMap<Arc<str>, Arc<Pattern>>,
|
|
411
|
+
/// The sources in the order they were compiled.
|
|
412
|
+
order: VecDeque<Arc<str>>,
|
|
413
|
+
/// The sum of what the patterns are counted as.
|
|
414
|
+
bytes: usize,
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
impl Cache {
|
|
418
|
+
fn get(&self, source: &str) -> Option<Arc<Pattern>> {
|
|
419
|
+
self.patterns.get(source).cloned()
|
|
420
|
+
}
|
|
421
|
+
|
|
422
|
+
/// Keeps a pattern, and returns the one to use: the pattern itself, or the one another thread kept for the same
|
|
423
|
+
/// source meanwhile.
|
|
424
|
+
fn insert(&mut self, pattern: Arc<Pattern>) -> Arc<Pattern> {
|
|
425
|
+
if pattern.retained > MAX_CACHED_PATTERN_BYTES {
|
|
426
|
+
return pattern;
|
|
427
|
+
}
|
|
428
|
+
if let Some(kept) = self.patterns.get(pattern.source.as_str()) {
|
|
429
|
+
return kept.clone();
|
|
430
|
+
}
|
|
431
|
+
let source: Arc<str> = Arc::from(pattern.source.as_str());
|
|
432
|
+
self.bytes += pattern.retained;
|
|
433
|
+
self.patterns.insert(source.clone(), pattern.clone());
|
|
434
|
+
self.order.push_back(source);
|
|
435
|
+
while self.patterns.len() > MAX_CACHED_PATTERNS || self.bytes > MAX_CACHED_BYTES {
|
|
436
|
+
let Some(oldest) = self.order.pop_front() else { break };
|
|
437
|
+
if let Some(evicted) = self.patterns.remove(&oldest) {
|
|
438
|
+
self.bytes -= evicted.retained;
|
|
439
|
+
}
|
|
440
|
+
}
|
|
441
|
+
pattern
|
|
442
|
+
}
|
|
443
|
+
}
|
|
444
|
+
|
|
445
|
+
static CACHE: LazyLock<Mutex<Cache>> = LazyLock::new(|| Mutex::new(Cache::default()));
|
|
446
|
+
|
|
447
|
+
fn cache() -> std::sync::MutexGuard<'static, Cache> {
|
|
448
|
+
CACHE.lock().unwrap_or_else(|e| e.into_inner())
|
|
449
|
+
}
|
|
450
|
+
|
|
451
|
+
/// Why a pattern does not compile.
|
|
452
|
+
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
|
453
|
+
pub(crate) enum PatternError {
|
|
454
|
+
/// It is not a valid ECMA-262 regular expression.
|
|
455
|
+
Invalid,
|
|
456
|
+
/// It is over a limit of size. It was not read to its end, so nothing is known of whether it is valid.
|
|
457
|
+
TooLarge(ecmaregex::TooLargeError),
|
|
458
|
+
}
|
|
459
|
+
|
|
460
|
+
impl PatternError {
|
|
461
|
+
/// The error of a schema whose keyword `keyword` holds the pattern `pattern`.
|
|
462
|
+
pub(crate) fn of(self, pattern: &str, keyword: &'static str) -> SchemaCompilationError {
|
|
463
|
+
match self {
|
|
464
|
+
PatternError::Invalid => SchemaCompilationError::of_kind(
|
|
465
|
+
format!("Invalid regular expression '{pattern}' in {keyword}."),
|
|
466
|
+
SchemaCompilationErrorKind::InvalidPattern,
|
|
467
|
+
),
|
|
468
|
+
PatternError::TooLarge(large) => {
|
|
469
|
+
let error =
|
|
470
|
+
PatternTooLargeError::new(pattern, keyword, large.limit.name(), large.value, large.to_string());
|
|
471
|
+
SchemaCompilationError::of_kind(error.to_string(), SchemaCompilationErrorKind::PatternTooLarge(error))
|
|
472
|
+
}
|
|
473
|
+
}
|
|
245
474
|
}
|
|
246
475
|
}
|
|
247
476
|
|
|
248
|
-
|
|
477
|
+
impl From<ecmaregex::Error> for PatternError {
|
|
478
|
+
fn from(e: ecmaregex::Error) -> PatternError {
|
|
479
|
+
match e {
|
|
480
|
+
ecmaregex::Error::TooLarge(large) => PatternError::TooLarge(large),
|
|
481
|
+
ecmaregex::Error::Invalid(_) => PatternError::Invalid,
|
|
482
|
+
}
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
|
|
486
|
+
/// A pattern in quotes for a message, or its first 120 bytes and its length when it is longer.
|
|
487
|
+
pub(crate) fn shown_pattern(pattern: &str) -> String {
|
|
488
|
+
const SHOWN: usize = 120;
|
|
489
|
+
if pattern.len() <= SHOWN {
|
|
490
|
+
return format!("'{pattern}'");
|
|
491
|
+
}
|
|
492
|
+
let mut cut = SHOWN;
|
|
493
|
+
while !pattern.is_char_boundary(cut) {
|
|
494
|
+
cut -= 1;
|
|
495
|
+
}
|
|
496
|
+
format!("'{}...' ({} bytes)", &pattern[..cut], pattern.len())
|
|
497
|
+
}
|
|
249
498
|
|
|
250
|
-
/// Compiles (or fetches from the process-wide cache) a pattern
|
|
251
|
-
pub(crate) fn compile(pattern: &str) ->
|
|
252
|
-
if let Some(p) =
|
|
253
|
-
return
|
|
499
|
+
/// Compiles (or fetches from the process-wide cache) a pattern.
|
|
500
|
+
pub(crate) fn compile(pattern: &str) -> Result<Arc<Pattern>, PatternError> {
|
|
501
|
+
if let Some(p) = cache().get(pattern) {
|
|
502
|
+
return Ok(p);
|
|
254
503
|
}
|
|
255
|
-
//
|
|
256
|
-
|
|
257
|
-
let
|
|
504
|
+
// The limits of size come first: nothing below reads a pattern that is over one. Validity is ECMA-262's: a
|
|
505
|
+
// pattern the engine's parser rejects is an error, whichever matcher would run it.
|
|
506
|
+
let facts = ecmaregex::check(pattern)?;
|
|
507
|
+
let matcher = match choose(pattern, &facts) {
|
|
258
508
|
Some(m) => m,
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
509
|
+
// Without the flag the pattern matches UTF-16 code units. `regex` matches characters, which is the same
|
|
510
|
+
// thing on a string with no character beyond the Basic Multilingual Plane.
|
|
511
|
+
None => match (!facts.unicode).then(|| linear(pattern, &facts)).flatten() {
|
|
512
|
+
Some(linear) => Matcher::RegexBmp(linear),
|
|
513
|
+
None => Matcher::Engine(ecmaregex::compile(pattern)?),
|
|
262
514
|
},
|
|
263
|
-
None => Matcher::Regress(regress),
|
|
264
515
|
};
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
516
|
+
// What the cache counts for the pattern (see `MAX_CACHED_BYTES`).
|
|
517
|
+
let mut retained = pattern.len() + 256;
|
|
518
|
+
if matches!(matcher, Matcher::Regex(_) | Matcher::RegexBmp(_) | Matcher::Engine(_)) {
|
|
519
|
+
retained += 4 * facts.size;
|
|
520
|
+
}
|
|
521
|
+
if matches!(matcher, Matcher::Regex(_) | Matcher::RegexBmp(_)) {
|
|
522
|
+
retained += (facts.linear_size as usize).min(3 * REGEX_SIZE_LIMIT);
|
|
523
|
+
}
|
|
524
|
+
Ok(cache().insert(Arc::new(Pattern { source: pattern.to_string(), matcher, retained })))
|
|
525
|
+
}
|
|
526
|
+
|
|
527
|
+
/// The pattern translated for the `regex` crate and compiled by it, when it is in the syntax both have and its
|
|
528
|
+
/// program is not large (see `ecmaregex::MAX_RE2_COST`: `regex` writes a counted repetition out, and takes time
|
|
529
|
+
/// proportional to its program to compile and, for every character of a text, to match).
|
|
530
|
+
fn linear(pattern: &str, facts: &ecmaregex::Facts) -> Option<Linear> {
|
|
531
|
+
if facts.re2_cost > ecmaregex::MAX_RE2_COST {
|
|
532
|
+
return None;
|
|
533
|
+
}
|
|
534
|
+
let translation = translate(pattern, facts.unicode)?;
|
|
535
|
+
let re = if ecmaregex::shallow(pattern) {
|
|
536
|
+
regex::RegexBuilder::new(&translation)
|
|
537
|
+
.size_limit(REGEX_SIZE_LIMIT)
|
|
538
|
+
.dfa_size_limit(REGEX_DFA_SIZE_LIMIT)
|
|
539
|
+
.build()
|
|
540
|
+
.ok()?
|
|
541
|
+
} else {
|
|
542
|
+
// `regex` reads 250 levels of nesting unless it is told otherwise, because what it does with a pattern goes
|
|
543
|
+
// a level deeper for each. A pattern that opens many groups is compiled on a thread with a stack for the
|
|
544
|
+
// 1,000 levels the engine's own limit allows (a group, its quantifier and a class inside it are three
|
|
545
|
+
// levels to `regex`), so that it is not the engine's for its nesting alone.
|
|
546
|
+
ecmaregex::deep(|| {
|
|
547
|
+
regex::RegexBuilder::new(&translation)
|
|
548
|
+
.size_limit(REGEX_SIZE_LIMIT)
|
|
549
|
+
.dfa_size_limit(REGEX_DFA_SIZE_LIMIT)
|
|
550
|
+
.nest_limit(4 * ecmaregex::MAX_NESTING as u32)
|
|
551
|
+
.build()
|
|
552
|
+
.ok()
|
|
553
|
+
})??
|
|
554
|
+
};
|
|
555
|
+
let engine = OnceLock::new();
|
|
556
|
+
if !facts.unicode || !ecmaregex::shallow(pattern) {
|
|
557
|
+
// The engine has the strings with a character beyond the Basic Multilingual Plane, so it is compiled now.
|
|
558
|
+
// So is that of a pattern that nests deeply, which is compiled on a stack of its own.
|
|
559
|
+
let _ = engine.set(Some(ecmaregex::compile(pattern).ok()?));
|
|
560
|
+
}
|
|
561
|
+
Some(Linear { re, cost: facts.linear_cost.max(1) as u64, engine })
|
|
268
562
|
}
|
|
269
563
|
|
|
270
|
-
fn choose(pattern: &str,
|
|
564
|
+
fn choose(pattern: &str, facts: &ecmaregex::Facts) -> Option<Matcher> {
|
|
271
565
|
// Unanchored (or start-anchored) `.*` finds an empty match in any string; `^.*$` does not (`.` stops at a line
|
|
272
566
|
// terminator), so it is not listed.
|
|
273
567
|
if matches!(pattern, "" | ".*" | "^.*" | ".*$" | "(.*)" | "^(.*)" | "[\\s\\S]*" | "^[\\s\\S]*" | "^[\\s\\S]*$") {
|
|
274
568
|
return Some(Matcher::Everything);
|
|
275
569
|
}
|
|
276
|
-
// Without the `u` flag, a pattern matches UTF-16 code units rather than characters: only
|
|
277
|
-
if !unicode {
|
|
570
|
+
// Without the `u` flag, a pattern matches UTF-16 code units rather than characters: only the engine has that.
|
|
571
|
+
if !facts.unicode {
|
|
278
572
|
return None;
|
|
279
573
|
}
|
|
280
574
|
match pattern {
|
|
@@ -288,7 +582,9 @@ fn choose(pattern: &str, unicode: bool) -> Option<Matcher> {
|
|
|
288
582
|
&& rest.len() > 1
|
|
289
583
|
&& !ends_with_escape(rest)
|
|
290
584
|
&& !rest.ends_with(['*', '+', '?', '}', '|', '(', '^'])
|
|
291
|
-
&& let
|
|
585
|
+
&& let Ok(rest_facts) = ecmaregex::check(rest)
|
|
586
|
+
&& rest_facts.unicode
|
|
587
|
+
&& let Some(m) = choose(rest, &rest_facts)
|
|
292
588
|
{
|
|
293
589
|
return Some(m);
|
|
294
590
|
}
|
|
@@ -313,8 +609,7 @@ fn choose(pattern: &str, unicode: bool) -> Option<Matcher> {
|
|
|
313
609
|
None => Matcher::Sequence(seq),
|
|
314
610
|
});
|
|
315
611
|
}
|
|
316
|
-
|
|
317
|
-
regex::Regex::new(&translated).ok().map(Matcher::Regex)
|
|
612
|
+
linear(pattern, facts).map(Matcher::Regex)
|
|
318
613
|
}
|
|
319
614
|
|
|
320
615
|
/// `^.{m,n}$` (and `^.*$`, `^.+$`, `^.{m}$`, `^.{m,}$`, each also as `^(.…)$`): the bounds on the length of a line.
|
|
@@ -393,12 +688,18 @@ const MAX_ALTERNATIVES: usize = 64;
|
|
|
393
688
|
/// `regex` crate's single automaton pass (measured on jsconfig's case-folded `lib` names).
|
|
394
689
|
const MAX_SEQUENCE_ALTERNATIVES: usize = 4;
|
|
395
690
|
|
|
691
|
+
/// The widest class sequence, in characters, and the longest literal, in bytes, that is searched for at every
|
|
692
|
+
/// position of a text. The search for a sequence takes the text times this, and these matchers have no limit, so a
|
|
693
|
+
/// wider one is left to the engine, which has the limit of a match. (The rule is the Go module's, so that the ports
|
|
694
|
+
/// give a pattern the same kind of matcher.)
|
|
695
|
+
const MAX_ANYWHERE_WIDTH: usize = 64;
|
|
696
|
+
|
|
396
697
|
/// Two or more alternatives once groups of alternatives (and optional groups) are expanded into whole alternatives
|
|
397
698
|
/// (`^a|b|c$`, `^([a|A]uto)|([n|N]one)$`, `^[Ee][Ss]2015(\.([Cc]ore|[Pp]roxy))?$`), each a literal or a class
|
|
398
699
|
/// sequence anchored where its own `^` and `$` say.
|
|
399
700
|
fn alternatives(p: &str) -> Option<Box<[Alternative]>> {
|
|
400
701
|
let c: Vec<char> = p.chars().collect();
|
|
401
|
-
let (parts, end) = expand(&c, 0)?;
|
|
702
|
+
let (parts, end) = expand(&c, 0, 0)?;
|
|
402
703
|
// A single alternative is only new here when a group was expanded (Sequence::parse takes the rest).
|
|
403
704
|
if end != c.len() || parts.is_empty() || (parts.len() == 1 && parts[0] == p) {
|
|
404
705
|
return None;
|
|
@@ -412,6 +713,10 @@ fn alternatives(p: &str) -> Option<Box<[Alternative]>> {
|
|
|
412
713
|
_ => (false, part),
|
|
413
714
|
};
|
|
414
715
|
if let Some(text) = literal_text(body) {
|
|
716
|
+
// A literal that is anchored at neither end is searched for at every position of the text.
|
|
717
|
+
if !start && !end && text.len() > MAX_ANYWHERE_WIDTH {
|
|
718
|
+
return None;
|
|
719
|
+
}
|
|
415
720
|
return Some(Alternative::Literal { text: text.into(), start, end });
|
|
416
721
|
}
|
|
417
722
|
let seq = Sequence::parse(&format!("^{body}{}", if end { "$" } else { "" }))?;
|
|
@@ -420,7 +725,11 @@ fn alternatives(p: &str) -> Option<Box<[Alternative]>> {
|
|
|
420
725
|
}
|
|
421
726
|
let width =
|
|
422
727
|
seq.items.iter().all(|i| i.min == i.max).then(|| seq.items.iter().map(|i| i.min as usize).sum())?;
|
|
423
|
-
|
|
728
|
+
if end {
|
|
729
|
+
return Some(Alternative::End { seq, width });
|
|
730
|
+
}
|
|
731
|
+
// A sequence that is not anchored is tried at every position of the text, and reads its width at each.
|
|
732
|
+
(width <= MAX_ANYWHERE_WIDTH).then_some(Alternative::Anywhere { seq, width })
|
|
424
733
|
})
|
|
425
734
|
.collect::<Option<_>>()?;
|
|
426
735
|
let sequences = alternatives.iter().any(|a| !matches!(a, Alternative::Literal { .. }));
|
|
@@ -432,10 +741,18 @@ fn ends_with_escape(t: &str) -> bool {
|
|
|
432
741
|
t.bytes().rev().take_while(|&b| b == b'\\').count() % 2 == 1
|
|
433
742
|
}
|
|
434
743
|
|
|
744
|
+
/// The deepest nesting of groups that is multiplied out. `expand` goes a level deeper for each, and a pattern may
|
|
745
|
+
/// nest a thousand deep, which the stack of a thread cannot be relied on to hold. A pattern that nests deeper than
|
|
746
|
+
/// this is left to the matchers after this one.
|
|
747
|
+
const MAX_EXPANDED_NESTING: usize = 32;
|
|
748
|
+
|
|
435
749
|
/// Expands the alternatives from `i` up to an unmatched `)` or the end: groups of alternatives multiply out, a group
|
|
436
750
|
/// quantified by `?` also contributes the empty alternative, and everything else is copied. `None` for lookarounds,
|
|
437
|
-
/// other quantified groups,
|
|
438
|
-
fn expand(c: &[char], mut i: usize) -> Option<(Vec<String>, usize)> {
|
|
751
|
+
/// other quantified groups, too many alternatives, or groups nested deeper than `MAX_EXPANDED_NESTING`.
|
|
752
|
+
fn expand(c: &[char], mut i: usize, depth: usize) -> Option<(Vec<String>, usize)> {
|
|
753
|
+
if depth > MAX_EXPANDED_NESTING {
|
|
754
|
+
return None;
|
|
755
|
+
}
|
|
439
756
|
let mut all = Vec::new();
|
|
440
757
|
let mut branch = vec![String::new()];
|
|
441
758
|
while i < c.len() {
|
|
@@ -454,7 +771,7 @@ fn expand(c: &[char], mut i: usize) -> Option<(Vec<String>, usize)> {
|
|
|
454
771
|
}
|
|
455
772
|
i += 2;
|
|
456
773
|
}
|
|
457
|
-
let (mut inner, next) = expand(c, i)?;
|
|
774
|
+
let (mut inner, next) = expand(c, i, depth + 1)?;
|
|
458
775
|
if c.get(next) != Some(&')') {
|
|
459
776
|
return None;
|
|
460
777
|
}
|
|
@@ -550,9 +867,10 @@ fn excluded_class_with_word(p: &str) -> Option<(CharSet, bool)> {
|
|
|
550
867
|
.then_some((excluded, bangs))
|
|
551
868
|
}
|
|
552
869
|
|
|
553
|
-
/// The `regex` format: a valid ECMA-262 regular expression (with the `u` flag).
|
|
870
|
+
/// The `regex` format: a valid ECMA-262 regular expression (with the `u` flag). A string over a limit of size is
|
|
871
|
+
/// not read to its end and is not valid.
|
|
554
872
|
pub(crate) fn is_valid_ecma_regex(s: &str) -> bool {
|
|
555
|
-
|
|
873
|
+
ecmaregex::valid(s)
|
|
556
874
|
}
|
|
557
875
|
|
|
558
876
|
// ---------------------------------------------------------------------------------------------------------------------
|
|
@@ -848,8 +1166,14 @@ impl SeparatedList {
|
|
|
848
1166
|
}
|
|
849
1167
|
}
|
|
850
1168
|
let final_ = self.final_.as_ref().expect("a separated list has a first or a final item");
|
|
1169
|
+
// Whether the text is ASCII is worked out once, for the whole of it. The final sequence is tried after
|
|
1170
|
+
// every item, and reading what is left of the text each time to see whether it is ASCII made the match take
|
|
1171
|
+
// time quadratic in the number of items.
|
|
1172
|
+
let ascii = s.is_ascii();
|
|
851
1173
|
loop {
|
|
852
|
-
if repeats >= self.min_repeats
|
|
1174
|
+
if repeats >= self.min_repeats
|
|
1175
|
+
&& (if ascii { final_.is_match_ascii(rest.as_bytes()) } else { final_.is_match_chars(rest) })
|
|
1176
|
+
{
|
|
853
1177
|
return true;
|
|
854
1178
|
}
|
|
855
1179
|
let Some(n) = self.repeated.consume(rest) else { return false };
|
|
@@ -895,7 +1219,9 @@ impl SeparatedList {
|
|
|
895
1219
|
let group = strip_group(&body[open..=close])?;
|
|
896
1220
|
let before = &body[..open];
|
|
897
1221
|
let after = &body[close + 2..];
|
|
898
|
-
|
|
1222
|
+
// The items of a text that is a class sequence. A `$` at the end of the text is not an item, and the list has
|
|
1223
|
+
// no place for it: `^x(,a$)*$` is not `^x(,a)*$`. Such a text is not a part of a list.
|
|
1224
|
+
let items = |t: &str| Sequence::parse(&format!("^{t}")).filter(|s| !s.to_end).map(|s| s.items.into_vec());
|
|
899
1225
|
let seq = |items: &[Item], to_end| Sequence { items: items.into(), to_end };
|
|
900
1226
|
let fixed = |items: &[Item]| !items.is_empty() && items.iter().all(|i| i.min == i.max && i.min > 0);
|
|
901
1227
|
let g = items(group)?;
|
|
@@ -923,7 +1249,10 @@ impl SeparatedList {
|
|
|
923
1249
|
let final_ = Sequence::parse(&format!("^{}$", unwrap_group(after)?))?;
|
|
924
1250
|
(1..g.len()).find_map(|k| {
|
|
925
1251
|
let (repeated, separator) = g.split_at(k);
|
|
926
|
-
(fixed(separator)
|
|
1252
|
+
(fixed(separator)
|
|
1253
|
+
&& Sequence::greedy_before(repeated, separator[0].set)
|
|
1254
|
+
&& final_stops_at(&final_, separator[0].set))
|
|
1255
|
+
.then(|| SeparatedList {
|
|
927
1256
|
first: None,
|
|
928
1257
|
repeated: seq(repeated, false),
|
|
929
1258
|
separator: seq(separator, false),
|
|
@@ -937,6 +1266,14 @@ impl SeparatedList {
|
|
|
937
1266
|
}
|
|
938
1267
|
}
|
|
939
1268
|
|
|
1269
|
+
/// Whether the final sequence of a list cannot read past the separator that follows an item: no item of it holds a
|
|
1270
|
+
/// character the first item of the separator holds. Tried at the start of an item, it then fails at that separator
|
|
1271
|
+
/// at the latest, so trying it after every item reads the text a bounded number of times. A final sequence that can
|
|
1272
|
+
/// read through separators would read to the end of the text from every item.
|
|
1273
|
+
fn final_stops_at(final_: &Sequence, separator: CharSet) -> bool {
|
|
1274
|
+
final_.items.iter().all(|item| item.set.disjoint(separator))
|
|
1275
|
+
}
|
|
1276
|
+
|
|
940
1277
|
/// The inside of `(...)` or `(?:...)`; `None` for other groups.
|
|
941
1278
|
fn strip_group(g: &str) -> Option<&str> {
|
|
942
1279
|
let inner = g.strip_prefix('(')?.strip_suffix(')')?;
|
|
@@ -1102,6 +1439,11 @@ const ECMA_SPACE: &str =
|
|
|
1102
1439
|
/// escapes of any non-alphanumeric character), and the translation has its meaning on strings within the BMP.
|
|
1103
1440
|
fn translate(p: &str, unicode: bool) -> Option<String> {
|
|
1104
1441
|
let c: Vec<char> = p.chars().collect();
|
|
1442
|
+
// Without the flag a character beyond the BMP is two code units, and what is around it (a quantifier, the ends
|
|
1443
|
+
// of a range) applies to one of them. That has no translation.
|
|
1444
|
+
if !unicode && c.iter().any(|&ch| ch as u32 > 0xFFFF) {
|
|
1445
|
+
return None;
|
|
1446
|
+
}
|
|
1105
1447
|
let mut out = String::with_capacity(p.len() * 2);
|
|
1106
1448
|
let mut i = 0;
|
|
1107
1449
|
while i < c.len() {
|
|
@@ -1225,23 +1567,29 @@ fn escape(c: &[char], i: usize, in_class: bool, unicode: bool) -> Option<(String
|
|
|
1225
1567
|
out.push_str("(?-u:\\b)");
|
|
1226
1568
|
i + 1
|
|
1227
1569
|
}
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
// Unicode
|
|
1233
|
-
//
|
|
1570
|
+
// `\B` has no translation. `(?-u:\B)` also holds between the bytes of a character that takes several, where
|
|
1571
|
+
// no match may start or end, and `regex` 1.13.1 then says there is no match where it finds one: for
|
|
1572
|
+
// `s*?(?-u:\B)|..$` on `k\u{17f}k`, `is_match` is false and `find` is not. The engine has it.
|
|
1573
|
+
'B' => return None,
|
|
1574
|
+
// A Unicode property is written as the ranges of the engine's own tables, and never by its name: the
|
|
1575
|
+
// `regex` crate has tables of its own, of the version of Unicode its release carries, and a name would
|
|
1576
|
+
// match what that version says. Without the `u` flag `\p` is the letter.
|
|
1234
1577
|
'p' | 'P' if unicode && c.get(i + 1) == Some(&'{') => {
|
|
1235
1578
|
let end = i + 1 + c[i + 1..].iter().position(|&x| x == '}')?;
|
|
1236
1579
|
let body: String = c[i + 2..end].iter().collect();
|
|
1237
|
-
|
|
1238
|
-
|
|
1580
|
+
let set = ecmaregex::property_set(&body)?;
|
|
1581
|
+
let negated = e == 'P';
|
|
1582
|
+
if set.ranges.is_empty() {
|
|
1583
|
+
// No character has the property: nothing, or with `\P` everything.
|
|
1584
|
+
out.push_str(if negated { r"[\x{0}-\x{10FFFF}]" } else { r"[^\x{0}-\x{10FFFF}]" });
|
|
1585
|
+
} else if in_class && !negated {
|
|
1586
|
+
push_ranges(&mut out, &set);
|
|
1587
|
+
} else {
|
|
1588
|
+
// A class of its own, which is also how a class holds the complement of a property.
|
|
1589
|
+
out.push_str(if negated { "[^" } else { "[" });
|
|
1590
|
+
push_ranges(&mut out, &set);
|
|
1591
|
+
out.push(']');
|
|
1239
1592
|
}
|
|
1240
|
-
out.push('\\');
|
|
1241
|
-
out.push(e);
|
|
1242
|
-
out.push('{');
|
|
1243
|
-
out.push_str(&body);
|
|
1244
|
-
out.push('}');
|
|
1245
1593
|
end + 1
|
|
1246
1594
|
}
|
|
1247
1595
|
'x' => {
|
|
@@ -1252,6 +1600,10 @@ fn escape(c: &[char], i: usize, in_class: bool, unicode: bool) -> Option<(String
|
|
|
1252
1600
|
// Without the `u` flag, `\u{...}` is not a code point escape.
|
|
1253
1601
|
'u' if unicode || c.get(i + 1) != Some(&'{') => {
|
|
1254
1602
|
let (code, next) = unicode_escape(c, i + 1)?;
|
|
1603
|
+
// Without the flag, two escapes that are a surrogate pair are two code units.
|
|
1604
|
+
if !unicode && code > 0xFFFF {
|
|
1605
|
+
return None;
|
|
1606
|
+
}
|
|
1255
1607
|
push_literal(&mut out, char::from_u32(code)?);
|
|
1256
1608
|
next
|
|
1257
1609
|
}
|
|
@@ -1271,6 +1623,42 @@ fn escape(c: &[char], i: usize, in_class: bool, unicode: bool) -> Option<(String
|
|
|
1271
1623
|
Some((out, next))
|
|
1272
1624
|
}
|
|
1273
1625
|
|
|
1626
|
+
/// The ranges of a set, each end spelled as an escape. A surrogate is no character to the `regex` crate, and no
|
|
1627
|
+
/// string holds one, so a range is written without the surrogates it spans. Two ranges that then touch, one ending
|
|
1628
|
+
/// at U+D7FF and the next starting at U+E000, are written as one: see [`class`] for what `regex` makes of the two.
|
|
1629
|
+
fn push_ranges(out: &mut String, set: &ecmaregex::CharSet) {
|
|
1630
|
+
// The range that has been read and not yet written.
|
|
1631
|
+
let mut pending: Option<(u32, u32)> = None;
|
|
1632
|
+
let write = |out: &mut String, (lo, hi): (u32, u32)| {
|
|
1633
|
+
if let (Some(lo), Some(hi)) = (char::from_u32(lo), char::from_u32(hi)) {
|
|
1634
|
+
push_literal(out, lo);
|
|
1635
|
+
if lo != hi {
|
|
1636
|
+
out.push('-');
|
|
1637
|
+
push_literal(out, hi);
|
|
1638
|
+
}
|
|
1639
|
+
}
|
|
1640
|
+
};
|
|
1641
|
+
for &[lo, hi] in set.ranges.as_chunks::<2>().0 {
|
|
1642
|
+
// The ends that are surrogates move out of the surrogates. A range of surrogates alone is nothing.
|
|
1643
|
+
let lo = if (0xD800..=0xDFFF).contains(&lo) { 0xE000 } else { lo as u32 };
|
|
1644
|
+
let hi = if (0xD800..=0xDFFF).contains(&hi) { 0xD7FF } else { hi as u32 };
|
|
1645
|
+
if lo > hi {
|
|
1646
|
+
continue;
|
|
1647
|
+
}
|
|
1648
|
+
match pending {
|
|
1649
|
+
Some((first, 0xD7FF)) if lo == 0xE000 => pending = Some((first, hi)),
|
|
1650
|
+
Some(range) => {
|
|
1651
|
+
write(out, range);
|
|
1652
|
+
pending = Some((lo, hi));
|
|
1653
|
+
}
|
|
1654
|
+
None => pending = Some((lo, hi)),
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1657
|
+
if let Some(range) = pending {
|
|
1658
|
+
write(out, range);
|
|
1659
|
+
}
|
|
1660
|
+
}
|
|
1661
|
+
|
|
1274
1662
|
/// `\u` escapes after the `u`: `XXXX` (joining a surrogate pair) or `{X...}`.
|
|
1275
1663
|
fn unicode_escape(c: &[char], i: usize) -> Option<(u32, usize)> {
|
|
1276
1664
|
if c.get(i) == Some(&'{') {
|
|
@@ -1354,6 +1742,13 @@ fn class(c: &[char], mut i: usize, unicode: bool) -> Option<(String, usize)> {
|
|
|
1354
1742
|
if members == 0 {
|
|
1355
1743
|
return None; // `[]` and `[^]` have no counterpart.
|
|
1356
1744
|
}
|
|
1745
|
+
// `regex` 1.13.1 (regex-syntax 0.8.11) negates wrongly a class with a range that ends at U+D7FF and another
|
|
1746
|
+
// that starts at U+E000, the characters on either side of the surrogates: the complement takes both in, so
|
|
1747
|
+
// `[^\x{D7FF}\x{E000}]` matches every character. A negated class that names both (every literal here is
|
|
1748
|
+
// spelled `\x{...}`, so a `^` is a negation) is left to the engine.
|
|
1749
|
+
if out.contains('^') && out.contains(r"\x{D7FF}") && out.contains(r"\x{E000}") {
|
|
1750
|
+
return None;
|
|
1751
|
+
}
|
|
1357
1752
|
out.push(']');
|
|
1358
1753
|
Some((out, i))
|
|
1359
1754
|
}
|
|
@@ -1484,6 +1879,20 @@ mod tests {
|
|
|
1484
1879
|
"^(é,)*a$",
|
|
1485
1880
|
"^(?=!+[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$",
|
|
1486
1881
|
"^(?=!+[^a]+$)(?=(.*\\w)).+$",
|
|
1882
|
+
// Lists with an anchor or an assertion inside them, and with an escaped anchor.
|
|
1883
|
+
"^x(,a$)*$",
|
|
1884
|
+
"^a(,a$)+$",
|
|
1885
|
+
"^x$(,a)*$",
|
|
1886
|
+
"^(a$)(,a)*$",
|
|
1887
|
+
"^$(,a)*$",
|
|
1888
|
+
"^(a,$)*b$",
|
|
1889
|
+
"^([a-z]+,$)+a$",
|
|
1890
|
+
"^a(^,a)*$",
|
|
1891
|
+
"^(^a,)*b$",
|
|
1892
|
+
"^a(,a\\b)*$",
|
|
1893
|
+
"^a(,a\\$)*$",
|
|
1894
|
+
"^a\\$(,a)*$",
|
|
1895
|
+
"^(a\\$,)*b$",
|
|
1487
1896
|
// Valid only without the `u` flag (identity escapes), so matching code units.
|
|
1488
1897
|
"^\\/[^\\*\\?\\&\\%]*(\\/\\*)?$",
|
|
1489
1898
|
"^[\\&\\@\\_]+$",
|
|
@@ -1493,9 +1902,10 @@ mod tests {
|
|
|
1493
1902
|
"\\uD83D\\uDE00|\\&",
|
|
1494
1903
|
];
|
|
1495
1904
|
|
|
1496
|
-
/// Every matcher agrees with
|
|
1905
|
+
/// Every matcher agrees with the engine's backtracking matcher on strings over an alphabet that exercises
|
|
1906
|
+
/// classes, anchors and non-ASCII.
|
|
1497
1907
|
#[test]
|
|
1498
|
-
fn
|
|
1908
|
+
fn matchers_agree_with_the_engine() {
|
|
1499
1909
|
let alphabet = [
|
|
1500
1910
|
"a", "b", "z", "A", "X", "Z", "0", "1", "5", "9", "_", "-", ".", ":", "/", "@", "#", "$", "*", "!", "{",
|
|
1501
1911
|
"}", "|", " ", "\n", "\u{a0}", "é", "µ", "😀", "\u{2028}", "x-", "es", "ES", "ms", "txt", "Au", "to", "No",
|
|
@@ -1509,12 +1919,21 @@ mod tests {
|
|
|
1509
1919
|
seed
|
|
1510
1920
|
};
|
|
1511
1921
|
for &p in PATTERNS {
|
|
1512
|
-
let
|
|
1922
|
+
let reference = ecmaregex::compile(p).unwrap();
|
|
1513
1923
|
let compiled = compile(p).unwrap();
|
|
1514
1924
|
for _ in 0..4000 {
|
|
1515
1925
|
let len = next() % 9;
|
|
1516
1926
|
let s: String = (0..len).map(|_| alphabet[(next() % alphabet.len() as u64) as usize]).collect();
|
|
1517
|
-
|
|
1927
|
+
// Without the `u` flag the engine matches the UTF-16 code units of the string.
|
|
1928
|
+
let found = reference.match_backtrack(&s, None).unwrap();
|
|
1929
|
+
for limit in [None, Some(Duration::from_secs(60)), Some(Duration::from_nanos(1))] {
|
|
1930
|
+
let mut budget = PatternBudget::new(PatternLimit::new(limit));
|
|
1931
|
+
let got = compiled.is_match(&s, &mut budget);
|
|
1932
|
+
// A limit of a nanosecond gives `regex` no string, so this is the engine for every pattern on
|
|
1933
|
+
// it. A match that reached its first check has no answer.
|
|
1934
|
+
assert!(got == found || budget.failed(), "{p:?} on {s:?} with a limit of {limit:?}");
|
|
1935
|
+
assert!(limit.is_some_and(|l| l.as_nanos() == 1) || !budget.failed(), "{p:?} on {s:?}");
|
|
1936
|
+
}
|
|
1518
1937
|
}
|
|
1519
1938
|
}
|
|
1520
1939
|
}
|
|
@@ -1555,9 +1974,34 @@ mod tests {
|
|
|
1555
1974
|
"^([a-zA-Z0-9]{2,3})(-[a-zA-Z0-9]{1,6})*$",
|
|
1556
1975
|
"^([a-zA-Z_$][a-zA-Z0-9_$]{0,39}\\.)*([a-zA-Z_$][a-zA-Z0-9_$]{0,39})$",
|
|
1557
1976
|
"^([a-z_$][a-z0-9_$]{0,39}\\.)*([a-zA-Z_$][a-zA-Z0-9_$]{0,39})$",
|
|
1977
|
+
// An escaped `$` or `^` is a literal character of the list.
|
|
1978
|
+
"^x(,a\\$)*$",
|
|
1979
|
+
"^x\\$(,a)*$",
|
|
1980
|
+
"^(a\\$,)*b$",
|
|
1981
|
+
"^x(,\\^a)*$",
|
|
1982
|
+
"^(a,)*b\\$$",
|
|
1983
|
+
"^x(,[$^]a)*$",
|
|
1558
1984
|
] {
|
|
1559
1985
|
assert!(matches!(compile(p).unwrap().matcher, Matcher::SeparatedList(_)), "{p}");
|
|
1560
1986
|
}
|
|
1987
|
+
// A list with an anchor or an assertion inside it has no shape.
|
|
1988
|
+
for p in [
|
|
1989
|
+
"^x(,a$)*$",
|
|
1990
|
+
"^x(?:,a$)+$",
|
|
1991
|
+
"^x$(,a)*$",
|
|
1992
|
+
"^(x$)(,a)*$",
|
|
1993
|
+
"^$(,a)*$",
|
|
1994
|
+
"^(a,$)*b$",
|
|
1995
|
+
"^x(^,a)*$",
|
|
1996
|
+
"^(^a,)*b$",
|
|
1997
|
+
"^x(,(a$))*$",
|
|
1998
|
+
"^x(,a\\b)*$",
|
|
1999
|
+
"^x\\B(,a)*$",
|
|
2000
|
+
"^x(,a(?=,|$))*$",
|
|
2001
|
+
"^x(?=,)(,a)*$",
|
|
2002
|
+
] {
|
|
2003
|
+
assert!(matches!(compile(p).unwrap().matcher, Matcher::Regex(_) | Matcher::Engine(_)), "{p}");
|
|
2004
|
+
}
|
|
1561
2005
|
assert!(matches!(
|
|
1562
2006
|
compile("^(?=[^!*,;{}[\\]~\\n]+$)(?=(.*\\w)).+$").unwrap().matcher,
|
|
1563
2007
|
Matcher::ExcludedClassWithWord { .. }
|
|
@@ -1568,8 +2012,240 @@ mod tests {
|
|
|
1568
2012
|
for p in ["^\\/[^\\*\\?\\&\\%]*(\\/\\*)?$", "a\\&.b", "^\\-a"] {
|
|
1569
2013
|
assert!(matches!(compile(p).unwrap().matcher, Matcher::RegexBmp(..)), "{p}");
|
|
1570
2014
|
}
|
|
1571
|
-
for p in ["^((\\.(?!\\.)\\/)?\\w+\\/?)+$", "^\\1(a)", "
|
|
1572
|
-
assert!(matches!(compile(p).unwrap().matcher, Matcher::
|
|
2015
|
+
for p in ["^((\\.(?!\\.)\\/)?\\w+\\/?)+$", "^\\1(a)", "\\Bs", "(?i:a)", "^[]", "\\cA"] {
|
|
2016
|
+
assert!(matches!(compile(p).unwrap().matcher, Matcher::Engine(_)), "{p}");
|
|
2017
|
+
}
|
|
2018
|
+
// Valid only without the `u` flag, and not in the syntax `regex` takes: the engine, by code unit.
|
|
2019
|
+
for p in ["^\\p{L}\\&", "^\\-?\u{10400}*$", "^\\-(?=a)"] {
|
|
2020
|
+
let compiled = compile(p).unwrap();
|
|
2021
|
+
assert!(matches!(&compiled.matcher, Matcher::Engine(engine) if !engine.unicode()), "{p}");
|
|
2022
|
+
}
|
|
2023
|
+
}
|
|
2024
|
+
}
|
|
2025
|
+
|
|
2026
|
+
#[cfg(test)]
|
|
2027
|
+
mod limits {
|
|
2028
|
+
use super::*;
|
|
2029
|
+
|
|
2030
|
+
/// Which matcher a pattern gets (the Go module's `TestAWideSearchIsLeftToTheEngine`). A sequence or a literal
|
|
2031
|
+
/// that would be searched for at every position of the text keeps its matcher up to 64 characters, and beyond
|
|
2032
|
+
/// that is left to a matcher that takes time proportional to the text or has the limit of a match.
|
|
2033
|
+
#[test]
|
|
2034
|
+
fn a_wide_search_is_left_to_the_engine() {
|
|
2035
|
+
let a = |n: usize| "a".repeat(n);
|
|
2036
|
+
for (pattern, alternatives) in [
|
|
2037
|
+
(r"[a-z]{64}[0-9]{0}|x".to_string(), true),
|
|
2038
|
+
(r"[a-z]{63}[0-9]|x".to_string(), true),
|
|
2039
|
+
(r"[a-z]{64}[0-9]|x".to_string(), false),
|
|
2040
|
+
(r"[a-z]{50000}[0-9]|x".to_string(), false),
|
|
2041
|
+
(r"^[a-z]{50000}[0-9]|x".to_string(), true),
|
|
2042
|
+
(r"[a-z]{50000}[0-9]$|^x".to_string(), true),
|
|
2043
|
+
(format!("{}|^x", a(64)), true),
|
|
2044
|
+
(format!("{}|^x", a(65)), false),
|
|
2045
|
+
(format!("^{}|^x", a(65)), true),
|
|
2046
|
+
(format!("{}$|^x", a(65)), true),
|
|
2047
|
+
] {
|
|
2048
|
+
let found = kind(&compile(&pattern).unwrap());
|
|
2049
|
+
assert_eq!(found == "alternatives", alternatives, "{pattern:.30} has the matcher {found}");
|
|
2050
|
+
assert!(alternatives || matches!(found, "regex" | "engine"), "{pattern:.30} has the matcher {found}");
|
|
2051
|
+
}
|
|
2052
|
+
// The final sequence of a list must stop at the separator.
|
|
2053
|
+
for (pattern, list) in [
|
|
2054
|
+
(r"^(?:[a-z]+,)*[a-z]{2}[0-9]$", true),
|
|
2055
|
+
(r"^(?:[a-z]+,)*[a-z,]+[0-9]$", false),
|
|
2056
|
+
(r"^(?:[a-z]+,)*[0-9]{1,3}$", true),
|
|
2057
|
+
] {
|
|
2058
|
+
let found = kind(&compile(pattern).unwrap());
|
|
2059
|
+
assert_eq!(found == "list", list, "{pattern} has the matcher {found}");
|
|
2060
|
+
}
|
|
2061
|
+
}
|
|
2062
|
+
|
|
2063
|
+
/// The `regex` crate is given no large program (the Go module's `TestTheRegexpPackageIsGivenNoLargeProgram`):
|
|
2064
|
+
/// it writes a counted repetition out, so the pattern goes to the engine when the copies are many.
|
|
2065
|
+
#[test]
|
|
2066
|
+
fn regex_is_given_no_large_program() {
|
|
2067
|
+
for (pattern, want) in [
|
|
2068
|
+
(r"a{1000}".to_string(), "regex"),
|
|
2069
|
+
(r"(?:a|b|ab)*c".to_string(), "regex"),
|
|
2070
|
+
(r"\p{L}{1000}".repeat(90), "engine"),
|
|
2071
|
+
(r"(?:[ab]{1000}){50}c".to_string(), "engine"),
|
|
2072
|
+
(r"(?:a{100}){101}".to_string(), "engine"),
|
|
2073
|
+
(r"(?:(?:a?){100}){100}b".to_string(), "engine"),
|
|
2074
|
+
] {
|
|
2075
|
+
assert_eq!(kind(&compile(&pattern).unwrap()), want, "{pattern:.40}");
|
|
2076
|
+
}
|
|
2077
|
+
}
|
|
2078
|
+
|
|
2079
|
+
/// A string too long for `regex` to answer within the limit of a match goes to the engine, which can be
|
|
2080
|
+
/// stopped (the Go module's `TestALongTextIsNotGivenToTheRegexpPackage`).
|
|
2081
|
+
#[test]
|
|
2082
|
+
fn a_long_string_is_not_given_to_regex() {
|
|
2083
|
+
let compiled = compile(r"^(?:[a-c\u00e9]{0,500}x)*$").unwrap();
|
|
2084
|
+
let Matcher::Regex(linear) = &compiled.matcher else { panic!("the pattern is on {}", kind(&compiled)) };
|
|
2085
|
+
assert!(linear.cost >= 500, "a program of {} instructions", linear.cost);
|
|
2086
|
+
let short = "\u{e9}abcx".to_string();
|
|
2087
|
+
let long = "\u{e9}abcx".repeat(100000);
|
|
2088
|
+
let second = PatternLimit::new(Some(Duration::from_secs(1)));
|
|
2089
|
+
let mut budget = PatternBudget::new(second);
|
|
2090
|
+
assert!(linear.fits(&short, &budget) && !linear.fits(&long, &budget), "the strings are not on each side");
|
|
2091
|
+
assert!(compiled.is_match(&short, &mut budget) && linear.engine.get().is_none(), "a short string");
|
|
2092
|
+
assert!(compiled.is_match(&long, &mut budget) && !budget.failed(), "a long string");
|
|
2093
|
+
assert!(matches!(linear.engine.get(), Some(Some(_))), "the engine was not made for the long string");
|
|
2094
|
+
// With no limit every string is `regex`'s.
|
|
2095
|
+
let mut unlimited = PatternBudget::new(PatternLimit::new(None));
|
|
2096
|
+
assert!(linear.fits(&long, &unlimited) && !compiled.is_match(&format!("{long}y"), &mut unlimited));
|
|
2097
|
+
}
|
|
2098
|
+
|
|
2099
|
+
/// The cache of compiled patterns is bounded in number and in size, and a pattern that has left it is still
|
|
2100
|
+
/// matched by whoever holds it.
|
|
2101
|
+
#[test]
|
|
2102
|
+
fn the_cache_of_patterns_is_bounded() {
|
|
2103
|
+
let mut budget = PatternBudget::new(PatternLimit::new(None));
|
|
2104
|
+
let first = compile("^cache-bound-0+$").unwrap();
|
|
2105
|
+
let held: Vec<Arc<Pattern>> =
|
|
2106
|
+
(0..3 * MAX_CACHED_PATTERNS).map(|n| compile(&format!("^cache-bound-{n}(?=x)x$")).unwrap()).collect();
|
|
2107
|
+
{
|
|
2108
|
+
let cache = cache();
|
|
2109
|
+
assert!(cache.patterns.len() <= MAX_CACHED_PATTERNS && cache.bytes <= MAX_CACHED_BYTES);
|
|
2110
|
+
assert_eq!(cache.patterns.len(), cache.order.len());
|
|
2111
|
+
assert_eq!(cache.bytes, cache.patterns.values().map(|p| p.retained).sum::<usize>());
|
|
2112
|
+
assert!(cache.get("^cache-bound-0+$").is_none(), "the oldest pattern is still in the cache");
|
|
2113
|
+
}
|
|
2114
|
+
assert!(first.is_match("cache-bound-000", &mut budget));
|
|
2115
|
+
assert!(held[0].is_match("cache-bound-0x", &mut budget) && !held[7].is_match("cache-bound-0x", &mut budget));
|
|
2116
|
+
// The same source compiles again, to a matcher of its own.
|
|
2117
|
+
let again = compile("^cache-bound-0+$").unwrap();
|
|
2118
|
+
assert!(!Arc::ptr_eq(&first, &again) && again.is_match("cache-bound-0", &mut budget));
|
|
2119
|
+
// Large patterns leave the cache by size, well before their number is the bound.
|
|
2120
|
+
let class = r"[\u0100\u0102\u0104\u0106]".repeat(1500);
|
|
2121
|
+
let large: Vec<Arc<Pattern>> = (0..200).map(|n| compile(&format!("^(?=large-{n}){class}$")).unwrap()).collect();
|
|
2122
|
+
assert!(large[0].retained > 256 << 10, "a large pattern is counted as {} bytes", large[0].retained);
|
|
2123
|
+
assert!(cache().bytes <= MAX_CACHED_BYTES);
|
|
2124
|
+
// A pattern too large for the cache is not kept at all.
|
|
2125
|
+
let huge = compile(&format!("^(?=huge){}$", "a".repeat(15000))).unwrap();
|
|
2126
|
+
assert!(huge.retained > MAX_CACHED_PATTERN_BYTES && cache().get(&huge.source).is_none());
|
|
2127
|
+
}
|
|
2128
|
+
|
|
2129
|
+
/// The `regex` crate compiles no program over its limit of size: a class of a Unicode property repeated often
|
|
2130
|
+
/// is the engine's, which holds the class once.
|
|
2131
|
+
#[test]
|
|
2132
|
+
fn regex_compiles_no_program_over_its_limit() {
|
|
2133
|
+
for (pattern, want) in [
|
|
2134
|
+
(r"^\p{L}+$", "regex"),
|
|
2135
|
+
(r"^[\p{L}\p{N}_-]{1,8}$", "regex"),
|
|
2136
|
+
(r"^[\p{L}\p{N}_-]{1,64}$", "engine"),
|
|
2137
|
+
(r"^[\p{Alphabetic}\p{Nd}]{1,255}$", "engine"),
|
|
2138
|
+
] {
|
|
2139
|
+
let compiled = compile(pattern).unwrap();
|
|
2140
|
+
assert_eq!(kind(&compiled), want, "{pattern}");
|
|
2141
|
+
let mut budget = PatternBudget::new(PatternLimit::new(Some(Duration::from_secs(1))));
|
|
2142
|
+
assert!(compiled.is_match("caf\u{e9}", &mut budget) && !compiled.is_match("caf\u{e9}!", &mut budget));
|
|
2143
|
+
}
|
|
2144
|
+
}
|
|
2145
|
+
|
|
2146
|
+
/// Alternatives of one character each are one class to a linear matcher, so a long alternation is a small
|
|
2147
|
+
/// program, and `regex` keeps a long string for it. And a pattern that nests more deeply than `regex` reads by
|
|
2148
|
+
/// default is still compiled by it, on a thread with a stack for the nesting.
|
|
2149
|
+
#[test]
|
|
2150
|
+
fn regex_keeps_a_long_alternation_and_a_deep_nest() {
|
|
2151
|
+
let alternatives: Vec<String> = (0..495u32).map(|k| char::from_u32(0x4E00 + k).unwrap().to_string()).collect();
|
|
2152
|
+
let compiled = compile(&format!("(?:{})*z", alternatives.join("|"))).unwrap();
|
|
2153
|
+
let Matcher::Regex(linear) = &compiled.matcher else { panic!("the alternation is on {}", kind(&compiled)) };
|
|
2154
|
+
assert!(linear.cost <= 4, "a program of {} instructions", linear.cost);
|
|
2155
|
+
let text = alternatives.concat().repeat(70);
|
|
2156
|
+
assert!(text.len() >= 100_000);
|
|
2157
|
+
let mut budget = PatternBudget::new(PatternLimit::new(Some(Duration::from_secs(1))));
|
|
2158
|
+
assert!(linear.fits(&text, &budget) && !compiled.is_match(&text, &mut budget) && !budget.failed());
|
|
2159
|
+
assert!(compiled.is_match(&format!("{text}z"), &mut budget) && linear.engine.get().is_none());
|
|
2160
|
+
|
|
2161
|
+
for depth in [300, 999] {
|
|
2162
|
+
let nested = format!("^{}a{}b$", "(".repeat(depth), "*)".repeat(depth));
|
|
2163
|
+
let compiled = compile(&nested).unwrap();
|
|
2164
|
+
assert_eq!(kind(&compiled), "regex", "{depth} nested groups");
|
|
2165
|
+
let start = std::time::Instant::now();
|
|
2166
|
+
assert!(
|
|
2167
|
+
compiled.is_match("aaaab", &mut budget)
|
|
2168
|
+
&& !compiled.is_match("\u{e9}aaaaaaaaaaaaaaaaaaaaaaaaa", &mut budget)
|
|
2169
|
+
);
|
|
2170
|
+
assert!(!budget.failed() && start.elapsed() < Duration::from_secs(20), "{depth} nested groups");
|
|
2171
|
+
}
|
|
2172
|
+
}
|
|
2173
|
+
|
|
2174
|
+
/// The translation for `regex` names no Unicode class: every property is written as the ranges of the engine's
|
|
2175
|
+
/// own tables, so the answer does not depend on the tables of the `regex` crate.
|
|
2176
|
+
#[test]
|
|
2177
|
+
fn the_translation_carries_no_unicode_class() {
|
|
2178
|
+
for p in [
|
|
2179
|
+
r"^\p{L}+$",
|
|
2180
|
+
r"\P{Lu}",
|
|
2181
|
+
r"[\p{Script=Greek}\p{Nd}]",
|
|
2182
|
+
r"[^\p{scx=Hira}]",
|
|
2183
|
+
r"\d\w\s.",
|
|
2184
|
+
r"\p{Any}\p{Assigned}\p{ASCII}",
|
|
2185
|
+
] {
|
|
2186
|
+
let compiled = compile(p).unwrap();
|
|
2187
|
+
assert_eq!(kind(&compiled), "regex", "{p}");
|
|
2188
|
+
let translated = translate(p, true).unwrap();
|
|
2189
|
+
for bad in [r"\p", r"\P", "(?i", "[[:", r"\d", r"\w", r"\s"] {
|
|
2190
|
+
assert!(!translated.contains(bad), "the translation of {p} contains {bad}");
|
|
2191
|
+
}
|
|
2192
|
+
}
|
|
2193
|
+
// The four scripts Unicode 17 added are in the tables, whatever the `regex` crate's own tables hold.
|
|
2194
|
+
let mut budget = PatternBudget::new(PatternLimit::new(None));
|
|
2195
|
+
for (p, text) in [
|
|
2196
|
+
(r"^\p{sc=Sidt}$", "\u{10940}"),
|
|
2197
|
+
(r"^\p{sc=Tols}$", "\u{11DB0}"),
|
|
2198
|
+
(r"^\p{sc=Berf}$", "\u{16EA0}"),
|
|
2199
|
+
(r"^\p{sc=Tayo}$", "\u{1E6C0}"),
|
|
2200
|
+
(r"^[\p{sc=Sidt}a]$", "\u{10940}"),
|
|
2201
|
+
(r"^[^\P{sc=Sidt}]$", "\u{10940}"),
|
|
2202
|
+
] {
|
|
2203
|
+
let compiled = compile(p).unwrap();
|
|
2204
|
+
assert_eq!(kind(&compiled), "regex", "{p}");
|
|
2205
|
+
assert!(compiled.is_match(text, &mut budget) && !compiled.is_match("a!", &mut budget), "{p}");
|
|
2206
|
+
}
|
|
2207
|
+
// The complement of a set whose ranges meet at the surrogates, which `regex` gets wrong when it is given
|
|
2208
|
+
// the two ranges: Unknown holds U+D7FC to U+D7FF and U+E000 to U+F8FF. And the same written by hand, which
|
|
2209
|
+
// is the engine's.
|
|
2210
|
+
for (p, want) in [
|
|
2211
|
+
(r"^\P{sc=Unknown}$", "regex"),
|
|
2212
|
+
(r"^[^\p{scx=Unknown}]$", "regex"),
|
|
2213
|
+
(r"^[^\p{Script=Unknown}a]$", "regex"),
|
|
2214
|
+
(r"^[^\ud7ff\ue000]$", "engine"),
|
|
2215
|
+
(r"^[^\ud7fc-\ud7ff\ue000-\uf8ff]$", "engine"),
|
|
2216
|
+
(r"^[^\p{Co}\ud7ff]$", "engine"),
|
|
2217
|
+
(r"^[^\p{Cn}\ue000]$", "engine"),
|
|
2218
|
+
] {
|
|
2219
|
+
let compiled = compile(p).unwrap();
|
|
2220
|
+
assert_eq!(kind(&compiled), want, "{p}");
|
|
2221
|
+
let engine = ecmaregex::compile(p).unwrap();
|
|
2222
|
+
for text in ["\u{D7FB}", "\u{D7FC}", "\u{D7FF}", "\u{E000}", "\u{E001}", "\u{F8FF}", "\u{F900}", "a", "b"] {
|
|
2223
|
+
assert_eq!(
|
|
2224
|
+
compiled.is_match(text, &mut budget),
|
|
2225
|
+
engine.match_backtrack(text, None).unwrap(),
|
|
2226
|
+
"{p} on {text:?}"
|
|
2227
|
+
);
|
|
2228
|
+
}
|
|
2229
|
+
assert!(!compiled.is_match("\u{E000}", &mut budget), "{p} matches U+E000");
|
|
2230
|
+
}
|
|
2231
|
+
// A set with a range on each side of the surrogates and none of them is written as one range, and a range
|
|
2232
|
+
// of surrogates alone as nothing.
|
|
2233
|
+
let mut written = String::new();
|
|
2234
|
+
push_ranges(&mut written, &ecmaregex::CharSet::new(&[0x61, 0x7A, 0xD700, 0xD7FF, 0xE000, 0xE0FF], false));
|
|
2235
|
+
assert_eq!(written, r"\x{61}-\x{7A}\x{D700}-\x{E0FF}");
|
|
2236
|
+
written.clear();
|
|
2237
|
+
push_ranges(&mut written, &ecmaregex::CharSet::new(&[0xD7FF, 0xDBFF, 0xDC00, 0xDFFF, 0xDFFF, 0xE000], false));
|
|
2238
|
+
assert_eq!(written, r"\x{D7FF}-\x{E000}");
|
|
2239
|
+
written.clear();
|
|
2240
|
+
push_ranges(&mut written, &ecmaregex::CharSet::new(&[0xD800, 0xDFFF], false));
|
|
2241
|
+
assert_eq!(written, "");
|
|
2242
|
+
// A property no character has, and its complement.
|
|
2243
|
+
for (p, yes, no) in
|
|
2244
|
+
[(r"^\P{sc=Hrkt}$", "a", "aa"), (r"^[\p{sc=Hrkt}a]$", "a", "b"), (r"^[^\P{sc=Hrkt}]?$", "", "a")]
|
|
2245
|
+
{
|
|
2246
|
+
let compiled = compile(p).unwrap();
|
|
2247
|
+
assert_eq!(kind(&compiled), "regex", "{p}");
|
|
2248
|
+
assert!(compiled.is_match(yes, &mut budget) && !compiled.is_match(no, &mut budget), "{p}");
|
|
1573
2249
|
}
|
|
1574
2250
|
}
|
|
1575
2251
|
}
|
|
@@ -1583,14 +2259,17 @@ mod corpus_patterns {
|
|
|
1583
2259
|
let Ok(path) = std::env::var("CORVUS_PATTERNS") else { return };
|
|
1584
2260
|
for line in std::fs::read_to_string(path).unwrap().lines() {
|
|
1585
2261
|
let (corpus, p) = line.split_once('\t').unwrap();
|
|
1586
|
-
let kind = super::compile(p)
|
|
1587
|
-
|
|
1588
|
-
|
|
1589
|
-
|
|
1590
|
-
|
|
1591
|
-
|
|
1592
|
-
|
|
1593
|
-
|
|
2262
|
+
let kind = super::compile(p).map(|c| super::kind(&c)).unwrap_or("invalid");
|
|
2263
|
+
// What the `regex` crate is given: the translation.
|
|
2264
|
+
let unicode = crate::ecmaregex::valid(p);
|
|
2265
|
+
let given = match kind {
|
|
2266
|
+
"regex" | "regex-bmp" => super::translate(p, unicode).unwrap_or_default(),
|
|
2267
|
+
_ => String::new(),
|
|
2268
|
+
};
|
|
2269
|
+
// And what the cache counts for the pattern, which for a pattern on `regex` holds the estimate of
|
|
2270
|
+
// that crate's program.
|
|
2271
|
+
let retained = super::compile(p).map_or(0, |c| c.retained);
|
|
2272
|
+
println!("{kind}\t{corpus}\t{p}\t{given}\t{retained}");
|
|
1594
2273
|
}
|
|
1595
2274
|
}
|
|
1596
2275
|
}
|