corvus_json_schema 0.1.3 → 0.2.0

data/ext/corvus_json_schema/vendor/corvus-json-schema/src/eval.rs

@@ -4,7 +4,7 @@
 //! the C# collecting mode (keyword order, paths, messages, which subschema results are committed or discarded).
 
 use std::collections::HashMap;
-use std::sync::OnceLock;
+use std::sync::{Mutex, OnceLock};
 
 use serde_json::{Number, Value};
 
@@ -14,7 +14,8 @@ use crate::formats::FormatKind;
 use crate::instance::{ArrayView, Instance, Kind, ObjectView, View};
 use crate::node::*;
 use crate::numbers::{Num, cmp, multiple_of, num_eq, number_text};
-use crate::options::FormatValidator;
+use crate::options::{FormatValidator, SchemaEvaluationDepthError, SchemaEvaluationError};
+use crate::pattern::{PatternBudget, PatternLimit};
 use crate::results::{JsonSchemaResultsCollector, Message, encode_pointer_segment};
 use crate::uri::resolve_pointer;
 
@@ -36,6 +37,11 @@ pub(crate) struct Program {
     pub annotation_sources: Vec<Option<AnnotationSource>>,
     pub annotations: OnceLock<Vec<Option<Vec<AnnotationEntry>>>>,
     pub assert_format_set: bool,
+    /// The limit of a match of a pattern, which every evaluation gives its matches.
+    pub pattern_limit: PatternLimit,
+    /// The budget of the matches made while the plans are compiled (a pattern against the property names the
+    /// schema declares). It is not used once the program is made.
+    pub compile_budget: Mutex<PatternBudget>,
     /// Fail-fast plans (with a dynamic scope kept, bodies push their resource on entry).
     plans: Vec<plan::Plan>,
 }
@@ -53,6 +59,7 @@ impl Program {
         documents: Vec<Box<Value>>,
         annotation_sources: Vec<Option<AnnotationSource>>,
         assert_format_set: bool,
+        pattern_limit: PatternLimit,
     ) -> Program {
         let fast_target = (0..nodes.len() as NodeId)
             .map(|id| {
@@ -98,6 +105,8 @@ impl Program {
             annotation_sources,
             annotations: OnceLock::new(),
             assert_format_set,
+            pattern_limit,
+            compile_budget: Mutex::new(PatternBudget::new(pattern_limit)),
             plans: Vec::new(),
         };
         program.plans = plan::compile_plans(&program);
@@ -313,9 +322,8 @@ fn str_hash(s: &str) -> u64 {
     const K: u64 = 0x9e37_79b9_7f4a_7c15;
     let b = s.as_bytes();
     let mut h = (b.len() as u64).wrapping_mul(K);
-    let mut chunks = b.chunks_exact(8);
-    for c in &mut chunks {
-        h = (h.rotate_left(5) ^ u64::from_le_bytes(c.try_into().unwrap())).wrapping_mul(K);
+    for c in b.as_chunks::<8>().0 {
+        h = (h.rotate_left(5) ^ u64::from_le_bytes(*c)).wrapping_mul(K);
     }
     // The tail as one word without copying it out: the last eight bytes when there are that many (overlapping the
     // chunks already hashed), else the overlapping first and last four, else the bytes themselves.
@@ -390,7 +398,7 @@ fn code_points(s: &str) -> u64 {
 
 fn base64_decode(s: &str) -> Option<Vec<u8>> {
     let b = s.as_bytes();
-    if b.len() % 4 != 0 {
+    if !b.len().is_multiple_of(4) {
         return None;
     }
     let val = |c: u8| -> Option<u32> {
@@ -548,6 +556,8 @@ pub(crate) struct Evaluator<'p, 'c> {
     scope: Vec<u32>,
     depth: u32,
     pub depth_exceeded: bool,
+    /// What every match of a pattern in this evaluation has, and the first match that reached a limit.
+    pub(crate) budget: PatternBudget,
     /// A string reused to evaluate property names against propertyNames (a `String`, not a `Value`: dropping a
     /// `Value` is an out-of-line call, which every evaluation paid).
     name_buffer: String,
@@ -581,7 +591,16 @@ macro_rules! and {
 impl<'p, 'c> Evaluator<'p, 'c> {
     pub fn new(p: &'p Program, c: Option<&'c mut JsonSchemaResultsCollector>) -> Self {
         let annotations: &'p [Option<Vec<AnnotationEntry>>] = if c.is_some() { p.annotations() } else { &[] };
-        Evaluator { p, annotations, c, scope: Vec::new(), depth: 0, depth_exceeded: false, name_buffer: String::new() }
+        Evaluator {
+            p,
+            annotations,
+            c,
+            scope: Vec::new(),
+            depth: 0,
+            depth_exceeded: false,
+            budget: PatternBudget::new(p.pattern_limit),
+            name_buffer: String::new(),
+        }
     }
 
     #[inline(always)]
@@ -598,8 +617,21 @@ impl<'p, 'c> Evaluator<'p, 'c> {
     pub fn validate<'x, I: Instance<'x>>(&mut self, x: I) -> bool {
         let root = self.p.fast_target[self.p.root as usize];
         // An evaluation that recursed in place beyond the maximum depth is not valid, whatever it came to: the
-        // branch that was abandoned counts as false, which a not above it turns into true.
-        self.fast(root, x) && !self.depth_exceeded
+        // branch that was abandoned counts as false, which a not above it turns into true. The same holds for a
+        // match of a pattern that reached its limit.
+        self.fast(root, x) && !self.depth_exceeded && !self.budget.failed()
+    }
+
+    /// What an evaluation that came to `ok` reports: no answer, and why, when a match of a pattern reached its
+    /// limit or the schema recursed in place beyond the maximum depth.
+    pub fn outcome(&mut self, ok: bool) -> Result<bool, SchemaEvaluationError> {
+        if let Some(e) = self.budget.take_failure() {
+            return Err(SchemaEvaluationError::PatternMatchLimit(e));
+        }
+        if self.depth_exceeded {
+            return Err(SchemaEvaluationError::DepthExceeded(SchemaEvaluationDepthError));
+        }
+        Ok(ok)
     }
 
     /// Fail-fast evaluation of a node, through its plan when there are plans.
@@ -723,15 +755,15 @@ impl<'p, 'c> Evaluator<'p, 'c> {
             }
             _ => {}
         }
-        if M::COLLECT {
-            if let Some(list) = &self.annotations[id as usize] {
-                for a in list {
-                    if a.strings_only && x.kind() != Kind::String {
-                        continue;
-                    }
-                    let v = &a.value;
-                    self.col().ignored_keyword(Message::Lazy(&|| v.to_string()), &a.keyword);
+        if M::COLLECT
+            && let Some(list) = &self.annotations[id as usize]
+        {
+            for a in list {
+                if a.strings_only && x.kind() != Kind::String {
+                    continue;
                 }
+                let v = &a.value;
+                self.col().ignored_keyword(Message::Lazy(&|| v.to_string()), &a.keyword);
             }
         }
         ok
@@ -742,24 +774,25 @@ impl<'p, 'c> Evaluator<'p, 'c> {
 
     fn eval_number<M: Mode>(&mut self, n: &'p SchemaNode, x: &Number) -> bool {
         let mut ok = true;
-        if n.assert_format && n.format_kind.is_numeric() {
-            if let Some(f) = &n.format {
-                let m = match self.p.formats.get(f) {
-                    Some(custom) => custom(&x.to_string()),
-                    None => n.format_kind.check_number(x),
-                };
-                let kind = n.format_kind.name();
-                check!(
-                    self,
-                    M,
-                    ok,
-                    m,
-                    Message::Lazy(&|| format!(
-                        "The value was expected to be in a supported format, and within bounds for '{kind}'"
-                    )),
-                    "format"
-                );
-            }
+        if n.assert_format
+            && n.format_kind.is_numeric()
+            && let Some(f) = &n.format
+        {
+            let m = match self.p.formats.get(f) {
+                Some(custom) => custom(&x.to_string()),
+                None => n.format_kind.check_number(x),
+            };
+            let kind = n.format_kind.name();
+            check!(
+                self,
+                M,
+                ok,
+                m,
+                Message::Lazy(&|| format!(
+                    "The value was expected to be in a supported format, and within bounds for '{kind}'"
+                )),
+                "format"
+            );
         }
         let xv = Num::of(x);
         use std::cmp::Ordering::*;
@@ -856,23 +889,24 @@ impl<'p, 'c> Evaluator<'p, 'c> {
                 self,
                 M,
                 ok,
-                p.is_match(s),
+                p.is_match(s, &mut self.budget),
                 Message::Lazy(&|| format!("Expected the value to match the regular expression{}", q(&p.source))),
                 "pattern"
             );
         }
-        if n.assert_format && !n.format_kind.is_numeric() {
-            if let Some(f) = &n.format {
-                let (m, message): (bool, Message) = match self.p.formats.get(f) {
-                    Some(custom) => (custom(s), Message::Lazy(&|| format!("Expected a string in the '{f}' format."))),
-                    None if n.format_kind == FormatKind::Unknown => (true, Message::None),
-                    None => (
-                        n.format_kind.check_string(s, n.dialect <= Dialect::Draft6),
-                        n.format_kind.message().map_or(Message::None, Message::Static),
-                    ),
-                };
-                check!(self, M, ok, m, message, "format");
-            }
+        if n.assert_format
+            && !n.format_kind.is_numeric()
+            && let Some(f) = &n.format
+        {
+            let (m, message): (bool, Message) = match self.p.formats.get(f) {
+                Some(custom) => (custom(s), Message::Lazy(&|| format!("Expected a string in the '{f}' format."))),
+                None if n.format_kind == FormatKind::Unknown => (true, Message::None),
+                None => (
+                    n.format_kind.check_string(s, n.dialect <= Dialect::Draft6),
+                    n.format_kind.message().map_or(Message::None, Message::Static),
+                ),
+            };
+            check!(self, M, ok, m, message, "format");
         }
         if n.assert_content {
             let m = content_ok(s, n.content);
@@ -963,7 +997,7 @@ impl<'p, 'c> Evaluator<'p, 'c> {
                 }
                 if let Some(pps) = &n.pattern_properties {
                     for pp in pps {
-                        if !pp.pattern.is_match(k) {
+                        if !pp.pattern.is_match(k, &mut self.budget) {
                             continue;
                         }
                         matched = true;
@@ -983,19 +1017,19 @@ impl<'p, 'c> Evaluator<'p, 'c> {
                         );
                     }
                 }
-                if let Some(ap) = n.additional_properties {
-                    if !matched {
-                        if let Some(b) = bits.as_deref_mut() {
-                            b.set(i);
-                        }
-                        and!(
-                            M,
-                            ok,
-                            self.eval_at::<M, I>(ap, &|| "additionalProperties".to_string(), v, &|| {
-                                encode_pointer_segment(k).into_owned()
-                            })
-                        );
+                if let Some(ap) = n.additional_properties
+                    && !matched
+                {
+                    if let Some(b) = bits.as_deref_mut() {
+                        b.set(i);
                     }
+                    and!(
+                        M,
+                        ok,
+                        self.eval_at::<M, I>(ap, &|| "additionalProperties".to_string(), v, &|| {
+                            encode_pointer_segment(k).into_owned()
+                        })
+                    );
                 }
                 if let Some(pn) = n.property_names {
                     let name = Value::String(k.to_string());
@@ -1185,10 +1219,10 @@ impl<'p, 'c> Evaluator<'p, 'c> {
                 };
                 if matched {
                     count += 1;
-                    if n.contains_marks_evaluated {
-                        if let Some(b) = bits.as_deref_mut() {
-                            b.set(i);
-                        }
+                    if n.contains_marks_evaluated
+                        && let Some(b) = bits.as_deref_mut()
+                    {
+                        b.set(i);
                     }
                 }
             }
@@ -1296,10 +1330,8 @@ impl<'p, 'c> Evaluator<'p, 'c> {
         if guarded {
             self.depth -= 1;
         }
-        if ok {
-            if let (Some(s), Some(b)) = (&scratch, bits) {
-                b.merge(s);
-            }
+        if ok && let (Some(s), Some(b)) = (&scratch, bits) {
+            b.merge(s);
         }
         (ok, scratch)
     }
@@ -1402,10 +1434,10 @@ impl<'p, 'c> Evaluator<'p, 'c> {
                     }
                 }
             }
-            if matched == 1 {
-                if let (Some(b), Some(o)) = (bits.as_deref_mut(), &only) {
-                    b.merge(o);
-                }
+            if matched == 1
+                && let (Some(b), Some(o)) = (bits.as_deref_mut(), &only)
+            {
+                b.merge(o);
             }
             let message = match matched {
                 0 => MATCHED_NO_SCHEMA,

data/ext/corvus_json_schema/vendor/corvus-json-schema/src/formats.rs

@@ -30,7 +30,8 @@ pub(crate) enum FormatKind {
     UriTemplate,
     JsonPointer,
     RelativeJsonPointer,
-    Regex,
+    /// The regex format, read in the grammar the dialect reads patterns with.
+    Regex(crate::pattern::Grammar),
     // Numeric formats (a Corvus extension).
     Byte,
     UInt16,
@@ -49,7 +50,7 @@ pub(crate) enum FormatKind {
 }
 
 impl FormatKind {
-    pub fn of(format: &str, dialect: Dialect) -> FormatKind {
+    pub fn of(format: &str, dialect: Dialect, grammar: crate::pattern::Grammar) -> FormatKind {
         use FormatKind::*;
         let at_least = |d: Dialect, k: FormatKind| if dialect >= d { k } else { Unknown };
         match format {
@@ -78,7 +79,7 @@ impl FormatKind {
             "json-pointer" => at_least(Dialect::Draft6, JsonPointer),
             "date" => at_least(Dialect::Draft7, Date),
             "time" => at_least(Dialect::Draft7, Time),
-            "regex" => at_least(Dialect::Draft7, Regex),
+            "regex" => at_least(Dialect::Draft7, Regex(grammar)),
             "relative-json-pointer" => at_least(Dialect::Draft7, RelativeJsonPointer),
             "idn-email" => at_least(Dialect::Draft7, IdnEmail),
             "idn-hostname" => at_least(Dialect::Draft7, IdnHostname),
@@ -133,7 +134,7 @@ impl FormatKind {
             UriTemplate => "uri-template",
             JsonPointer => "json-pointer",
             RelativeJsonPointer => "relative-json-pointer",
-            Regex => "regex",
+            Regex(_) => "regex",
             Byte => "byte",
             UInt16 => "uint16",
             UInt32 => "uint32",
@@ -175,7 +176,7 @@ impl FormatKind {
             RelativeJsonPointer => {
                 "Expected a Relative JSON Pointer. (https://json-schema.org/draft/2020-12/relative-json-pointer)."
             }
-            Regex => "Expected a regular expression specification.",
+            Regex(_) => "Expected a regular expression specification.",
             _ => return None,
         })
     }
@@ -208,7 +209,7 @@ impl FormatKind {
             UriTemplate => URI_TEMPLATE_RE.is_match(s),
             JsonPointer => JSON_POINTER_RE.is_match(s),
             RelativeJsonPointer => RELATIVE_JSON_POINTER_RE.is_match(s),
-            Regex => crate::pattern::is_valid_ecma_regex(s),
+            Regex(grammar) => crate::pattern::is_valid_ecma_regex(s, grammar),
             _ => true,
         }
     }
@@ -264,7 +265,7 @@ impl FormatKind {
 }
 
 fn is_leap_year(y: u32) -> bool {
-    y % 4 == 0 && (y % 100 != 0 || y % 400 == 0)
+    y.is_multiple_of(4) && (!y.is_multiple_of(100) || y.is_multiple_of(400))
 }
 
 fn digits(s: &[u8]) -> Option<u32> {
@@ -773,7 +774,20 @@ fn idn_label_ok(label: &str) -> bool {
     true
 }
 
+/// The most bytes an internationalized host name has. A name is 253 characters at most as A-labels, and no label
+/// is shorter as an A-label than it has characters: a letter of ASCII is itself there, any other character is a
+/// digit or more, and `xn--` comes before them. So a name has 253 characters at most, each of four bytes at most.
+const MAX_IDN_HOSTNAME_BYTES: usize = 4 * 253;
+
+/// Checks an internationalized host name. A string over [`MAX_IDN_HOSTNAME_BYTES`] is not one, and is not read: the
+/// check decodes and encodes Punycode and walks around every joiner and every katakana middle dot, which take time
+/// that grows with the square of a label, so it must not be given a label of any length.
 pub(crate) fn idn_hostname(s: &str) -> bool {
+    s.len() <= MAX_IDN_HOSTNAME_BYTES && idn_hostname_of_any_length(s)
+}
+
+/// The check itself, for a string of any length.
+fn idn_hostname_of_any_length(s: &str) -> bool {
     if s.is_empty() {
         return false;
     }
@@ -932,6 +946,87 @@ mod tests {
         assert!(DURATION_RE.is_match("P4DT12H30M5S") && !DURATION_RE.is_match("PT1D"));
     }
 
+    /// The check of an internationalized host name decodes and encodes Punycode, looks for a kana for every katakana
+    /// middle dot and walks the neighbours of every joiner, each of which takes time that grows with the square of a
+    /// label. It did so for a label of any length: a host name of 100,000 different letters took 4 seconds to
+    /// refuse. A string over the bound is refused at once and not read. The time allowed here is generous.
+    #[test]
+    fn a_long_string_is_not_an_internationalized_host_name() {
+        const N: usize = 1_000_000;
+        let distinct: String = (0..N).map(|i| char::from_u32(0x4e00 + (i % 20000) as u32).unwrap()).collect();
+        for (name, text) in [
+            ("katakana middle dots after a kana", format!("\u{30a2}{}", "\u{30fb}".repeat(N))),
+            ("different letters", distinct),
+            ("one letter", "\u{e9}".repeat(N)),
+            ("joiners", format!("\u{628}{}\u{628}", "\u{64b}\u{200c}".repeat(N / 2))),
+            ("a long A-label", format!("xn--{}-{}", "a".repeat(N / 2), "b9".repeat(N / 4))),
+            ("labels", format!("{}a", "a.".repeat(N / 2))),
+        ] {
+            for (format, text) in [(FormatKind::IdnHostname, text.clone()), (FormatKind::IdnEmail, format!("a@{text}"))]
+            {
+                let start = std::time::Instant::now();
+                let valid = format.check_string(&text, false);
+                let took = start.elapsed();
+                assert!(
+                    !valid && took < std::time::Duration::from_secs(2),
+                    "{format:?}, {name}: valid {valid} in {took:?}"
+                );
+            }
+        }
+    }
+
+    /// What the bound rests on: every host name the check accepts has 253 characters at most, so 1,012 bytes at
+    /// most. Names are built from labels of letters that take one to four bytes, on both sides of the bound, and
+    /// the check is compared with the check that has no bound.
+    #[test]
+    fn no_internationalized_host_name_is_over_the_bound() {
+        // A small generator of its own, so that the test needs no crate: the numbers only have to be spread.
+        let mut state = 20261011u64;
+        let mut next = |below: usize| {
+            state = state.wrapping_mul(6364136223846793005).wrapping_add(1442695040888963407);
+            ((state >> 33) as usize) % below
+        };
+        let letters =
+            ['a', 'z', '0', '\u{e9}', '\u{3b1}', '\u{4e00}', '\u{30a2}', '\u{20000}', '\u{2a700}', '\u{10428}'];
+        let (mut accepted, mut longest, mut over) = (0, 0, 0);
+        for round in 0..20000 {
+            let mut name = String::new();
+            // A label of one letter of four bytes is the most bytes a name can have for its length as an A-label,
+            // so every fourth name is made of those.
+            let wide = round % 4 == 0;
+            let labels = 1 + next(1 + round % 500);
+            for label in 0..labels {
+                if wide {
+                    name.push(letters[7 + next(2)]);
+                } else {
+                    for _ in 0..1 + next(3) {
+                        name.push(letters[next(letters.len())]);
+                    }
+                }
+                if label + 1 < labels {
+                    name.push('.');
+                }
+            }
+            let (bounded, unbounded) = (idn_hostname(&name), idn_hostname_of_any_length(&name));
+            assert_eq!(bounded, unbounded, "{name} ({} bytes)", name.len());
+            if unbounded {
+                accepted += 1;
+                longest = longest.max(name.len());
+            }
+            if name.len() > MAX_IDN_HOSTNAME_BYTES {
+                over += 1;
+            }
+        }
+        assert!(
+            accepted >= 1000 && over >= 1000 && (100..=MAX_IDN_HOSTNAME_BYTES).contains(&longest),
+            "{accepted} names accepted, the longest of {longest} bytes, {over} over the bound"
+        );
+        // The longest names there are: 253 ASCII characters, and a name over that by one.
+        let label = "a".repeat(63);
+        let at = format!("{label}.{label}.{label}.{}", "a".repeat(61));
+        assert!(at.len() == 253 && idn_hostname(&at) && !idn_hostname(&format!("{at}a")));
+    }
+
     /// The straightforward (allocating) readings of the IP address formats.
     fn reference_ipv4(s: &str) -> bool {
         let parts: Vec<&str> = s.split('.').collect();