corvus_json_schema 0.1.3 → 0.1.4

data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/dfa.rs

@@ -0,0 +1,366 @@
+//! A deterministic automaton over ASCII text, for the patterns that need no backtracking (`dfa.go`).
+//!
+//! The strings a schema tests are short and nearly always ASCII, and most patterns in schemas are small. For such a
+//! pattern every state set the text can lead to is worked out once, and a match is then one table read per byte.
+//!
+//! The automaton reads bytes below 0x80 only. A text with any other byte is handed back to the backtracking matcher
+//! from the start, so nothing here needs to know UTF-8 or the Unicode classes: a class is its ASCII members. A
+//! pattern with a word boundary, or whose automaton has more states than a byte can number, has no automaton. The
+//! automaton is built on the first match, not when the pattern is compiled, so a schema pays nothing for the
+//! patterns its instances never reach.
+
+use std::collections::HashMap;
+
+use super::parse::{Node, NodeKind, UNBOUNDED};
+
+// The states every automaton has. DEAD: no match can follow. ACCEPT: a match was found. BAIL: the text is not
+// ASCII.
+const DFA_DEAD: u8 = 0;
+const DFA_ACCEPT: u8 = 1;
+const DFA_BAIL: u8 = 2;
+const DFA_FIRST: usize = 3;
+
+// An automaton has at most this many states (they are numbered in a byte), built from at most this many states of
+// the pattern, in at most this many steps (a step is one state of the pattern looked at while a set is made). The
+// steps bound what a pattern too large for an automaton costs before that is known, to about a millisecond.
+const MAX_DFA_STATES: usize = 256;
+const MAX_NFA_STATES: usize = 512;
+const MAX_DFA_STEPS: i64 = 400_000;
+
+#[derive(Clone, Copy, PartialEq, Eq)]
+enum NfaKind {
+    /// Reads one byte of its set and goes to `out`.
+    Set,
+    /// Goes to `out` and to `alt`, reading nothing.
+    Split,
+    /// `Bol` and `Eol` go to `out` at the start and at the end of the text.
+    Bol,
+    Eol,
+    Match,
+}
+
+#[derive(Clone, Copy)]
+struct NfaState {
+    kind: NfaKind,
+    set: [u64; 2],
+    out: usize,
+    alt: usize,
+}
+
+impl NfaState {
+    fn new(kind: NfaKind, out: usize) -> NfaState {
+        NfaState { kind, set: [0; 2], out, alt: 0 }
+    }
+}
+
+#[derive(Default)]
+struct NfaBuilder {
+    states: Vec<NfaState>,
+    /// The pattern has something an automaton over bytes cannot decide, or is too large.
+    failed: bool,
+}
+
+impl NfaBuilder {
+    fn add(&mut self, s: NfaState) -> usize {
+        if self.states.len() >= MAX_NFA_STATES {
+            self.failed = true;
+            return 0;
+        }
+        self.states.push(s);
+        self.states.len() - 1
+    }
+
+    /// Adds the states of a node that go on to `next`, and returns the state they start at.
+    fn compile(&mut self, n: &Node, mut next: usize) -> usize {
+        if self.failed {
+            return 0;
+        }
+        match n.kind {
+            NodeKind::Empty => next,
+            NodeKind::Char => {
+                let mut set = [0u64; 2];
+                if (0..128).contains(&n.r) {
+                    set[(n.r >> 6) as usize] = 1 << (n.r & 63);
+                }
+                self.add(NfaState { set, ..NfaState::new(NfaKind::Set, next) })
+            }
+            NodeKind::Set => self.add(NfaState { set: n.class().ascii, ..NfaState::new(NfaKind::Set, next) }),
+            NodeKind::Cat => {
+                for sub in n.subs.iter().rev() {
+                    next = self.compile(sub, next);
+                }
+                next
+            }
+            NodeKind::Alt => {
+                let mut start = self.compile(&n.subs[n.subs.len() - 1], next);
+                for sub in n.subs[..n.subs.len() - 1].iter().rev() {
+                    let out = self.compile(sub, next);
+                    start = self.add(NfaState { alt: start, ..NfaState::new(NfaKind::Split, out) });
+                }
+                start
+            }
+            NodeKind::Group => self.compile(&n.subs[0], next),
+            NodeKind::Repeat => {
+                let sub = &n.subs[0];
+                if n.max == UNBOUNDED {
+                    // The loop, then the copies that must match before it.
+                    let loop_ = self.add(NfaState { alt: next, ..NfaState::new(NfaKind::Split, 0) });
+                    if self.failed {
+                        return 0;
+                    }
+                    let out = self.compile(sub, loop_);
+                    self.states[loop_].out = out;
+                    next = loop_;
+                } else {
+                    // The copies that may match, innermost last.
+                    let mut i = n.min;
+                    while i < n.max && !self.failed {
+                        let out = self.compile(sub, next);
+                        next = self.add(NfaState { alt: next, ..NfaState::new(NfaKind::Split, out) });
+                        i += 1;
+                    }
+                }
+                let mut i = 0;
+                while i < n.min && !self.failed {
+                    next = self.compile(sub, next);
+                    i += 1;
+                }
+                next
+            }
+            NodeKind::Bol => self.add(NfaState::new(NfaKind::Bol, next)),
+            NodeKind::Eol => self.add(NfaState::new(NfaKind::Eol, next)),
+            // A word boundary, or anything that needs the backtracking matcher.
+            _ => {
+                self.failed = true;
+                0
+            }
+        }
+    }
+}
+
+/// The automaton. A state is a byte, a byte of the text is one of a few classes (bytes the pattern does not tell
+/// apart), and `trans[state << shift | class]` is the state after the byte.
+pub(crate) struct Dfa {
+    class: [u8; 256],
+    trans: Vec<u8>,
+    shift: u32,
+    start: u8,
+    /// For each state: the pattern matches if the text ends there.
+    at_end: Vec<bool>,
+    /// The pattern matches the empty text.
+    empty: bool,
+}
+
+impl Dfa {
+    /// Reports whether the pattern matches the text. `None` when the text is not ASCII, which this automaton does
+    /// not decide.
+    #[inline]
+    pub(crate) fn is_match(&self, text: &[u8]) -> Option<bool> {
+        if text.is_empty() {
+            return Some(self.empty);
+        }
+        let mut s = self.start;
+        if s == DFA_ACCEPT {
+            return Some(true);
+        }
+        let (trans, shift) = (&self.trans[..], self.shift);
+        for &c in text {
+            s = trans[(s as usize) << shift | self.class[c as usize] as usize];
+            if (s as usize) < DFA_FIRST {
+                return if s == DFA_BAIL { None } else { Some(s == DFA_ACCEPT) };
+            }
+        }
+        Some(self.at_end[s as usize])
+    }
+}
+
+/// Makes the automaton's states from sets of the pattern's states.
+struct DfaBuilder<'a> {
+    nfa: &'a [NfaState],
+    // Scratch for closure: the states visited, by generation, and the stack.
+    mark: Vec<u32>,
+    generation: u32,
+    stack: Vec<usize>,
+    /// The steps left (see `MAX_DFA_STEPS`).
+    steps: i64,
+}
+
+impl DfaBuilder<'_> {
+    /// Adds to `set` every state that reading nothing reaches from the states in `from`, keeping the states that
+    /// read a byte, the end anchors still to pass, and the match. A start anchor is passed only `at_start`, and an
+    /// end anchor only `at_end`.
+    fn closure(&mut self, set: &mut Vec<usize>, from: &[usize], at_start: bool, at_end: bool) {
+        self.generation += 1;
+        self.stack.clear();
+        self.stack.extend_from_slice(from);
+        while let Some(i) = self.stack.pop() {
+            self.steps -= 1;
+            if self.mark[i] == self.generation {
+                continue;
+            }
+            self.mark[i] = self.generation;
+            let s = &self.nfa[i];
+            match s.kind {
+                NfaKind::Split => {
+                    self.stack.push(s.out);
+                    self.stack.push(s.alt);
+                }
+                NfaKind::Bol => {
+                    if at_start {
+                        self.stack.push(s.out);
+                    }
+                }
+                NfaKind::Eol => {
+                    if at_end {
+                        self.stack.push(s.out);
+                    } else {
+                        set.push(i);
+                    }
+                }
+                _ => set.push(i),
+            }
+        }
+    }
+
+    fn has_match(&self, set: &[usize]) -> bool {
+        set.iter().any(|&i| self.nfa[i].kind == NfaKind::Match)
+    }
+}
+
+/// A set as a map key: its states in increasing order.
+fn set_key(set: &mut [usize]) -> Vec<u16> {
+    set.sort_unstable();
+    set.iter().map(|&s| s as u16).collect()
+}
+
+/// Builds the automaton of a pattern for an unanchored search, or `None` when it has none.
+pub(crate) fn build_dfa(root: &Node) -> Option<Dfa> {
+    let mut nb = NfaBuilder::default();
+    let match_ = nb.add(NfaState::new(NfaKind::Match, 0));
+    let start = nb.compile(root, match_);
+    if nb.failed {
+        return None;
+    }
+    let nfa = &nb.states[..];
+    let mut b = DfaBuilder { nfa, mark: vec![0; nfa.len()], generation: 0, stack: Vec::new(), steps: MAX_DFA_STEPS };
+
+    // The classes of bytes: two bytes are in one class when every set of the pattern has both or neither. Class 0
+    // is the bytes that are not ASCII.
+    let mut d = Dfa { class: [0; 256], trans: Vec::new(), shift: 0, start: 0, at_end: Vec::new(), empty: false };
+    let mut classes = 1usize;
+    let mut signatures: HashMap<Vec<u8>, u8> = HashMap::new();
+    for c in 0..128usize {
+        let mut signature = Vec::with_capacity(nfa.len() / 8 + 1);
+        let (mut acc, mut n) = (0u8, 0u8);
+        for s in nfa {
+            if s.kind != NfaKind::Set {
+                continue;
+            }
+            acc = acc << 1 | ((s.set[c >> 6] >> (c & 63)) & 1) as u8;
+            n += 1;
+            if n == 8 {
+                signature.push(acc);
+                (acc, n) = (0, 0);
+            }
+        }
+        signature.push(acc);
+        let class = *signatures.entry(signature).or_insert_with(|| {
+            classes += 1;
+            (classes - 1) as u8
+        });
+        d.class[c] = class;
+    }
+    while 1usize << d.shift < classes {
+        d.shift += 1;
+    }
+    // One byte of each class, to move a set by.
+    let mut sample = [0usize; 129];
+    for c in (0..128usize).rev() {
+        sample[d.class[c] as usize] = c;
+    }
+
+    // The search is unanchored: a match may start at any position, so after every byte the states the pattern
+    // starts in (not at the start of the text) join the set.
+    let mut restart = Vec::new();
+    b.closure(&mut restart, &[start], false, false);
+    let mut at_both = Vec::new();
+    b.closure(&mut at_both, &[start], true, true);
+    d.empty = b.has_match(&at_both);
+
+    let mut ids: HashMap<Vec<u16>, u8> = HashMap::new();
+    let mut sets: Vec<Vec<usize>> = vec![Vec::new(); DFA_FIRST];
+    let stride = 1usize << d.shift;
+    d.trans = vec![0; DFA_FIRST * stride];
+    d.at_end = vec![false; DFA_FIRST];
+    for class in 0..stride {
+        d.trans[DFA_ACCEPT as usize * stride + class] = DFA_ACCEPT;
+        d.trans[DFA_BAIL as usize * stride + class] = DFA_BAIL;
+    }
+    // Gives a set its state. `None` when there are too many.
+    fn intern(
+        b: &mut DfaBuilder<'_>,
+        d: &mut Dfa,
+        ids: &mut HashMap<Vec<u16>, u8>,
+        sets: &mut Vec<Vec<usize>>,
+        stride: usize,
+        set: &mut [usize],
+    ) -> Option<u8> {
+        if b.has_match(set) {
+            return Some(DFA_ACCEPT);
+        }
+        if set.is_empty() {
+            return Some(DFA_DEAD);
+        }
+        let key = set_key(set);
+        if let Some(&id) = ids.get(&key) {
+            return Some(id);
+        }
+        if sets.len() >= MAX_DFA_STATES {
+            return None;
+        }
+        let id = sets.len() as u8;
+        ids.insert(key, id);
+        sets.push(set.to_vec());
+        d.trans.resize(d.trans.len() + stride, 0);
+        // At the end of the text the end anchors are passed.
+        let ends: Vec<usize> = set.iter().filter(|&&i| b.nfa[i].kind == NfaKind::Eol).map(|&i| b.nfa[i].out).collect();
+        let mut closed = Vec::new();
+        if !ends.is_empty() {
+            b.closure(&mut closed, &ends, false, true);
+        }
+        d.at_end.push(!ends.is_empty() && b.has_match(&closed));
+        Some(id)
+    }
+    let mut first = Vec::new();
+    b.closure(&mut first, &[start], true, false);
+    d.start = intern(&mut b, &mut d, &mut ids, &mut sets, stride, &mut first)?;
+    let (mut moved, mut next) = (Vec::new(), Vec::new());
+    let mut id = DFA_FIRST;
+    while id < sets.len() {
+        for class in 0..stride {
+            if class == 0 || class >= classes {
+                // Not ASCII (or no such class).
+                d.trans[id * stride + class] = DFA_BAIL;
+                continue;
+            }
+            let c = *sample.get(class).expect("a class has a byte");
+            moved.clear();
+            for &i in &sets[id] {
+                let s = &nfa[i];
+                if s.kind == NfaKind::Set && (s.set[c >> 6] >> (c & 63)) & 1 != 0 {
+                    moved.push(s.out);
+                }
+            }
+            moved.extend_from_slice(&restart);
+            next.clear();
+            b.closure(&mut next, &moved, false, false);
+            let to = intern(&mut b, &mut d, &mut ids, &mut sets, stride, &mut next)?;
+            if b.steps < 0 {
+                return None;
+            }
+            d.trans[id * stride + class] = to;
+        }
+        id += 1;
+    }
+    Some(d)
+}

data/ext/corvus_json_schema/vendor/corvus-json-schema/src/ecmaregex/fold.rs

@@ -0,0 +1,138 @@
+//! Case-insensitive matching (`fold.go`).
+//!
+//! It exists only inside a modifier group such as `(?i:...)`. ECMA-262 defines it through a Canonicalize function.
+//! Two characters match when they canonicalize to the same character. The function differs between the two
+//! grammars. With the u flag it is Unicode simple case folding. With no flag it is the single character uppercase
+//! mapping of a UTF-16 code unit, which never maps a character outside ASCII into ASCII.
+//!
+//! The parser applies the equivalence to every literal and class as it reads them, so a case-insensitive group is
+//! matched with plain classes. Only a backreference needs the function when matching.
+//!
+//! The folding and the uppercase mapping come from the tables of `ucd`, never from the standard library, so a
+//! pattern matches the same texts whichever toolchain built the program.
+
+use std::collections::HashMap;
+use std::sync::{Arc, LazyLock};
+
+use super::Rune;
+use super::charset::CharSet;
+use super::ucd;
+
+/// The characters of the Basic Multilingual Plane whose uppercase form is more than one character, as inclusive
+/// pairs. Canonicalize leaves them alone.
+static MULTI_UPPER: LazyLock<CharSet> = LazyLock::new(|| {
+    CharSet::new(
+        &[
+            0xDF, 0xDF, 0x149, 0x149, 0x1F0, 0x1F0, 0x390, 0x390, 0x3B0, 0x3B0, 0x587, 0x587, 0x1E96, 0x1E9A, 0x1F50,
+            0x1F50, 0x1F52, 0x1F52, 0x1F54, 0x1F54, 0x1F56, 0x1F56, 0x1F80, 0x1FAF, 0x1FB2, 0x1FB4, 0x1FB6, 0x1FB7,
+            0x1FBC, 0x1FBC, 0x1FC2, 0x1FC4, 0x1FC6, 0x1FC7, 0x1FCC, 0x1FCC, 0x1FD2, 0x1FD3, 0x1FD6, 0x1FD7, 0x1FE2,
+            0x1FE4, 0x1FE6, 0x1FE7, 0x1FF2, 0x1FF4, 0x1FF6, 0x1FF7, 0x1FFC, 0x1FFC, 0xFB00, 0xFB06, 0xFB13, 0xFB17,
+        ],
+        false,
+    )
+});
+
+/// The Canonicalize function of ECMA-262 for case-insensitive matching.
+pub(crate) fn canonicalize(r: Rune, unicode_mode: bool) -> Rune {
+    if unicode_mode {
+        // Every character of a simple case folding class folds to the same character.
+        return ucd::fold(r);
+    }
+    // A character beyond the Basic Multilingual Plane is two code units, and neither has an uppercase form.
+    if r > 0xFFFF || MULTI_UPPER.contains(r) {
+        return r;
+    }
+    let upper = ucd::to_upper(r);
+    if r >= 128 && upper < 128 || upper > 0xFFFF {
+        return r;
+    }
+    upper
+}
+
+/// The characters that are equivalent to some other character, each with all the members of its equivalence class.
+pub(crate) struct FoldTable {
+    points: Vec<Rune>,
+    /// For each point, the number of its class in `classes`.
+    class_of: Vec<u32>,
+    classes: Vec<Vec<Rune>>,
+}
+
+fn new_fold_table(classes: HashMap<Rune, Vec<Rune>>) -> FoldTable {
+    let mut t = FoldTable { points: Vec::new(), class_of: Vec::new(), classes: Vec::new() };
+    let mut by_member: HashMap<Rune, u32> = HashMap::new();
+    for (_, class) in classes {
+        if class.len() < 2 {
+            continue;
+        }
+        for &member in &class {
+            t.points.push(member);
+            by_member.insert(member, t.classes.len() as u32);
+        }
+        t.classes.push(class);
+    }
+    t.points.sort_unstable();
+    t.class_of = t.points.iter().map(|point| by_member[point]).collect();
+    t
+}
+
+static UNICODE_FOLD_TABLE: LazyLock<FoldTable> = LazyLock::new(|| {
+    // A class is a character that others fold to, with those others.
+    let mut classes: HashMap<Rune, Vec<Rune>> = HashMap::new();
+    for r in ucd::folding() {
+        let key = ucd::fold(r);
+        classes.entry(key).or_insert_with(|| vec![key]).push(r);
+    }
+    new_fold_table(classes)
+});
+
+static LEGACY_FOLD_TABLE: LazyLock<FoldTable> = LazyLock::new(|| {
+    let mut classes: HashMap<Rune, Vec<Rune>> = HashMap::new();
+    for r in 0..=0xFFFF {
+        classes.entry(canonicalize(r, false)).or_default().push(r);
+    }
+    new_fold_table(classes)
+});
+
+/// The table of the grammar in use. It is built on first use, which only a pattern with a case-insensitive group
+/// reaches.
+fn fold_classes(unicode_mode: bool) -> &'static FoldTable {
+    if unicode_mode { &UNICODE_FOLD_TABLE } else { &LEGACY_FOLD_TABLE }
+}
+
+/// The set of every character equivalent to some member of the set.
+///
+/// It also returns the number of characters with a case variant that the set holds, which is the work it did.
+pub(crate) fn fold_closure(set: &Arc<CharSet>, unicode_mode: bool) -> (Arc<CharSet>, usize) {
+    let table = fold_classes(unicode_mode);
+    let mut extra: Vec<Rune> = Vec::new();
+    let mut variants = 0;
+    for &[lo, hi] in set.ranges.as_chunks::<2>().0 {
+        let mut k = table.points.partition_point(|&p| p < lo);
+        while k < table.points.len() && table.points[k] <= hi {
+            variants += 1;
+            for &member in &table.classes[table.class_of[k] as usize] {
+                if !set.contains(member) {
+                    extra.push(member);
+                    extra.push(member);
+                }
+            }
+            k += 1;
+        }
+    }
+    if extra.is_empty() {
+        return (set.clone(), variants);
+    }
+    extra.extend_from_slice(&set.ranges);
+    (CharSet::shared(&extra, false), variants)
+}
+
+/// With the u flag and case-insensitive matching, `\w` and `\b` also count the two characters that fold to an ASCII
+/// letter (U+017F, the long s, and U+212A, the Kelvin sign) as word characters.
+const FOLD_WORD_PAIRS: &[Rune] = &[0x30, 0x39, 0x41, 0x5A, 0x5F, 0x5F, 0x61, 0x7A, 0x17F, 0x17F, 0x212A, 0x212A];
+
+pub(crate) static FOLD_WORD_SET: LazyLock<Arc<CharSet>> = LazyLock::new(|| CharSet::shared(FOLD_WORD_PAIRS, false));
+pub(crate) static NOT_FOLD_WORD_SET: LazyLock<Arc<CharSet>> = LazyLock::new(|| CharSet::shared(FOLD_WORD_PAIRS, true));
+
+pub(crate) fn is_fold_word_rune(r: Rune) -> bool {
+    (0..128).contains(&r) && super::charset::is_word_byte(r as u8) || r == 0x17F || r == 0x212A
+}