schemurai 2.3.0 → 3.0.1

data/ext/schemurai_native/src/memo_scope.rs

@@ -0,0 +1,240 @@
+//! Exact reachable scope dependencies, shared across schema components.
+use super::*;
+use std::borrow::Cow;
+use std::collections::HashSet;
+use std::sync::Arc;
+
+const MAX_NAMES: usize = 65536;
+
+#[derive(Default)]
+pub(super) struct Projection {
+    names: Vec<Option<Arc<[usize]>>>,
+    // If materialized closures exceed the budget, retain the linear dependency
+    // graph and compute those exact projections on demand, never all names.
+    graph: Option<Vec<Vec<usize>>>,
+}
+impl Projection {
+    pub(super) fn compile(edges: &[Vec<usize>], parents: &[Vec<usize>], count: usize) -> Self {
+        Self::bounded(edges, parents, count, MAX_NAMES)
+    }
+    fn bounded(edges: &[Vec<usize>], parents: &[Vec<usize>], count: usize, limit: usize) -> Self {
+        let mut seen = vec![false; edges.len()];
+        let order = graph::postorder(edges, &mut seen);
+        let mut component = vec![usize::MAX; edges.len()];
+        let mut members = vec![];
+        for root in order.into_iter().rev() {
+            if component[root] != usize::MAX {
+                continue;
+            }
+            let id = members.len();
+            let mut nodes = vec![];
+            let mut stack = vec![root];
+            component[root] = id;
+            while let Some(node) = stack.pop() {
+                nodes.push(node);
+                for &parent in &parents[node] {
+                    if component[parent] == usize::MAX {
+                        component[parent] = id;
+                        stack.push(parent);
+                    }
+                }
+            }
+            members.push(nodes);
+        }
+        // Kosaraju components are ordered from sources to sinks. Every member
+        // of a cycle has the same reachable reference names; solve it once.
+        let mut closures: Vec<Option<Arc<[usize]>>> = vec![None; members.len()];
+        let mut interned: HashSet<Arc<[usize]>> = HashSet::new();
+        let mut slots = 0;
+        for (id, nodes) in members.iter().enumerate().rev() {
+            let mut names = vec![];
+            let mut children = HashSet::new();
+            let mut dependencies = HashSet::new();
+            let mut complete = true;
+            for &node in nodes {
+                if node >= count {
+                    names.push(node - count);
+                }
+                for &child in &edges[node] {
+                    let child = component[child];
+                    if child != id && children.insert(child) {
+                        match &closures[child] {
+                            Some(dependency) => {
+                                if dependencies.insert(dependency.as_ptr()) {
+                                    names.extend(dependency.iter().copied());
+                                }
+                            }
+                            None => complete = false,
+                        }
+                        if names.len() > limit {
+                            names.sort_unstable();
+                            names.dedup();
+                            if names.len() > limit {
+                                complete = false;
+                                break;
+                            }
+                        }
+                    }
+                }
+                // Bound temporary concatenation too. Deduplicate overlapping
+                // dependencies before choosing the on-demand representation.
+                if names.len() > limit {
+                    names.sort_unstable();
+                    names.dedup();
+                    if names.len() > limit {
+                        complete = false;
+                        break;
+                    }
+                }
+            }
+            if !complete {
+                continue;
+            }
+            names.sort_unstable();
+            names.dedup();
+            if let Some(existing) = interned.get(names.as_slice()) {
+                closures[id] = Some(existing.clone());
+            } else if names.len() <= limit - slots {
+                slots += names.len();
+                let names: Arc<[usize]> = names.into();
+                interned.insert(names.clone());
+                closures[id] = Some(names);
+            }
+        }
+        let names: Vec<_> = component[..count]
+            .iter()
+            .map(|&id| closures[id].clone())
+            .collect();
+        let graph = names.iter().any(Option::is_none).then(|| edges.to_vec());
+        Self { names, graph }
+    }
+    pub(super) fn names(
+        &self,
+        node: usize,
+        count: usize,
+        anchors: usize,
+        guarded: bool,
+        budget: &budget::Budget,
+    ) -> Option<Cow<'_, [usize]>> {
+        if self.names.is_empty() {
+            // Hand-built engines in key tests have no compiled dependency graph.
+            return Some(Cow::Owned((0..anchors + usize::from(guarded)).collect()));
+        }
+        if let Some(names) = &self.names[node] {
+            return Some(Cow::Borrowed(names));
+        }
+        let graph = self.graph.as_ref()?;
+        let mut seen = vec![false; graph.len()];
+        let mut stack = vec![node];
+        let mut names = vec![];
+        seen[node] = true;
+        while let Some(node) = stack.pop() {
+            if !budget.checkpoint() {
+                return None;
+            }
+            if node >= count {
+                names.push(node - count);
+            }
+            for &child in &graph[node] {
+                if !seen[child] {
+                    seen[child] = true;
+                    stack.push(child);
+                }
+            }
+        }
+        names.sort_unstable();
+        Some(Cow::Owned(names))
+    }
+    pub(super) fn memory_usage(&self) -> usize {
+        let mut seen = HashSet::new();
+        self.names.capacity() * std::mem::size_of::<Option<Arc<[usize]>>>()
+            + self
+                .names
+                .iter()
+                .flatten()
+                .filter(|names| seen.insert(names.as_ptr()))
+                .map(|names| (names.len() + 2) * std::mem::size_of::<usize>())
+                .sum::<usize>()
+            + self.graph.as_ref().map_or(0, |graph| {
+                graph.capacity() * std::mem::size_of::<Vec<usize>>()
+                    + graph
+                        .iter()
+                        .map(|row| row.capacity() * std::mem::size_of::<usize>())
+                        .sum::<usize>()
+            })
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn parents(edges: &[Vec<usize>]) -> Vec<Vec<usize>> {
+        let mut parents = vec![vec![]; edges.len()];
+        for (node, children) in edges.iter().enumerate() {
+            for &child in children {
+                parents[child].push(node);
+            }
+        }
+        parents
+    }
+    #[test]
+    fn projects_cycles_and_transitive_dynamic_dispatch_targets() {
+        // Four nodes, two dynamic names and the recursive dispatch vertex.
+        let edges = vec![
+            vec![1],
+            vec![0, 4],
+            vec![5],
+            vec![],
+            vec![2],
+            vec![3],
+            vec![0],
+        ];
+        for limit in [0, 1, MAX_NAMES] {
+            let projection = Projection::bounded(&edges, &parents(&edges), 4, limit);
+            let names = |node| {
+                projection
+                    .names(node, 4, 2, true, &budget::Budget::default())
+                    .unwrap()
+                    .into_owned()
+            };
+            assert_eq!(names(0), [0, 1]);
+            assert_eq!(names(1), [0, 1]);
+            assert_eq!(names(2), [1]);
+            assert!(names(3).is_empty());
+        }
+    }
+    #[test]
+    fn retains_recursive_dispatch_only_where_reachable() {
+        let edges = vec![vec![1], vec![4], vec![3], vec![], vec![0]];
+        let projection = Projection::compile(&edges, &parents(&edges), 3);
+        let names = |node| {
+            projection
+                .names(node, 3, 1, true, &budget::Budget::default())
+                .unwrap()
+                .into_owned()
+        };
+        assert_eq!(names(0), [1]);
+        assert_eq!(names(1), [1]);
+        assert_eq!(names(2), [0]);
+    }
+
+    #[test]
+    fn shares_equal_closures_and_retains_exact_projection_after_budget_exhaustion() {
+        let edges = vec![vec![1, 2], vec![3], vec![3], vec![4], vec![], vec![]];
+        let projection = Projection::bounded(&edges, &parents(&edges), 4, MAX_NAMES);
+        assert!(projection.graph.is_none());
+        assert!(Arc::ptr_eq(
+            projection.names[0].as_ref().unwrap(),
+            projection.names[3].as_ref().unwrap()
+        ));
+        let projection = Projection::bounded(&edges, &parents(&edges), 4, 0);
+        assert!(projection.graph.is_some());
+        assert_eq!(
+            &*projection
+                .names(0, 4, 1, false, &budget::Budget::default())
+                .unwrap(),
+            &[0]
+        );
+    }
+}

data/ext/schemurai_native/src/number.rs

@@ -96,14 +96,13 @@ pub(super) fn integer_fingerprint(integer: Value) -> u64 {
     let mut words = [0_u64; 16];
     let sign = unsafe { sn_integer1024(integer, words.as_mut_ptr()) };
     if sign.abs() > 1 {
-        // No finite Float can equal this magnitude. Mix bounded low words,
-        // sign, and magnitude size; full equality resolves all collisions,
-        // including integers that differ only in unsampled high digits.
-        let len = words
-            .iter()
-            .rposition(|&word| word != 0)
-            .map_or(0, |index| index + 1);
-        return integer_bucket(&words[..len], sign, unsafe { sn_integer_bytes(integer) });
+        // No finite Float can equal this magnitude, so the full native digit
+        // hash needs no Float normalization. Sampling only low digits makes
+        // mixed/nested uniqueItems quadratic for differences above bit 1024
+        // and collapses large enum indexes into a single candidate bucket.
+        // The adapter hashes the original Bignum without Ruby dispatch or
+        // allocation; exact equality still decides every hash collision.
+        return mix(big_only_fingerprint(integer).unwrap() ^ TYPE_NUMBER as u64);
     }
     exact_integer_fingerprint(&words, sign)
 }

data/ext/schemurai_native/src/pattern.rs

@@ -4,29 +4,147 @@
 use super::*;
 use regex_automata::{
     Anchored, Input, MatchKind,
-    dfa::{Automaton, StartKind, dense},
-    meta::Regex,
+    dfa::{Automaton, OverlappingState, StartKind, dense},
+    meta::{Cache, Regex},
     nfa::thompson,
 };
 use regex_syntax::ast::{Ast, RepetitionKind, RepetitionRange};
 
+// Each validator owns its search caches. Compiled regex/DFA data stays shared.
+#[derive(Default)]
+pub struct Caches(std::cell::RefCell<Vec<Option<Box<Cache>>>>);
+impl Caches {
+    fn is_match(&self, index: usize, regex: &Regex, bytes: &[u8]) -> bool {
+        // Each runner is tied to one engine, whose regex indices are fixed at
+        // compilation. Allocate only the caches this validator actually uses.
+        let mut caches = self.0.borrow_mut();
+        if caches.len() <= index {
+            caches.resize_with(index + 1, || None);
+        }
+        let cache = caches[index].get_or_insert_with(|| Box::new(regex.create_cache()));
+        regex
+            .search_half_with(cache, &Input::new(bytes).earliest(true))
+            .is_some()
+    }
+}
+#[unsafe(no_mangle)]
+pub extern "C" fn sn_caches_new() -> *mut Caches {
+    Box::into_raw(Box::new(Caches::default()))
+}
+/// # Safety
+/// Caches must be owned, returned by sn_caches_new, and freed exactly once.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn sn_caches_free(caches: *mut Caches) {
+    drop(unsafe { Box::from_raw(caches) });
+}
+/// # Safety
+/// Caches must be live; call under the GVL or while all Ractors are stopped.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn sn_caches_size(caches: *const Caches) -> usize {
+    // GC may inspect a stopped runner while its search has borrowed a cache.
+    let Ok(caches) = (unsafe { &*caches }).0.try_borrow() else {
+        return std::mem::size_of::<Caches>();
+    };
+    std::mem::size_of::<Caches>()
+        + caches.capacity() * std::mem::size_of::<Option<Box<Cache>>>()
+        + caches
+            .iter()
+            .flatten()
+            .map(|cache| std::mem::size_of::<Cache>() + cache.memory_usage())
+            .sum::<usize>()
+}
+
 #[derive(Debug)]
 pub(super) enum Pattern {
     Unsupported,
     Dfa(Box<dense::DFA<Vec<u32>>>),
-    Compiled(Regex),
+    Compiled { regex: Regex, index: usize },
+}
+
+// Preserve every overlapping match while scanning a key with one automaton.
+// Keep individual matchers for unsupported compilation/search configurations.
+#[derive(Debug)]
+pub(super) struct PatternGroup {
+    dfa: dense::DFA<Vec<u32>>,
+    all: u64,
+}
+impl PatternGroup {
+    pub(super) fn compile(names: &[Value], patterns: &[(Pattern, usize)]) -> Option<Self> {
+        if !(2..=64).contains(&patterns.len())
+            || !patterns
+                .iter()
+                .all(|(pattern, _)| matches!(pattern, Pattern::Dfa(_)))
+        {
+            return None;
+        }
+        // Existing DFA compilation already proved syntax, encoding, compatibility
+        // and full start anchoring. Reuse exactly the same translation.
+        let sources: Option<Vec<_>> = names
+            .iter()
+            .map(|&name| {
+                std::str::from_utf8(unsafe { string_bytes(name) })
+                    .ok()
+                    .and_then(translate)
+            })
+            .collect();
+        let dfa = dense::Builder::new()
+            .configure(
+                dense::Config::new()
+                    .match_kind(MatchKind::All)
+                    .start_kind(StartKind::Anchored)
+                    .dfa_size_limit(Some(1 << 20))
+                    .determinize_size_limit(Some(1 << 22)),
+            )
+            .thompson(thompson::Config::new().nfa_size_limit(Some(1 << 20)))
+            .build_many(&sources?)
+            .ok()?;
+        let all = u64::MAX >> (64 - patterns.len());
+        Some(Self { dfa, all })
+    }
+    pub(super) fn matches(&self, value: Value) -> Option<u64> {
+        let mut data = std::ptr::null();
+        let mut len = 0;
+        let encoding = unsafe { sn_pattern_view(value, &mut data, &mut len) };
+        if encoding == ENCODING_OTHER {
+            return None;
+        }
+        let bytes = unsafe { byte_view(data, len) };
+        if !valid_encoding(bytes, encoding) {
+            return None;
+        }
+        let input = Input::new(bytes).anchored(Anchored::Yes);
+        let mut state = OverlappingState::start();
+        let mut matches = 0;
+        loop {
+            self.dfa
+                .try_search_overlapping_fwd(&input, &mut state)
+                .ok()?;
+            let Some(found) = state.get_match() else {
+                break;
+            };
+            matches |= 1_u64 << found.pattern().as_usize();
+            if matches == self.all {
+                break;
+            }
+        }
+        Some(matches)
+    }
+    pub(super) fn memory_usage(&self) -> usize {
+        self.dfa.memory_usage()
+    }
 }
 
 impl Pattern {
     // Select the crate matcher once for a homogeneous array. The borrowed
     // Ruby slices are used synchronously; classification and encoding are
     // still checked on every element, including non-string instances.
-    pub(super) fn eval_items(&self, required: i32, values: &[Value]) -> Verdict {
-        self.visit_invalid_items(required, values, |_, _| Invalid)
+    pub(super) fn eval_items(&self, caches: &Caches, required: i32, values: &[Value]) -> Verdict {
+        self.visit_invalid_items(caches, required, values, |_, _| Invalid)
     }
 
     pub(super) fn visit_invalid_items(
         &self,
+        caches: &Caches,
         required: i32,
         values: &[Value],
         invalid: impl FnMut(usize, Value) -> Verdict,
@@ -39,8 +157,8 @@ impl Pattern {
                     Err(_) => Unsupported,
                 }
             }),
-            Self::Compiled(regex) => scan_items(required, values, invalid, |bytes| {
-                verdict(regex.is_match(bytes))
+            Self::Compiled { regex, index } => scan_items(required, values, invalid, |bytes| {
+                verdict(caches.is_match(*index, regex, bytes))
             }),
             Self::Unsupported => scan_items(required, values, invalid, |_| Unsupported),
         }
@@ -49,12 +167,12 @@ impl Pattern {
     pub(super) fn memory_usage(&self) -> usize {
         match self {
             Self::Dfa(dfa) => std::mem::size_of_val(&**dfa) + dfa.memory_usage(),
-            Self::Compiled(regex) => regex.memory_usage(),
+            Self::Compiled { regex, .. } => regex.memory_usage(),
             Self::Unsupported => 0,
         }
     }
 
-    pub(super) fn compile(value: Value) -> Self {
+    pub(super) fn compile(value: Value, cache_count: &mut usize) -> Self {
         if unsafe { sn_kind(value) } != TYPE_STRING {
             return Self::Unsupported;
         }
@@ -105,11 +223,15 @@ impl Pattern {
                     .hybrid_cache_capacity(1 << 20),
             )
             .build(&source)
-            .map(Self::Compiled)
+            .map(|regex| {
+                let index = *cache_count;
+                *cache_count += 1;
+                Self::Compiled { regex, index }
+            })
             .unwrap_or(Self::Unsupported)
     }
 
-    pub(super) fn eval(&self, value: Value) -> Verdict {
+    pub(super) fn eval(&self, caches: &Caches, value: Value) -> Verdict {
         if matches!(self, Self::Unsupported) {
             return Unsupported;
         }
@@ -120,17 +242,14 @@ impl Pattern {
             return Unsupported;
         }
         let bytes = unsafe { std::slice::from_raw_parts(data, len) };
-        self.eval_bytes(bytes, encoding)
+        self.eval_bytes(caches, bytes, encoding)
     }
 
-    pub(super) fn eval_bytes(&self, bytes: &[u8], encoding: i32) -> Verdict {
+    pub(super) fn eval_bytes(&self, caches: &Caches, bytes: &[u8], encoding: i32) -> Verdict {
         if encoding == ENCODING_OTHER || matches!(self, Self::Unsupported) {
             return Unsupported;
         }
-        if encoding != ENCODING_SEVEN_BIT
-            && ((encoding == ENCODING_ASCII && !bytes.is_ascii())
-                || std::str::from_utf8(bytes).is_err())
-        {
+        if !valid_encoding(bytes, encoding) {
             return Unsupported;
         }
         let matched = match self {
@@ -140,13 +259,27 @@ impl Pattern {
                 Ok(matched) => matched.is_some(),
                 Err(_) => return Unsupported,
             },
-            Self::Compiled(regex) => regex.is_match(bytes),
+            Self::Compiled { regex, index } => caches.is_match(*index, regex, bytes),
             Self::Unsupported => return Unsupported,
         };
         if matched { Valid } else { Invalid }
     }
 }
 
+// Keep short checks in std and skip scanning when Ruby already proved ASCII.
+// The compatibility SIMD API retains early rejection of malformed UTF-8.
+fn valid_encoding(bytes: &[u8], encoding: i32) -> bool {
+    if encoding == ENCODING_SEVEN_BIT {
+        true
+    } else if encoding == ENCODING_ASCII {
+        bytes.is_ascii()
+    } else if bytes.len() < 64 {
+        std::str::from_utf8(bytes).is_ok()
+    } else {
+        simdutf8::compat::from_utf8(bytes).is_ok()
+    }
+}
+
 fn scan_items(
     required: i32,
     values: &[Value],
@@ -173,10 +306,7 @@ fn scan_items(
             return Unsupported;
         }
         let bytes = unsafe { std::slice::from_raw_parts(data, len) };
-        if encoding != ENCODING_SEVEN_BIT
-            && ((encoding == ENCODING_ASCII && !bytes.is_ascii())
-                || std::str::from_utf8(bytes).is_err())
-        {
+        if !valid_encoding(bytes, encoding) {
             return Unsupported;
         }
         let result = matches(bytes);
@@ -286,3 +416,42 @@ fn compatible(ast: &Ast) -> bool {
         _ => true,
     }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn encoding_validation_matches_std_across_simd_blocks_and_tails() {
+        let original = "日𐀀é".repeat(32).into_bytes();
+        for length in [63, 64, 65, 127, 128, 129, original.len()] {
+            for at in 0..length {
+                for byte in 0..=255 {
+                    let mut input = original[..length].to_vec();
+                    input[at] = byte;
+                    assert_eq!(
+                        valid_encoding(&input, ENCODING_UTF8),
+                        std::str::from_utf8(&input).is_ok(),
+                        "length {length}, position {at}, byte {byte}"
+                    );
+                    assert_eq!(valid_encoding(&input, ENCODING_ASCII), input.is_ascii());
+                }
+            }
+        }
+        assert!(valid_encoding(&original, ENCODING_UTF8));
+        assert!(valid_encoding(b"already classified", ENCODING_SEVEN_BIT));
+    }
+
+    #[test]
+    fn reports_memory_when_a_search_cache_is_borrowed() {
+        let caches = Caches::default();
+        let regex = Regex::new("value[0-9]+").unwrap();
+        assert!(caches.is_match(0, &regex, b"value1"));
+        assert!(unsafe { sn_caches_size(&caches) } > std::mem::size_of::<Caches>());
+        let _search = caches.0.borrow_mut();
+        assert_eq!(
+            unsafe { sn_caches_size(&caches) },
+            std::mem::size_of::<Caches>()
+        );
+    }
+}