schemurai 3.0.0 → 3.0.2

data/ext/schemurai_native/src/pattern.rs

@@ -1,61 +1,25 @@
 //! Crate-backed matching over borrowed Ruby string bytes.
 //! The compatibility adapter translates VM shorthands and absolute anchors;
-//! regex-syntax parses syntax, and regex-automata performs every search.
+//! regex-syntax parses syntax and regex-automata handles the fast searches.
+//! Other cases request a libruby match after the Rust evaluation returns.
 use super::*;
 use regex_automata::{
     Anchored, Input, MatchKind,
     dfa::{Automaton, OverlappingState, StartKind, dense},
-    meta::{Cache, Regex},
+    meta::Regex,
     nfa::thompson,
 };
 use regex_syntax::ast::{Ast, RepetitionKind, RepetitionRange};
 
-// Each validator owns its search caches. Compiled regex/DFA data stays shared.
-#[derive(Default)]
-pub struct Caches(std::cell::RefCell<Vec<Option<Box<Cache>>>>);
-impl Caches {
-    fn is_match(&self, index: usize, regex: &Regex, bytes: &[u8]) -> bool {
-        // Each runner is tied to one engine, whose regex indices are fixed at
-        // compilation. Allocate only the caches this validator actually uses.
-        let mut caches = self.0.borrow_mut();
-        if caches.len() <= index {
-            caches.resize_with(index + 1, || None);
-        }
-        let cache = caches[index].get_or_insert_with(|| Box::new(regex.create_cache()));
-        regex
-            .search_half_with(cache, &Input::new(bytes).earliest(true))
-            .is_some()
-    }
-}
-#[unsafe(no_mangle)]
-pub extern "C" fn sn_caches_new() -> *mut Caches {
-    Box::into_raw(Box::new(Caches::default()))
-}
-/// # Safety
-/// Caches must be owned, returned by sn_caches_new, and freed exactly once.
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn sn_caches_free(caches: *mut Caches) {
-    drop(unsafe { Box::from_raw(caches) });
-}
-/// # Safety
-/// Caches must be live; call under the GVL or while all Ractors are stopped.
-#[unsafe(no_mangle)]
-pub unsafe extern "C" fn sn_caches_size(caches: *const Caches) -> usize {
-    // GC may inspect a stopped runner while its search has borrowed a cache.
-    let Ok(caches) = (unsafe { &*caches }).0.try_borrow() else {
-        return std::mem::size_of::<Caches>();
-    };
-    std::mem::size_of::<Caches>()
-        + caches.capacity() * std::mem::size_of::<Option<Box<Cache>>>()
-        + caches
-            .iter()
-            .flatten()
-            .map(|cache| std::mem::size_of::<Cache>() + cache.memory_usage())
-            .sum::<usize>()
-}
+use super::runtime::Caches;
 
 #[derive(Debug)]
-pub(super) enum Pattern {
+pub(super) struct Pattern {
+    source: Value,
+    matcher: Matcher,
+}
+#[derive(Debug)]
+enum Matcher {
     Unsupported,
     Dfa(Box<dense::DFA<Vec<u32>>>),
     Compiled { regex: Regex, index: usize },
@@ -73,7 +37,7 @@ impl PatternGroup {
         if !(2..=64).contains(&patterns.len())
             || !patterns
                 .iter()
-                .all(|(pattern, _)| matches!(pattern, Pattern::Dfa(_)))
+                .all(|(pattern, _)| matches!(pattern.matcher, Matcher::Dfa(_)))
         {
             return None;
         }
@@ -109,10 +73,7 @@ impl PatternGroup {
             return None;
         }
         let bytes = unsafe { byte_view(data, len) };
-        if encoding != ENCODING_SEVEN_BIT
-            && ((encoding == ENCODING_ASCII && !bytes.is_ascii())
-                || std::str::from_utf8(bytes).is_err())
-        {
+        if !valid_encoding(bytes, encoding) {
             return None;
         }
         let input = Input::new(bytes).anchored(Anchored::Yes);
@@ -141,10 +102,6 @@ impl Pattern {
     // Select the crate matcher once for a homogeneous array. The borrowed
     // Ruby slices are used synchronously; classification and encoding are
     // still checked on every element, including non-string instances.
-    pub(super) fn eval_items(&self, caches: &Caches, required: i32, values: &[Value]) -> Verdict {
-        self.visit_invalid_items(caches, required, values, |_, _| Invalid)
-    }
-
     pub(super) fn visit_invalid_items(
         &self,
         caches: &Caches,
@@ -152,30 +109,97 @@ impl Pattern {
         values: &[Value],
         invalid: impl FnMut(usize, Value) -> Verdict,
     ) -> Verdict {
-        match self {
-            Self::Dfa(dfa) => scan_items(required, values, invalid, |bytes| {
-                match dfa.try_search_fwd(&Input::new(bytes).anchored(Anchored::Yes).earliest(true))
+        match &self.matcher {
+            Matcher::Dfa(dfa) => scan_items(
+                required,
+                values,
+                invalid,
+                |bytes| match dfa
+                    .try_search_fwd(&Input::new(bytes).anchored(Anchored::Yes).earliest(true))
                 {
                     Ok(matched) => verdict(matched.is_some()),
                     Err(_) => Unsupported,
-                }
-            }),
-            Self::Compiled { regex, index } => scan_items(required, values, invalid, |bytes| {
-                verdict(caches.is_match(*index, regex, bytes))
-            }),
-            Self::Unsupported => scan_items(required, values, invalid, |_| Unsupported),
+                },
+                |value| caches.ruby_match(self.source, value, true),
+            ),
+            Matcher::Compiled { regex, index } => scan_items(
+                required,
+                values,
+                invalid,
+                |bytes| verdict(caches.is_match(*index, regex, bytes)),
+                |value| caches.ruby_match(self.source, value, true),
+            ),
+            Matcher::Unsupported => scan_items(
+                required,
+                values,
+                invalid,
+                |_| Unsupported,
+                |value| caches.ruby_match(self.source, value, true),
+            ),
         }
     }
 
     pub(super) fn memory_usage(&self) -> usize {
-        match self {
-            Self::Dfa(dfa) => std::mem::size_of_val(&**dfa) + dfa.memory_usage(),
-            Self::Compiled { regex, .. } => regex.memory_usage(),
-            Self::Unsupported => 0,
+        match &self.matcher {
+            Matcher::Dfa(dfa) => std::mem::size_of_val(&**dfa) + dfa.memory_usage(),
+            Matcher::Compiled { regex, .. } => regex.memory_usage(),
+            Matcher::Unsupported => 0,
         }
     }
 
     pub(super) fn compile(value: Value, cache_count: &mut usize) -> Self {
+        Self {
+            source: value,
+            matcher: Matcher::compile(value, cache_count),
+        }
+    }
+
+    pub(super) fn eval(&self, caches: &Caches, value: Value) -> Verdict {
+        self.eval_value(caches, value, false)
+    }
+    pub(super) fn assertion(&self, caches: &Caches, value: Value) -> Verdict {
+        self.eval_value(caches, value, true)
+    }
+    fn eval_value(&self, caches: &Caches, value: Value, assertion: bool) -> Verdict {
+        if matches!(self.matcher, Matcher::Unsupported) {
+            return caches.ruby_match(self.source, value, assertion);
+        }
+        let mut data = std::ptr::null();
+        let mut len = 0;
+        let encoding = unsafe { sn_pattern_view(value, &mut data, &mut len) };
+        if encoding == ENCODING_OTHER {
+            return caches.ruby_match(self.source, value, assertion);
+        }
+        let bytes = unsafe { byte_view(data, len) };
+        let result = self.matcher.eval_bytes(caches, bytes, encoding);
+        if result == Unsupported {
+            caches.ruby_match(self.source, value, assertion)
+        } else {
+            result
+        }
+    }
+
+    pub(super) fn eval_bytes(
+        &self,
+        caches: &Caches,
+        value: Value,
+        bytes: &[u8],
+        encoding: i32,
+    ) -> Verdict {
+        let result = self.matcher.eval_bytes(caches, bytes, encoding);
+        if result == Unsupported {
+            caches.ruby_match(self.source, value, true)
+        } else {
+            result
+        }
+    }
+    pub(super) fn invalid(&self, caches: &Caches, value: Value) -> bool {
+        caches.invalid_regexp(self.source, value)
+    }
+}
+
+impl Matcher {
+    fn compile(value: Value, cache_count: &mut usize) -> Self {
         if unsafe { sn_kind(value) } != TYPE_STRING {
             return Self::Unsupported;
         }
@@ -234,28 +258,11 @@ impl Pattern {
             .unwrap_or(Self::Unsupported)
     }
 
-    pub(super) fn eval(&self, caches: &Caches, value: Value) -> Verdict {
-        if matches!(self, Self::Unsupported) {
-            return Unsupported;
-        }
-        let mut data = std::ptr::null();
-        let mut len = 0;
-        let encoding = unsafe { sn_pattern_view(value, &mut data, &mut len) };
-        if encoding == ENCODING_OTHER {
-            return Unsupported;
-        }
-        let bytes = unsafe { std::slice::from_raw_parts(data, len) };
-        self.eval_bytes(caches, bytes, encoding)
-    }
-
-    pub(super) fn eval_bytes(&self, caches: &Caches, bytes: &[u8], encoding: i32) -> Verdict {
+    fn eval_bytes(&self, caches: &Caches, bytes: &[u8], encoding: i32) -> Verdict {
         if encoding == ENCODING_OTHER || matches!(self, Self::Unsupported) {
             return Unsupported;
         }
-        if encoding != ENCODING_SEVEN_BIT
-            && ((encoding == ENCODING_ASCII && !bytes.is_ascii())
-                || std::str::from_utf8(bytes).is_err())
-        {
+        if !valid_encoding(bytes, encoding) {
             return Unsupported;
         }
         let matched = match self {
@@ -272,18 +279,37 @@ impl Pattern {
     }
 }
 
+// Keep short checks in std and skip scanning when Ruby already proved ASCII.
+// The compatibility SIMD API retains early rejection of malformed UTF-8.
+fn valid_encoding(bytes: &[u8], encoding: i32) -> bool {
+    if encoding == ENCODING_SEVEN_BIT {
+        true
+    } else if encoding == ENCODING_ASCII {
+        bytes.is_ascii()
+    } else if bytes.len() < 64 {
+        std::str::from_utf8(bytes).is_ok()
+    } else {
+        simdutf8::compat::from_utf8(bytes).is_ok()
+    }
+}
+
 fn scan_items(
     required: i32,
     values: &[Value],
     mut invalid: impl FnMut(usize, Value) -> Verdict,
     mut matches: impl FnMut(&[u8]) -> Verdict,
+    mut fallback: impl FnMut(Value) -> Verdict,
 ) -> Verdict {
     for (index, &value) in values.iter().enumerate() {
         let mut data = std::ptr::null();
         let mut len = 0;
         let encoding = unsafe { sn_pattern_item_view(value, &mut data, &mut len) };
         if encoding == -TYPE_UNSUPPORTED {
-            return Unsupported;
+            let result = invalid(index, value);
+            if result != Valid {
+                return result;
+            }
+            continue;
         }
         if encoding < 0 {
             if required != 0 {
@@ -294,17 +320,21 @@ fn scan_items(
             }
             continue;
         }
-        if encoding == ENCODING_OTHER {
-            return Unsupported;
-        }
-        let bytes = unsafe { std::slice::from_raw_parts(data, len) };
-        if encoding != ENCODING_SEVEN_BIT
-            && ((encoding == ENCODING_ASCII && !bytes.is_ascii())
-                || std::str::from_utf8(bytes).is_err())
-        {
-            return Unsupported;
-        }
-        let result = matches(bytes);
+        let result = if encoding == ENCODING_OTHER {
+            Unsupported
+        } else {
+            let bytes = unsafe { byte_view(data, len) };
+            if valid_encoding(bytes, encoding) {
+                matches(bytes)
+            } else {
+                Unsupported
+            }
+        };
+        let result = if result == Unsupported {
+            fallback(value)
+        } else {
+            result
+        };
         match result {
             Unsupported => return Unsupported,
             Invalid => {
@@ -417,15 +447,23 @@ mod tests {
     use super::*;
 
     #[test]
-    fn reports_memory_when_a_search_cache_is_borrowed() {
-        let caches = Caches::default();
-        let regex = Regex::new("value[0-9]+").unwrap();
-        assert!(caches.is_match(0, &regex, b"value1"));
-        assert!(unsafe { sn_caches_size(&caches) } > std::mem::size_of::<Caches>());
-        let _search = caches.0.borrow_mut();
-        assert_eq!(
-            unsafe { sn_caches_size(&caches) },
-            std::mem::size_of::<Caches>()
-        );
+    fn encoding_validation_matches_std_across_simd_blocks_and_tails() {
+        let original = "日𐀀é".repeat(32).into_bytes();
+        for length in [63, 64, 65, 127, 128, 129, original.len()] {
+            for at in 0..length {
+                for byte in 0..=255 {
+                    let mut input = original[..length].to_vec();
+                    input[at] = byte;
+                    assert_eq!(
+                        valid_encoding(&input, ENCODING_UTF8),
+                        std::str::from_utf8(&input).is_ok(),
+                        "length {length}, position {at}, byte {byte}"
+                    );
+                    assert_eq!(valid_encoding(&input, ENCODING_ASCII), input.is_ascii());
+                }
+            }
+        }
+        assert!(valid_encoding(&original, ENCODING_UTF8));
+        assert!(valid_encoding(b"already classified", ENCODING_SEVEN_BIT));
     }
 }

data/ext/schemurai_native/src/pointer.rs

@@ -0,0 +1,97 @@
+//! Native UTF-8 pointer text with deferred names that need Ruby string semantics.
+use super::*;
+
+#[derive(Clone, Default, Eq, Hash, PartialEq)]
+pub(super) struct Path {
+    pub(super) text: String,
+    pub(super) names: Vec<(usize, Value)>,
+}
+impl Path {
+    pub(super) fn checkpoint(&self) -> (usize, usize) {
+        (self.text.len(), self.names.len())
+    }
+    pub(super) fn truncate(&mut self, checkpoint: (usize, usize)) {
+        self.text.truncate(checkpoint.0);
+        self.names.truncate(checkpoint.1);
+    }
+    pub(super) fn push_str(&mut self, text: &str) {
+        self.text.push_str(text);
+    }
+    pub(super) fn index(&mut self, index: usize) {
+        use std::fmt::Write;
+        write!(self.text, "/{index}").unwrap();
+    }
+    pub(super) fn name(&mut self, value: Value) -> Result<(), Verdict> {
+        if unsafe { sn_kind(value) } != TYPE_STRING {
+            return Err(Unsupported);
+        }
+        self.text.push('/');
+        let mut data = std::ptr::null();
+        let mut len = 0;
+        let encoding = unsafe { sn_string_view(value, &mut data, &mut len) };
+        let bytes = unsafe { byte_view(data, len) };
+        if matches!(encoding, ENCODING_UTF8 | ENCODING_ASCII)
+            && (encoding != ENCODING_ASCII || bytes.is_ascii())
+            && let Ok(name) = std::str::from_utf8(bytes)
+        {
+            for character in name.chars() {
+                match character {
+                    '~' => self.text.push_str("~0"),
+                    '/' => self.text.push_str("~1"),
+                    _ => self.text.push(character),
+                }
+            }
+        } else {
+            self.names.push((self.text.len(), value));
+        }
+        Ok(())
+    }
+    pub(super) fn render(
+        &self,
+        caches: &runtime::Caches,
+        keyword: Option<&str>,
+    ) -> Result<Rendered, Verdict> {
+        if self.names.is_empty() {
+            let text = keyword.map_or_else(
+                || self.text.clone(),
+                |keyword| format!("{}/{keyword}", self.text),
+            );
+            Ok(Rendered::Native(text))
+        } else if let Some(keyword) = keyword {
+            let mut path = self.clone();
+            path.push_str("/");
+            path.push_str(keyword);
+            caches.pointer(&path).map(Rendered::Ruby)
+        } else {
+            caches.pointer(self).map(Rendered::Ruby)
+        }
+    }
+    pub(super) fn memory_usage(&self) -> usize {
+        self.text.capacity() + self.names.capacity() * std::mem::size_of::<(usize, Value)>()
+    }
+}
+#[derive(Clone)]
+pub(super) enum Rendered {
+    Native(String),
+    Ruby(Value),
+}
+impl Rendered {
+    pub(super) fn bytes(&self) -> &[u8] {
+        match self {
+            Self::Native(text) => text.as_bytes(),
+            Self::Ruby(_) => &[],
+        }
+    }
+    pub(super) fn value(&self) -> Value {
+        match self {
+            Self::Native(_) => unsafe { sn_nil() },
+            Self::Ruby(value) => *value,
+        }
+    }
+    pub(super) fn memory_usage(&self) -> usize {
+        match self {
+            Self::Native(text) => text.capacity(),
+            Self::Ruby(_) => 0,
+        }
+    }
+}

data/ext/schemurai_native/src/runtime/properties.rs

@@ -0,0 +1,162 @@
+//! Opaque evaluation buffers for annotation merges that require Ruby semantics.
+use super::*;
+use crate::tracking::{RuntimeLocations, Tracked};
+use std::cell::RefCell;
+
+struct Part {
+    runtime: Option<RuntimeLocations>,
+    keys: Vec<Value>,
+    items: Vec<usize>,
+}
+impl Part {
+    fn new(engine: &Engine, tracked: &Tracked) -> Self {
+        Self {
+            runtime: tracked.runtime_locations(),
+            keys: tracked.native_properties(engine),
+            items: tracked.native_items(),
+        }
+    }
+    fn mark(&self) {
+        if let Some(runtime) = self.runtime {
+            runtime.mark();
+        }
+        for &key in &self.keys {
+            unsafe {
+                sn_pin(key);
+            }
+        }
+    }
+    fn memory_usage(&self) -> usize {
+        self.keys.capacity() * std::mem::size_of::<Value>()
+            + self.items.capacity() * std::mem::size_of::<usize>()
+    }
+}
+#[derive(Default)]
+pub(super) struct Properties {
+    pending: RefCell<Option<(Value, [Part; 2])>>,
+    completed: RefCell<HashMap<Value, RuntimeLocations>>,
+}
+impl Properties {
+    pub(super) fn reset(&self) {
+        self.pending.borrow_mut().take();
+    }
+    pub(super) fn clear(&self) {
+        self.reset();
+        *self.completed.borrow_mut() = HashMap::new();
+    }
+    pub(super) fn memory_usage(&self) -> usize {
+        self.pending.try_borrow().map_or(0, |pending| {
+            pending.as_ref().map_or(0, |(_, parts)| {
+                parts.iter().map(Part::memory_usage).sum::<usize>()
+            })
+        }) + self.completed.try_borrow().map_or(0, |results| {
+            results.capacity() * std::mem::size_of::<(Value, RuntimeLocations)>()
+        })
+    }
+    pub(super) fn mark(&self) {
+        if let Ok(pending) = self.pending.try_borrow()
+            && let Some((_, parts)) = &*pending
+        {
+            for part in parts {
+                part.mark();
+            }
+        }
+        if let Ok(completed) = self.completed.try_borrow() {
+            for runtime in completed.values() {
+                runtime.mark();
+            }
+        }
+    }
+}
+impl Caches {
+    pub(crate) fn merge_properties(
+        &self,
+        engine: &Engine,
+        left: &Tracked,
+        right: &Tracked,
+    ) -> Result<RuntimeLocations, Verdict> {
+        let ordinal = self.next_call()?;
+        if self.observe(ordinal, unsafe { sn_nil() }, RUNTIME_PROPERTY_MERGE) == Unsupported {
+            *self.properties.pending.borrow_mut() =
+                Some((ordinal, [Part::new(engine, left), Part::new(engine, right)]));
+            return Err(Unsupported);
+        }
+        self.properties
+            .completed
+            .borrow()
+            .get(&ordinal)
+            .copied()
+            .ok_or(Unsupported)
+    }
+}
+/// # Safety
+/// Caches are idle with a pending merge. Side is 0/1 and outputs are writable.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn sn_properties_part(
+    caches: *const Caches,
+    side: usize,
+    properties: *mut usize,
+    items: *mut usize,
+) -> Value {
+    let pending = unsafe { &*caches }.properties.pending.borrow();
+    let part = &pending.as_ref().expect("property merge").1[side];
+    unsafe {
+        *properties = part.keys.len();
+        *items = part.items.len();
+    }
+    part.runtime
+        .map_or_else(|| unsafe { sn_nil() }, |runtime| runtime.buffer)
+}
+/// # Safety
+/// Same pending merge requirements as sn_properties_part; index is in bounds.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn sn_properties_key(
+    caches: *const Caches,
+    side: usize,
+    index: usize,
+) -> Value {
+    unsafe { &*caches }
+        .properties
+        .pending
+        .borrow()
+        .as_ref()
+        .expect("property merge")
+        .1[side]
+        .keys[index]
+}
+/// # Safety
+/// Same pending merge requirements as sn_properties_part; index is in bounds.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn sn_properties_item(
+    caches: *const Caches,
+    side: usize,
+    index: usize,
+) -> usize {
+    unsafe { &*caches }
+        .properties
+        .pending
+        .borrow()
+        .as_ref()
+        .expect("property merge")
+        .1[side]
+        .items[index]
+}
+/// # Safety
+/// Caches are idle; locations is readable and all handles are rooted by C first.
+#[unsafe(no_mangle)]
+pub unsafe extern "C" fn sn_properties_result(
+    caches: *const Caches,
+    locations: *const RuntimeLocations,
+) {
+    let properties = &unsafe { &*caches }.properties;
+    let ordinal = properties
+        .pending
+        .borrow()
+        .as_ref()
+        .expect("property merge")
+        .0;
+    properties
+        .completed
+        .borrow_mut()
+        .insert(ordinal, unsafe { *locations });
+}