schemurai 2.2.2 → 2.3.0

data/ext/schemurai_native/src/pattern.rs

@@ -0,0 +1,288 @@
+//! Crate-backed matching over borrowed Ruby string bytes.
+//! The compatibility adapter translates VM shorthands and absolute anchors;
+//! regex-syntax parses syntax, and regex-automata performs every search.
+use super::*;
+use regex_automata::{
+    Anchored, Input, MatchKind,
+    dfa::{Automaton, StartKind, dense},
+    meta::Regex,
+    nfa::thompson,
+};
+use regex_syntax::ast::{Ast, RepetitionKind, RepetitionRange};
+
+#[derive(Debug)]
+pub(super) enum Pattern {
+    Unsupported,
+    Dfa(Box<dense::DFA<Vec<u32>>>),
+    Compiled(Regex),
+}
+
+impl Pattern {
+    // Select the crate matcher once for a homogeneous array. The borrowed
+    // Ruby slices are used synchronously; classification and encoding are
+    // still checked on every element, including non-string instances.
+    pub(super) fn eval_items(&self, required: i32, values: &[Value]) -> Verdict {
+        self.visit_invalid_items(required, values, |_, _| Invalid)
+    }
+
+    pub(super) fn visit_invalid_items(
+        &self,
+        required: i32,
+        values: &[Value],
+        invalid: impl FnMut(usize, Value) -> Verdict,
+    ) -> Verdict {
+        match self {
+            Self::Dfa(dfa) => scan_items(required, values, invalid, |bytes| {
+                match dfa.try_search_fwd(&Input::new(bytes).anchored(Anchored::Yes).earliest(true))
+                {
+                    Ok(matched) => verdict(matched.is_some()),
+                    Err(_) => Unsupported,
+                }
+            }),
+            Self::Compiled(regex) => scan_items(required, values, invalid, |bytes| {
+                verdict(regex.is_match(bytes))
+            }),
+            Self::Unsupported => scan_items(required, values, invalid, |_| Unsupported),
+        }
+    }
+
+    pub(super) fn memory_usage(&self) -> usize {
+        match self {
+            Self::Dfa(dfa) => std::mem::size_of_val(&**dfa) + dfa.memory_usage(),
+            Self::Compiled(regex) => regex.memory_usage(),
+            Self::Unsupported => 0,
+        }
+    }
+
+    pub(super) fn compile(value: Value) -> Self {
+        if unsafe { sn_kind(value) } != TYPE_STRING {
+            return Self::Unsupported;
+        }
+        let mut data = std::ptr::null();
+        let mut len = 0;
+        let encoding = unsafe { sn_string_view(value, &mut data, &mut len) };
+        if encoding == ENCODING_OTHER {
+            return Self::Unsupported;
+        }
+        let bytes = unsafe { std::slice::from_raw_parts(data, len) };
+        if encoding == ENCODING_ASCII && !bytes.is_ascii() {
+            return Self::Unsupported;
+        }
+        let Some(source) = std::str::from_utf8(bytes).ok().and_then(translate) else {
+            return Self::Unsupported;
+        };
+        let Ok(ast) = regex_syntax::ast::parse::Parser::new().parse(&source) else {
+            return Self::Unsupported;
+        };
+        if !compatible(&ast) {
+            return Self::Unsupported;
+        }
+        // Use a single immutable DFA for fully start-anchored validity. The
+        // meta engine retains its literal prefilters for unanchored searches.
+        // Bound both the DFA table and auxiliary determinization memory.
+        if let Ok(hir) = regex_syntax::hir::translate::Translator::new().translate(&source, &ast)
+            && hir
+                .properties()
+                .look_set_prefix()
+                .contains(regex_syntax::hir::Look::Start)
+            && let Ok(dfa) = dense::Builder::new()
+                .configure(
+                    dense::Config::new()
+                        .match_kind(MatchKind::LeftmostFirst)
+                        .start_kind(StartKind::Anchored)
+                        .dfa_size_limit(Some(1 << 16))
+                        .determinize_size_limit(Some(1 << 18)),
+                )
+                .thompson(thompson::Config::new().nfa_size_limit(Some(1 << 20)))
+                .build(&source)
+        {
+            return Self::Dfa(Box::new(dfa));
+        }
+        Regex::builder()
+            .configure(
+                Regex::config()
+                    .nfa_size_limit(Some(1 << 20))
+                    .hybrid_cache_capacity(1 << 20),
+            )
+            .build(&source)
+            .map(Self::Compiled)
+            .unwrap_or(Self::Unsupported)
+    }
+
+    pub(super) fn eval(&self, value: Value) -> Verdict {
+        if matches!(self, Self::Unsupported) {
+            return Unsupported;
+        }
+        let mut data = std::ptr::null();
+        let mut len = 0;
+        let encoding = unsafe { sn_pattern_view(value, &mut data, &mut len) };
+        if encoding == ENCODING_OTHER {
+            return Unsupported;
+        }
+        let bytes = unsafe { std::slice::from_raw_parts(data, len) };
+        self.eval_bytes(bytes, encoding)
+    }
+
+    pub(super) fn eval_bytes(&self, bytes: &[u8], encoding: i32) -> Verdict {
+        if encoding == ENCODING_OTHER || matches!(self, Self::Unsupported) {
+            return Unsupported;
+        }
+        if encoding != ENCODING_SEVEN_BIT
+            && ((encoding == ENCODING_ASCII && !bytes.is_ascii())
+                || std::str::from_utf8(bytes).is_err())
+        {
+            return Unsupported;
+        }
+        let matched = match self {
+            Self::Dfa(dfa) => match dfa
+                .try_search_fwd(&Input::new(bytes).anchored(Anchored::Yes).earliest(true))
+            {
+                Ok(matched) => matched.is_some(),
+                Err(_) => return Unsupported,
+            },
+            Self::Compiled(regex) => regex.is_match(bytes),
+            Self::Unsupported => return Unsupported,
+        };
+        if matched { Valid } else { Invalid }
+    }
+}
+
+fn scan_items(
+    required: i32,
+    values: &[Value],
+    mut invalid: impl FnMut(usize, Value) -> Verdict,
+    mut matches: impl FnMut(&[u8]) -> Verdict,
+) -> Verdict {
+    for (index, &value) in values.iter().enumerate() {
+        let mut data = std::ptr::null();
+        let mut len = 0;
+        let encoding = unsafe { sn_pattern_item_view(value, &mut data, &mut len) };
+        if encoding == -TYPE_UNSUPPORTED {
+            return Unsupported;
+        }
+        if encoding < 0 {
+            if required != 0 {
+                let result = invalid(index, value);
+                if result != Valid {
+                    return result;
+                }
+            }
+            continue;
+        }
+        if encoding == ENCODING_OTHER {
+            return Unsupported;
+        }
+        let bytes = unsafe { std::slice::from_raw_parts(data, len) };
+        if encoding != ENCODING_SEVEN_BIT
+            && ((encoding == ENCODING_ASCII && !bytes.is_ascii())
+                || std::str::from_utf8(bytes).is_err())
+        {
+            return Unsupported;
+        }
+        let result = matches(bytes);
+        match result {
+            Unsupported => return Unsupported,
+            Invalid => {
+                let result = invalid(index, value);
+                if result != Valid {
+                    return result;
+                }
+            }
+            Valid => {}
+        }
+    }
+    Valid
+}
+
+// This is a VM compatibility translation, not a regex parser. Unsupported
+// escapes, class set operations and special groups retain Ruby's behavior.
+fn translate(source: &str) -> Option<String> {
+    let mut output = String::with_capacity(source.len());
+    let mut characters = source.chars().peekable();
+    let mut in_class = false;
+    while let Some(character) = characters.next() {
+        match character {
+            '\\' => {
+                let escaped = characters.next()?;
+                let class = match escaped {
+                    'd' => Some("0-9"),
+                    'D' => Some("^0-9"),
+                    'w' => Some("A-Za-z0-9_"),
+                    'W' => Some("^A-Za-z0-9_"),
+                    's' => Some(
+                        r"\x{0009}-\x{000D}\x{0020}\x{00A0}\x{1680}\x{2000}-\x{200A}\x{2028}\x{2029}\x{202F}\x{205F}\x{3000}\x{FEFF}",
+                    ),
+                    'S' => Some(
+                        r"^\x{0009}-\x{000D}\x{0020}\x{00A0}\x{1680}\x{2000}-\x{200A}\x{2028}\x{2029}\x{202F}\x{205F}\x{3000}\x{FEFF}",
+                    ),
+                    _ => None,
+                };
+                if let Some(class) = class {
+                    if !in_class {
+                        output.push('[');
+                    }
+                    output.push_str(class);
+                    if !in_class {
+                        output.push(']');
+                    }
+                } else if ".^$?+*(){}[]|\\-afnrtv".contains(escaped)
+                    || (!in_class && matches!(escaped, 'A' | 'z'))
+                {
+                    output.push('\\');
+                    output.push(escaped);
+                } else {
+                    return None;
+                }
+            }
+            '[' => {
+                if in_class {
+                    return None;
+                }
+                in_class = true;
+                output.push(character);
+            }
+            ']' => {
+                in_class = false;
+                output.push(character);
+            }
+            '&' if in_class => return None,
+            '-' | '~' if in_class && characters.peek() == Some(&character) => return None,
+            '(' if !in_class && characters.peek() == Some(&'?') => {
+                characters.next();
+                if characters.next() != Some(':') {
+                    return None;
+                }
+                output.push_str("(?:");
+            }
+            '^' if !in_class => output.push_str(r"\A"),
+            '$' if !in_class => output.push_str(r"\z"),
+            _ => output.push(character),
+        }
+    }
+    Some(output)
+}
+
+fn compatible(ast: &Ast) -> bool {
+    match ast {
+        Ast::Repetition(repetition) => {
+            // Ruby treats adjacent repetition operators as possessive or
+            // rejects them, while regex-syntax can parse nested repetitions.
+            if matches!(*repetition.ast, Ast::Repetition(_)) {
+                return false;
+            }
+            let bounded = match repetition.op.kind {
+                RepetitionKind::Range(
+                    RepetitionRange::Exactly(n) | RepetitionRange::AtLeast(n),
+                ) => n <= 4096,
+                RepetitionKind::Range(RepetitionRange::Bounded(_, max)) => max <= 4096,
+                _ => true,
+            };
+            bounded && compatible(&repetition.ast)
+        }
+        Ast::Group(group) => compatible(&group.ast),
+        Ast::Alternation(alternation) => alternation.asts.iter().all(compatible),
+        Ast::Concat(concat) => concat.asts.iter().all(compatible),
+        Ast::Flags(_) | Ast::ClassUnicode(_) | Ast::ClassPerl(_) => false,
+        _ => true,
+    }
+}

data/ext/schemurai_native/src/scope.rs

@@ -0,0 +1,210 @@
+//! Resource/anchor lookup is compiled once. Evaluation only follows native
+//! tables and stack frames, including when speculative branches collect annotations.
+use super::*;
+use tracking::JsonKey;
+#[derive(Clone, Copy, Default)]
+pub(super) struct Metadata {
+    pub resource: Value,
+    pub enters_scope: bool,
+    pub recursive_anchor: bool,
+    pub dynamic_anchor: Option<Value>,
+    pub primary_binding: Option<usize>,
+}
+#[derive(Clone, Copy, Debug)]
+enum ReferenceKind {
+    Static,
+    Recursive,
+    Dynamic(usize),
+}
+#[derive(Clone, Copy, Debug)]
+pub(super) struct Reference {
+    target: usize,
+    kind: ReferenceKind,
+}
+#[derive(Clone, Copy, Default)]
+pub(super) struct Context<'a> {
+    pub active: Option<&'a RefFrame<'a>>,
+    pub scope: Option<&'a ScopeFrame<'a>>,
+}
+pub(super) struct ScopeFrame<'a> {
+    pub resource: Value,
+    pub parent: Option<&'a ScopeFrame<'a>>,
+    pub primary_binding: Option<usize>,
+}
+impl Engine {
+    pub(super) fn compile_scope(
+        &mut self,
+        program: Value,
+        index: usize,
+        flags: i64,
+    ) -> Result<(), Value> {
+        let node = self.field(program, b"node\0")?;
+        let resource = self.field(node, b"resource\0")?;
+        let recursive = self.field(program, b"recursive_anchor?\0")?;
+        let dynamic = self.field(program, b"dynamic_anchor\0")?;
+        self.metadata[index] = Metadata {
+            resource,
+            enters_scope: flags & i64::from(TRACKS_DYNAMIC_SCOPE) != 0,
+            recursive_anchor: unsafe { sn_true(recursive) } != 0,
+            primary_binding: None,
+            dynamic_anchor: (unsafe { sn_same(dynamic, sn_nil()) } == 0).then_some(dynamic),
+        };
+        if !self.resources.contains_key(&resource) {
+            // Reserve the resource before compiling its root: roots can reach
+            // the current child through a recursive or dynamic reference.
+            self.resources.insert(resource, usize::MAX);
+            let mut root = 0;
+            if unsafe { sn_resource_program(self.compiler, resource, &mut root) } != 0 {
+                return Err(root);
+            }
+            self.retain(root);
+            let root = self.compile(root)?;
+            self.resources.insert(resource, root);
+        }
+        Ok(())
+    }
+    pub(super) fn compile_reference(
+        &mut self,
+        operand: Value,
+        opcode: i32,
+        target: usize,
+    ) -> Result<Op, Value> {
+        let fragment = self.field(operand, b"fragment\0")?;
+        let kind = if opcode == OP_RECURSIVE_REF
+            && unsafe { sn_same(fragment, sn_nil()) } == 0
+            && unsafe { sn_string_bytes(fragment) } == 0
+            && self.metadata[target].recursive_anchor
+        {
+            ReferenceKind::Recursive
+        } else if opcode == OP_DYNAMIC_REF {
+            let anchor = self.metadata[target].dynamic_anchor;
+            match anchor {
+                Some(anchor)
+                    if unsafe { sn_same(fragment, sn_nil()) } == 0
+                        && unsafe { sn_string_bytes(fragment) } != 0
+                        && unsafe { *sn_string_data(fragment) } != b'/'
+                        && unsafe { sn_string_equal(fragment, anchor) } != 0 =>
+                {
+                    let name = JsonKey::new(fragment);
+                    let index = self
+                        .anchor_names
+                        .iter()
+                        .position(|&candidate| candidate == name)
+                        .unwrap_or_else(|| {
+                            let index = self.anchor_names.len();
+                            self.anchor_names.push(name);
+                            index
+                        });
+                    ReferenceKind::Dynamic(index)
+                }
+                _ => ReferenceKind::Static,
+            }
+        } else {
+            ReferenceKind::Static
+        };
+        Ok(Op::Ref(Reference { target, kind }))
+    }
+    pub(super) fn bind_dynamic_anchors(&mut self) -> Result<(), Value> {
+        loop {
+            let pending: Vec<_> = self
+                .resources
+                .keys()
+                .flat_map(|&resource| {
+                    (0..self.anchor_names.len()).map(move |name| (resource, name))
+                })
+                .filter(|key| !self.dynamic_bindings.contains_key(key))
+                .collect();
+            if pending.is_empty() {
+                self.anchor_tables = vec![vec![]; self.anchor_names.len()];
+                for (&(resource, name), &target) in &self.dynamic_bindings {
+                    if let Some(target) = target {
+                        self.anchor_tables[name].push((resource, target));
+                    }
+                }
+                if self.anchor_tables.len() == 1 {
+                    for metadata in &mut self.metadata {
+                        metadata.primary_binding =
+                            self.anchor_tables[0]
+                                .iter()
+                                .find_map(|&(resource, target)| {
+                                    (resource == metadata.resource).then_some(target)
+                                });
+                    }
+                }
+                return Ok(());
+            }
+            for (resource, name) in pending {
+                let mut target = 0;
+                if unsafe {
+                    sn_anchor_program(
+                        self.compiler,
+                        resource,
+                        self.anchor_names[name].0,
+                        &mut target,
+                    )
+                } != 0
+                {
+                    return Err(target);
+                }
+                self.retain(target);
+                let target = if unsafe { sn_same(target, sn_nil()) } != 0 {
+                    None
+                } else {
+                    Some(self.compile(target)?)
+                };
+                self.dynamic_bindings.insert((resource, name), target);
+            }
+        }
+    }
+    pub(super) fn enter_scope<'a>(
+        &self,
+        node: usize,
+        active: Context<'a>,
+        frame: &'a ScopeFrame<'a>,
+    ) -> Context<'a> {
+        if self.metadata[node].enters_scope
+            && active
+                .scope
+                .is_none_or(|scope| scope.resource != frame.resource)
+        {
+            Context {
+                scope: Some(frame),
+                ..active
+            }
+        } else {
+            active
+        }
+    }
+    pub(super) fn reference_target(&self, reference: Reference, context: Context<'_>) -> usize {
+        if matches!(reference.kind, ReferenceKind::Dynamic(0)) && self.anchor_tables.len() == 1 {
+            return context
+                .scope
+                .and_then(|frame| frame.primary_binding)
+                .unwrap_or(reference.target);
+        }
+        let mut target = reference.target;
+        let mut current = context.scope;
+        // Linked frames walk inner to outer. Keep replacing a match so the
+        // outermost matching resource wins, just as the VM's scope.each does.
+        while let Some(frame) = current {
+            match reference.kind {
+                ReferenceKind::Static => break,
+                ReferenceKind::Recursive => {
+                    if let Some(&root) = self.resources.get(&frame.resource)
+                        && self.metadata[root].recursive_anchor
+                    {
+                        target = root;
+                    }
+                }
+                ReferenceKind::Dynamic(name) => {
+                    if let Some(Some(binding)) = self.dynamic_bindings.get(&(frame.resource, name))
+                    {
+                        target = *binding;
+                    }
+                }
+            }
+            current = frame.parent;
+        }
+        target
+    }
+}