schemurai 2.3.0 → 3.0.0

data/ext/schemurai_native/src/tracking.rs

@@ -24,18 +24,139 @@ impl Hash for JsonKey {
 }
 pub(super) type KeySet = HashSet<JsonKey, BuildHasherDefault<BucketHasher>>;
 
+// Store only nonempty words. The common single-word case needs no allocation,
+// even when its property IDs are near the end of a large compiled engine.
+#[derive(Clone, Default)]
+enum PropertyWords {
+    #[default]
+    Empty,
+    Single(usize, u64),
+    Dense(usize, Vec<u64>),
+    Many(HashMap<usize, u64>),
+}
+#[derive(Clone, Default)]
+struct PropertyBits {
+    words: PropertyWords,
+    count: usize,
+}
+impl PropertyBits {
+    fn word(&self, index: usize) -> u64 {
+        match &self.words {
+            PropertyWords::Empty => 0,
+            PropertyWords::Single(at, bits) => {
+                if *at == index {
+                    *bits
+                } else {
+                    0
+                }
+            }
+            PropertyWords::Dense(first, words) => index
+                .checked_sub(*first)
+                .and_then(|offset| words.get(offset))
+                .copied()
+                .unwrap_or(0),
+            PropertyWords::Many(words) => words.get(&index).copied().unwrap_or(0),
+        }
+    }
+    fn insert_word(&mut self, index: usize, bits: u64) {
+        let previous = self.word(index);
+        self.count += (bits & !previous).count_ones() as usize;
+        match &mut self.words {
+            PropertyWords::Empty => self.words = PropertyWords::Single(index, bits),
+            PropertyWords::Single(at, existing) if *at == index => *existing |= bits,
+            PropertyWords::Single(at, existing) if at.abs_diff(index) < 128 => {
+                let first = (*at).min(index);
+                let mut words = vec![0; at.abs_diff(index) + 1];
+                words[*at - first] = *existing;
+                words[index - first] = bits;
+                self.words = PropertyWords::Dense(first, words);
+            }
+            PropertyWords::Single(at, existing) => {
+                self.words = PropertyWords::Many(HashMap::from([(*at, *existing), (index, bits)]));
+            }
+            PropertyWords::Dense(first, words)
+                if index >= *first && index - *first < words.len() =>
+            {
+                words[index - *first] |= bits;
+            }
+            PropertyWords::Dense(first, words)
+                if index.max(*first + words.len() - 1) - index.min(*first) < 128 =>
+            {
+                if index < *first {
+                    let offset = *first - index;
+                    words.splice(0..0, std::iter::repeat_n(0, offset));
+                    *first = index;
+                } else {
+                    words.resize(index - *first + 1, 0);
+                }
+                words[index - *first] |= bits;
+            }
+            PropertyWords::Dense(first, words) => {
+                let mut sparse: HashMap<_, _> = words
+                    .iter()
+                    .enumerate()
+                    .filter(|&(_, &word)| word != 0)
+                    .map(|(offset, &word)| (*first + offset, word))
+                    .collect();
+                sparse.insert(index, bits);
+                self.words = PropertyWords::Many(sparse);
+            }
+            PropertyWords::Many(words) => *words.entry(index).or_default() |= bits,
+        }
+    }
+    fn insert(&mut self, index: usize) {
+        self.insert_word(index / 64, 1_u64 << (index % 64));
+    }
+    fn contains(&self, index: usize) -> bool {
+        self.word(index / 64) & (1_u64 << (index % 64)) != 0
+    }
+    fn merge(&mut self, other: Self) {
+        if matches!(self.words, PropertyWords::Empty) {
+            *self = other;
+            return;
+        }
+        match other.words {
+            PropertyWords::Empty => {}
+            PropertyWords::Single(index, bits) => self.insert_word(index, bits),
+            PropertyWords::Dense(first, words) => {
+                for (offset, bits) in words.into_iter().enumerate() {
+                    if bits != 0 {
+                        self.insert_word(first + offset, bits);
+                    }
+                }
+            }
+            PropertyWords::Many(words) => {
+                for (index, bits) in words {
+                    self.insert_word(index, bits);
+                }
+            }
+        }
+    }
+}
+
+#[derive(Clone)]
 pub(super) struct Tracked {
     pub status: Verdict,
     properties: KeySet,
-    known_properties: Vec<u64>,
+    known_properties: PropertyBits,
     items: Vec<u64>,
 }
 impl Tracked {
+    // Bound retained annotation slots independently of the number of results.
+    pub(super) fn memo_slots(&self) -> usize {
+        self.properties.capacity()
+            + self.items.capacity()
+            + match &self.known_properties.words {
+                PropertyWords::Empty | PropertyWords::Single(..) => 0,
+                PropertyWords::Dense(_, words) => words.capacity(),
+                PropertyWords::Many(words) => words.capacity(),
+            }
+    }
     pub(super) fn new(status: Verdict) -> Self {
         Self {
             status,
             properties: KeySet::default(),
-            known_properties: vec![],
+            known_properties: PropertyBits::default(),
             items: vec![],
         }
     }
@@ -52,24 +173,13 @@ impl Tracked {
             .is_some_and(|word| word & (1_u64 << (index % 64)) != 0)
     }
     fn known_property(&mut self, index: usize) {
-        let word = index / 64;
-        if self.known_properties.len() <= word {
-            self.known_properties.resize(word + 1, 0);
-        }
-        self.known_properties[word] |= 1_u64 << (index % 64);
+        self.known_properties.insert(index);
     }
     fn contains_known_property(&self, index: usize) -> bool {
-        self.known_properties
-            .get(index / 64)
-            .is_some_and(|word| word & (1_u64 << (index % 64)) != 0)
+        self.known_properties.contains(index)
     }
     pub(super) fn property_count(&self) -> usize {
-        self.properties.len()
-            + self
-                .known_properties
-                .iter()
-                .map(|word| word.count_ones() as usize)
-                .sum::<usize>()
+        self.properties.len() + self.known_properties.count
     }
     pub(super) fn item_count(&self) -> usize {
         self.items
@@ -86,7 +196,7 @@ impl Tracked {
         } else {
             self.properties.extend(other.properties);
         }
-        merge_bits(&mut self.known_properties, other.known_properties);
+        self.known_properties.merge(other.known_properties);
         merge_bits(&mut self.items, other.items);
         Valid
     }
@@ -132,7 +242,7 @@ fn merge_bits(target: &mut Vec<u64>, source: Vec<u64>) {
         }
     }
 }
-impl Engine {
+impl Execution<'_> {
     pub(super) fn eval_tracked(
         &self,
         node: usize,
@@ -143,6 +253,9 @@ impl Engine {
         if depth > MAX_EVALUATION_DEPTH {
             return Tracked::new(Unsupported);
         }
+        if !self.budget.checkpoint() {
+            return Tracked::new(Unsupported);
+        }
         let kind = if value & self.fixnum_flag != 0 {
             TYPE_INTEGRAL_NUMBER
         } else {
@@ -160,6 +273,36 @@ impl Engine {
         depth: usize,
         active: scope::Context<'_>,
         kind: i32,
+    ) -> Tracked {
+        if !self.budget.checkpoint() {
+            return Tracked::new(Unsupported);
+        }
+        let Some(key) = self
+            .memo
+            .key(self, node, value, depth, active, &self.budget)
+        else {
+            if self.budget.hit.get().0 != 0 {
+                return Tracked::new(Unsupported);
+            }
+            return self.eval_tracked_uncached(node, value, depth, active, kind);
+        };
+        if self.budget.hit.get().0 != 0 || !self.budget.checkpoint() {
+            return Tracked::new(Unsupported);
+        }
+        if let Some(result) = self.memo.tracked(key) {
+            return result;
+        }
+        let result = self.eval_tracked_uncached(node, value, depth, active, kind);
+        self.memo.record_tracked(key, &result);
+        result
+    }
+    fn eval_tracked_uncached(
+        &self,
+        node: usize,
+        value: Value,
+        depth: usize,
+        active: scope::Context<'_>,
+        kind: i32,
     ) -> Tracked {
         let frame = scope::ScopeFrame {
             resource: self.metadata[node].resource,
@@ -178,6 +321,9 @@ impl Engine {
         }
         let mut evaluation = Tracked::new(Valid);
         for op in &self.nodes[node] {
+            if !self.budget.checkpoint() {
+                return Tracked::new(Unsupported);
+            }
             let result = match op {
                 Op::Ref(target) => {
                     if RefFrame::contains(active.active, node, value) {
@@ -203,6 +349,9 @@ impl Engine {
                 Op::All(nodes) => {
                     let mut result = Valid;
                     for &child in nodes {
+                        if !self.budget.checkpoint() {
+                            return Tracked::new(Unsupported);
+                        }
                         result = evaluation.merge(self.eval_tracked_kind(
                             child,
                             value,
@@ -217,26 +366,19 @@ impl Engine {
                     result
                 }
                 Op::Any(nodes) | Op::One(nodes) => {
-                    let mut matched = Tracked::new(Valid);
-                    let mut count = 0;
-                    let mut result = Valid;
-                    for &(child, multiplicity) in nodes {
-                        let branch = self.eval_tracked_kind(child, value, depth + 1, active, kind);
-                        if branch.status == Unsupported {
-                            result = Unsupported;
-                            break;
-                        }
-                        if branch.status == Valid {
-                            count += multiplicity;
-                            matched.merge(branch);
-                        }
-                    }
-                    if result != Valid {
-                        result
-                    } else if count == 0 || (matches!(op, Op::One(_)) && count != 1) {
-                        Invalid
+                    let branch_kind = if matches!(op, Op::Any(_)) {
+                        evaluation::BranchKind::Any
                     } else {
-                        evaluation.merge(matched)
+                        evaluation::BranchKind::One
+                    };
+                    match evaluation::scan_branches(
+                        nodes,
+                        evaluation::BranchPolicy::Complete,
+                        |child| self.eval_tracked_kind(child, value, depth + 1, active, kind),
+                    ) {
+                        Err(status) => status,
+                        Ok(result) if !branch_kind.accepts(result.matches) => Invalid,
+                        Ok(result) => evaluation.merge(result.coverage),
                     }
                 }
                 Op::Not(child) => match self.eval(*child, value, depth + 1, active) {
@@ -304,17 +446,23 @@ impl Engine {
         active: scope::Context<'_>,
         evaluation: &mut Tracked,
     ) -> Verdict {
+        if !self.budget.checkpoint() {
+            return Unsupported;
+        }
         let len = unsafe { sn_len(value) };
         if bounds(len, r.min, r.max) == Invalid {
             return Invalid;
         }
         if r.unique {
-            let result = unique(value, depth, self.fixnum_flag);
+            let result = unique(value, depth, self.fixnum_flag, &self.budget);
             if result != Valid {
                 return result;
             }
         }
         for (index, &child) in r.prefix.iter().take(len).enumerate() {
+            if !self.budget.checkpoint() {
+                return Unsupported;
+            }
             let result = self.eval(child, unsafe { sn_entry(value, index) }, depth, active);
             if result != Valid {
                 return result;
@@ -322,6 +470,9 @@ impl Engine {
             evaluation.item(index);
         }
         for index in 0..len {
+            if !self.budget.checkpoint() {
+                return Unsupported;
+            }
             let child = if r.list {
                 r.items.get(index).copied().or(r.additional)
             } else if index >= r.prefix.len() {
@@ -338,17 +489,14 @@ impl Engine {
             }
         }
         if let Some(child) = r.contains {
-            let mut count = 0;
-            for index in 0..len {
-                match self.eval(child, unsafe { sn_entry(value, index) }, depth, active) {
-                    Valid => {
-                        count += 1;
-                        evaluation.item(index);
-                    }
-                    Invalid => {}
-                    Unsupported => return Unsupported,
-                }
-            }
+            let count = match evaluation::scan_contains(
+                len,
+                |index| self.eval(child, unsafe { sn_entry(value, index) }, depth, active),
+                |index| evaluation.item(index),
+            ) {
+                Ok(count) => count,
+                Err(status) => return status,
+            };
             if bounds(count, Some(r.min_contains), r.max_contains) == Invalid {
                 return Invalid;
             }
@@ -358,6 +506,9 @@ impl Engine {
                 return Invalid;
             }
             for index in 0..len {
+                if !self.budget.checkpoint() {
+                    return Unsupported;
+                }
                 if evaluation.contains_item(index) {
                     continue;
                 }
@@ -378,47 +529,47 @@ impl Engine {
         active: scope::Context<'_>,
         evaluation: &mut Tracked,
     ) -> Verdict {
+        if !self.budget.checkpoint() {
+            return Unsupported;
+        }
         let len = unsafe { sn_len(value) };
         if bounds(len, r.min, r.max) == Invalid {
             return Invalid;
         }
-        for &key in &r.required {
-            if missing(value, key, self.undefined) {
-                return Invalid;
-            }
+        let status = r.check_required(value, self.undefined, &self.budget);
+        if status != Valid {
+            return status;
         }
-        let mut found = 0;
-        for (index, &(key, child)) in r.properties.iter().enumerate() {
-            let item = unsafe { sn_lookup(value, key) };
-            if item == self.undefined {
-                if r.required_properties[index] {
-                    return Invalid;
+        let found = match r.visit_properties(
+            value,
+            len,
+            self.undefined,
+            &self.budget,
+            |index, child, item| {
+                let result = self.record_value(child, item, depth, active);
+                if result == Valid {
+                    evaluation.known_property(r.property_indices[index]);
                 }
-                continue;
-            }
-            found += 1;
-            let result = self.record_value(child, item, depth, active);
-            if result != Valid {
-                return result;
-            }
-            evaluation.known_property(r.property_indices[index]);
-        }
+                result
+            },
+        ) {
+            Ok(found) => found,
+            Err(status) => return status,
+        };
         if r.names.is_some() || !r.patterns.is_empty() || (r.additional.is_some() && found < len) {
             let result = self.tracked_object_walk(r, value, depth, active, evaluation, false);
             if result != Valid {
                 return result;
             }
         }
-        for (name, required) in &r.dependent_required {
-            if !missing(value, *name, self.undefined) {
-                for &key in required {
-                    if missing(value, key, self.undefined) {
-                        return Invalid;
-                    }
-                }
-            }
+        let status = r.check_dependent_required(value, self.undefined, &self.budget);
+        if status != Valid {
+            return status;
         }
         for &(key, child) in &r.dependent_schemas {
+            if !self.budget.checkpoint() {
+                return Unsupported;
+            }
             if !missing(value, key, self.undefined) {
                 let result = evaluation.merge(self.eval_tracked(child, value, depth, active));
                 if result != Valid {
@@ -464,7 +615,7 @@ impl Engine {
     }
 }
 struct TrackingContext<'a> {
-    engine: &'a Engine,
+    engine: &'a Execution<'a>,
     rules: &'a Object,
     depth: usize,
     active: scope::Context<'a>,
@@ -474,6 +625,9 @@ struct TrackingContext<'a> {
 unsafe extern "C" fn tracking_entry(context: *mut c_void, key: Value, value: Value) -> i32 {
     catch_unwind(AssertUnwindSafe(|| {
         let c = unsafe { &mut *context.cast::<TrackingContext>() };
+        if !c.engine.budget.checkpoint() {
+            return Unsupported as i32;
+        }
         if unsafe { sn_kind(key) } != TYPE_STRING {
             return Unsupported as i32;
         }
@@ -494,26 +648,27 @@ unsafe extern "C" fn tracking_entry(context: *mut c_void, key: Value, value: Val
                     return result as i32;
                 }
             }
-            let mut matched = c.rules.property_keys.contains(&key_location);
-            for (pattern, child) in &c.rules.patterns {
-                match pattern.eval(key) {
-                    Unsupported => return Unsupported as i32,
-                    Invalid => {}
+            let result = c.rules.visit_value_rules::<false>(
+                c.engine.caches,
+                &c.engine.budget,
+                key,
+                Some(key_location),
+                |_, _, child| match c.engine.eval(child, value, c.depth, c.active) {
                     Valid => {
-                        matched = true;
-                        let result = c.engine.eval(*child, value, c.depth, c.active);
-                        if result != Valid {
-                            return result as i32;
-                        }
                         if let Some(index) = known {
                             c.evaluation.known_property(index);
                         } else {
                             c.evaluation.properties.insert(key_location);
                         }
+                        Ok(())
                     }
-                }
-            }
-            if matched { None } else { c.rules.additional }
+                    status => Err(status),
+                },
+            );
+            return match result {
+                Ok(_) => Valid as i32,
+                Err(status) => status as i32,
+            };
         };
         if let Some(child) = child {
             let result = c.engine.eval(child, value, c.depth, c.active);

data/lib/schemurai/backend.rb

@@ -28,6 +28,7 @@ module Schemurai
 
     module_function def load_vm!
       require_relative VM_FEATURE
+      require_relative "vm/limited_evaluator"
     end
   end
 end