schemurai 3.0.0 → 3.0.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8fde4b5a2e6ccf979c4948fd13476c8250acf053346de28c4d62b32ed0c92e4f
-  data.tar.gz: 7e3a5fb2dfa17195b1a45cde045f7256a1c5f3687f19f8d2c7b821980ffe2ab1
+  metadata.gz: f82df02813a6067fc242975190077fb66c9d104866c0e373497ad639f1ba12e3
+  data.tar.gz: d8ba792bfd5a7fbbdf2295bfa98f26e5804c702ad249ebf6efd3e8ba683af76e
 SHA512:
-  metadata.gz: 42c0e94ce30932d09c9f1617592338f2386076eca4d176f1e1a9461557ea723abf512f94d7c68fb52a392021e9b45cba4ab329ee3669754b6d0a996f0319a07f
-  data.tar.gz: 68b8c4e585cd40e11d4d7fbae8d9345f52a4bfca437704b86999eab79f219cf52e276b623faf0271cd49c4044c596f544399be77ba8a8fd36946f20524708cc2
+  metadata.gz: ce2508c012b1bf512bd51c7d2d67f1feacb6935e78d73577e0769cb2dd4664b12eaa6fbf58b57f3027f6d46b30aa684fde9cfda15ced02b24eaf004488be6a18
+  data.tar.gz: 1282ae16eed2abaecb7f352c9030e5d6ef1417c55c4c24b4c5e4860214a3e01fa2cbeed81a9aea506e43b57e62373f53f2f669e4d3e46b2db02bc43064ab9140

data/ext/schemurai_native/Cargo.lock

@@ -82,7 +82,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
 
 [[package]]
 name = "schemurai-native"
-version = "3.0.0"
+version = "3.0.1"
 dependencies = [
  "base64-simd",
  "base64-turbo",
@@ -90,8 +90,15 @@ dependencies = [
  "regex-automata",
  "regex-syntax",
  "ryu",
+ "simdutf8",
 ]
 
+[[package]]
+name = "simdutf8"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
+
 [[package]]
 name = "vsimd"
 version = "0.8.0"

data/ext/schemurai_native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "schemurai-native"
-version = "3.0.0"
+version = "3.0.1"
 edition = "2024"
 rust-version = "1.93"
 publish = false
@@ -12,6 +12,7 @@ memchr = "2.8.3"
 regex-automata = "0.4.18"
 regex-syntax = "0.8.11"
 ryu = "1.0.23"
+simdutf8 = { version = "0.1.5", features = ["aarch64_neon"] }
 
 [lib]
 crate-type = ["staticlib"]

data/ext/schemurai_native/src/content.rs

@@ -167,6 +167,17 @@ fn decode_json_direct(bytes: &[u8], length: usize) -> Verdict {
     decode_json_turbo(bytes, length, &mut scratch)
 }
 
+#[cfg(target_arch = "aarch64")]
+fn decode_json_turbo(bytes: &[u8], length: usize, scratch: &mut [u8]) -> Verdict {
+    let capacity = base64_turbo::STANDARD.decoded_len_estimate(bytes.len());
+    if decode_groups(bytes, &mut scratch[..capacity], length) {
+        json(&scratch[..length])
+    } else {
+        Invalid
+    }
+}
+
+#[cfg(not(target_arch = "aarch64"))]
 fn decode_json_turbo(bytes: &[u8], length: usize, scratch: &mut [u8]) -> Verdict {
     // The slice API requires an initialized upper-bound buffer, including
     // up to two padding bytes beyond the exact decoded length. Only the
@@ -195,6 +206,13 @@ fn decode_json_stream(bytes: &[u8], length: usize) -> Verdict {
 
 // Decode borrowed bytes into the caller's fixed stack scratch. The STANDARD
 // variant also checks each chunk's alphabet, padding and unused trailing bits.
+#[cfg(target_arch = "aarch64")]
+fn decode_groups(bytes: &[u8], output: &mut [u8], expected: usize) -> bool {
+    use base64_simd::AsOut;
+    matches!(base64_simd::STANDARD.decode(bytes, output.as_out()), Ok(decoded) if decoded.len() == expected)
+}
+
+#[cfg(not(target_arch = "aarch64"))]
 fn decode_groups(bytes: &[u8], output: &mut [u8], expected: usize) -> bool {
     matches!(
         base64_turbo::STANDARD.decode_slice(bytes, output),

data/ext/schemurai_native/src/format.rs

@@ -118,16 +118,100 @@ fn date_exact(bytes: &[u8]) -> bool {
 fn date_time(bytes: &[u8]) -> bool {
     bytes.len() >= 20 && date(bytes) && matches!(bytes[10], b'T' | b't') && time(&bytes[11..])
 }
+// Keep the SIMD body out of the shared format dispatchers. Local x64 benchmarks
+// and disassembly showed that inlining it shifted the unchanged JSON Pointer
+// loop into a slower instruction layout; outlining retains the UUID SIMD gain.
+#[inline(never)]
 fn uuid(bytes: &[u8]) -> bool {
-    bytes.len() == 36
-        && bytes.iter().enumerate().all(|(index, byte)| {
-            if matches!(index, 8 | 13 | 18 | 23) {
-                *byte == b'-'
-            } else {
-                byte.is_ascii_hexdigit()
-            }
-        })
+    if bytes.len() != 36 || [8, 13, 18, 23].iter().any(|&at| bytes[at] != b'-') {
+        return false;
+    }
+    #[cfg(any(
+        all(target_arch = "x86_64", target_feature = "sse2"),
+        all(
+            target_arch = "aarch64",
+            target_feature = "neon",
+            target_endian = "little"
+        )
+    ))]
+    {
+        // The first two vectors contain all four separators. Their exact
+        // positions were checked above; only those lanes may bypass hex checks.
+        hex_block(&bytes[..16], (1 << 8) | (1 << 13))
+            && hex_block(&bytes[16..32], (1 << 2) | (1 << 7))
+            && bytes[32..].iter().all(u8::is_ascii_hexdigit)
+    }
+    #[cfg(not(any(
+        all(target_arch = "x86_64", target_feature = "sse2"),
+        all(
+            target_arch = "aarch64",
+            target_feature = "neon",
+            target_endian = "little"
+        )
+    )))]
+    bytes
+        .iter()
+        .enumerate()
+        .all(|(index, byte)| matches!(index, 8 | 13 | 18 | 23) || byte.is_ascii_hexdigit())
 }
+
+#[cfg(all(target_arch = "x86_64", target_feature = "sse2"))]
+fn hex_block(bytes: &[u8], separators: u16) -> bool {
+    use core::arch::x86_64::{
+        _mm_cmpeq_epi8, _mm_loadu_si128, _mm_movemask_epi8, _mm_or_si128, _mm_set1_epi8,
+        _mm_setzero_si128, _mm_sub_epi8, _mm_subs_epu8,
+    };
+    // The caller supplies sixteen readable bytes. SSE2 is a target baseline;
+    // unsigned wrapping subtraction rejects bytes below the ASCII ranges too.
+    let valid = unsafe {
+        let word = _mm_loadu_si128(bytes.as_ptr().cast());
+        let digits = _mm_sub_epi8(word, _mm_set1_epi8(b'0' as i8));
+        let letters = _mm_sub_epi8(
+            _mm_or_si128(word, _mm_set1_epi8(32)),
+            _mm_set1_epi8(b'a' as i8),
+        );
+        let digits = _mm_cmpeq_epi8(_mm_subs_epu8(digits, _mm_set1_epi8(9)), _mm_setzero_si128());
+        let letters = _mm_cmpeq_epi8(
+            _mm_subs_epu8(letters, _mm_set1_epi8(5)),
+            _mm_setzero_si128(),
+        );
+        _mm_movemask_epi8(_mm_or_si128(digits, letters)) as u16
+    };
+    valid | separators == u16::MAX
+}
+
+#[cfg(all(
+    target_arch = "aarch64",
+    target_feature = "neon",
+    target_endian = "little"
+))]
+fn hex_block(bytes: &[u8], separators: u16) -> bool {
+    use core::arch::aarch64::{
+        vcleq_u8, vdupq_n_u8, vget_lane_u64, vld1q_u8, vorrq_u8, vreinterpret_u64_u8,
+        vreinterpretq_u16_u8, vshrn_n_u16, vsubq_u8,
+    };
+    // Each load remains within the sixteen-byte block. Narrowing packs two
+    // four-bit flags per byte in little-endian lane order.
+    let valid = unsafe {
+        let word = vld1q_u8(bytes.as_ptr());
+        let digits = vcleq_u8(vsubq_u8(word, vdupq_n_u8(b'0')), vdupq_n_u8(9));
+        let letters = vcleq_u8(
+            vsubq_u8(vorrq_u8(word, vdupq_n_u8(32)), vdupq_n_u8(b'a')),
+            vdupq_n_u8(5),
+        );
+        let flags = vorrq_u8(digits, letters);
+        let packed = vshrn_n_u16::<4>(vreinterpretq_u16_u8(flags));
+        vget_lane_u64::<0>(vreinterpret_u64_u8(packed))
+    };
+    let mut ignored = 0_u64;
+    for lane in 0..16 {
+        if separators & (1 << lane) != 0 {
+            ignored |= 0xf << (lane * 4);
+        }
+    }
+    valid | ignored == u64::MAX
+}
+
 fn relative_pointer(bytes: &[u8]) -> bool {
     let Some(end) = digits_end(bytes, 0) else {
         return false;
@@ -355,3 +439,29 @@ fn ipv6(bytes: &[u8]) -> bool {
     }
     if compressed { groups < 8 } else { groups == 8 }
 }
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn uuid_checks_every_byte_and_separator_without_accepting_non_ascii() {
+        let original = *b"123e4567-e89b-12d3-a456-426614174000";
+        for at in 0..36 {
+            for byte in 0..=255 {
+                let mut input = original;
+                input[at] = byte;
+                let expected = if matches!(at, 8 | 13 | 18 | 23) {
+                    byte == b'-'
+                } else {
+                    byte.is_ascii_hexdigit()
+                };
+                assert_eq!(uuid(&input), expected, "position {at}, byte {byte}");
+            }
+        }
+        assert!(!uuid(&original[..35]));
+        let mut longer = original.to_vec();
+        longer.push(b'0');
+        assert!(!uuid(&longer));
+    }
+}

data/ext/schemurai_native/src/integer_scan.rs

@@ -0,0 +1,132 @@
+//! Validate complete blocks of borrowed, header-verified immediate integers.
+//! Stop before any uncertain block so the ordinary evaluator retains ordering.
+use super::Value;
+
+pub(super) fn valid_prefix(values: &[Value], min: i64, max: i64, fixnum_flag: Value) -> usize {
+    if fixnum_flag != 1 || values.len() < 16 {
+        return 0;
+    }
+    // Comparing tagged values preserves signed integer order. Clamp the
+    // interval before tagging, since some schema limits exceed Fixnum range.
+    let min = min.max(-(1_i64 << 62));
+    let max = max.min((1_i64 << 62) - 1);
+    if min > max {
+        return 0;
+    }
+    let min = (min << 1) | 1;
+    let max = (max << 1) | 1;
+    #[cfg(target_arch = "x86_64")]
+    if std::is_x86_feature_detected!("avx2") {
+        // Detection includes OS support. The kernel only loads full blocks.
+        return unsafe { avx2_prefix(values, min, max) };
+    }
+    #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
+    return neon_prefix(values, min, max);
+    #[allow(unreachable_code)]
+    {
+        let _ = (min, max);
+        0
+    }
+}
+
+#[cfg(target_arch = "x86_64")]
+#[target_feature(enable = "avx2")]
+unsafe fn avx2_prefix(values: &[Value], min: i64, max: i64) -> usize {
+    use core::arch::x86_64::{
+        _mm256_and_si256, _mm256_andnot_si256, _mm256_cmpeq_epi64, _mm256_cmpgt_epi64,
+        _mm256_loadu_si256, _mm256_movemask_epi8, _mm256_or_si256, _mm256_set1_epi64x,
+    };
+    let lower = _mm256_set1_epi64x(min);
+    let upper = _mm256_set1_epi64x(max);
+    let tag = _mm256_set1_epi64x(1);
+    let mut consumed = 0;
+    for block in values.as_chunks::<4>().0 {
+        // Four VALUE words provide all 32 readable bytes. No Ruby object is
+        // dereferenced; the header-validated low bit identifies each Fixnum.
+        let valid = unsafe {
+            let word = _mm256_loadu_si256(block.as_ptr().cast());
+            let tagged = _mm256_cmpeq_epi64(_mm256_and_si256(word, tag), tag);
+            let outside = _mm256_or_si256(
+                _mm256_cmpgt_epi64(lower, word),
+                _mm256_cmpgt_epi64(word, upper),
+            );
+            _mm256_movemask_epi8(_mm256_andnot_si256(outside, tagged)) == -1
+        };
+        if !valid {
+            break;
+        }
+        consumed += 4;
+    }
+    consumed
+}
+
+#[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
+fn neon_prefix(values: &[Value], min: i64, max: i64) -> usize {
+    use core::arch::aarch64::{
+        vandq_u64, vceqq_u64, vcgeq_s64, vcleq_s64, vdupq_n_s64, vdupq_n_u64, vgetq_lane_u64,
+        vld1q_u64, vreinterpretq_s64_u64,
+    };
+    // This function is compiled only when the target guarantees NEON.
+    let (lower, upper, tag) = unsafe { (vdupq_n_s64(min), vdupq_n_s64(max), vdupq_n_u64(1)) };
+    let mut consumed = 0;
+    for block in values.as_chunks::<2>().0 {
+        // Baseline NEON; the unaligned load stays within two borrowed VALUEs.
+        let valid = unsafe {
+            let word = vld1q_u64(block.as_ptr().cast());
+            let tagged = vceqq_u64(vandq_u64(word, tag), tag);
+            let signed = vreinterpretq_s64_u64(word);
+            let bounds = vandq_u64(vcgeq_s64(signed, lower), vcleq_s64(signed, upper));
+            let valid = vandq_u64(tagged, bounds);
+            vgetq_lane_u64::<0>(valid) & vgetq_lane_u64::<1>(valid) == u64::MAX
+        };
+        if !valid {
+            break;
+        }
+        consumed += 2;
+    }
+    consumed
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    fn tag(value: i64) -> Value {
+        ((value << 1) | 1) as Value
+    }
+
+    #[test]
+    fn skips_only_valid_fixnums_before_the_first_uncertain_block() {
+        for len in 0..65 {
+            let original: Vec<_> = (0..len).map(|at| tag((at % 33) as i64 - 16)).collect();
+            for at in 0..len {
+                for value in [0, 2, tag(-17), tag(17), tag(-(1 << 62)), tag((1 << 62) - 1)] {
+                    let mut values = original.clone();
+                    values[at] = value;
+                    let prefix = valid_prefix(&values, -16, 16, 1);
+                    assert!(prefix <= at, "length {len}, position {at}, prefix {prefix}");
+                    assert_eq!(&values[..prefix], &original[..prefix]);
+                }
+            }
+        }
+    }
+
+    #[test]
+    fn clamps_schema_limits_without_overflow_and_obeys_disabled_hints() {
+        let values = [tag(-(1 << 62)), tag((1 << 62) - 1)].repeat(32);
+        let prefix = valid_prefix(&values, i64::MIN, i64::MAX, 1);
+        #[cfg(target_arch = "x86_64")]
+        if std::is_x86_feature_detected!("avx2") {
+            assert_eq!(prefix, values.len());
+        }
+        #[cfg(all(target_arch = "aarch64", target_feature = "neon"))]
+        assert_eq!(prefix, values.len());
+        assert!(prefix <= values.len());
+        assert_eq!(valid_prefix(&values, i64::MIN, i64::MAX, 0), 0);
+        assert_eq!(valid_prefix(&values, 1 << 62, i64::MAX, 1), 0);
+        assert_eq!(valid_prefix(&values, i64::MIN, -(1 << 62) - 1, 1), 0);
+        assert_eq!(valid_prefix(&values, 1, 0, 1), 0);
+        assert_eq!(valid_prefix(&values, -(1 << 62) + 1, i64::MAX, 1), 0);
+        assert_eq!(valid_prefix(&values, i64::MIN, (1 << 62) - 2, 1), 0);
+    }
+}

data/ext/schemurai_native/src/lib.rs

@@ -13,6 +13,7 @@ mod evaluation;
 mod format;
 mod graph;
 mod instance_graph;
+mod integer_scan;
 mod memo;
 mod memo_scope;
 mod number;
@@ -1658,7 +1659,12 @@ impl Execution<'_> {
             }
             _ => None,
         };
-        for &value in values {
+        let prefix = if self.budget.timed() {
+            0
+        } else {
+            integer_scan::valid_prefix(values, min, max, self.fixnum_flag)
+        };
+        for &value in &values[prefix..] {
             if !self.budget.checkpoint() {
                 return Unsupported;
             }

data/ext/schemurai_native/src/pattern.rs

@@ -109,10 +109,7 @@ impl PatternGroup {
             return None;
         }
         let bytes = unsafe { byte_view(data, len) };
-        if encoding != ENCODING_SEVEN_BIT
-            && ((encoding == ENCODING_ASCII && !bytes.is_ascii())
-                || std::str::from_utf8(bytes).is_err())
-        {
+        if !valid_encoding(bytes, encoding) {
             return None;
         }
         let input = Input::new(bytes).anchored(Anchored::Yes);
@@ -252,10 +249,7 @@ impl Pattern {
         if encoding == ENCODING_OTHER || matches!(self, Self::Unsupported) {
             return Unsupported;
         }
-        if encoding != ENCODING_SEVEN_BIT
-            && ((encoding == ENCODING_ASCII && !bytes.is_ascii())
-                || std::str::from_utf8(bytes).is_err())
-        {
+        if !valid_encoding(bytes, encoding) {
             return Unsupported;
         }
         let matched = match self {
@@ -272,6 +266,20 @@ impl Pattern {
     }
 }
 
+// Keep short checks in std and skip scanning when Ruby already proved ASCII.
+// The compatibility SIMD API retains early rejection of malformed UTF-8.
+fn valid_encoding(bytes: &[u8], encoding: i32) -> bool {
+    if encoding == ENCODING_SEVEN_BIT {
+        true
+    } else if encoding == ENCODING_ASCII {
+        bytes.is_ascii()
+    } else if bytes.len() < 64 {
+        std::str::from_utf8(bytes).is_ok()
+    } else {
+        simdutf8::compat::from_utf8(bytes).is_ok()
+    }
+}
+
 fn scan_items(
     required: i32,
     values: &[Value],
@@ -298,10 +306,7 @@ fn scan_items(
             return Unsupported;
         }
         let bytes = unsafe { std::slice::from_raw_parts(data, len) };
-        if encoding != ENCODING_SEVEN_BIT
-            && ((encoding == ENCODING_ASCII && !bytes.is_ascii())
-                || std::str::from_utf8(bytes).is_err())
-        {
+        if !valid_encoding(bytes, encoding) {
             return Unsupported;
         }
         let result = matches(bytes);
@@ -416,6 +421,27 @@ fn compatible(ast: &Ast) -> bool {
 mod tests {
     use super::*;
 
+    #[test]
+    fn encoding_validation_matches_std_across_simd_blocks_and_tails() {
+        let original = "日𐀀é".repeat(32).into_bytes();
+        for length in [63, 64, 65, 127, 128, 129, original.len()] {
+            for at in 0..length {
+                for byte in 0..=255 {
+                    let mut input = original[..length].to_vec();
+                    input[at] = byte;
+                    assert_eq!(
+                        valid_encoding(&input, ENCODING_UTF8),
+                        std::str::from_utf8(&input).is_ok(),
+                        "length {length}, position {at}, byte {byte}"
+                    );
+                    assert_eq!(valid_encoding(&input, ENCODING_ASCII), input.is_ascii());
+                }
+            }
+        }
+        assert!(valid_encoding(&original, ENCODING_UTF8));
+        assert!(valid_encoding(b"already classified", ENCODING_SEVEN_BIT));
+    }
+
     #[test]
     fn reports_memory_when_a_search_cache_is_borrowed() {
         let caches = Caches::default();

data/lib/schemurai/version.rb

@@ -2,5 +2,5 @@
 
 module Schemurai
   # The installed Schemurai version.
-  VERSION = "3.0.0"
+  VERSION = "3.0.1"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: schemurai
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.0.1
 platform: ruby
 authors:
 - Kuya KOHARA (oakcask)
@@ -51,6 +51,7 @@ files:
 - ext/schemurai_native/src/format.rs
 - ext/schemurai_native/src/graph.rs
 - ext/schemurai_native/src/instance_graph.rs
+- ext/schemurai_native/src/integer_scan.rs
 - ext/schemurai_native/src/lib.rs
 - ext/schemurai_native/src/memo.rs
 - ext/schemurai_native/src/memo_scope.rs
@@ -94,7 +95,7 @@ licenses:
 metadata:
   homepage_uri: https://github.com/oakcask/schemurai
   source_code_uri: https://github.com/oakcask/schemurai/tree/main
-  documentation_uri: https://rubydoc.info/gems/schemurai/3.0.0
+  documentation_uri: https://rubydoc.info/gems/schemurai/3.0.1
   rubygems_mfa_required: 'true'
 rdoc_options:
 - "--main"