fast_regexp 0.6.2 → 0.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ece32247dfc6514f567a262e759bf5bc28a63d45703547316851dabdd19f5d27
-  data.tar.gz: f68fc5c53b8544fff834dabc4e1d2e516d0a57d6c529e273b416b33f8f6194f4
+  metadata.gz: a08e75e08540db9629b91a3c82a460a901a857fa8d40db6180d8efec1fd2eb9e
+  data.tar.gz: cdf0f2d5a053346c412369b1751605b618c2e9fc6f8594dfd6a4c86b08e6a56f
 SHA512:
-  metadata.gz: f787fa035f0a0ffa54d717af678068d832831884743210f68094565e57545c2e7da80af7e246af8a255416bbf235092364c850925d926e05fac704e1039957bc
-  data.tar.gz: 7cc68035055b11e1f2aa2f89c61e558cd6e5b7e40e69b0b3b793c67b572877abad4800a1258520e65f7b25b14323160e4dd70a13813aff9b8639c622cb51a008
+  metadata.gz: 474a1577c07cda797f422caa596e35c2831d33c0acd8dfaf1dd5d1397f9c036118264bb814e0fb32a96c7e43db32f797fce1d0c8f2a36eeb187417ca45d9ccaa
+  data.tar.gz: 76298cd1b9e8a2648f48e695eb4d1199d12a24b5dc384b8d6f7acc9495769a8ef605403c0f9d4b0ec37192408850770ff72e7827be56f99124bc3f3855798d6f

data/README.md

@@ -244,7 +244,18 @@ In-depth docs live under [`docs/`](docs/README.md), organized via the
 - **Tutorial:** [Getting started](docs/tutorials/getting-started.md)
 - **How-to:** [Migrate from stdlib `::Regexp`](docs/how-to/migrate-from-stdlib-regexp.md), [Handle unsupported syntax](docs/how-to/handle-unsupported-syntax.md)
 - **Reference:** [`Fast::Regexp`](docs/reference/fast-regexp.md), [`MatchData`](docs/reference/fast-regexp-matchdata.md), [`Set`](docs/reference/fast-regexp-set.md)
-- **Explainers:** [Engine fallback](docs/explainers/engine-fallback.md)
+- **Explainers:** [Engine fallback](docs/explainers/engine-fallback.md), [Benchmarks](docs/explainers/benchmarks.md)
+
+## Performance
+
+Matching runs 8–10x stdlib `::Regexp` on a hit or a miss, scanning 2–5x,
+`gsub` with a template 7–13x; reading a `MatchData` costs the same objects
+as `::MatchData`, and `match?`, `=~`, `===` and `Set#match?` allocate
+nothing. Compiling a pattern is the one place rust/regex is slower (it
+builds its automaton up front), so compile once and keep the object. Every
+operation, with the stdlib equivalent beside it, is on the
+[benchmarks page](docs/explainers/benchmarks.md); reproduce with
+`bundle exec ruby benchmark/operations.rb`.
 
 ## Development
 
@@ -253,6 +264,7 @@ bin/setup     # install deps
 bin/console   # interactive prompt to play around
 rake compile  # (re)compile extension
 rake spec     # run tests
+bundle exec ruby benchmark/operations.rb   # the benchmarks page's table
 ```
 
 ## Contributing

data/ext/fast_regexp/src/lib.rs

@@ -1,15 +1,25 @@
 use magnus::{
-    function, method,
+    function, gc, method,
     scan_args::{get_kwargs, scan_args},
-    value::ReprValue,
-    Error, Module, Object, RArray, RClass, RHash, RString, Ruby, Symbol, TryConvert, Value,
+    value::{Opaque, ReprValue},
+    DataTypeFunctions, Error, Integer, Module, Object, RArray, RClass, RHash, RString, Ruby,
+    Symbol, TryConvert, TypedData, Value,
 };
 use regex::bytes::{NoExpand, Regex, RegexBuilder, RegexSet, RegexSetBuilder};
 use std::collections::HashMap;
-use std::sync::Arc;
-
-fn haystack_bytes(haystack: &RString) -> Vec<u8> {
-    unsafe { haystack.as_slice() }.to_vec()
+use std::sync::{Arc, OnceLock};
+
+/// A haystack's bytes, read in place for the duration of one native call.
+///
+/// Sound because the call hands nothing back to Ruby while the borrow lives:
+/// the GVL is held and no block is yielded, so no Ruby code can mutate the
+/// string, and a GC triggered by the result objects the call builds neither
+/// frees nor moves it — a method argument is rooted and pinned on the VM
+/// stack for the whole call. Paths that *do* run Ruby code mid-call (the
+/// block forms of `sub`/`gsub`) read a frozen snapshot instead; see
+/// [`FastMatchData`].
+fn bytes(haystack: &RString) -> &[u8] {
+    unsafe { haystack.as_slice() }
 }
 
 fn utf8_string(ruby: &Ruby, bytes: &[u8]) -> RString {
@@ -17,31 +27,31 @@ fn utf8_string(ruby: &Ruby, bytes: &[u8]) -> RString {
 }
 
 fn arg_error(message: impl Into<String>) -> Error {
-    Error::new(
-        Ruby::get().unwrap().exception_arg_error(),
-        message.into(),
-    )
+    Error::new(Ruby::get().unwrap().exception_arg_error(), message.into())
 }
 
 fn type_error(message: impl Into<String>) -> Error {
-    Error::new(
-        Ruby::get().unwrap().exception_type_error(),
-        message.into(),
-    )
+    Error::new(Ruby::get().unwrap().exception_type_error(), message.into())
 }
 
 type CaptureOffset = Option<(usize, usize)>;
 
-/// Inner state shared between a compiled regex and any [`FastMatchData`] it
-/// produces — wrapped in `Arc` so positions, names, and haystack bytes can be
-/// shared cheaply across many matches (e.g. from `#scan_matches`).
+/// Inner state shared between a compiled regex and every [`FastMatchData`]
+/// it produces, behind an `Arc`.
 struct RegexInner {
     regex: Regex,
     /// `name -> capture index`. Empty when the pattern has no named captures.
     names: HashMap<String, usize>,
-    /// Ordered list of capture-group names (None for unnamed groups), matching
-    /// the indices produced by `regex.captures_iter()`.
+    /// Capture-group names in index order (`None` for unnamed groups).
     name_index: Vec<Option<String>>,
+    /// `name_index` as one Ruby Array of interned frozen Strings (nil for
+    /// unnamed groups), so `names` and `named_captures` hand out the same
+    /// String objects on every call. Built on first use, from a method of a
+    /// live wrapper, so something marks it from the moment it exists; built
+    /// into the Array as each name is interned, because the interning table
+    /// holds its strings only weakly and a GC between two interns would
+    /// otherwise free the first. Marked by every wrapper holding this `Arc`.
+    ruby_names: OnceLock<Opaque<RArray>>,
 }
 
 impl RegexInner {
@@ -51,26 +61,71 @@ impl RegexInner {
             .build()
             .map_err(|e| arg_error(e.to_string()))?;
 
-        let name_index: Vec<Option<String>> =
-            regex.capture_names().map(|n| n.map(String::from)).collect();
         let mut names = HashMap::new();
-        for (idx, name) in name_index.iter().enumerate() {
-            if let Some(n) = name {
-                names.insert(n.clone(), idx);
-            }
+        let mut name_index = Vec::with_capacity(regex.captures_len());
+        for (idx, name) in regex.capture_names().enumerate() {
+            name_index.push(name.map(|n| {
+                names.insert(n.to_owned(), idx);
+                n.to_owned()
+            }));
         }
 
         Ok(Self {
             regex,
             names,
             name_index,
+            ruby_names: OnceLock::new(),
         })
     }
+
+    fn mark(&self, marker: &gc::Marker) {
+        if let Some(names) = self.ruby_names.get() {
+            marker.mark(*names);
+        }
+    }
+
+    /// The interned names Array (see the field), built on first use.
+    fn ruby_names(&self, ruby: &Ruby) -> RArray {
+        ruby.get_inner(*self.ruby_names.get_or_init(|| {
+            let names = ruby.ary_new_capa(self.name_index.len());
+            for name in &self.name_index {
+                let pushed = match name {
+                    Some(name) => names.push(ruby.str_new(name).to_interned_str()),
+                    None => names.push(()),
+                };
+                pushed.expect("pushing onto a fresh, unfrozen Array");
+            }
+            Opaque::from(names)
+        }))
+    }
+
+    /// Capture-group names in declaration order, unnamed groups skipped.
+    fn names(&self, ruby: &Ruby) -> RArray {
+        let names = self.ruby_names(ruby);
+        // SAFETY: the Array is reachable from the wrapper this runs on behalf
+        // of and no Ruby code runs while the slice is read.
+        let values = unsafe { names.as_slice() };
+        ruby.ary_from_iter(values.iter().skip(1).filter(|v| !v.is_nil()).copied())
+    }
+
+    /// Every group's `(start, end)` for one match, index 0 the whole match.
+    fn offsets(&self, caps: &regex::bytes::Captures) -> Vec<CaptureOffset> {
+        (0..self.regex.captures_len())
+            .map(|i| caps.get(i).map(|m| (m.start(), m.end())))
+            .collect()
+    }
 }
 
-#[magnus::wrap(class = "Fast::Regexp::Native", free_immediately, size)]
+#[derive(TypedData)]
+#[magnus(class = "Fast::Regexp::Native", free_immediately, size, mark)]
 pub struct FastRegexp(Arc<RegexInner>);
 
+impl DataTypeFunctions for FastRegexp {
+    fn mark(&self, marker: &gc::Marker) {
+        self.0.mark(marker);
+    }
+}
+
 impl FastRegexp {
     pub fn new(args: &[Value]) -> Result<Self, Error> {
         let args = scan_args::<(String,), (), (), (), RHash, ()>(args)?;
@@ -83,61 +138,47 @@ impl FastRegexp {
         Ok(Self(Arc::new(RegexInner::from_pattern(&pattern, unicode)?)))
     }
 
-    fn build_match_data(
-        &self,
-        haystack: Arc<Vec<u8>>,
-        offsets: Vec<CaptureOffset>,
-    ) -> FastMatchData {
+    fn match_data(&self, haystack: RString, captures: Vec<CaptureOffset>) -> FastMatchData {
         FastMatchData {
-            haystack,
-            captures: offsets,
+            haystack: haystack.into(),
+            captures,
             inner: self.0.clone(),
         }
     }
 
+    /// First match as a MatchData, or nil. The search runs over the live
+    /// haystack; only a hit pays for the frozen snapshot the MatchData keeps.
     pub fn rmatch(&self, haystack: RString) -> Option<FastMatchData> {
-        let regex = &self.0.regex;
-        let bytes = haystack_bytes(&haystack);
-
-        let offsets = regex.captures(&bytes).map(|caps| {
-            (0..regex.captures_len())
-                .map(|i| caps.get(i).map(|m| (m.start(), m.end())))
-                .collect::<Vec<_>>()
-        })?;
+        let inner = &self.0;
+        let captures = inner
+            .regex
+            .captures(bytes(&haystack))
+            .map(|caps| inner.offsets(&caps))?;
+        Some(self.match_data(RString::new_frozen(haystack), captures))
+    }
 
-        Some(self.build_match_data(Arc::new(bytes), offsets))
+    /// Byte offset of the first match, or nil — `=~` without a MatchData.
+    pub fn find(&self, haystack: RString) -> Option<usize> {
+        self.0.regex.find(bytes(&haystack)).map(|m| m.start())
     }
 
     pub fn scan(ruby: &Ruby, rb_self: &Self, haystack: RString) -> Result<RArray, Error> {
         let regex = &rb_self.0.regex;
-        let bytes = haystack_bytes(&haystack);
+        let bytes = bytes(&haystack);
 
         if regex.captures_len() == 1 {
-            let ranges: Vec<(usize, usize)> = regex
-                .find_iter(&bytes)
-                .map(|m| (m.start(), m.end()))
-                .collect();
-            let result = ruby.ary_new_capa(ranges.len());
-            for (s, e) in ranges {
-                result.push(utf8_string(ruby, &bytes[s..e]))?;
+            let result = ruby.ary_new();
+            for m in regex.find_iter(bytes) {
+                result.push(utf8_string(ruby, m.as_bytes()))?;
             }
             Ok(result)
         } else {
-            let groups: Vec<Vec<CaptureOffset>> = regex
-                .captures_iter(&bytes)
-                .map(|caps| {
-                    caps.iter()
-                        .skip(1)
-                        .map(|c| c.map(|m| (m.start(), m.end())))
-                        .collect()
-                })
-                .collect();
-            let result = ruby.ary_new_capa(groups.len());
-            for group_ranges in groups {
-                let group = ruby.ary_new_capa(group_ranges.len());
-                for range in group_ranges {
-                    match range {
-                        Some((s, e)) => group.push(utf8_string(ruby, &bytes[s..e]))?,
+            let result = ruby.ary_new();
+            for caps in regex.captures_iter(bytes) {
+                let group = ruby.ary_new_capa(regex.captures_len() - 1);
+                for m in caps.iter().skip(1) {
+                    match m {
+                        Some(m) => group.push(utf8_string(ruby, m.as_bytes()))?,
                         None => group.push(())?,
                     }
                 }
@@ -147,32 +188,29 @@ impl FastRegexp {
         }
     }
 
+    /// Every match as a MatchData, all sharing one frozen snapshot of the
+    /// haystack (taken only if there is at least one match).
     pub fn scan_matches(ruby: &Ruby, rb_self: &Self, haystack: RString) -> Result<RArray, Error> {
-        let regex = &rb_self.0.regex;
-        let bytes = haystack_bytes(&haystack);
-        let n_groups = regex.captures_len();
-
-        let all: Vec<Vec<CaptureOffset>> = regex
-            .captures_iter(&bytes)
-            .map(|caps| {
-                (0..n_groups)
-                    .map(|i| caps.get(i).map(|m| (m.start(), m.end())))
-                    .collect()
-            })
+        let inner = &rb_self.0;
+        let all: Vec<Vec<CaptureOffset>> = inner
+            .regex
+            .captures_iter(bytes(&haystack))
+            .map(|caps| inner.offsets(&caps))
             .collect();
 
-        let shared = Arc::new(bytes);
         let result = ruby.ary_new_capa(all.len());
-        for offsets in all {
-            result.push(rb_self.build_match_data(shared.clone(), offsets))?;
+        if all.is_empty() {
+            return Ok(result);
+        }
+        let snapshot = RString::new_frozen(haystack);
+        for captures in all {
+            result.push(rb_self.match_data(snapshot, captures))?;
         }
         Ok(result)
     }
 
     pub fn is_match(&self, haystack: RString) -> bool {
-        let regex = &self.0.regex;
-        let bytes = haystack_bytes(&haystack);
-        regex.is_match(&bytes)
+        self.0.regex.is_match(bytes(&haystack))
     }
 
     pub fn sub_str(
@@ -183,12 +221,11 @@ impl FastRegexp {
         literal: bool,
     ) -> RString {
         let regex = &rb_self.0.regex;
-        let bytes = haystack_bytes(&haystack);
-        let repl = haystack_bytes(&replacement);
-        let out: Vec<u8> = if literal {
-            regex.replace(&bytes, NoExpand(&repl)).into_owned()
+        let (bytes, repl) = (bytes(&haystack), bytes(&replacement));
+        let out = if literal {
+            regex.replace(bytes, NoExpand(repl))
         } else {
-            regex.replace(&bytes, &repl[..]).into_owned()
+            regex.replace(bytes, repl)
         };
         utf8_string(ruby, &out)
     }
@@ -201,16 +238,70 @@ impl FastRegexp {
         literal: bool,
     ) -> RString {
         let regex = &rb_self.0.regex;
-        let bytes = haystack_bytes(&haystack);
-        let repl = haystack_bytes(&replacement);
-        let out: Vec<u8> = if literal {
-            regex.replace_all(&bytes, NoExpand(&repl)).into_owned()
+        let (bytes, repl) = (bytes(&haystack), bytes(&replacement));
+        let out = if literal {
+            regex.replace_all(bytes, NoExpand(repl))
         } else {
-            regex.replace_all(&bytes, &repl[..]).into_owned()
+            regex.replace_all(bytes, repl)
         };
         utf8_string(ruby, &out)
     }
 
+    /// The block forms of `sub` (`limit` 1) and `gsub` (no limit) in one
+    /// pass: each match is yielded as a MatchData and the block's result
+    /// (via `to_s`) is spliced in for it. The block runs arbitrary Ruby, so
+    /// everything is read from a frozen snapshot of the haystack rather than
+    /// the live string — the same snapshot every yielded MatchData keeps.
+    fn replace_block(
+        ruby: &Ruby,
+        rb_self: &Self,
+        haystack: RString,
+        limit: Option<usize>,
+    ) -> Result<RString, Error> {
+        let inner = &rb_self.0;
+        if !inner.regex.is_match(bytes(&haystack)) {
+            // No match: a copy of the haystack, encoding and all, as
+            // `String#sub`/`#gsub` return (copy-on-write, nothing copied yet).
+            return Ok(RString::new_shared(haystack));
+        }
+        let snapshot = RString::new_frozen(haystack);
+        let bytes = bytes(&snapshot);
+        // Built as a UTF-8 String from the start: the haystack's own bytes go
+        // in raw (`cat`), each block result through Ruby's encoding
+        // negotiation (`buf_append`, as `String#gsub` appends), so an
+        // incompatible replacement raises Encoding::CompatibilityError rather
+        // than leaving invalid bytes behind. The block's result is taken as
+        // `rb_obj_as_string` takes it: a String as is, anything else via
+        // `to_s`, falling back to `Object#to_s` when that returns a non-String.
+        let out = utf8_string(ruby, &[]);
+        let mut cursor = 0;
+        for caps in inner
+            .regex
+            .captures_iter(bytes)
+            .take(limit.unwrap_or(usize::MAX))
+        {
+            let whole = caps.get(0).expect("group 0 is the match");
+            out.cat(&bytes[cursor..whole.start()]);
+            let replacement: Value =
+                ruby.yield_value(rb_self.match_data(snapshot, inner.offsets(&caps)))?;
+            out.buf_append(replacement.to_r_string()?)?;
+            cursor = whole.end();
+        }
+        out.cat(&bytes[cursor..]);
+        // Keep the snapshot reachable from this frame until the last read of
+        // `bytes` above, whatever the yielded MatchData objects' fate.
+        std::hint::black_box(snapshot);
+        Ok(out)
+    }
+
+    pub fn sub_block(ruby: &Ruby, rb_self: &Self, haystack: RString) -> Result<RString, Error> {
+        Self::replace_block(ruby, rb_self, haystack, Some(1))
+    }
+
+    pub fn gsub_block(ruby: &Ruby, rb_self: &Self, haystack: RString) -> Result<RString, Error> {
+        Self::replace_block(ruby, rb_self, haystack, None)
+    }
+
     pub fn pattern(&self) -> &str {
         self.0.regex.as_str()
     }
@@ -220,27 +311,46 @@ impl FastRegexp {
         self.0.regex.captures_len() - 1
     }
 
-    pub fn names(ruby: &Ruby, rb_self: &Self) -> Result<RArray, Error> {
-        let arr = ruby.ary_new();
-        for name in rb_self.0.name_index.iter().skip(1).flatten() {
-            arr.push(name.as_str())?;
-        }
-        Ok(arr)
+    pub fn names(ruby: &Ruby, rb_self: &Self) -> RArray {
+        rb_self.0.names(ruby)
     }
 }
 
-#[magnus::wrap(class = "Fast::Regexp::Native::MatchData", free_immediately, size)]
+/// One match: capture offsets into a frozen snapshot of the haystack. The
+/// snapshot is `rb_str_new_frozen` of the caller's string — the string
+/// itself when it was already frozen, otherwise a frozen copy-on-write
+/// sibling sharing its buffer — so a later mutation of the caller's string
+/// can't reach it, and nothing was copied to guarantee that.
+#[derive(TypedData)]
+#[magnus(
+    class = "Fast::Regexp::Native::MatchData",
+    free_immediately,
+    size,
+    mark
+)]
 pub struct FastMatchData {
-    haystack: Arc<Vec<u8>>,
+    haystack: Opaque<RString>,
     /// Index 0 is the whole match. Subsequent entries are capture groups in
     /// order; `None` indicates a group that did not participate.
     captures: Vec<CaptureOffset>,
     inner: Arc<RegexInner>,
 }
 
+impl DataTypeFunctions for FastMatchData {
+    fn mark(&self, marker: &gc::Marker) {
+        marker.mark(self.haystack);
+        self.inner.mark(marker);
+    }
+}
+
 impl FastMatchData {
+    fn haystack(&self, ruby: &Ruby) -> RString {
+        ruby.get_inner(self.haystack)
+    }
+
     fn slice(&self, ruby: &Ruby, range: (usize, usize)) -> RString {
-        utf8_string(ruby, &self.haystack[range.0..range.1])
+        let haystack = self.haystack(ruby);
+        utf8_string(ruby, &bytes(&haystack)[range.0..range.1])
     }
 
     fn capture_index(&self, idx: i64) -> Option<usize> {
@@ -255,21 +365,40 @@ impl FastMatchData {
 
     /// Returns `Some(Some(range))` for a participating group, `Some(None)` for
     /// a known-but-non-participating group, `None` for an out-of-range index
-    /// or an unknown capture name.
+    /// or an unknown capture name. Dispatches on the key's type rather than
+    /// attempting conversions: a failed conversion raises (and allocates an
+    /// exception) on the way to being caught.
     fn resolve(&self, key: Value) -> Result<Option<CaptureOffset>, Error> {
-        if let Ok(i) = <i64 as TryConvert>::try_convert(key) {
-            return Ok(self.capture_index(i).map(|i| self.captures[i]));
+        if let Some(index) = Integer::from_value(key) {
+            return Ok(self
+                .capture_index(index.to_i64()?)
+                .map(|i| self.captures[i]));
+        }
+        let by_name = |name: &str| self.inner.names.get(name).map(|&i| self.captures[i]);
+        if let Some(symbol) = Symbol::from_value(key) {
+            return Ok(by_name(&symbol.name()?));
         }
-        if let Ok(sym) = <Symbol as TryConvert>::try_convert(key) {
-            let name = sym.name()?.into_owned();
-            return Ok(self.inner.names.get(&name).map(|&i| self.captures[i]));
+        if let Some(name) = RString::from_value(key) {
+            // A UTF-8 (or ASCII) name is read in place; any other encoding
+            // is transcoded, and one that can't be names no group.
+            return Ok(match unsafe { name.as_str() } {
+                Ok(name) => by_name(name),
+                Err(_) => name.to_string().ok().and_then(|name| by_name(&name)),
+            });
         }
-        if let Ok(name) = <String as TryConvert>::try_convert(key) {
-            return Ok(self.inner.names.get(&name).map(|&i| self.captures[i]));
+        // Anything else that converts to an Integer (a Float, a `to_int`
+        // object) indexes like one, as `::MatchData#[]` allows; anything
+        // that converts to a String (`to_str`) names a group, as this class
+        // has always allowed.
+        if let Ok(index) = i64::try_convert(key) {
+            return Ok(self.capture_index(index).map(|i| self.captures[i]));
+        }
+        match RString::try_convert(key) {
+            Ok(name) => Ok(by_name(&name.to_string()?)),
+            Err(_) => Err(type_error(
+                "no implicit conversion of capture key into Integer, String, or Symbol",
+            )),
         }
-        Err(type_error(
-            "no implicit conversion of capture key into Integer, String, or Symbol",
-        ))
     }
 
     fn whole(&self) -> (usize, usize) {
@@ -283,49 +412,47 @@ impl FastMatchData {
         }
     }
 
-    pub fn to_a(ruby: &Ruby, rb_self: &Self) -> Result<RArray, Error> {
-        let arr = ruby.ary_new_capa(rb_self.captures.len());
-        for cap in &rb_self.captures {
+    fn slices(&self, ruby: &Ruby, captures: &[CaptureOffset]) -> Result<RArray, Error> {
+        let arr = ruby.ary_new_capa(captures.len());
+        for cap in captures {
             match cap {
-                Some(r) => arr.push(rb_self.slice(ruby, *r))?,
+                Some(r) => arr.push(self.slice(ruby, *r))?,
                 None => arr.push(())?,
             }
         }
         Ok(arr)
     }
 
+    pub fn to_a(ruby: &Ruby, rb_self: &Self) -> Result<RArray, Error> {
+        rb_self.slices(ruby, &rb_self.captures)
+    }
+
     pub fn captures(ruby: &Ruby, rb_self: &Self) -> Result<RArray, Error> {
-        let arr = ruby.ary_new_capa(rb_self.captures.len() - 1);
-        for cap in rb_self.captures.iter().skip(1) {
-            match cap {
-                Some(r) => arr.push(rb_self.slice(ruby, *r))?,
-                None => arr.push(())?,
-            }
-        }
-        Ok(arr)
+        rb_self.slices(ruby, &rb_self.captures[1..])
     }
 
     pub fn named_captures(ruby: &Ruby, rb_self: &Self) -> Result<RHash, Error> {
         let hash = ruby.hash_new();
+        let names = rb_self.inner.ruby_names(ruby);
+        // SAFETY: the Array is reachable from this match's wrapper and no
+        // Ruby code runs while the slice is read.
+        let names = unsafe { names.as_slice() };
         // Iterate in declaration order so the hash preserves regex order.
-        for (idx, name) in rb_self.inner.name_index.iter().enumerate() {
-            if let Some(name) = name {
-                let value: Value = match rb_self.captures[idx] {
-                    Some(r) => rb_self.slice(ruby, r).as_value(),
-                    None => ruby.qnil().as_value(),
-                };
-                hash.aset(name.as_str(), value)?;
+        for (idx, name) in names.iter().enumerate() {
+            if name.is_nil() {
+                continue;
             }
+            let value: Value = match rb_self.captures[idx] {
+                Some(r) => rb_self.slice(ruby, r).as_value(),
+                None => ruby.qnil().as_value(),
+            };
+            hash.aset(*name, value)?;
         }
         Ok(hash)
     }
 
-    pub fn names(ruby: &Ruby, rb_self: &Self) -> Result<RArray, Error> {
-        let arr = ruby.ary_new();
-        for name in rb_self.inner.name_index.iter().skip(1).flatten() {
-            arr.push(name.as_str())?;
-        }
-        Ok(arr)
+    pub fn names(ruby: &Ruby, rb_self: &Self) -> RArray {
+        rb_self.inner.names(ruby)
     }
 
     pub fn size(&self) -> usize {
@@ -334,22 +461,23 @@ impl FastMatchData {
 
     pub fn pre_match(ruby: &Ruby, rb_self: &Self) -> RString {
         let (s, _) = rb_self.whole();
-        utf8_string(ruby, &rb_self.haystack[..s])
+        rb_self.slice(ruby, (0, s))
     }
 
     pub fn post_match(ruby: &Ruby, rb_self: &Self) -> RString {
         let (_, e) = rb_self.whole();
-        utf8_string(ruby, &rb_self.haystack[e..])
+        let haystack = rb_self.haystack(ruby);
+        let len = bytes(&haystack).len();
+        rb_self.slice(ruby, (e, len))
     }
 
     pub fn whole_match(ruby: &Ruby, rb_self: &Self) -> RString {
         rb_self.slice(ruby, rb_self.whole())
     }
 
+    /// The frozen snapshot the match was taken over (see the struct doc).
     pub fn string(ruby: &Ruby, rb_self: &Self) -> RString {
-        let s = utf8_string(ruby, &rb_self.haystack);
-        s.freeze();
-        s
+        rb_self.haystack(ruby)
     }
 
     pub fn byteoffset(ruby: &Ruby, rb_self: &Self, key: Value) -> Result<Value, Error> {
@@ -384,7 +512,8 @@ impl FastMatchData {
 
     pub fn inspect(ruby: &Ruby, rb_self: &Self) -> RString {
         let (s, e) = rb_self.whole();
-        let matched = String::from_utf8_lossy(&rb_self.haystack[s..e]);
+        let haystack = rb_self.haystack(ruby);
+        let matched = String::from_utf8_lossy(&bytes(&haystack)[s..e]);
         ruby.str_new(&format!("#<Fast::Regexp::MatchData {:?}>", matched))
     }
 }
@@ -410,15 +539,11 @@ impl FastRegexpSet {
     }
 
     pub fn matches(&self, haystack: RString) -> Vec<usize> {
-        let set = &self.0;
-        let bytes = haystack_bytes(&haystack);
-        set.matches(&bytes).iter().collect()
+        self.0.matches(bytes(&haystack)).iter().collect()
     }
 
     pub fn is_match(&self, haystack: RString) -> bool {
-        let set = &self.0;
-        let bytes = haystack_bytes(&haystack);
-        set.is_match(&bytes)
+        self.0.is_match(bytes(&haystack))
     }
 
     pub fn patterns(&self) -> Vec<String> {
@@ -429,15 +554,19 @@ impl FastRegexpSet {
 #[magnus::init]
 pub fn init(ruby: &Ruby) -> Result<(), Error> {
     let object_class = ruby.class_object();
-    // Fast::Regexp must already be defined as a class by lib/fast_regexp.rb
-    // before this extension is required — the Ruby façade owns that constant
-    // and delegates to the Native class registered below.
+    // Fast::Regexp and Fast::Regexp::MatchData must already be defined by
+    // lib/fast_regexp.rb before this extension is required — the Ruby façade
+    // owns those constants; the Native classes registered below sit under
+    // them, and Native::MatchData subclasses the façade's MatchData so a
+    // native match is a Fast::Regexp::MatchData with no wrapper around it.
     let fast_module: magnus::RModule = object_class.const_get("Fast")?;
     let regexp_facade: RClass = fast_module.const_get("Regexp")?;
+    let match_data_facade: RClass = regexp_facade.const_get("MatchData")?;
     let regexp_class = regexp_facade.define_class("Native", object_class)?;
 
     regexp_class.define_singleton_method("_native_new", function!(FastRegexp::new, -1))?;
     regexp_class.define_method("_native_match", method!(FastRegexp::rmatch, 1))?;
+    regexp_class.define_method("_native_find", method!(FastRegexp::find, 1))?;
     regexp_class.define_method("match?", method!(FastRegexp::is_match, 1))?;
     regexp_class.define_method("scan", method!(FastRegexp::scan, 1))?;
     regexp_class.define_method("scan_matches", method!(FastRegexp::scan_matches, 1))?;
@@ -446,8 +575,10 @@ pub fn init(ruby: &Ruby) -> Result<(), Error> {
     regexp_class.define_method("names", method!(FastRegexp::names, 0))?;
     regexp_class.define_method("_native_sub", method!(FastRegexp::sub_str, 3))?;
     regexp_class.define_method("_native_gsub", method!(FastRegexp::gsub_str, 3))?;
+    regexp_class.define_method("_native_sub_block", method!(FastRegexp::sub_block, 1))?;
+    regexp_class.define_method("_native_gsub_block", method!(FastRegexp::gsub_block, 1))?;
 
-    let match_data_class = regexp_class.define_class("MatchData", object_class)?;
+    let match_data_class = regexp_class.define_class("MatchData", match_data_facade)?;
     match_data_class.define_method("[]", method!(FastMatchData::aref, 1))?;
     match_data_class.define_method("to_a", method!(FastMatchData::to_a, 0))?;
     match_data_class.define_method("captures", method!(FastMatchData::captures, 0))?;

data/lib/fast_regexp/version.rb

@@ -2,6 +2,6 @@
 
 module Fast
   class Regexp
-    VERSION = "0.6.2"
+    VERSION = "0.7.0"
   end
 end

data/lib/fast_regexp.rb

@@ -21,6 +21,62 @@ module Fast
       candidates = [File.join(base, abi, "fast_regexp"), File.join(base, "fast_regexp")]
       candidates.find { |stem| NATIVE_EXTENSIONS.any? { |ext| File.exist?(stem + ext) } }
     end
+
+    # One match, whichever engine produced it. On the fast path the match
+    # *is* a `Fast::Regexp::Native::MatchData`, a subclass of this one
+    # defined by the extension (so it must exist before the extension
+    # loads); on the stdlib path it's an instance of this class wrapping the
+    # `::MatchData`. Same public surface either way.
+    class MatchData
+      include Enumerable
+
+      attr_reader :backend
+
+      def initialize(backend)
+        @backend = backend
+      end
+
+      def native? = false
+      def stdlib? = true
+
+      def native = nil
+      def stdlib = @backend
+
+      def [](key) = @backend[key]
+      def to_a = @backend.to_a
+      def captures = @backend.captures
+      def named_captures = @backend.named_captures
+      def names = @backend.names
+      def size = @backend.size
+      alias_method :length, :size
+      def pre_match = @backend.pre_match
+      def post_match = @backend.post_match
+      def to_s = @backend.to_s
+      alias_method :match, :to_s
+
+      # The haystack the match was taken over: `::MatchData#string` is already
+      # a frozen snapshot, the same thing the native subclass keeps.
+      def string = @backend.string
+
+      # Byte-based offsets. `::MatchData#byteoffset` exists since Ruby 3.2;
+      # its `byte_begin`/`byte_end` don't (3.4 added `bytebegin`/`byteend`),
+      # so those two derive from `byteoffset` here.
+      def byteoffset(key) = @backend.byteoffset(key)
+      def byte_begin(key) = byteoffset(key)[0]
+      def byte_end(key) = byteoffset(key)[1]
+
+      def each(&block) = to_a.each(&block)
+      def values_at(*indices) = indices.map { |i| self[i] }
+
+      def ==(other)
+        other.is_a?(MatchData) && to_a == other.to_a && string == other.string
+      end
+      alias_method :eql?, :==
+
+      def hash = [to_a, string].hash
+
+      def inspect = "#<Fast::Regexp::MatchData #{to_s.inspect}>"
+    end
   end
 end
 
@@ -30,6 +86,22 @@ require native
 
 module Fast
   class Regexp
+    class Native
+      class MatchData
+        def native? = true
+        def stdlib? = false
+
+        def native = self
+        def stdlib = nil
+        def backend = self
+
+        # Immutable, so a copy is the object itself (the wrapped class has no
+        # allocator for `dup`/`clone` to go through).
+        def dup = self
+        def clone(freeze: nil) = self
+      end
+    end
+
     RUBY_FLAG_MAP = {
       ::Regexp::IGNORECASE => "i",
       ::Regexp::EXTENDED => "x",
@@ -96,8 +168,9 @@ module Fast
 
     def match(haystack)
       haystack = coerce_string(haystack)
-      raw = fast? ? @backend._native_match(haystack) : @backend.match(haystack)
-      raw && MatchData.new(raw, haystack)
+      return @backend._native_match(haystack) if fast?
+      m = @backend.match(haystack)
+      m && MatchData.new(m)
     end
 
     def match?(haystack)
@@ -113,8 +186,10 @@ module Fast
     # also returns bytes here for API consistency).
     def =~(other)
       return nil unless other.respond_to?(:to_str)
-      m = match(other.to_str)
-      m && m.byte_begin(0)
+      haystack = other.to_str
+      return @backend._native_find(haystack) if fast?
+      m = @backend.match(haystack)
+      m && m.byteoffset(0)[0]
     end
 
     def scan(haystack)
@@ -123,22 +198,17 @@ module Fast
 
     def scan_matches(haystack)
       haystack = coerce_string(haystack)
-      if fast?
-        @backend.scan_matches(haystack).map { |m| MatchData.new(m, haystack) }
-      else
-        results = []
-        haystack.scan(@backend) { results << MatchData.new(::Regexp.last_match, haystack) }
-        results
-      end
+      return @backend.scan_matches(haystack) if fast?
+      results = []
+      haystack.scan(@backend) { results << MatchData.new(::Regexp.last_match) }
+      results
     end
 
     def sub(haystack, replacement = nil, literal: false, &block)
       haystack = coerce_string(haystack)
-      if block
+      if block_given?
         raise ArgumentError, "wrong number of arguments (given 2, expected 1 with block)" if replacement
-        m = match(haystack)
-        return haystack.dup unless m
-        "#{m.pre_match}#{block.call(m)}#{m.post_match}"
+        fast? ? @backend._native_sub_block(haystack, &block) : stdlib_sub_with_block(haystack, &block)
       else
         raise ArgumentError, "wrong number of arguments (given 1, expected 2)" if replacement.nil?
         replacement = coerce_string(replacement)
@@ -152,9 +222,9 @@ module Fast
 
     def gsub(haystack, replacement = nil, literal: false, &block)
       haystack = coerce_string(haystack)
-      if block
+      if block_given?
         raise ArgumentError, "wrong number of arguments (given 2, expected 1 with block)" if replacement
-        fast? ? fast_gsub_with_block(haystack, &block) : stdlib_gsub_with_block(haystack, &block)
+        fast? ? @backend._native_gsub_block(haystack, &block) : stdlib_gsub_with_block(haystack, &block)
       else
         raise ArgumentError, "wrong number of arguments (given 1, expected 2)" if replacement.nil?
         replacement = coerce_string(replacement)
@@ -254,29 +324,14 @@ module Fast
       end
     end
 
-    # Fast path: rust/regex has no native iterate-with-replace, so we scan all
-    # match positions up-front, then splice the result by byte offset in one
-    # pass. Stays UTF-8 since haystack and match slices are.
-    def fast_gsub_with_block(haystack)
-      matches = scan_matches(haystack)
-      return haystack.dup if matches.empty?
-
-      out = String.new(encoding: Encoding::UTF_8)
-      cursor = 0
-      matches.each do |m|
-        bs, be = m.byteoffset(0)
-        out << haystack.byteslice(cursor, bs - cursor) if bs > cursor
-        out << yield(m).to_s
-        cursor = be
-      end
-      out << haystack.byteslice(cursor, haystack.bytesize - cursor) if cursor < haystack.bytesize
-      out
+    # Stdlib path: String#sub/#gsub already do single-pass iterate-and-replace
+    # and set $~ inside the block, so wrap the current ::MatchData and yield.
+    def stdlib_sub_with_block(haystack)
+      haystack.sub(@backend) { yield(MatchData.new(::Regexp.last_match)).to_s }
     end
 
-    # Stdlib path: String#gsub already does single-pass iterate-and-replace
-    # and sets $~ inside the block, so wrap the current ::MatchData and yield.
     def stdlib_gsub_with_block(haystack)
-      haystack.gsub(@backend) { yield(MatchData.new(::Regexp.last_match, haystack)).to_s }
+      haystack.gsub(@backend) { yield(MatchData.new(::Regexp.last_match)).to_s }
     end
 
     def coerce_string(value)
@@ -284,53 +339,5 @@ module Fast
       return value.to_str if value.respond_to?(:to_str)
       raise TypeError, "no implicit conversion of #{value.class} into String"
     end
-
-    # Wraps either a Fast::Regexp::Native::MatchData or a stdlib ::MatchData
-    # so callers see one type regardless of which backend ran.
-    class MatchData
-      include Enumerable
-
-      attr_reader :backend, :string
-
-      def initialize(backend, haystack)
-        @backend = backend
-        @string = haystack
-      end
-
-      def native? = @backend.is_a?(Fast::Regexp::Native::MatchData)
-      def stdlib? = !native?
-
-      def native = native? ? @backend : nil
-      def stdlib = stdlib? ? @backend : nil
-
-      def [](key) = @backend[key]
-      def to_a = @backend.to_a
-      def captures = @backend.captures
-      def named_captures = @backend.named_captures
-      def names = @backend.names
-      def size = @backend.size
-      alias_method :length, :size
-      def pre_match = @backend.pre_match
-      def post_match = @backend.post_match
-      def to_s = @backend.to_s
-
-      # Byte-based offsets. Both backends expose these (stdlib MatchData has
-      # `byteoffset` / `byte_begin` / `byte_end` since Ruby 3.2).
-      def byteoffset(key) = @backend.byteoffset(key)
-      def byte_begin(key) = @backend.byte_begin(key)
-      def byte_end(key) = @backend.byte_end(key)
-
-      def each(&block) = to_a.each(&block)
-      def values_at(*indices) = indices.map { |i| self[i] }
-
-      def ==(other)
-        other.is_a?(MatchData) && to_a == other.to_a && string == other.string
-      end
-      alias_method :eql?, :==
-
-      def hash = [to_a, string].hash
-
-      def inspect = "#<Fast::Regexp::MatchData #{to_s.inspect}>"
-    end
   end
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fast_regexp
 version: !ruby/object:Gem::Version
-  version: 0.6.2
+  version: 0.7.0
 platform: ruby
 authors:
 - Eric Jacobs
@@ -11,7 +11,7 @@ cert_chain: []
 date: 1980-01-02 00:00:00.000000000 Z
 dependencies: []
 description: 'Ruby bindings to rust/regex with a Fast::Regexp API: MatchData, sub/gsub,
-  ===, =~, named captures, and GVL release for thread/fiber-friendly matching.'
+  ===, =~, & named captures.'
 email:
 - eric@ebj.dev
 executables: []
@@ -33,6 +33,7 @@ licenses:
 - MIT
 metadata:
   bug_tracker_uri: https://github.com/jetpks/fast_regexp/issues
+  changelog_uri: https://github.com/jetpks/fast_regexp/blob/main/CHANGELOG.md
   homepage_uri: https://github.com/jetpks/fast_regexp
   source_code_uri: https://github.com/jetpks/fast_regexp
 rdoc_options: []