scrubber_rb 0.1.0

data/ext/scrubber_rb/src/lib.rs

@@ -0,0 +1,247 @@
+//! Ruby bindings for the scrubber_rb engine.
+//!
+//! This file is the only place that touches Ruby objects. Everything below it
+//! is plain Rust that knows nothing about VALUEs, which is what makes the
+//! engine testable with `cargo test` and safe to run with the GVL released.
+//!
+//! Encoding is handled on the Ruby side: we take and return byte strings and
+//! `lib/scrubber/instance.rb` restores the caller's encoding. That keeps the
+//! FFI surface to bytes and integers.
+
+mod detectors;
+mod engine;
+mod nogvl;
+mod offsets;
+mod pattern;
+mod replace;
+
+use magnus::{
+    exception::ExceptionClass, function, method, prelude::*, value::ReprValue, Error, RArray,
+    RModule, RString, Ruby, Value,
+};
+
+use engine::{BuildError, CustomSpec, Engine};
+use nogvl::GVL_THRESHOLD;
+
+/// The `Scrubber::Native` object: an immutable compiled engine.
+#[magnus::wrap(class = "Scrubber::Native", free_immediately, size)]
+struct Native {
+    engine: Engine,
+}
+
+impl Native {
+    /// `Scrubber::Native.new(detectors, customs, replacement, hash_salt)`
+    ///
+    /// * `detectors` - array of detector-key strings
+    /// * `customs`   - array of `[name, regexp_source, regexp_options]` triples
+    /// * `replacement` - `"label"` / `"mask"` / `"hash"` / `"remove"`
+    /// * `hash_salt` - string, or nil
+    fn new(
+        detectors: Vec<String>,
+        customs: Vec<(String, String, i32)>,
+        replacement: String,
+        hash_salt: Option<String>,
+    ) -> Result<Native, Error> {
+        let specs: Vec<CustomSpec> = customs
+            .into_iter()
+            .map(|(name, source, options)| CustomSpec {
+                name,
+                source,
+                options,
+            })
+            .collect();
+
+        let engine = Engine::build(&detectors, &specs, &replacement, hash_salt)
+            .map_err(build_error_to_ruby)?;
+        Ok(Native { engine })
+    }
+
+    /// Redact `input`, returning a new binary string, or `nil` when nothing
+    /// matched so the caller can skip rebuilding an identical string.
+    fn scrub(ruby: &Ruby, rb_self: &Native, input: RString) -> Result<Value, Error> {
+        // Copy out of Ruby memory first: everything after this point may run
+        // without the GVL, and may not touch a VALUE.
+        let bytes = unsafe { input.as_slice() }.to_vec();
+        let engine = &rb_self.engine;
+
+        let outcome = if bytes.len() >= GVL_THRESHOLD {
+            nogvl::without_gvl(|| engine.scrub(&bytes))
+        } else {
+            nogvl::guarded(|| engine.scrub(&bytes))
+        };
+
+        match outcome {
+            Ok(Some(out)) => Ok(ruby.str_from_slice(&out).as_value()),
+            Ok(None) => Ok(ruby.qnil().as_value()),
+            Err(panic) => Err(internal_error(ruby, &nogvl::panic_message(panic.as_ref()))),
+        }
+    }
+
+    /// Locate matches without replacing them.
+    ///
+    /// Returns `[[type, begin, end, preview], ...]`. Offsets are character
+    /// offsets when `char_offsets` is true (the caller checked the string is
+    /// UTF-8) and byte offsets otherwise, which for single-byte encodings is
+    /// the same thing.
+    fn detect(
+        ruby: &Ruby,
+        rb_self: &Native,
+        input: RString,
+        char_offsets: bool,
+    ) -> Result<RArray, Error> {
+        let bytes = unsafe { input.as_slice() }.to_vec();
+        let engine = &rb_self.engine;
+
+        let outcome = if bytes.len() >= GVL_THRESHOLD {
+            nogvl::without_gvl(|| collect_matches(engine, &bytes, char_offsets))
+        } else {
+            nogvl::guarded(|| collect_matches(engine, &bytes, char_offsets))
+        };
+
+        let found = match outcome {
+            Ok(found) => found,
+            Err(panic) => return Err(internal_error(ruby, &nogvl::panic_message(panic.as_ref()))),
+        };
+
+        let out = ruby.ary_new_capa(found.len());
+        for (kind, begin, end, preview) in found {
+            let row = ruby.ary_new_capa(4);
+            row.push(ruby.str_new(&kind))?;
+            row.push(begin)?;
+            row.push(end)?;
+            row.push(ruby.str_new(&preview))?;
+            out.push(row)?;
+        }
+        Ok(out)
+    }
+
+    /// How many compiled patterns this engine holds. Used by specs and
+    /// `Scrubber::Instance#inspect`.
+    fn rule_count(&self) -> usize {
+        self.engine.rule_count()
+    }
+}
+
+/// Pure-Rust half of `detect`, safe to run without the GVL.
+fn collect_matches(
+    engine: &Engine,
+    bytes: &[u8],
+    char_offsets: bool,
+) -> Vec<(String, usize, usize, String)> {
+    let hits = engine.scan(bytes);
+    if hits.is_empty() {
+        return Vec::new();
+    }
+
+    let positions: Vec<usize> = if char_offsets {
+        let mut wanted = Vec::with_capacity(hits.len() * 2);
+        for hit in &hits {
+            wanted.push(hit.start);
+            wanted.push(hit.end);
+        }
+        // `wanted` is already ascending: hits are in document order and
+        // non-overlapping.
+        offsets::to_char_offsets(bytes, &wanted)
+    } else {
+        hits.iter().flat_map(|h| [h.start, h.end]).collect()
+    };
+
+    hits.iter()
+        .enumerate()
+        .map(|(i, hit)| {
+            (
+                engine.kind_of(hit).to_string(),
+                positions[i * 2],
+                positions[i * 2 + 1],
+                engine.preview(bytes, hit),
+            )
+        })
+        .collect()
+}
+
+fn error_class(ruby: &Ruby, name: &str) -> ExceptionClass {
+    ruby.class_object()
+        .const_get::<_, RModule>("Scrubber")
+        .and_then(|m| m.const_get::<_, ExceptionClass>(name))
+        .unwrap_or_else(|_| ruby.exception_runtime_error())
+}
+
+fn internal_error(ruby: &Ruby, message: &str) -> Error {
+    Error::new(
+        error_class(ruby, "InternalError"),
+        format!(
+            "{message}. This is a bug in scrubber_rb - please report it at \
+             https://github.com/TheSoloHacker47/scrubber-rb/issues (redact your input first)."
+        ),
+    )
+}
+
+fn build_error_to_ruby(err: BuildError) -> Error {
+    let Ok(ruby) = Ruby::get() else {
+        // Unreachable from a Ruby thread; keeps the signature total.
+        return Error::new(
+            magnus::exception::runtime_error(),
+            "scrubber_rb: no Ruby VM on this thread",
+        );
+    };
+    let class = match &err {
+        BuildError::UnknownDetector { .. } => "UnknownDetectorError",
+        BuildError::UnsupportedPattern { .. } | BuildError::InvalidPattern { .. } => {
+            "UnsupportedPatternError"
+        }
+        BuildError::UnknownStrategy { .. } => "ConfigurationError",
+    };
+    Error::new(error_class(&ruby, class), err.to_string())
+}
+
+/// Detector keys, so Ruby doesn't have to keep a second copy of the list in
+/// sync with the registry.
+fn default_detectors(ruby: &Ruby) -> Result<RArray, Error> {
+    let out = ruby.ary_new_capa(detectors::DEFAULTS.len());
+    for key in detectors::DEFAULTS {
+        out.push(ruby.str_new(key))?;
+    }
+    Ok(out)
+}
+
+fn india_detectors(ruby: &Ruby) -> Result<RArray, Error> {
+    let out = ruby.ary_new_capa(detectors::INDIA.len());
+    for key in detectors::INDIA {
+        out.push(ruby.str_new(key))?;
+    }
+    Ok(out)
+}
+
+fn all_detectors(ruby: &Ruby) -> Result<RArray, Error> {
+    let keys = detectors::all_keys();
+    let out = ruby.ary_new_capa(keys.len());
+    for key in keys {
+        out.push(ruby.str_new(key))?;
+    }
+    Ok(out)
+}
+
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    let module = ruby.define_module("Scrubber")?;
+
+    // Error hierarchy lives here so the native extension can raise it before
+    // any Ruby file has been loaded.
+    let base = module.define_error("Error", ruby.exception_standard_error())?;
+    module.define_error("UnknownDetectorError", base)?;
+    module.define_error("UnsupportedPatternError", base)?;
+    module.define_error("ConfigurationError", base)?;
+    module.define_error("InternalError", base)?;
+
+    let native = module.define_class("Native", ruby.class_object())?;
+    native.define_singleton_method("new", function!(Native::new, 4))?;
+    native.define_method("scrub", method!(Native::scrub, 1))?;
+    native.define_method("detect", method!(Native::detect, 2))?;
+    native.define_method("rule_count", method!(Native::rule_count, 0))?;
+
+    native.define_singleton_method("default_detectors", function!(default_detectors, 0))?;
+    native.define_singleton_method("india_detectors", function!(india_detectors, 0))?;
+    native.define_singleton_method("all_detectors", function!(all_detectors, 0))?;
+
+    Ok(())
+}

data/ext/scrubber_rb/src/nogvl.rs

@@ -0,0 +1,84 @@
+//! Releasing the GVL around long scans.
+//!
+//! Scanning a 50MB log file takes long enough that holding the GVL would stall
+//! every other thread in a Puma worker. The rule that makes this safe: the
+//! input is copied into a Rust-owned buffer *before* the GVL is released, and
+//! nothing inside the closure touches a Ruby object.
+
+use std::ffi::c_void;
+use std::panic::{catch_unwind, AssertUnwindSafe};
+
+/// Inputs at least this large are scanned with the GVL released. Below it the
+/// release/reacquire round trip costs more than the scan.
+pub const GVL_THRESHOLD: usize = 64 * 1024;
+
+/// Run `func` with the GVL released.
+///
+/// # Safety contract
+///
+/// `func` must not call any Ruby C API function or touch any `VALUE`. Callers
+/// in this crate satisfy that by operating only on an owned `Vec<u8>`.
+pub fn without_gvl<F, R>(func: F) -> std::thread::Result<R>
+where
+    F: FnOnce() -> R,
+{
+    struct Payload<F, R> {
+        func: Option<F>,
+        result: Option<std::thread::Result<R>>,
+    }
+
+    unsafe extern "C" fn trampoline<F, R>(data: *mut c_void) -> *mut c_void
+    where
+        F: FnOnce() -> R,
+    {
+        let payload = &mut *(data as *mut Payload<F, R>);
+        if let Some(func) = payload.func.take() {
+            // A panic must not unwind across the C frame Ruby put us in, so it
+            // is caught here and re-raised on the Ruby side as
+            // Scrubber::InternalError.
+            payload.result = Some(catch_unwind(AssertUnwindSafe(func)));
+        }
+        std::ptr::null_mut()
+    }
+
+    let mut payload = Payload {
+        func: Some(func),
+        result: None,
+    };
+
+    unsafe {
+        rb_sys::rb_thread_call_without_gvl(
+            Some(trampoline::<F, R>),
+            std::ptr::addr_of_mut!(payload) as *mut c_void,
+            // No unblocking function: the scan is pure CPU work with no
+            // syscalls to interrupt, and it always terminates (the regex engine
+            // is linear-time). Ruby will simply defer the interrupt.
+            None,
+            std::ptr::null_mut(),
+        );
+    }
+
+    payload
+        .result
+        .take()
+        .unwrap_or_else(|| Err(Box::new("scan closure never ran")))
+}
+
+/// Run `func`, converting a panic into a `Result` instead of an abort.
+pub fn guarded<F, R>(func: F) -> std::thread::Result<R>
+where
+    F: FnOnce() -> R,
+{
+    catch_unwind(AssertUnwindSafe(func))
+}
+
+/// Human-readable text for whatever `catch_unwind` handed back.
+pub fn panic_message(payload: &(dyn std::any::Any + Send)) -> String {
+    if let Some(s) = payload.downcast_ref::<&str>() {
+        (*s).to_string()
+    } else if let Some(s) = payload.downcast_ref::<String>() {
+        s.clone()
+    } else {
+        "unknown panic in the scrubber_rb native extension".to_string()
+    }
+}

data/ext/scrubber_rb/src/offsets.rs

@@ -0,0 +1,118 @@
+//! Byte offset -> character offset mapping.
+//!
+//! Rust works in bytes; `Scrubber#detect` promises offsets that index the Ruby
+//! string. For UTF-8 those differ the moment anyone writes an emoji or a word
+//! in Devanagari, so we convert here rather than making callers guess
+//! (behaviour contract S11).
+
+/// Convert byte offsets to character offsets in one pass over `bytes`.
+///
+/// `bytes` may contain invalid UTF-8; each invalid byte counts as one
+/// character, which is exactly how Ruby counts them in a UTF-8 string.
+/// `wanted` must be sorted ascending. Offsets past the end map to the total
+/// character count.
+pub fn to_char_offsets(bytes: &[u8], wanted: &[usize]) -> Vec<usize> {
+    let mut out = Vec::with_capacity(wanted.len());
+    if wanted.is_empty() {
+        return out;
+    }
+
+    let mut next = 0usize;
+    let mut byte_idx = 0usize;
+    let mut char_idx = 0usize;
+
+    while byte_idx <= bytes.len() {
+        while next < wanted.len() && wanted[next] <= byte_idx {
+            // `<=` rather than `==` so an offset landing inside a multi-byte
+            // sequence (only reachable via a caller bug) degrades to the
+            // character that contains it instead of running off the end.
+            out.push(char_idx);
+            next += 1;
+        }
+        if next == wanted.len() || byte_idx == bytes.len() {
+            break;
+        }
+        byte_idx += char_len_at(bytes, byte_idx);
+        char_idx += 1;
+    }
+
+    // Anything past the end of the string clamps to the character count.
+    while out.len() < wanted.len() {
+        out.push(char_idx);
+    }
+    out
+}
+
+/// Length in bytes of the character starting at `i`, treating an invalid
+/// sequence as a single one-byte character.
+fn char_len_at(bytes: &[u8], i: usize) -> usize {
+    let b = bytes[i];
+    let expected = match b {
+        0x00..=0x7f => 1,
+        0xc2..=0xdf => 2,
+        0xe0..=0xef => 3,
+        0xf0..=0xf4 => 4,
+        _ => return 1, // continuation byte or invalid lead: one "character"
+    };
+    if expected == 1 {
+        return 1;
+    }
+    // Verify the continuation bytes are actually there; if not, the lead byte
+    // stands alone.
+    for k in 1..expected {
+        match bytes.get(i + k) {
+            Some(c) if (0x80..=0xbf).contains(c) => {}
+            _ => return 1,
+        }
+    }
+    expected
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn ascii_offsets_are_identity() {
+        let s = "hello world";
+        assert_eq!(to_char_offsets(s.as_bytes(), &[0, 6, 11]), vec![0, 6, 11]);
+    }
+
+    #[test]
+    fn multibyte_offsets_count_characters() {
+        // "🎉 नमस्ते nik@example.com"
+        let s = "🎉 नमस्ते nik@example.com";
+        let byte_start = s.find("nik@").unwrap();
+        let char_start = s.chars().take_while(|c| *c != 'n').count();
+        assert_eq!(
+            to_char_offsets(s.as_bytes(), &[byte_start]),
+            vec![char_start]
+        );
+    }
+
+    #[test]
+    fn handles_invalid_bytes_as_single_characters() {
+        let mut bytes = b"ab".to_vec();
+        bytes.push(0xff);
+        bytes.extend_from_slice("cd".as_bytes());
+        // a=0, b=1, 0xff=2, c=3, d=4
+        assert_eq!(to_char_offsets(&bytes, &[0, 2, 3, 5]), vec![0, 2, 3, 5]);
+    }
+
+    #[test]
+    fn truncated_sequence_does_not_run_off_the_end() {
+        let bytes = vec![0xe0, 0xa4]; // start of a 3-byte sequence, truncated
+        assert_eq!(to_char_offsets(&bytes, &[0, 1, 2]), vec![0, 1, 2]);
+    }
+
+    #[test]
+    fn offsets_past_the_end_clamp() {
+        let s = "abc";
+        assert_eq!(to_char_offsets(s.as_bytes(), &[1, 99]), vec![1, 3]);
+    }
+
+    #[test]
+    fn empty_request_is_empty() {
+        assert!(to_char_offsets(b"abc", &[]).is_empty());
+    }
+}