gigatoken 0.3.0 → 0.4.0

data/benches/pretokenize_profile.rs

@@ -0,0 +1,36 @@
+//! Profiling target for the `FastR50kPretokenizer` hot loop in isolation: a plain
+//! single-pass `main` (no criterion, no BPE encode) that `black_box`es every
+//! yielded pretoken slice, so the slice production can't be optimized away.
+
+use gigatoken_rs::pretokenize::FastR50kPretokenizer;
+use std::hint::black_box;
+use std::time::Instant;
+
+mod common;
+fn main() {
+    let input = common::load_owt_input(None);
+    let size_gb = input.len() as f64 / 1e9;
+
+    // Feed the entire buffer to one pretokenizer in a single pass — this matches
+    // the real encode path (`pretokenize_as_iter(text.as_bytes())`), which does
+    // not pre-split on newlines.
+    let buf: &[u8] = &input;
+
+    eprintln!("Pretokenizing (fast_scalar, single-threaded, whole buffer)...");
+    let start = Instant::now();
+    let mut total_tokens: usize = 0;
+    // Hand each real pretoken slice to black_box so the bounds computation can't
+    // be optimized down to a counter.
+    let mut iter = FastR50kPretokenizer::new(buf);
+    for pretoken in iter {
+        black_box(pretoken);
+        total_tokens += 1;
+    }
+    let elapsed = start.elapsed().as_secs_f64();
+    let throughput_gb = size_gb / elapsed;
+
+    eprintln!(
+        "{total_tokens} tokens in {elapsed:.2}s — {throughput_gb:.2} GB/s ({:.0} MB/s)",
+        throughput_gb * 1000.0
+    );
+}

data/benches/pretokenize_profile_all.rs

@@ -0,0 +1,52 @@
+//! Per-scheme variant of `pretokenize_profile`: same single-pass loop over
+//! OWT with every yielded pretoken black_boxed, scheme selected via the
+//! SCHEME env var (r50k | cl100k | olmo3 | qwen2 | qwen3_5 | deepseek_v3).
+//! r50k is also what ByteLevel tokenizers like ModernBERT resolve to. Used
+//! for interleaved A/B runs of the mask-scanner schemes.
+
+use gigatoken_rs::pretokenize::{
+    FastCl100kPretokenizer, FastDeepSeekV3Pretokenizer, FastOlmo3Pretokenizer,
+    FastQwen2Pretokenizer, FastQwen35Pretokenizer, FastR50kPretokenizer,
+};
+use std::hint::black_box;
+use std::time::Instant;
+
+mod common;
+
+macro_rules! drive {
+    ($ty:ty, $buf:expr) => {{
+        let mut total_tokens: usize = 0;
+        let mut iter = <$ty>::new($buf);
+        while let Some(pretoken) = iter.next() {
+            black_box(pretoken);
+            total_tokens += 1;
+        }
+        total_tokens
+    }};
+}
+
+fn main() {
+    let input = common::load_owt_input(None);
+    let size_gb = input.len() as f64 / 1e9;
+    let buf: &[u8] = &input;
+    let scheme = std::env::var("SCHEME").unwrap_or_else(|_| "r50k".to_string());
+
+    eprintln!("Pretokenizing ({scheme}, single-threaded, whole buffer)...");
+    let start = Instant::now();
+    let total_tokens = match scheme.as_str() {
+        "r50k" => drive!(FastR50kPretokenizer, buf),
+        "cl100k" => drive!(FastCl100kPretokenizer, buf),
+        "olmo3" => drive!(FastOlmo3Pretokenizer, buf),
+        "qwen2" => drive!(FastQwen2Pretokenizer, buf),
+        "qwen3_5" => drive!(FastQwen35Pretokenizer, buf),
+        "deepseek_v3" => drive!(FastDeepSeekV3Pretokenizer, buf),
+        other => panic!("unknown SCHEME {other:?}"),
+    };
+    let elapsed = start.elapsed().as_secs_f64();
+    let throughput_gb = size_gb / elapsed;
+
+    eprintln!(
+        "{total_tokens} tokens in {elapsed:.2}s — {throughput_gb:.2} GB/s ({:.0} MB/s)",
+        throughput_gb * 1000.0
+    );
+}

data/benches/simdutf_transcode.rs

@@ -0,0 +1,82 @@
+use criterion::{Criterion, Throughput, criterion_group, criterion_main};
+use rayon::prelude::*;
+use std::hint::black_box;
+
+const TARGET_BENCH_SIZE: usize = 100_000_000; // ~100 MB
+
+/// Load OWT data, truncated to a UTF-8-safe boundary near `max_bytes`.
+fn load_owt(max_bytes: usize) -> Vec<u8> {
+    let data_dir = std::env::home_dir().unwrap().join("data");
+    let all_bytes =
+        std::fs::read(data_dir.join("owt_train.txt")).expect("Could not read ~/data/owt_train.txt");
+    let mut end = max_bytes.min(all_bytes.len());
+    while end > 0 && std::str::from_utf8(&all_bytes[..end]).is_err() {
+        end -= 1;
+    }
+    all_bytes[..end].to_vec()
+}
+
+fn simdutf_transcode_benches(c: &mut Criterion) {
+    let input = load_owt(TARGET_BENCH_SIZE);
+    let input_len = input.len() as u64;
+    eprintln!("Benchmark input size: {:.1} MB", input_len as f64 / 1e6);
+
+    let mut group = c.benchmark_group("simdutf_transcode");
+    group.throughput(Throughput::Bytes(input_len));
+    group.sample_size(10);
+
+    for num_threads in [1, 2, 4, 8, 12, 16] {
+        let pool = rayon::ThreadPoolBuilder::new()
+            .num_threads(num_threads)
+            .build()
+            .unwrap();
+
+        let chunk_size = input.len() / num_threads;
+
+        // Pre-compute chunk boundaries aligned to UTF-8 char boundaries.
+        let mut boundaries = Vec::with_capacity(num_threads + 1);
+        boundaries.push(0);
+        for i in 1..num_threads {
+            let mut pos = chunk_size * i;
+            while pos < input.len() && input[pos] & 0b1100_0000 == 0b1000_0000 {
+                pos += 1;
+            }
+            boundaries.push(pos);
+        }
+        boundaries.push(input.len());
+
+        // Pre-allocate one destination buffer per thread.
+        let mut dst_bufs: Vec<Vec<u32>> = boundaries
+            .windows(2)
+            .map(|w| vec![0u32; simdutf::utf32_length_from_utf8(&input[w[0]..w[1]])])
+            .collect();
+
+        group.bench_function(format!("utf8_to_utf32_{num_threads}t"), |b| {
+            b.iter(|| {
+                pool.install(|| {
+                    let total: usize = boundaries
+                        .par_windows(2)
+                        .zip(&mut dst_bufs)
+                        .map(|(w, dst)| {
+                            let chunk = &input[w[0]..w[1]];
+                            let written = unsafe {
+                                simdutf::convert_valid_utf8_to_utf32(
+                                    chunk.as_ptr(),
+                                    chunk.len(),
+                                    dst.as_mut_ptr(),
+                                )
+                            };
+                            black_box(written)
+                        })
+                        .sum();
+                    black_box(total);
+                })
+            });
+        });
+    }
+
+    group.finish();
+}
+
+criterion_group!(benches, simdutf_transcode_benches);
+criterion_main!(benches);

data/benches/unicode.rs

@@ -0,0 +1,88 @@
+use criterion::{Criterion, criterion_group, criterion_main};
+use icu::properties::{CodePointMapDataBorrowed, props::EnumeratedProperty};
+use std::hint::black_box;
+
+use rand::{self, Rng, RngExt};
+
+// pub fn fibonacci(n: u64) -> u64 {
+//     let mut a = 0;
+//     let mut b = 1;
+//     for _ in 0..n {
+//         let c = a + b;
+//         a = b;
+//         b = c;
+//     }
+//     a
+// }
+
+// Removed dependency since icu is ~95% faster
+// use unicode_properties::{GeneralCategoryGroup, UnicodeGeneralCategory};
+// pub fn unicode_properties_classify(c: char) -> bool {
+//     c.general_category_group() == GeneralCategoryGroup::Letter
+// }
+
+pub fn icu4x_classify(c: char) -> bool {
+    icu::properties::props::GeneralCategoryGroup::Letter
+        .contains(icu::properties::props::GeneralCategory::for_char(c))
+}
+
+pub fn icu4x_classify_table(c: char) -> bool {
+    let gc: CodePointMapDataBorrowed<icu::properties::props::GeneralCategory> =
+        icu::properties::CodePointMapData::new();
+    icu::properties::props::GeneralCategoryGroup::Letter.contains(gc.get(c))
+}
+
+pub fn criterion_benchmark(c: &mut Criterion) {
+    // c.bench_function("fib 20", |b| b.iter(|| fibonacci(black_box(20))));
+    let mut group = c.benchmark_group("unicode_classify");
+
+    let chars_input: Vec<char> = rand::rng()
+        .sample_iter::<char, _>(rand::distr::StandardUniform)
+        .take(4096)
+        .collect();
+    group.bench_with_input("icu4x", chars_input.as_slice(), |b, chars: &[char]| {
+        b.iter(|| {
+            for c in chars {
+                icu4x_classify(*c);
+            }
+        });
+    });
+    // group.bench_with_input(
+    //     "unicode_properties",
+    //     chars_input.as_slice(),
+    //     |b, chars: &[char]| {
+    //         b.iter(|| {
+    //             for c in chars {
+    //                 unicode_properties_classify(*c);
+    //             }
+    //         });
+    //     },
+    // );
+    group.bench_with_input(
+        "icu4x table",
+        chars_input.as_slice(),
+        |b, chars: &[char]| {
+            b.iter(|| {
+                for c in chars {
+                    icu4x_classify_table(*c);
+                }
+            });
+        },
+    );
+
+    // c.bench_function("unicode classify letter", |b| {
+    //     b.iter_batched(
+    //         || {},
+    //         |chars: Vec<char>| {
+    //             for c in chars {
+    //                 unicode_classify(c);
+    //             }
+    //         },
+    //         criterion::BatchSize::SmallInput,
+    //     )
+    // });
+    // c.bench_function("unicode pretokenize")
+}
+
+criterion_group!(benches, criterion_benchmark);
+criterion_main!(benches);

data/ext/gigatoken/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "gigatoken-rb"
-version = "0.3.0"
+version = "0.4.0"
 edition = "2021"
 
 [lib]

data/ext/gigatoken/src/error.rs

@@ -1,20 +1,40 @@
-//! `Gigatoken::Error` is defined in Ruby (`lib/gigatoken.rb`), loaded before
-//! this extension is required — looked up fresh at each raise site rather
-//! than cached, so no magnus `Value` needs a GC-registered static home.
+//! `Gigatoken::Error` and its subclasses are defined in Ruby
+//! (`lib/gigatoken.rb`), loaded before this extension is required — looked up
+//! fresh at each raise site rather than cached, so no magnus `Value` needs a
+//! GC-registered static home.
 
 use magnus::{exception::ExceptionClass, prelude::*, Error, RModule, Ruby};
 
-fn error_class(ruby: &Ruby) -> Result<ExceptionClass, Error> {
+fn error_class(ruby: &Ruby, name: &str) -> Result<ExceptionClass, Error> {
     ruby.class_object()
         .const_get::<_, RModule>("Gigatoken")?
-        .const_get("Error")
+        .const_get(name)
 }
 
-/// Raise `Gigatoken::Error` with `message` (core load/encode failures surface
-/// through this — never a Rust panic across the Ruby boundary).
-pub fn raise(ruby: &Ruby, message: impl Into<String>) -> Error {
-    match error_class(ruby) {
+fn raise_as(ruby: &Ruby, name: &str, message: impl Into<String>) -> Error {
+    match error_class(ruby, name) {
         Ok(class) => Error::new(class, message.into()),
         Err(e) => e,
     }
 }
+
+/// Raise `Gigatoken::Error` with `message` — what a failure that is neither
+/// the caller's document nor the model itself surfaces through (never a Rust
+/// panic across the Ruby boundary). Prefer [`input_error`] or [`model_error`]
+/// where one of them fits.
+pub fn raise(ruby: &Ruby, message: impl Into<String>) -> Error {
+    raise_as(ruby, "Error", message)
+}
+
+/// Raise `Gigatoken::InputError`: a document the tokenizer cannot take —
+/// invalid UTF-8 on the SentencePiece path, an id outside the vocabulary in
+/// `decode`.
+pub fn input_error(ruby: &Ruby, message: impl Into<String>) -> Error {
+    raise_as(ruby, "InputError", message)
+}
+
+/// Raise `Gigatoken::ModelError`: a tokenizer that cannot be loaded — bad or
+/// hostile JSON, an unknown pretokenizer scheme, a malformed `.tiktoken`.
+pub fn model_error(ruby: &Ruby, message: impl Into<String>) -> Error {
+    raise_as(ruby, "ModelError", message)
+}

data/ext/gigatoken/src/gvl.rs

@@ -36,15 +36,27 @@
 //! `VALUE` or thread-local state, so running one OS thread over instead of
 //! another changes nothing about its safety, and the types involved satisfy
 //! `Send`/`Sync` on their own merits.
+//!
+//! Interrupts. `rb_nogvl` ends by checking interrupts and raises any that
+//! are pending — a `Timeout`, `Thread#kill`, `Interrupt`, an `Async`
+//! timeout — by longjmping out of itself, over every Rust frame in between.
+//! Both entry points here therefore call it under `magnus::rb_sys::protect`
+//! (see [`run`]), so the raise comes back as an ordinary `Error` and every
+//! guard, `InputDocs` and Ruby String lock the caller holds is released by
+//! ordinary Rust unwinding on the way out. [`without_gvl_cancellable`] goes
+//! one further and supplies a real unblock function, so a pending interrupt
+//! cancels the work in flight instead of waiting for it.
 
 use std::any::Any;
 use std::ffi::c_void;
 use std::os::raw::c_int;
 use std::panic::{self, AssertUnwindSafe};
+use std::sync::atomic::{AtomicBool, Ordering};
 
+use magnus::Error;
 use rb_sys::rb_nogvl;
 
-/// `RB_NOGVL_OFFLOAD_SAFE` (`ruby/thread.h:84` in a current Ruby checkout;
+/// `RB_NOGVL_OFFLOAD_SAFE` (`ruby/thread.h:73` in Ruby 4.0.7's own headers;
 /// introduced by Ruby's `Fiber::Scheduler#blocking_operation_wait` support,
 /// first released in Ruby 3.4.0). Defined locally rather than taken from
 /// `rb_sys::` bindings: `rb-sys` bindgens its constants from the *building*
@@ -61,62 +73,184 @@ const RB_NOGVL_OFFLOAD_SAFE: c_int = 0x4;
 /// value, or a caught panic payload to re-raise once we're back on ordinary
 /// (non-`extern "C"`) Rust stack frames. Unwinding a panic directly across
 /// the `extern "C"` trampoline `rb_nogvl` calls into is undefined behavior;
-/// catching it here and resuming it from `without_gvl` below turns that into
-/// an ordinary Rust panic, which magnus's own `method!`/`function!` call
-/// trampolines already wrap in `catch_unwind` and convert into a fatal Ruby
-/// exception (`magnus::error::Error::from_panic`) — the same outcome any
-/// other panicking native method already gets, just carried safely across
-/// the extra C boundary this one call adds.
+/// catching it here and resuming it from [`Slot::take`] below turns that
+/// into an ordinary Rust panic, which magnus's own `method!`/`function!`
+/// call trampolines already wrap in `catch_unwind` and convert into a fatal
+/// Ruby exception (`magnus::error::Error::from_panic`) — the same outcome
+/// any other panicking native method already gets, just carried safely
+/// across the extra C boundary this one call adds.
 enum Outcome<R> {
     Value(R),
     Panic(Box<dyn Any + Send + 'static>),
 }
 
+/// The one thing `rb_nogvl`'s callback and [`run`] share: the callback takes
+/// the closure out of `input` and leaves its result in `output`. The rule is
+/// that nothing travels through `rb_nogvl`'s *return value* — the caller owns
+/// this slot, in an ordinary Rust frame.
+///
+/// That rule is what makes the raise `rb_nogvl` performs on its way out
+/// leak-free. The `protect` closure in [`run`] holds the single FFI call and
+/// owns nothing at all, so the longjmp skips no destructor that matters:
+/// whatever is in flight is either here — in a frame that unwinds normally
+/// afterwards — or in one of the caller's own frames, which unwind with it.
+/// It is also the only way to tell "the callback ran" from "it never did",
+/// which a fiber scheduler cancelling the offloaded operation before its
+/// worker pool picks it up makes a real case (io-event's
+/// `worker_pool_work_wait`, `ext/io/event/worker_pool.c`).
+struct Slot<F, R> {
+    input: Option<F>,
+    output: Option<Outcome<R>>,
+}
+
+impl<F, R> Slot<F, R> {
+    fn new(f: F) -> Self {
+        Self {
+            input: Some(f),
+            output: None,
+        }
+    }
+
+    /// The callback's value, or `None` if it never ran. A panic it caught
+    /// resumes here, on an ordinary Rust frame.
+    fn take(&mut self) -> Option<R> {
+        match self.output.take() {
+            Some(Outcome::Value(value)) => Some(value),
+            Some(Outcome::Panic(payload)) => panic::resume_unwind(payload),
+            None => None,
+        }
+    }
+}
+
 unsafe extern "C" fn call_without_gvl<F, R>(arg: *mut c_void) -> *mut c_void
 where
     F: FnOnce() -> R + Send,
     R: Send,
 {
-    // SAFETY: `arg` is the `*mut Option<F>` handed to `rb_nogvl` below, valid
-    // for the duration of that (synchronous) call, and this is the only
-    // place it's dereferenced.
-    let closure = unsafe { (*(arg as *mut Option<F>)).take() }
+    // SAFETY: `arg` is the `*mut Slot<F, R>` handed to `rb_nogvl` by `run`,
+    // valid for the duration of that (synchronous) call, and this is the
+    // only place it's dereferenced.
+    let slot = unsafe { &mut *(arg as *mut Slot<F, R>) };
+    let closure = slot
+        .input
+        .take()
         .expect("without_gvl callback invoked more than once");
-    let outcome = match panic::catch_unwind(AssertUnwindSafe(closure)) {
+    slot.output = Some(match panic::catch_unwind(AssertUnwindSafe(closure)) {
         Ok(value) => Outcome::Value(value),
         Err(payload) => Outcome::Panic(payload),
-    };
-    Box::into_raw(Box::new(outcome)) as *mut c_void
+    });
+    std::ptr::null_mut()
+}
+
+/// Run `slot`'s closure with the GVL released, under `rb_protect`.
+///
+/// `rb_nogvl` finishes by checking interrupts and raising any that are
+/// pending, by longjmping out of itself. `protect` catches that and returns
+/// it as an `Error` — including the `Tag::Fatal` a `Thread#kill` jumps with,
+/// which magnus resumes with `rb_jump_tag` once the caller's frames have
+/// unwound. The closure below is deliberately trivial: one FFI call and a
+/// `nil`, owning nothing the longjmp could strand.
+fn run<F, R>(
+    slot: &mut Slot<F, R>,
+    ubf: rb_sys::rb_unblock_function_t,
+    data2: *mut c_void,
+) -> Result<(), Error>
+where
+    F: FnOnce() -> R + Send,
+    R: Send,
+{
+    let arg = slot as *mut Slot<F, R> as *mut c_void;
+    let nil: rb_sys::VALUE = rb_sys::Qnil.into();
+    magnus::rb_sys::protect(|| {
+        // SAFETY: `arg` points at `slot`, which outlives this synchronous
+        // call; the callback is the only reader of it (see `Slot`).
+        unsafe {
+            rb_nogvl(
+                Some(call_without_gvl::<F, R>),
+                arg,
+                ubf,
+                data2,
+                RB_NOGVL_OFFLOAD_SAFE,
+            )
+        };
+        nil
+    })?;
+    Ok(())
+}
+
+/// `rb_nogvl`'s unblock function: Ruby calls it from another thread when an
+/// interrupt is pending for this one, and a fiber scheduler calls it to
+/// cancel an offloaded operation (`rb_fiber_scheduler_blocking_operation_cancel`
+/// "marks it as cancelled and calls the unblock function", Ruby 4.0.7's
+/// `ruby/fiber/scheduler.h:455-457` — which is how an `Async` timeout reaches
+/// an encode running on io-event's worker pool). It does exactly one thing:
+/// set the flag the core's encode loops poll (`src/batch.rs`), so the encode
+/// stops at the next document boundary instead of running to completion.
+unsafe extern "C" fn set_cancel(arg: *mut c_void) {
+    // SAFETY: `arg` is the `&AtomicBool` `without_gvl_cancellable` passed as
+    // `data2`, living in its frame for the whole `rb_nogvl` call — the only
+    // window in which Ruby may call this.
+    unsafe { &*(arg as *const AtomicBool) }.store(true, Ordering::Relaxed);
 }
 
 /// Run `f` with the GVL released: other Ruby threads may run while `f`
 /// executes, and — under a fiber scheduler with a worker pool — the calling
 /// fiber yields to the reactor while `f` runs on a background thread. `f`
 /// must not touch any Ruby object (`VALUE`) — only plain Rust data — per the
-/// Ruby C API's contract for this call. A panic inside `f` is caught and
-/// re-raised here rather than left to unwind across the C trampoline.
-pub fn without_gvl<F, R>(f: F) -> R
+/// Ruby C API's contract for this call. An interrupt that arrives meanwhile
+/// is delivered once `f` has finished, as an `Err`; use
+/// [`without_gvl_cancellable`] for work that can stop early.
+pub fn without_gvl<F, R>(f: F) -> Result<R, Error>
 where
     F: FnOnce() -> R + Send,
     R: Send,
 {
-    let mut slot = Some(f);
-    let arg = &mut slot as *mut Option<F> as *mut c_void;
-    let result = unsafe {
-        rb_nogvl(
-            Some(call_without_gvl::<F, R>),
-            arg,
-            None,
-            std::ptr::null_mut(),
-            RB_NOGVL_OFFLOAD_SAFE,
-        )
-    };
-    // SAFETY: `result` is the `Box::into_raw(Box::new(Outcome<R>))` pointer
-    // produced by the callback above, which always runs exactly once before
-    // `rb_nogvl` returns.
-    let outcome = *unsafe { Box::from_raw(result as *mut Outcome<R>) };
-    match outcome {
-        Outcome::Value(value) => value,
-        Outcome::Panic(payload) => panic::resume_unwind(payload),
+    let mut slot = Slot::new(f);
+    run(&mut slot, None, std::ptr::null_mut())?;
+    Ok(match slot.take() {
+        Some(value) => value,
+        // A fiber scheduler cancelled the offloaded operation before its
+        // worker pool ever started it, and nothing was raised (that would
+        // have come back as an `Err` above). The closure is still in the
+        // slot and still ours to run: do that here, GVL and all, rather
+        // than invent a result.
+        None => slot.input.take().expect("the callback left the closure")(),
+    })
+}
+
+/// [`without_gvl`] for work that can stop early: `attempt` builds the
+/// closure, which gets a cancellation token to poll and returns `None` if it
+/// saw the token set and cut its work short. An interrupt arriving during
+/// the run sets that token through [`set_cancel`], so the caller sees the
+/// interrupt instead of waiting out the whole batch.
+///
+/// `attempt` is a factory, not the closure itself, because a cancelled run
+/// sometimes has to be redone: Ruby calls the unblock function for every
+/// interrupt, including ones that never raise (a trap handler, a
+/// `Thread#wakeup`), and returning the half-encoded batch those produce
+/// would be a silently truncated result. The redo runs uninterruptibly —
+/// plain [`without_gvl`], which is exactly how this path behaved before
+/// cancellation existed — so a chatty signal handler can cost a batch one
+/// extra pass, never an unbounded number of them.
+pub fn without_gvl_cancellable<A, F, R>(mut attempt: A) -> Result<R, Error>
+where
+    A: FnMut() -> F,
+    F: FnOnce(&AtomicBool) -> Option<R> + Send,
+    R: Send,
+{
+    let cancel = AtomicBool::new(false);
+    let f = attempt();
+    let mut slot = Slot::new(|| f(&cancel));
+    run(
+        &mut slot,
+        Some(set_cancel),
+        &cancel as *const AtomicBool as *mut c_void,
+    )?;
+    if let Some(Some(value)) = slot.take() {
+        return Ok(value);
     }
+    let f = attempt();
+    let never_cancelled = AtomicBool::new(false);
+    without_gvl(|| f(&never_cancelled))
+        .map(|value| value.expect("nothing sets the token of an uncancellable run"))
 }

data/ext/gigatoken/src/lib.rs

@@ -1,3 +1,4 @@
+use gigatoken_rs::input::file_source::load_file;
 use gigatoken_rs::load_tokenizer::hf::{self, HfTokenizer};
 use gigatoken_rs::pretokenize::PretokenizerType;
 use magnus::{Error, Module, RString, Ruby, Value, function};
@@ -26,9 +27,9 @@ mod sentencepiece;
 mod sources;
 mod tokenizer;
 
-use error::raise;
+use error::{model_error, raise};
 use sentencepiece::SentencePieceTokenizer;
-use tokenizer::BPETokenizer;
+use tokenizer::{binary_string, BPETokenizer};
 
 // The gigatoken core crate exposes no version constant of its own, so this
 // is the ext crate's (gigatoken-rb's) version — see the builder report.
@@ -66,7 +67,20 @@ fn load_hf_json(ruby: &Ruby, data: RString) -> Result<Value, Error> {
     match hf::load_hf_slice(bytes) {
         Ok(HfTokenizer::Bpe(tokenizer)) => Ok(ruby.into_value(BPETokenizer::from_tokenizer(tokenizer))),
         Ok(HfTokenizer::SentencePiece(tokenizer)) => Ok(ruby.into_value(SentencePieceTokenizer::from_tokenizer(tokenizer))),
-        Err(e) => Err(raise(ruby, e.to_string())),
+        Err(e) => Err(model_error(ruby, e.to_string())),
+    }
+}
+
+/// One file's contents as a binary String, decompressed by extension the way
+/// the native file sources do it (`.gz`, `.zst`/`.zstd`, plain — see the core's
+/// `load_file`). The CLI's Ruby-side split reads through here so both sides of
+/// `gigatoken validate` see the same bytes with one decoder between them
+/// (`lib/gigatoken/cli/support.rb`); the encode paths never touch it, which is
+/// why it keeps the GVL.
+fn read_input(ruby: &Ruby, path: String) -> Result<RString, Error> {
+    match load_file(std::path::Path::new(&path)) {
+        Ok(file) => Ok(binary_string(ruby, file.as_bytes())),
+        Err(e) => Err(raise(ruby, format!("{path}: {e}"))),
     }
 }
 
@@ -77,6 +91,7 @@ fn init(ruby: &Ruby) -> Result<(), Error> {
     native.define_module_function("crate_version", function!(crate_version, 0))?;
     native.define_module_function("load_hf_json", function!(load_hf_json, 1))?;
     native.define_module_function("pretokenizer_names", function!(pretokenizer_names, 0))?;
+    native.define_module_function("read_input", function!(read_input, 1))?;
     native.define_module_function("set_max_cache_bytes", function!(set_max_cache_bytes, 1))?;
     native.define_module_function("get_max_cache_bytes", function!(get_max_cache_bytes, 0))?;
     sources::init(ruby, native)?;