gigatoken 0.3.0 → 0.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 41dcafd85415b4ee0ee40e7c77414557b1c6913676d4846e0d5934f689f3657a
-  data.tar.gz: fddc7d15a5a7e1b9cf9be794a0468294dc31c251ad90dd540d16e9ede344b3a2
+  metadata.gz: 1fd5d74bb0d8cb5be9f7cf9fa077cc9c695ab5279436f3ecf3db4374952be586
+  data.tar.gz: 5953850c0513e14bb6aceccf4205b774c1db027f6d4d0f4508d90107f0d483ef
 SHA512:
-  metadata.gz: 20b5c1fdb98dda4eff0fa0323adbf72d334ddeff672b9b1a3ae1085fdc1454ff1b00158b2aa7ea876be39188c7873c57b7df9d5aeeee99d641f3da22f88699ce
-  data.tar.gz: 1c57fc8322d4bb6531cec42def1e6194506742d9491df4b8c2daaf1fb5a1beb29877520935654a955c684d28fcab3e76ba81b5288849092dec5d217d48d190c7
+  metadata.gz: 9b85351d2af130d4f3b0b96c1f654b94ca5a865bd8865b6bd7d29ce2c5d297f052e8ee4a7ee40391573b88b3d81df5ba3aff8fd6edf64e6e03dee596644f1ae9
+  data.tar.gz: c8aca888f7b6477e406fa1234633e8e334a3242d9056951153c2306a6cc5b5eda8d46636c00bfd2c515ecf309130ffaf3543ced5305cc01e00473d8cf2966a73

data/Cargo.lock

@@ -771,7 +771,7 @@ dependencies = [
 
 [[package]]
 name = "gigatoken-rb"
-version = "0.3.0"
+version = "0.4.0"
 dependencies = [
  "gigatoken",
  "magnus",

data/Cargo.toml

@@ -70,6 +70,13 @@ debug = true
 strip = "none"
 split-debuginfo = "packed"
 
+# Frame pointers make Instruments' stack unwinding reliable, but a per-profile
+# `rustflags` here is nightly-only and would need a `.cargo/config.toml` opt-in
+# that the gem does not ship — which made the source gem unbuildable. The
+# profile keeps the debug info; scripts/profile-cpu.fish exports RUSTFLAGS for
+# its own build instead (this profile has its own target subdirectory, so that
+# never reaches a release build). Mangling stays at the toolchain default (v0);
+# `cargo instruments` demangles v0 symbols itself.
 [profile.profiling]
 inherits = "release"
 debug = 2
@@ -77,9 +84,6 @@ lto = false
 codegen-units = 1
 strip = "none"
 split-debuginfo = "packed"
-# Frame pointers make Instruments' stack unwinding reliable. Mangling is left at
-# the toolchain default (v0); `cargo instruments` demangles v0 symbols itself.
-rustflags = ["-C", "force-frame-pointers=yes"]
 
 [profile.profiling.package."*"]
 debug = 2

data/README.md

@@ -57,9 +57,9 @@ Gigatoken::Tokenizer.from_tiktoken("cl100k_base.tiktoken", pretokenizer: "gpt4",
 Gigatoken::Tokenizer.from_json(File.binread("tokenizer.json"))
 ```
 
-A `.tiktoken` file holds mergeable ranks only — its pretokenization scheme and special tokens live in the code that defines the encoding, not the file — so `pretokenizer:` is a required keyword (one of `Gigatoken::Native.pretokenizer_names`: `gpt2`/`r50k`, `gpt4`/`cl100k`, `qwen2`, `qwen35`, `olmo3`, `deepseek_v3`, `o200k`, `nemotron`, `kimi`) and `special_tokens:` defaults to none. Nothing is guessed: an unknown scheme raises `Gigatoken::Error` naming the valid ones, and `Tokenizer.load` on a `.tiktoken` path with no `pretokenizer:` raises rather than silently picking one.
+A `.tiktoken` file holds mergeable ranks only — its pretokenization scheme and special tokens live in the code that defines the encoding, not the file — so `pretokenizer:` is a required keyword (one of `Gigatoken::Native.pretokenizer_names`: `gpt2`/`r50k`, `gpt4`/`cl100k`, `qwen2`, `qwen35`, `olmo3`, `deepseek_v3`, `o200k`, `nemotron`, `kimi`) and `special_tokens:` defaults to none. Nothing is guessed: an unknown scheme raises `Gigatoken::ModelError` naming the valid ones, and `Tokenizer.load` on a `.tiktoken` path with no `pretokenizer:` raises rather than silently picking one.
 
-SentencePiece-BPE models (Llama, Gemma, Mistral — any `tokenizer.json` with `byte_fallback: true`) load through the same entry points and pick the right backend automatically. One difference: the SentencePiece core decodes text, so it validates input and raises `Gigatoken::Error` on invalid UTF-8 instead of guessing.
+SentencePiece-BPE models (Llama, Gemma, Mistral — any `tokenizer.json` with `byte_fallback: true`) load through the same entry points and pick the right backend automatically. One difference: the SentencePiece core decodes text, so it validates input and raises `Gigatoken::InputError` on invalid UTF-8 instead of guessing.
 
 ### Packaged tiktoken encodings
 
@@ -71,7 +71,7 @@ Gigatoken::Tokenizer.load("cl100k_base")          # same result — packaged nam
                                                   # checked before the Hub-repo-id shape
 ```
 
-`p50k_base` and `p50k_edit` are deliberately not packaged: both load the same non-dense ranks (id 50256 is left free for `<|endoftext|>`), and the rank loader rejects non-dense ranks. Both entry points raise `Gigatoken::Error` explaining that, rather than `load` falling through to the Hub for a name that happens to look like a legacy repo id.
+`p50k_base` and `p50k_edit` are deliberately not packaged: both load the same non-dense ranks (id 50256 is left free for `<|endoftext|>`), and the rank loader rejects non-dense ranks. Both entry points raise `Gigatoken::ModelError` explaining that, rather than `load` falling through to the Hub for a name that happens to look like a legacy repo id.
 
 `encode` on a packaged tokenizer honours its special-token table: text containing `<|endoftext|>` (or any other literal special-token string) is tokenized as that special token, not as ordinary text. That matches [`tiktoken`](https://github.com/openai/tiktoken)'s `encode_with_special_tokens`, not its plain `encode`, which treats the same literal as ordinary text — a difference worth knowing if you're tokenizing untrusted input. To get tiktoken's non-honouring default instead, build a tokenizer from the same rank file with an empty special-token table:
 
@@ -117,6 +117,12 @@ packed.token_count  # => total tokens
 packed[3]           # => document 3's ids as an Array, on demand
 ```
 
+### Errors
+
+Everything the library raises is a `Gigatoken::Error`, never a raw Rust panic — and, under it, one of three: `Gigatoken::ModelError` when a tokenizer can't be loaded (bad or hostile JSON, a missing file or directory, an unknown or unpackable encoding name, a malformed `.tiktoken`), `Gigatoken::InputError` when a document or an id can't be taken (a String that won't transcode, invalid UTF-8 on the SentencePiece path, an id outside the vocabulary in `decode`), and `Gigatoken::HubError` for everything `Gigatoken::Hub` raises (HTTP status, transport, timeout, the repo-id and header checks). `rescue Gigatoken::Error` catches all three.
+
+`encode` and `encode_batch` honour the String's encoding tag: UTF-8, US-ASCII and binary go through byte-wise, and a real non-UTF-8 encoding (ISO-8859-1, UTF-16LE, …) is transcoded first, so it gives the same ids as the same text read as UTF-8. See [the reference](docs/reference/tokenizer.md#input-encodings).
+
 ### Async
 
 `encode_batch` and `encode_files` release the GVL for the whole encode; the parallelism runs on the engine's rayon pool, not Ruby threads. Under `Async`, give the fiber scheduler a worker pool (`ASYNC_SCHEDULER_WORKER_POOL=true`) and the calling fiber yields to the reactor too. Design notes: [docs/how-to/run-under-async.md](docs/how-to/run-under-async.md).

data/benches/common/mod.rs

@@ -0,0 +1,105 @@
+//! Shared input loading for the bench targets. Lives in a subdirectory so
+//! cargo does not treat it as a bench target; each bench pulls it in with
+//! `mod common;`.
+
+use std::time::Instant;
+
+/// Re-enable transparent huge pages for this process. The encode paths
+/// madvise their big tables and buffers to 2 MiB pages (they far exceed
+/// 4 KiB dTLB coverage, and Zen drops software prefetches that miss the
+/// TLB), but some session managers launch children with
+/// PR_SET_THP_DISABLE, which silently vetoes MADV_HUGEPAGE; clear it so
+/// the bench measures the tokenizer, not the launcher's memory policy.
+/// No-op off Linux.
+pub fn allow_thp() {
+    #[cfg(target_os = "linux")]
+    // SAFETY: prctl(PR_SET_THP_DISABLE, 0) only clears a per-process flag.
+    unsafe {
+        libc::prctl(libc::PR_SET_THP_DISABLE, 0, 0, 0, 0);
+    }
+}
+
+/// Hint 2 MiB pages for a buffer's reserved capacity, BEFORE it is first
+/// written (the ordering is what makes the hint effective: pages fault in
+/// huge only if the madvise precedes the first touch). Multi-GB bench
+/// buffers otherwise saturate the dTLB alongside the encode's own tables.
+/// No-op off Linux.
+#[allow(unused_variables, clippy::missing_safety_doc)]
+pub fn madvise_hugepage_capacity<T>(v: &mut Vec<T>) {
+    #[cfg(target_os = "linux")]
+    if v.capacity() > 0 {
+        // Align the start inward: malloc's mmap chunks carry a 16-byte
+        // header, and an unaligned madvise start is EINVAL (silent no-op).
+        const PAGE: usize = 4096;
+        let addr = v.as_mut_ptr() as usize;
+        let start = (addr + PAGE - 1) & !(PAGE - 1);
+        let end = addr + v.capacity() * std::mem::size_of::<T>();
+        if end > start {
+            // SAFETY: the range is one live allocation; the hint neither
+            // reads nor writes it.
+            unsafe {
+                libc::madvise(start as *mut libc::c_void, end - start, libc::MADV_HUGEPAGE);
+            }
+        }
+    }
+}
+
+/// Load the benchmark input from `~/data/owt_train.txt`, truncated to a
+/// UTF-8 character boundary.
+///
+/// ENCODE_MB caps the input for fast profiling iterations (only that many
+/// bytes are read from disk, so the read does not dominate a profile of the
+/// encode loop). When it is unset, `default_mb` applies; `None` reads the
+/// whole file.
+pub fn load_owt_input(default_mb: Option<usize>) -> Vec<u8> {
+    let owt_path = std::env::home_dir().unwrap().join("data/owt_train.txt");
+    eprintln!("Reading {owt_path:?}...");
+    let t0 = Instant::now();
+
+    let cap_mb = std::env::var("ENCODE_MB")
+        .ok()
+        .map(|mb| {
+            mb.trim()
+                .parse::<usize>()
+                .expect("ENCODE_MB must be an integer")
+        })
+        .or(default_mb);
+    let mut data = match cap_mb {
+        Some(mb) => {
+            use std::io::Read;
+            let max_bytes = mb * 1_000_000;
+            let file =
+                std::fs::File::open(&owt_path).expect("Could not open ~/data/owt_train.txt");
+            let mut data = Vec::with_capacity(max_bytes);
+            madvise_hugepage_capacity(&mut data);
+            file.take(max_bytes as u64)
+                .read_to_end(&mut data)
+                .expect("read failed");
+            data
+        }
+        None => {
+            let len = std::fs::metadata(&owt_path)
+                .expect("Could not stat ~/data/owt_train.txt")
+                .len() as usize;
+            use std::io::Read;
+            let mut data = Vec::with_capacity(len + 1);
+            madvise_hugepage_capacity(&mut data);
+            std::fs::File::open(&owt_path)
+                .expect("Could not open ~/data/owt_train.txt")
+                .read_to_end(&mut data)
+                .expect("read failed");
+            data
+        }
+    };
+    // Back up to a UTF-8 character boundary (a byte cap can split a
+    // multibyte character).
+    if let Err(e) = std::str::from_utf8(&data) {
+        data.truncate(e.valid_up_to());
+    }
+    eprintln!(
+        "Read {:.2} GB in {:.1}s",
+        data.len() as f64 / 1e9,
+        t0.elapsed().as_secs_f64()
+    );
+    data
+}

data/benches/encode.rs

@@ -0,0 +1,79 @@
+//! Whole-file parallel encode benchmark. For both backends the entire input
+//! is ONE document handed to the library's parallel encode path
+//! (`encode_docs_ragged` / `sp_encode_docs_ragged`) — the same chunking
+//! policy, safe-boundary document splitting, and (for BPE) persistent worker
+//! pool as `encode_batch` / `encode_files` — so this measures gigatoken's
+//! own parallelism, not a bench-local split.
+//!
+//! Run with: cargo bench --bench encode                 (full OWT)
+//!           ENCODE_MB=500 cargo bench --bench encode
+//!           TOKENIZER_JSON=data/qwen3_5_tokenizer.json cargo bench --bench encode
+//!           TOKENIZER_JSON=meta-llama/Llama-3.1-8B cargo bench --bench encode
+//!             (a HuggingFace repo id: resolved via the standard HF cache,
+//!              downloaded into it on a miss)
+
+use gigatoken_rs::load_tokenizer::hf::{HfTokenizer, load_hf_slice};
+use gigatoken_rs::load_tokenizer::hub;
+use gigatoken_rs::{WorkerPool, encode_docs_ragged, sp_encode_docs_ragged};
+use std::hint::black_box;
+use std::path::PathBuf;
+use std::time::Instant;
+
+mod common;
+
+fn main() {
+    common::allow_thp();
+    let tokenizer_json =
+        std::env::var("TOKENIZER_JSON").unwrap_or_else(|_| "data/gpt2_tokenizer.json".to_string());
+    let tokenizer_path = {
+        let path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(&tokenizer_json);
+        if !path.exists() && hub::looks_like_repo_id(&tokenizer_json) {
+            hub::hub_file(&tokenizer_json, "tokenizer.json", "main")
+                .expect("Could not fetch tokenizer.json from the HuggingFace Hub")
+        } else {
+            path
+        }
+    };
+    eprintln!("Loading tokenizer from {tokenizer_path:?}...");
+    let data = std::fs::read(&tokenizer_path).expect("Could not read tokenizer.json");
+    let tokenizer = load_hf_slice(&data).expect("Could not load tokenizer");
+
+    let input = common::load_owt_input(None);
+    let size_gb = input.len() as f64 / 1e9;
+
+    let workers = WorkerPool::new();
+    let (n_tokens, elapsed) = match &tokenizer {
+        HfTokenizer::Bpe(tokenizer) => {
+            eprintln!(
+                "Encoding (ByteLevel BPE, 1 document, {} threads)...",
+                rayon::current_num_threads()
+            );
+            let start = Instant::now();
+            let (ids, lens) = encode_docs_ragged(&workers, tokenizer, &[&input]);
+            black_box((&ids, lens));
+            (ids.len(), start.elapsed().as_secs_f64())
+        }
+        HfTokenizer::SentencePiece(tokenizer) => {
+            // Trailing partial UTF-8 char (from the ENCODE_MB byte cap) is
+            // dropped; SP encoding takes str input.
+            let text = match std::str::from_utf8(&input) {
+                Ok(text) => text,
+                Err(e) => std::str::from_utf8(&input[..e.valid_up_to()]).unwrap(),
+            };
+            eprintln!(
+                "Encoding (SentencePiece, 1 document, {} threads)...",
+                rayon::current_num_threads()
+            );
+            let start = Instant::now();
+            let (ids, lens) = sp_encode_docs_ragged(tokenizer, &[text]);
+            black_box((&ids, lens));
+            (ids.len(), start.elapsed().as_secs_f64())
+        }
+    };
+    let throughput_gb = size_gb / elapsed;
+
+    eprintln!(
+        "{n_tokens} tokens in {elapsed:.2}s — {throughput_gb:.2} GB/s ({:.0} MB/s)",
+        throughput_gb * 1000.0
+    );
+}

data/benches/encode_doc.rs

@@ -0,0 +1,60 @@
+//! Whole-document multithreaded encode benchmark, mirroring
+//! `BPETokenizer.encode_files` on a single plain-text file: the entire input
+//! is ONE document handed to the library's parallel encode path
+//! (`encode_docs_ragged`), which splits it at pretoken-safe boundaries
+//! (token-identical to a serial pass), encodes with a persistent worker
+//! pool, and gathers one flat id buffer.
+//!
+//! One round per process by default: a first pass over a fresh dataset is
+//! the workload that matters, and everything a second in-process round
+//! reuses — worker pretoken caches, allocator arenas, faulted pages, grown
+//! buffer capacities — makes later rounds unrealistically fast. Restart the
+//! binary to collect independent samples; ENCODE_ROUNDS>1 remains available
+//! for cache-behavior experiments.
+//!
+//! Run with: cargo bench --bench encode_doc              (2 GB default)
+//!           ENCODE_MB=500 cargo bench --bench encode_doc
+//!           TOKENIZER_JSON=data/qwen3_5_tokenizer.json cargo bench --bench encode_doc
+
+use gigatoken_rs::load_tokenizer::hf::load_hf_bpe;
+use gigatoken_rs::{WorkerPool, encode_docs_ragged};
+use std::hint::black_box;
+use std::path::PathBuf;
+use std::time::Instant;
+
+mod common;
+
+const DEFAULT_MB: usize = 2000;
+
+fn main() {
+    common::allow_thp();
+    let tokenizer_json = std::env::var("TOKENIZER_JSON")
+        .unwrap_or_else(|_| "data/olmo3_tokenizer.json".to_string());
+    let tokenizer_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(&tokenizer_json);
+    eprintln!("Loading tokenizer from {tokenizer_path:?}...");
+    let tokenizer = load_hf_bpe(&tokenizer_path).expect("Could not load tokenizer");
+
+    let input = common::load_owt_input(Some(DEFAULT_MB));
+    let size_mb = input.len() as f64 / 1e6;
+    eprintln!("1 document, {} threads\n", rayon::current_num_threads());
+
+    let rounds: usize = std::env::var("ENCODE_ROUNDS")
+        .ok()
+        .and_then(|v| v.trim().parse().ok())
+        .unwrap_or(1);
+    for round in 0..rounds {
+        // Fresh worker pool per round so extra rounds at least start with
+        // cold pretoken caches (the pool retains one forked tokenizer per
+        // rayon thread).
+        let workers = WorkerPool::new();
+        let t0 = Instant::now();
+        let (flat, lens) = encode_docs_ragged(&workers, &tokenizer, &[&input]);
+        black_box((&flat, lens));
+        let elapsed = t0.elapsed().as_secs_f64();
+        eprintln!(
+            "round {round}: {} tokens in {elapsed:.3}s — {:.0} MB/s",
+            flat.len(),
+            size_mb / elapsed
+        );
+    }
+}

data/benches/encode_st.rs

@@ -0,0 +1,105 @@
+use gigatoken_rs::load_tokenizer::hf::{HfTokenizer, load_hf_slice};
+use gigatoken_rs::pretokenize::FastR50kPretokenizer;
+use std::path::PathBuf;
+use std::time::Instant;
+
+mod common;
+fn main() {
+    common::allow_thp();
+    // ENCODE_TOKENIZER overrides the tokenizer.json: a local path (e.g.
+    // data/qwen3_tokenizer.json to bench the qwen2-scheme encode path) or a
+    // HuggingFace repo id (e.g. Qwen/Qwen2-1.5B-Instruct), served from the
+    // standard HF cache and downloaded into it on a miss. Encoding then runs
+    // through the scheme dispatch instead of the hardcoded r50k pretokenizer.
+    let tokenizer_override = std::env::var("ENCODE_TOKENIZER").ok().map(|value| {
+        let path = PathBuf::from(&value);
+        if !path.exists() && gigatoken_rs::load_tokenizer::hub::looks_like_repo_id(&value) {
+            gigatoken_rs::load_tokenizer::hub::hub_file(&value, "tokenizer.json", "main")
+                .expect("Could not fetch tokenizer.json from the HuggingFace Hub")
+        } else {
+            path
+        }
+    });
+    let tokenizer_path = tokenizer_override.clone().unwrap_or_else(|| {
+        PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("data/gpt2_tokenizer.json")
+    });
+    eprintln!("Loading tokenizer from {tokenizer_path:?}...");
+    let data = std::fs::read(&tokenizer_path).expect("Could not read tokenizer.json");
+    // byte_fallback configs (Llama/gemma-style SentencePiece) dispatch to the
+    // SP encode path; everything else to ByteLevel BPE, as in benches/encode.
+    let tokenizer = load_hf_slice(&data).expect("Could not load tokenizer");
+
+    let input = common::load_owt_input(None);
+    let size_gb = input.len() as f64 / 1e9;
+    // Encode the whole buffer in one pass (matches real usage; the pretokenizer
+    // handles newlines itself, so pre-splitting into lines is unnecessary).
+    let buf: &[u8] = &input;
+
+    // ENCODE_PASSES=N re-encodes the same buffer N times; passes after the
+    // first run with a fully warm pretoken cache, isolating the hit path.
+    let passes: usize = std::env::var("ENCODE_PASSES")
+        .ok()
+        .and_then(|v| v.trim().parse().ok())
+        .unwrap_or(1);
+    eprintln!("Encoding (single-threaded)...");
+    // Flat output buffer, reserved once from the batch engine's
+    // bytes-per-token estimate (see batch::encode_chunk) and reused across
+    // passes; the token count is its length.
+    let mut out: Vec<u32> = Vec::with_capacity(input.len() / 4 + 16);
+    common::madvise_hugepage_capacity(&mut out);
+    // ENCODE_DUMP=path writes the final pass's tokens as little-endian u32s,
+    // for byte-exact A/B comparison across encode-path changes.
+    let dump_path = std::env::var("ENCODE_DUMP").ok();
+    match tokenizer {
+        HfTokenizer::Bpe(mut tokenizer) => {
+            for pass in 0..passes {
+                out.clear();
+                let start = Instant::now();
+                if tokenizer_override.is_some() {
+                    let pretokens = tokenizer.pretokenizer_type().pretokenize(buf);
+                    tokenizer.memoized_encode_flat(pretokens, &mut out);
+                } else {
+                    tokenizer.memoized_encode_flat(FastR50kPretokenizer::new(buf), &mut out);
+                }
+                report_pass(pass, out.len(), size_gb, start);
+            }
+            let (sl, sc, ll, lc, lkb, al, ac) = tokenizer.cache_mem_stats();
+            eprintln!(
+                "cache: short {sl} entries (cap {sc}), long {ll} (cap {lc}, {lkb} key bytes), arena {al} tokens (cap {ac})"
+            );
+        }
+        HfTokenizer::SentencePiece(tokenizer) => {
+            // Trailing partial UTF-8 char (from the ENCODE_MB byte cap) is
+            // dropped; SP encoding takes str input.
+            let text = match std::str::from_utf8(buf) {
+                Ok(text) => text,
+                Err(e) => std::str::from_utf8(&buf[..e.valid_up_to()]).unwrap(),
+            };
+            let mut state = gigatoken_rs::EncodeState::new();
+            for pass in 0..passes {
+                out.clear();
+                let start = Instant::now();
+                let out_ref = &mut out;
+                tokenizer.encode_raw_cb(&mut state, text, &mut |tokens| {
+                    out_ref.extend(tokens.iter().map(|t| t.0))
+                });
+                report_pass(pass, out.len(), size_gb, start);
+            }
+            eprintln!("cache: {} units", state.cache_size());
+        }
+    }
+    if let Some(path) = dump_path {
+        let bytes: Vec<u8> = out.iter().flat_map(|t| t.to_le_bytes()).collect();
+        std::fs::write(&path, bytes).expect("Could not write ENCODE_DUMP");
+        eprintln!("dumped {} tokens to {path}", out.len());
+    }
+}
+
+fn report_pass(pass: usize, total_tokens: usize, size_gb: f64, start: Instant) {
+    let elapsed = start.elapsed().as_secs_f64();
+    let throughput_gb = size_gb / elapsed;
+    eprintln!(
+        "pass {pass}: {total_tokens} tokens in {elapsed:.2}s — {throughput_gb:.2} GB/s ({:.0} MB/s)",
+        throughput_gb * 1000.0
+    );
+}

data/benches/encode_st_sp.rs

@@ -0,0 +1,51 @@
+//! Single-threaded SentencePiece encode throughput on OWT, mirroring
+//! `encode_st` (GPT-2 byte-level) for a like-for-like comparison.
+//!
+//! Select the tokenizer with SP_TOKENIZER=tinyllama (default) or sp4096;
+//! cap the input with ENCODE_MB like `encode_st`.
+
+use gigatoken_rs::load_tokenizer::hf::load_hf_sentencepiece;
+use std::path::PathBuf;
+use std::time::Instant;
+
+mod common;
+
+fn main() {
+    let which = std::env::var("SP_TOKENIZER").unwrap_or_else(|_| "tinyllama".to_string());
+    let file = match which.as_str() {
+        "tinyllama" => "data/tinyllama_tokenizer.json",
+        "sp4096" => "data/fineweb_4096_bpe_tokenizer.json",
+        other => panic!("unknown SP_TOKENIZER {other:?}"),
+    };
+    let tokenizer_path = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join(file);
+    eprintln!("Loading {which} tokenizer from {tokenizer_path:?}...");
+    let tokenizer = load_hf_sentencepiece(&tokenizer_path).expect("Could not load tokenizer");
+
+    let input = common::load_owt_input(None);
+    let size_gb = input.len() as f64 / 1e9;
+    let text = std::str::from_utf8(&input).expect("input must be UTF-8");
+
+    eprintln!("Encoding (single-threaded)...");
+    // Count-only callback, mirroring encode_st's measurement of the GPT-2
+    // path (the full encode runs; only output materialization is skipped).
+    let passes: usize = std::env::var("SP_PASSES")
+        .ok()
+        .and_then(|v| v.parse().ok())
+        .unwrap_or(2);
+    let mut state = gigatoken_rs::EncodeState::new();
+    for pass in 1..=passes {
+        let mut total_tokens: usize = 0;
+        let start = Instant::now();
+        tokenizer.encode_raw_cb(&mut state, text, &mut |tokens: &[_]| {
+            total_tokens += tokens.len();
+        });
+        let elapsed = start.elapsed().as_secs_f64();
+        let throughput_gb = size_gb / elapsed;
+        eprintln!(
+            "pass {pass} (cache {}): {total_tokens} tokens in {elapsed:.2}s — {throughput_gb:.2} GB/s ({:.0} MB/s), {} cached units",
+            if pass == 1 { "cold" } else { "warm" },
+            throughput_gb * 1000.0,
+            state.cache_size()
+        );
+    }
+}

data/benches/pretokenize.rs

@@ -0,0 +1,151 @@
+use criterion::{criterion_group, criterion_main, Criterion, Throughput};
+#[cfg(all(
+    target_arch = "x86_64",
+    target_feature = "avx512bw",
+    target_feature = "avx512vl"
+))]
+use gigatoken_rs::pretokenize::reference::avx512::Avx512PretokenizerIter;
+use gigatoken_rs::pretokenize::{
+    reference::combinator::pretokens_iterator, FastCl100kPretokenizer, FastQwen2Pretokenizer,
+    FastQwen35Pretokenizer, FastR50kPretokenizer, PretokenizerIter,
+};
+use gigatoken_rs::pretokenize::reference::simd::SimdPretokIter;
+use std::hint::black_box;
+
+const TARGET_BENCH_SIZE: usize = 100_000_000; // ~100 MB
+
+/// Load OWT data, truncated to a UTF-8-safe boundary near `max_bytes`.
+fn load_owt(max_bytes: usize) -> Vec<u8> {
+    let data_dir = std::env::home_dir().unwrap().join("data");
+    let all_bytes =
+        std::fs::read(data_dir.join("owt_train.txt")).expect("Could not read ~/data/owt_train.txt");
+    let mut end = max_bytes.min(all_bytes.len());
+    // Back up to a UTF-8 character boundary
+    while end > 0 && !std::str::from_utf8(&all_bytes[..end]).is_ok() {
+        end -= 1;
+    }
+    all_bytes[..end].to_vec()
+}
+
+fn pretokenize_benches(c: &mut Criterion) {
+    let input = load_owt(TARGET_BENCH_SIZE);
+    let input_len = input.len() as u64;
+    eprintln!("Benchmark input size: {:.1} MB", input_len as f64 / 1e6);
+
+    let mut group = c.benchmark_group("pretokenize");
+    group.throughput(Throughput::Bytes(input_len));
+    group.sample_size(10);
+
+    group.bench_function("state_machine", |b| {
+        b.iter(|| {
+            let count = PretokenizerIter::new(&input).count();
+            black_box(count);
+        });
+    });
+
+    group.bench_function("winnow", |b| {
+        b.iter(|| {
+            let mut input_str = unsafe { std::str::from_utf8_unchecked(&input) };
+            let count = pretokens_iterator(&mut input_str).count();
+            black_box(count);
+        });
+    });
+
+    #[cfg(all(
+        target_arch = "x86_64",
+        target_feature = "avx512bw",
+        target_feature = "avx512vl"
+    ))]
+    group.bench_function("avx512", |b| {
+        b.iter(|| {
+            let mut iter = Avx512PretokenizerIter::new(&input);
+            let mut count = 0;
+            while iter.next().is_some() {
+                count += 1;
+            }
+            black_box(count);
+        });
+    });
+    group.bench_function("simd", |b| {
+        b.iter(|| {
+            let count = SimdPretokIter::new(&input).count();
+            black_box(count);
+        });
+    });
+
+    group.bench_function("fast_scalar", |b| {
+        b.iter(|| {
+            let mut iter = FastR50kPretokenizer::new(&input);
+            let mut count = 0;
+            while iter.next().is_some() {
+                count += 1;
+            }
+            black_box(count);
+        });
+    });
+
+    group.bench_function("cl100k_fast_scalar", |b| {
+        b.iter(|| {
+            let mut iter = FastCl100kPretokenizer::new(&input);
+            let mut count = 0;
+            while iter.next().is_some() {
+                count += 1;
+            }
+            black_box(count);
+        });
+    });
+
+    group.bench_function("qwen2_fast_scalar", |b| {
+        b.iter(|| {
+            let mut iter = FastQwen2Pretokenizer::new(&input);
+            let mut count = 0;
+            while iter.next().is_some() {
+                count += 1;
+            }
+            black_box(count);
+        });
+    });
+
+    group.bench_function("qwen3_5_fast_scalar", |b| {
+        b.iter(|| {
+            let mut iter = FastQwen35Pretokenizer::new(&input);
+            let mut count = 0;
+            while iter.next().is_some() {
+                count += 1;
+            }
+            black_box(count);
+        });
+    });
+
+    let re = fancy_regex::Regex::new(
+        r"'(?:[sdmt]|ll|ve|re)| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+",
+    )
+    .unwrap();
+
+    group.bench_function("regex", |b| {
+        b.iter(|| {
+            let text = unsafe { std::str::from_utf8_unchecked(&input) };
+            let count = re.find_iter(text).count();
+            black_box(count);
+        });
+    });
+
+    // Backtracking-compatible equivalent of the possessive cl100k pattern
+    let re_cl100k = fancy_regex::Regex::new(
+        r"'(?i:[sdmt]|ll|ve|re)|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\s\p{L}\p{N}]+[\r\n]*|\s+$|\s*[\r\n]|\s+(?!\S)|\s+",
+    )
+    .unwrap();
+
+    group.bench_function("cl100k_regex", |b| {
+        b.iter(|| {
+            let text = unsafe { std::str::from_utf8_unchecked(&input) };
+            let count = re_cl100k.find_iter(text).count();
+            black_box(count);
+        });
+    });
+
+    group.finish();
+}
+
+criterion_group!(benches, pretokenize_benches);
+criterion_main!(benches);