gigatoken 0.1.0

data/Cargo.toml

@@ -0,0 +1,135 @@
+[workspace]
+members = [".", "ext/gigatoken"]
+
+[package]
+name = "gigatoken"
+version = "0.9.0"
+edition = "2024"
+license = "MIT"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+[lib]
+name = "gigatoken_rs"
+crate-type = ["cdylib", "lib"]
+
+[dependencies]
+dashmap = "6.1"
+indicatif = "0.18"
+itertools = "0.15"
+priority-queue = "2.0"
+# For now uses abi3-py310, should use version-specific ABI in the future
+pyo3 = { version = "0.29", features = ["abi3-py310", "eyre"], optional = true }
+numpy = { version = "0.29", optional = true }
+rayon = "1.11"
+rustc-hash = "2.1"
+memmap2 = { version = "0.9", features = ["stable_deref_trait"] }
+base64 = "0.22"
+icu = { version = "2.2", features = ["compiled_data", "datagen"] }
+eyre = "0.6.12"
+sonic-rs = "0.5.8"
+memchr = "2.8.0"
+aho-corasick = "1.1"
+serde = { version = "1.0.228", features = ["derive"] }
+flate2 = "1.1"
+zstd = "0.13"
+spm_precompiled = "0.1.4"
+simdutf = "0.7.0"
+winnow = { version = "1", features = ["simd"] }
+parquet = { version = "59.1.0", default-features = false, features = [
+    "arrow",
+    "snap",
+    "zstd",
+    "flate2",
+    "lz4",
+    "brotli",
+    "flate2-rust_backend",
+] }
+arrow-array = "59"
+arrow-schema = "59"
+ureq = { version = "3.3.0", default-features = false, features = ["rustls"] }
+
+# madvise/prctl on Linux (pretoken cache table, batch gather, bench THP
+# setup); already in the tree transitively via memmap2/pyo3.
+[target.'cfg(target_os = "linux")'.dependencies]
+libc = "0.2"
+
+[features]
+default = ["python"]
+python = ["dep:pyo3", "dep:numpy"]
+
+[profile.release]
+lto = "fat"
+
+# Profiling parity: benches keep the release codegen (fat LTO, same inlining)
+# but carry full debug info so profilers can resolve inlined frames and source
+# lines. Debug info does not change generated code; parity is verified by
+# comparing bench throughput against a stripped release build (within noise).
+[profile.bench]
+inherits = "release"
+debug = true
+strip = "none"
+split-debuginfo = "packed"
+
+[profile.profiling]
+inherits = "release"
+debug = 2
+lto = false
+codegen-units = 1
+strip = "none"
+split-debuginfo = "packed"
+# Frame pointers make Instruments' stack unwinding reliable. Mangling is left at
+# the toolchain default (v0); `cargo instruments` demangles v0 symbols itself.
+rustflags = ["-C", "force-frame-pointers=yes"]
+
+[profile.profiling.package."*"]
+debug = 2
+strip = "none"
+
+[dev-dependencies]
+criterion = "0.8"
+fancy-regex = "0.18.0"
+rand = "0.10"
+simdutf = "0.7.0"
+tempfile = "3.27.0"
+
+
+[[bench]]
+name = "pretokenize"
+harness = false
+
+[[bench]]
+name = "encode"
+harness = false
+test = false
+
+[[bench]]
+name = "encode_st"
+harness = false
+test = false
+
+[[bench]]
+name = "encode_st_sp"
+harness = false
+test = false
+
+[[bench]]
+name = "encode_doc"
+harness = false
+test = false
+
+[[bench]]
+name = "pretokenize_profile"
+harness = false
+test = false
+
+[[bench]]
+name = "pretokenize_profile_all"
+harness = false
+test = false
+
+[[bench]]
+name = "simdutf_transcode"
+harness = false
+
+[lints.rust]
+unused = "allow"

data/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2026 Marcel Rød
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -0,0 +1,141 @@
+# gigatoken-rb
+
+**12 GB/s / 2.8 billion tokens per second in Ruby.**
+
+Ruby bindings for [marcelroed/gigatoken](https://github.com/marcelroed/gigatoken), the fastest open-source BPE tokenizer around — running **1.6x faster than upstream's own Python package**, on the same Rust engine.
+
+| | Corpus | MB/s (median) | Gtok/s (median) |
+|---|---|---|---|
+| **gigatoken** (this gem, Ruby) | 11.9 GB | **12,278** | **2.78** |
+| gigatoken (Python wheel, upstream) | 11.9 GB | 7,400 | 1.68 |
+| tiktoken (Python) | 1.35 GB | 69.7 | 0.0158 |
+| tiktoken_ruby | 1.35 GB | 30.7 | 0.0070 |
+| tokenizers gem (ankane) | 1.35 GB | 10.0 | 0.0023 |
+| tokenizers (Python, Hugging Face) | 1.35 GB | 5.6 | 0.0013 |
+
+Mac Studio M4 Max, OpenWebText, GPT-2 tokenizer; every library produces the same tokenization, gigatoken just does it faster. **340x faster** than the fastest existing Ruby gem (tiktoken_ruby) and **1,050x faster** than the tokenizers gem.  Full methodology, exact counts, and the caveats that matter: [docs/rb/benchmarks.md](docs/rb/benchmarks.md).
+
+## Install
+
+```bash
+gem install gigatoken
+```
+
+Precompiled native gems ship for Apple Silicon macOS (`arm64-darwin`) and x86_64/aarch64 Linux — on those platforms RubyGems grabs the binary automatically, no Rust toolchain, no compile wait. In a Bundler project it's one command:
+
+```bash
+bundle add gigatoken
+```
+
+(or drop `gem "gigatoken"` into the Gemfile yourself).
+
+Anywhere else (or with `--platform ruby` to opt out of the binary), the extension builds from source. That needs a Rust toolchain: `rust-toolchain.toml` pins the nightly, and `rustup` fetches it automatically on first build.
+
+## Use
+
+```ruby
+require "gigatoken"
+
+tok = Gigatoken::Tokenizer.load("openai-community/gpt2")
+
+tok.encode("Hello, world!")             # => [15496, 11, 995, 0]
+tok.decode([15496, 11, 995, 0])         # => "Hello, world!"
+tok.encode_batch(["Hello!", "Another"]) # => [[15496, 0], [6610]]
+
+tok.vocab_size                          # => 50257
+tok.special_tokens                      # => {"<|endoftext|>" => 50256}
+```
+
+`load` takes a `tokenizer.json` path, a directory holding one, a HuggingFace Hub repo id, or a `.tiktoken` mergeable-ranks file, and dispatches on shape. Hub downloads run over socketry's `async-http` — no Python anywhere. Know what you have? Skip the dispatch:
+
+```ruby
+Gigatoken::Tokenizer.from_file("tokenizer.json")
+Gigatoken::Tokenizer.from_hub("openai-community/gpt2", revision: "main")
+Gigatoken::Tokenizer.from_tiktoken("vocab.tiktoken")
+Gigatoken::Tokenizer.from_json(File.binread("tokenizer.json"))
+```
+
+SentencePiece-BPE models (Llama, Gemma, Mistral — any `tokenizer.json` with `byte_fallback: true`) load through the same entry points and pick the right backend automatically. One difference: the SentencePiece core decodes text, so it validates input and raises `Gigatoken::Error` on invalid UTF-8 instead of guessing.
+
+### Tokenize files without leaving Rust
+
+`encode_files` reads and tokenizes files entirely on the native side — document contents never materialize as Ruby objects. `.gz` and `.zst` decompress transparently.
+
+```ruby
+tok.encode_files("owt_train.txt", separator: "<|endoftext|>")
+
+jsonl   = Gigatoken::Native::JsonlFileSource.new(["docs.jsonl"], field: "text")
+parquet = Gigatoken::Native::ParquetFileSource.new(["docs.parquet"], column: "text")
+tok.encode_files(jsonl)
+```
+
+### Packed results
+
+Pass `packed: true` to `encode_batch` or `encode_files` and results land in a single `IO::Buffer` of u32 token ids instead of a ragged Array of Arrays — no per-token Ruby allocation, the fastest way out of the engine:
+
+```ruby
+packed = tok.encode_files("owt_train.txt", packed: true, separator: "<|endoftext|>")
+
+packed.buffer       # => one IO::Buffer, every document's ids back to back
+packed.lens         # => [12, 8, 41, ...] tokens per document
+packed.token_count  # => total tokens
+packed[3]           # => document 3's ids as an Array, on demand
+```
+
+### Async
+
+`encode_batch` and `encode_files` release the GVL for the whole encode; the parallelism runs on the engine's rayon pool, not Ruby threads. Under `Async`, give the fiber scheduler a worker pool (`ASYNC_SCHEDULER_WORKER_POOL=true`) and the calling fiber yields to the reactor too. Design notes: [docs/rb/async.md](docs/rb/async.md).
+
+## CLI
+
+```bash
+gigatoken bench openai-community/gpt2 owt_train.txt --doc-separator "<|endoftext|>"
+gigatoken validate openai-community/gpt2 owt_train.txt --doc-separator "<|endoftext|>"
+```
+
+`bench` reports MB/s and Mtok/s (`--packed` for the fused packed path, `--no-parallel` for the serial core). `validate` confirms native split-and-encode agrees with a Ruby-side split through `encode_batch`.
+
+## Development
+
+```bash
+bundle install
+bundle exec rake compile    # native extension (Rust nightly, via rust-toolchain.toml)
+bundle exec rspec
+bundle exec standardrb
+```
+
+The Ruby layer is fiber-first throughout — no `Thread`, no `Mutex`; all parallelism lives in the core's rayon pool. CI runs ubuntu + macos × Ruby 3.3/3.4/4.0, and `release.yml` cross-builds the precompiled native gems (arm64-darwin, x86_64-linux, aarch64-linux).
+
+## Fork status
+
+This fork exists because I need fast tokenization in Ruby. The Rust core is changed as little as possible from upstream. Most of the python shell has been removed from this fork, but you can still find it [upstream](https://github.com/marcelroed/gigatoken).
+
+Not ported/no current plans:
+- the HF/tiktoken Python compat shims
+- padded-batch matrices
+- and BPE training
+
+SentencePiece works but — matching upstream — is less optimized than the BPE path.
+
+## Citation
+
+The engine is Marcel Rød's gigatoken. If it shows up in your research, cite that:
+
+```bibtex
+@software{roed2026gigatoken,
+  author = {Marcel R{\o}d},
+  title = {{G}igatoken: SIMD and Cache Hierarchies for 1000x Faster Byte-Pair Encoding Tokenization on Modern CPUs},
+  url = {https://github.com/marcelroed/gigatoken},
+  year = {2026},
+}
+```
+
+---
+
+<details>
+<summary>AI Use Disclosure</summary>
+
+The Rust engine is upstream's — see <a href="https://github.com/marcelroed/gigatoken#readme">upstream's AI-use disclosure</a> for how that was built (majority hand-crafted, AI-assisted toward the end).
+
+The Ruby port in this fork is 100% AI generated using Fable 5 and Sonnet 5 via [space-architect](https://github.com/jetpks/space-architect) over ~24 hours.
+</details>

data/exe/gigatoken

@@ -0,0 +1,9 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# The packed path uses IO::Buffer, which warns as experimental.
+Warning[:experimental] = false
+
+require "gigatoken/cli"
+
+Dry::CLI.new(Gigatoken::CLI::Commands).call

data/ext/gigatoken/Cargo.toml

@@ -0,0 +1,24 @@
+[package]
+name = "gigatoken-rb"
+version = "0.1.0"
+edition = "2021"
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+# `rb-sys`: lets us reach `magnus::rb_sys::{AsRawValue, protect}` for the raw
+# rb-sys VALUE conversions and locktmp calls this crate needs directly (see
+# tokenizer.rs) that magnus's safe API doesn't expose.
+magnus = { version = "0.8", features = ["rb-sys"] }
+gigatoken = { path = "../..", default-features = false }
+rayon = "1.11"
+mimalloc = { version = "0.1", default-features = false }
+# Same version requirement and feature set magnus-0.8.2 already puts on its
+# own rb-sys dependency (see magnus-0.8.2/Cargo.toml), so Cargo resolves both
+# to the one rb-sys already in Cargo.lock rather than a second copy.
+rb-sys = { version = ">=0.9.113", default-features = false, features = [
+    "bindgen-rbimpls",
+    "bindgen-deprecated-types",
+    "stable-api",
+] }

data/ext/gigatoken/extconf.rb

@@ -0,0 +1,19 @@
+require "mkmf"
+require "rb_sys/mkmf"
+
+# Hide install_name_tool from rb_sys's fixup_libnames (rb_sys-0.9.128's
+# mkmf.rb): on some hosts `install_name_tool -id "" $(DLLIB)` corrupts the
+# compiled bundle's LC_ID_DYLIB load command (dyld then refuses to load it:
+# "load command #N string extends beyond end of load command"), reproduced
+# independent of this crate on a trivial unrelated cdylib. Skipping the
+# rewrite leaves the bundle's original (absolute) install name in place,
+# which `Kernel#require`'s dlopen does not care about.
+def find_executable(bin, path = nil)
+  return nil if bin == "install_name_tool"
+  super
+end
+
+# Target basename must match the compiled artifact: the crate's [package]
+# name is "gigatoken-rb" (the workspace root package already owns
+# "gigatoken"), so Cargo's default lib name is "gigatoken_rb".
+create_rust_makefile("gigatoken/gigatoken_rb")

data/ext/gigatoken/src/error.rs

@@ -0,0 +1,20 @@
+//! `Gigatoken::Error` is defined in Ruby (`lib/gigatoken.rb`), loaded before
+//! this extension is required — looked up fresh at each raise site rather
+//! than cached, so no magnus `Value` needs a GC-registered static home.
+
+use magnus::{exception::ExceptionClass, prelude::*, Error, RModule, Ruby};
+
+fn error_class(ruby: &Ruby) -> Result<ExceptionClass, Error> {
+    ruby.class_object()
+        .const_get::<_, RModule>("Gigatoken")?
+        .const_get("Error")
+}
+
+/// Raise `Gigatoken::Error` with `message` (core load/encode failures surface
+/// through this — never a Rust panic across the Ruby boundary).
+pub fn raise(ruby: &Ruby, message: impl Into<String>) -> Error {
+    match error_class(ruby) {
+        Ok(class) => Error::new(class, message.into()),
+        Err(e) => e,
+    }
+}

data/ext/gigatoken/src/gvl.rs

@@ -0,0 +1,122 @@
+//! Release Ruby's GVL (global VM lock) for the duration of a CPU-bound Rust
+//! closure, so other Ruby threads/fibers keep running while the core worker
+//! pool chews through a batch encode.
+//!
+//! Uses `rb_sys::rb_nogvl` (this crate's direct `rb-sys` dependency, the same
+//! version magnus already resolves — see `ext/gigatoken/Cargo.toml`) with the
+//! `RB_NOGVL_OFFLOAD_SAFE` flag, rather than plain `rb_thread_call_without_gvl`.
+//! Under a `Fiber.scheduler` built with a worker pool (`Async::Scheduler.new`
+//! with `ASYNC_SCHEDULER_WORKER_POOL=true` or an explicit `worker_pool:`,
+//! `~/architect/src/github.com/socketry/async/lib/async/scheduler.rb:31-101`),
+//! `OFFLOAD_SAFE` tells Ruby it's safe to hand `func`/`data1` to the
+//! scheduler's `blocking_operation_wait` hook instead of just blocking this OS
+//! thread: the scheduler runs the closure on an `IO::Event::WorkerPool`
+//! background thread while transferring the calling *fiber* (not just other
+//! Threads) back to the reactor, so other fibers on this thread keep running
+//! for the encode's duration
+//! (`~/architect/src/github.com/socketry/io-event/ext/io/event/worker_pool.c:309-316`).
+//! Without such a scheduler (or with one lacking a worker pool), `rb_nogvl`
+//! degrades to exactly today's behavior: release the GVL, block this thread.
+//! See `docs/rb/async-design.md` and `docs/rb/async.md` for the full design
+//! and gotchas (worker pool is opt-in, defaults to one background worker).
+//!
+//! `func`/`data1` may now run on a different OS thread than the caller (the
+//! scheduler's worker pool), where `rb_thread_call_without_gvl` always ran
+//! them on the calling thread itself. `without_gvl` therefore requires
+//! `F: Send` (the closure crosses onto the worker thread) and `R: Send` (its
+//! result crosses back) — the compiler enforces the cross-thread contract
+//! instead of leaving it to call-site audit. Every current `without_gvl`
+//! call site (`tokenizer.rs`'s `encode_batch`/`encode_files`) already shares
+//! its `&Tokenizer`/`&WorkerPool` across `rayon` worker threads inside
+//! `encode_docs_ragged`. The BPE batch paths' input marshal (`tokenizer.rs`'s
+//! `marshal_inputs`) borrows heap `RString` buffers zero-copy where it can
+//! (locked via `rb_str_locktmp`, or unlocked when the string is frozen) and
+//! falls back to an owned `Vec<u8>` copy otherwise — either way, only raw
+//! byte slices are ever captured by the closure passed here, never a Ruby
+//! `VALUE` or thread-local state, so running one OS thread over instead of
+//! another changes nothing about its safety, and the types involved satisfy
+//! `Send`/`Sync` on their own merits.
+
+use std::any::Any;
+use std::ffi::c_void;
+use std::os::raw::c_int;
+use std::panic::{self, AssertUnwindSafe};
+
+use rb_sys::rb_nogvl;
+
+/// `RB_NOGVL_OFFLOAD_SAFE` (`ruby/thread.h:84` in a current Ruby checkout;
+/// introduced by Ruby's `Fiber::Scheduler#blocking_operation_wait` support,
+/// first released in Ruby 3.4.0). Defined locally rather than taken from
+/// `rb_sys::` bindings: `rb-sys` bindgens its constants from the *building*
+/// Ruby's own headers, and this gem's floor (`gigatoken.gemspec`, `>=
+/// 3.3.0`) may build against 3.3 headers that don't declare this macro at
+/// all. Verified against Ruby 3.3.0's `thread.c` (`rb_nogvl`, ~line 1508):
+/// it only tests `RB_NOGVL_UBF_ASYNC_SAFE`/`RB_NOGVL_INTR_FAIL` against
+/// `flags`, so an unrecognized bit here is silently ignored — passing it on
+/// 3.3 degrades to plain blocking `rb_nogvl`, identical to the old
+/// `rb_thread_call_without_gvl`. No `RUBY_VERSION` gate is needed.
+const RB_NOGVL_OFFLOAD_SAFE: c_int = 0x4;
+
+/// The result of running `f` inside `call_without_gvl`: either its return
+/// value, or a caught panic payload to re-raise once we're back on ordinary
+/// (non-`extern "C"`) Rust stack frames. Unwinding a panic directly across
+/// the `extern "C"` trampoline `rb_nogvl` calls into is undefined behavior;
+/// catching it here and resuming it from `without_gvl` below turns that into
+/// an ordinary Rust panic, which magnus's own `method!`/`function!` call
+/// trampolines already wrap in `catch_unwind` and convert into a fatal Ruby
+/// exception (`magnus::error::Error::from_panic`) — the same outcome any
+/// other panicking native method already gets, just carried safely across
+/// the extra C boundary this one call adds.
+enum Outcome<R> {
+    Value(R),
+    Panic(Box<dyn Any + Send + 'static>),
+}
+
+unsafe extern "C" fn call_without_gvl<F, R>(arg: *mut c_void) -> *mut c_void
+where
+    F: FnOnce() -> R + Send,
+    R: Send,
+{
+    // SAFETY: `arg` is the `*mut Option<F>` handed to `rb_nogvl` below, valid
+    // for the duration of that (synchronous) call, and this is the only
+    // place it's dereferenced.
+    let closure = unsafe { (*(arg as *mut Option<F>)).take() }
+        .expect("without_gvl callback invoked more than once");
+    let outcome = match panic::catch_unwind(AssertUnwindSafe(closure)) {
+        Ok(value) => Outcome::Value(value),
+        Err(payload) => Outcome::Panic(payload),
+    };
+    Box::into_raw(Box::new(outcome)) as *mut c_void
+}
+
+/// Run `f` with the GVL released: other Ruby threads may run while `f`
+/// executes, and — under a fiber scheduler with a worker pool — the calling
+/// fiber yields to the reactor while `f` runs on a background thread. `f`
+/// must not touch any Ruby object (`VALUE`) — only plain Rust data — per the
+/// Ruby C API's contract for this call. A panic inside `f` is caught and
+/// re-raised here rather than left to unwind across the C trampoline.
+pub fn without_gvl<F, R>(f: F) -> R
+where
+    F: FnOnce() -> R + Send,
+    R: Send,
+{
+    let mut slot = Some(f);
+    let arg = &mut slot as *mut Option<F> as *mut c_void;
+    let result = unsafe {
+        rb_nogvl(
+            Some(call_without_gvl::<F, R>),
+            arg,
+            None,
+            std::ptr::null_mut(),
+            RB_NOGVL_OFFLOAD_SAFE,
+        )
+    };
+    // SAFETY: `result` is the `Box::into_raw(Box::new(Outcome<R>))` pointer
+    // produced by the callback above, which always runs exactly once before
+    // `rb_nogvl` returns.
+    let outcome = *unsafe { Box::from_raw(result as *mut Outcome<R>) };
+    match outcome {
+        Outcome::Value(value) => value,
+        Outcome::Panic(payload) => panic::resume_unwind(payload),
+    }
+}

data/ext/gigatoken/src/lib.rs

@@ -0,0 +1,50 @@
+use gigatoken_rs::load_tokenizer::hf::{self, HfTokenizer};
+use magnus::{Error, Module, RString, Ruby, Value, function};
+
+// XZM-WORKAROUND: macOS 26's xzm malloc zone SIGTRAPs on multi-GB Rust chunk
+// frees (`_xzm_reclaim_mark_used_locked` assertion); routing Rust allocations
+// through mimalloc avoids the xzm zone entirely.
+#[global_allocator]
+static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
+
+mod error;
+mod gvl;
+mod sentencepiece;
+mod sources;
+mod tokenizer;
+
+use error::raise;
+use sentencepiece::SentencePieceTokenizer;
+use tokenizer::BPETokenizer;
+
+// The gigatoken core crate exposes no version constant of its own, so this
+// is the ext crate's (gigatoken-rb's) version — see the builder report.
+fn crate_version() -> String {
+    env!("CARGO_PKG_VERSION").to_string()
+}
+
+/// Load a tokenizer from in-memory HuggingFace `tokenizer.json` contents.
+/// Returns a `SentencePieceTokenizer` when the model uses `byte_fallback`, a
+/// `BPETokenizer` otherwise — the same split as pyo3's `load_hf_json` and
+/// the two classes' own `from_hf_json` constructors.
+fn load_hf_json(ruby: &Ruby, data: RString) -> Result<Value, Error> {
+    // SAFETY: read-only, for the duration of this synchronous call.
+    let bytes = unsafe { data.as_slice() };
+    match hf::load_hf_slice(bytes) {
+        Ok(HfTokenizer::Bpe(tokenizer)) => Ok(ruby.into_value(BPETokenizer::from_tokenizer(tokenizer))),
+        Ok(HfTokenizer::SentencePiece(tokenizer)) => Ok(ruby.into_value(SentencePieceTokenizer::from_tokenizer(tokenizer))),
+        Err(e) => Err(raise(ruby, e.to_string())),
+    }
+}
+
+#[magnus::init]
+fn init(ruby: &Ruby) -> Result<(), Error> {
+    let gigatoken = ruby.define_module("Gigatoken")?;
+    let native = gigatoken.define_module("Native")?;
+    native.define_module_function("crate_version", function!(crate_version, 0))?;
+    native.define_module_function("load_hf_json", function!(load_hf_json, 1))?;
+    sources::init(ruby, native)?;
+    tokenizer::init(ruby, native)?;
+    sentencepiece::init(ruby, native)?;
+    Ok(())
+}