xberg 1.0.0.pre.rc.7 → 1.0.0.pre.rc.9

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8dd187e353e4926f840bca1b2fd8ac9aa099ff3b844580d4e325f78f1f20a547
-  data.tar.gz: 3d421a8ab9b4beb0539e4ba899af07c1ffea3b235c40d6ffc9633f6a80a6b69b
+  metadata.gz: c426c771373b092f070bb61bb9479785b38fce29336a8ccca7a99203715351f6
+  data.tar.gz: 5b206ba76956c60c6ff3eb3d567a8792fecf71be02203354dc9a5d99edb4359b
 SHA512:
-  metadata.gz: 8c640729c97cf6cc2a290b30823908cb46a3c74d2a032f03c222d8c6666e7a1c8451f0d957e0aa8b3a38fc47a4cc89eebfeb46361ede09323e5218e66af617cb
-  data.tar.gz: 8f6a497814c9215b7aeef1c326477e0b17ccfa2b0f30f10023f9488c99453ff51e7412299a957c8c7f99368e29773723e27ce9a01683f653792fdc9cc5495d35
+  metadata.gz: 1934ce27152766156b6446d41ee34e257469374ee6436f01c5a9b08e43fc35e505a07b32069d2b6a75071f7eda215d030e52e39bfcf9d0ee66cd857b773e303f
+  data.tar.gz: b0067920fc491c15012fa0117193d027081ac365d8b8b31c04eb92de7fd3d9ed34c88f019d47fc365620bbc8ff79e427cd54ee48ec6a6da93394ecdeab9f74f5

data/ext/xberg_rb/native/Cargo.lock

@@ -1486,9 +1486,9 @@ checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2"
 
 [[package]]
 name = "defmt"
-version = "1.1.0"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a6e524506490a1953d237cb87b1cfc1e46f88c18f10a22dfe0f507dc6bfc7f7f"
+checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1"
 dependencies = [
  "bitflags 1.3.2",
  "defmt-macros",
@@ -1496,12 +1496,11 @@ dependencies = [
 
 [[package]]
 name = "defmt-macros"
-version = "1.1.0"
+version = "1.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f0a27770e9c8f719a79d8b638281f4d828f77d8fd61e0bd94451b9b85e576a0b"
+checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8"
 dependencies = [
  "defmt-parser",
- "proc-macro-error2",
  "proc-macro2",
  "quote",
  "syn 2.0.118",
@@ -3174,6 +3173,36 @@ dependencies = [
  "windows-sys 0.45.0",
 ]
 
+[[package]]
+name = "jni"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5efd9a482cf3a427f00d6b35f14332adc7902ce91efb778580e180ff90fa3498"
+dependencies = [
+ "cfg-if",
+ "combine",
+ "jni-macros",
+ "jni-sys 0.4.1",
+ "log",
+ "simd_cesu8",
+ "thiserror 2.0.18",
+ "walkdir",
+ "windows-link",
+]
+
+[[package]]
+name = "jni-macros"
+version = "0.22.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a00109accc170f0bdb141fed3e393c565b6f5e072365c3bd58f5b062591560a3"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "rustc_version",
+ "simd_cesu8",
+ "syn 2.0.118",
+]
+
 [[package]]
 name = "jni-sys"
 version = "0.3.1"
@@ -3997,9 +4026,9 @@ dependencies = [
 
 [[package]]
 name = "num-bigint"
-version = "0.4.7"
+version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c863e9ab5e7bf9c99ba75e1050f1e4d624ae87ed3532d6238ffbdc7b585dbbe6"
+checksum = "c89e69e7e0f03bea5ef08013795c25018e101932225a656383bd384495ecc367"
 dependencies = [
  "num-integer",
  "num-traits",
@@ -4096,16 +4125,16 @@ checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3"
 
 [[package]]
 name = "office_oxide"
-version = "0.1.2"
+version = "0.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "866a8fa5e1a756f8cc9f9f614031ad7e33adc7f6972eab74a35b0c799fe08da5"
+checksum = "8900703278aea7d2329f5c490b62653306b1083efc7993ee18e3e86dcfd5afeb"
 dependencies = [
  "atoi_simd 0.18.1",
  "encoding_rs",
  "fast-float2",
  "libc",
  "log",
- "quick-xml 0.40.1",
+ "quick-xml 0.41.0",
  "serde",
  "serde_json",
  "thiserror 2.0.18",
@@ -4321,9 +4350,9 @@ dependencies = [
 
 [[package]]
 name = "pdf_oxide"
-version = "0.3.70"
+version = "0.3.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3be9e4752f2319dec7c079ca96ec141e1be70faac82b21cb06a53bb6c2e0034e"
+checksum = "3b7fe525ccb850888a423bd211c1efd1af704761a7084e83edae1b99d0c7d2b4"
 dependencies = [
  "aes 0.9.1",
  "base64 0.22.1",
@@ -4646,28 +4675,6 @@ dependencies = [
  "toml_edit",
 ]
 
-[[package]]
-name = "proc-macro-error-attr2"
-version = "2.0.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "96de42df36bb9bba5542fe9f1a054b8cc87e172759a1868aa05c1f3acc89dfc5"
-dependencies = [
- "proc-macro2",
- "quote",
-]
-
-[[package]]
-name = "proc-macro-error2"
-version = "2.0.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "11ec05c52be0a07b08061f7dd003e7d7092e0472bc731b4af7bb1ef876109802"
-dependencies = [
- "proc-macro-error-attr2",
- "proc-macro2",
- "quote",
- "syn 2.0.118",
-]
-
 [[package]]
 name = "proc-macro2"
 version = "1.0.106"
@@ -4824,16 +4831,6 @@ dependencies = [
  "memchr",
 ]
 
-[[package]]
-name = "quick-xml"
-version = "0.40.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2474bd2e5029e7ccb6abb2ba48cf2383a333851dedf495901544281590c7da7f"
-dependencies = [
- "memchr",
- "serde",
-]
-
 [[package]]
 name = "quick-xml"
 version = "0.41.0"
@@ -5290,7 +5287,7 @@ dependencies = [
  "quinn",
  "rustls",
  "rustls-pki-types",
- "rustls-platform-verifier",
+ "rustls-platform-verifier 0.7.0",
  "serde",
  "serde_json",
  "serde_urlencoded",
@@ -5493,6 +5490,15 @@ version = "2.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d"
 
+[[package]]
+name = "rustc_version"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cfcb3a22ef46e85b45de6ee7e79d063319ebb6594faafcf1c225ea92ab6e9b92"
+dependencies = [
+ "semver",
+]
+
 [[package]]
 name = "rustfft"
 version = "6.4.1"
@@ -5566,7 +5572,28 @@ checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784"
 dependencies = [
  "core-foundation 0.10.1",
  "core-foundation-sys",
- "jni",
+ "jni 0.21.1",
+ "log",
+ "once_cell",
+ "rustls",
+ "rustls-native-certs",
+ "rustls-platform-verifier-android",
+ "rustls-webpki",
+ "security-framework",
+ "security-framework-sys",
+ "webpki-root-certs",
+ "windows-sys 0.61.2",
+]
+
+[[package]]
+name = "rustls-platform-verifier"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "26d1e2536ce4f35f4846aa13bff16bd0ff40157cdb14cc056c7b14ba41233ba0"
+dependencies = [
+ "core-foundation 0.10.1",
+ "core-foundation-sys",
+ "jni 0.22.4",
  "log",
  "once_cell",
  "rustls",
@@ -5725,6 +5752,12 @@ dependencies = [
  "libc",
 ]
 
+[[package]]
+name = "semver"
+version = "1.0.28"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd"
+
 [[package]]
 name = "seq-macro"
 version = "0.3.6"
@@ -5953,6 +5986,16 @@ version = "0.3.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214"
 
+[[package]]
+name = "simd_cesu8"
+version = "1.1.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "94f90157bb87cddf702797c5dadfa0be7d266cdf49e22da2fcaa32eff75b2c33"
+dependencies = [
+ "rustc_version",
+ "simdutf8",
+]
+
 [[package]]
 name = "simd_helpers"
 version = "0.1.0"
@@ -7022,9 +7065,9 @@ checksum = "009994f150cc0cd50ff54917d5bc8bffe8cad10ca10d81c34da2ec421ae61782"
 
 [[package]]
 name = "tree-sitter-language-pack"
-version = "1.12.3"
+version = "1.12.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "32def7d253a2662db9a32196440b07bb0e9731a2133dbad0ed42239f94d548ea"
+checksum = "d49e30d1927ccc08599c054380bda399817781f74eb9184007e90606c7ff3a2c"
 dependencies = [
  "ahash 0.8.12",
  "cc",
@@ -7084,15 +7127,15 @@ checksum = "2896d95c02a80c6d6a5d6e953d479f5ddf2dfdb6a244441010e373ac0fb88971"
 
 [[package]]
 name = "unhwp"
-version = "0.5.1"
+version = "0.5.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4b0109dc52c6db66836c28af0d86f051b0a846f8c7a0a483fd9debd90f431279"
+checksum = "9db2add52da5275c7e850c5b2da1d8e580b223cd18a97807afcbd1026ffe3525"
 dependencies = [
  "cfb 0.10.0",
  "flate2",
  "pulldown-cmark 0.12.2",
  "pulldown-cmark-to-cmark",
- "quick-xml 0.37.5",
+ "quick-xml 0.41.0",
  "rayon",
  "regex",
  "serde",
@@ -7248,7 +7291,7 @@ dependencies = [
  "percent-encoding",
  "rustls",
  "rustls-pki-types",
- "rustls-platform-verifier",
+ "rustls-platform-verifier 0.6.2",
  "serde",
  "serde_json",
  "socks",
@@ -8020,9 +8063,9 @@ dependencies = [
 
 [[package]]
 name = "xberg"
-version = "1.0.0-rc.7"
+version = "1.0.0-rc.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae067a49ec38ff99d501d0e4bc745f5a1332dd22a86b3085083e219531249191"
+checksum = "627033553a27835485fee775b7b10cf8ed4e162f12d0df4e077ffb45846944f3"
 dependencies = [
  "ahash 0.8.12",
  "async-trait",
@@ -8138,9 +8181,9 @@ dependencies = [
 
 [[package]]
 name = "xberg-gliner"
-version = "1.0.0-rc.7"
+version = "1.0.0-rc.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2d29a5fcf85c2cc09f22be902f04c7e57e80d8902b044750293c7974c54c8cc"
+checksum = "86e43aeca6ee9066bb9a457c8f5649705b8e04d0e04645103ad60fdbdd8596b6"
 dependencies = [
  "ndarray 0.17.2",
  "ort",
@@ -8152,9 +8195,9 @@ dependencies = [
 
 [[package]]
 name = "xberg-libheif"
-version = "1.0.0-rc.7"
+version = "1.0.0-rc.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2913e23c08a64d6ecb3a3318fbc615f1f046a8b33f40a849a42a484862c594b8"
+checksum = "47142402f9ab8e1f1041acb5d8fbd3dae397fd9a6b4c65f22be9c4434c81731b"
 dependencies = [
  "cfg-if",
  "enumn",
@@ -8165,9 +8208,9 @@ dependencies = [
 
 [[package]]
 name = "xberg-paddle-ocr"
-version = "1.0.0-rc.7"
+version = "1.0.0-rc.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "797b2d23626c27d87aaa2831d23066f3260940c2ec740d540014299d07537c1c"
+checksum = "d0d77debe5b69db07cb3a184bf240c3e33414bf37540f27c0586fcfc0dc47b7d"
 dependencies = [
  "geo-clipper",
  "geo-types",
@@ -8182,7 +8225,7 @@ dependencies = [
 
 [[package]]
 name = "xberg-rb"
-version = "1.0.0-rc.7"
+version = "1.0.0-rc.9"
 dependencies = [
  "async-trait",
  "crawlberg",
@@ -8196,9 +8239,9 @@ dependencies = [
 
 [[package]]
 name = "xberg-tesseract"
-version = "1.0.0-rc.7"
+version = "1.0.0-rc.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9e290213d7492421f7e0e83bb43e747aeb13c05df3a113328e0496cb3c248607"
+checksum = "7ec0056f3798302ed20cf62cea73a1eac49c987758d7bd52585a55f8e2b8648a"
 dependencies = [
  "cc",
  "cmake",

data/ext/xberg_rb/native/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "xberg-rb"
-version = "1.0.0-rc.7"
+version = "1.0.0-rc.9"
 edition = "2024"
 license = "MIT"
 description = "High-performance document intelligence library"
@@ -49,17 +49,17 @@ xml = ["xberg/xml"]
 
 [dependencies]
 async-trait = "0.1"
-crawlberg = { default-features = false, version = "1.0.2" }
+crawlberg = { default-features = false, version = "1.0.3" }
 magnus = "0.8"
 rb-sys = ">=0.9, <0.9.128"
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 tokio = { version = "1", features = ["rt-multi-thread"] }
 [target.'cfg(not(any(target_os = "windows", all(target_os = "macos", target_arch = "x86_64"))))'.dependencies]
-xberg = { version = "1.0.0-rc.7", features = ["full", "pdf", "ocr", "paddle-ocr", "paddle-ocr-types", "layout-detection", "layout-types", "embeddings", "embedding-presets", "reranker", "reranker-presets", "chunking", "keywords-yake", "keywords-rake", "language-detection", "html", "tree-sitter", "office", "email", "archives", "stopwords", "auto-rotate", "auto-rotate-types", "tokio-runtime", "api", "api-types", "mcp", "liter-llm", "quality", "svg", "transcription", "transcription-types", "classification", "captioning", "ner-onnx", "ner-llm", "diff", "markdown-footnotes", "redaction", "enrichment", "heuristics", "presets", "structured", "url-ingestion"] }
+xberg = { version = "1.0.0-rc.9", features = ["full", "pdf", "ocr", "paddle-ocr", "paddle-ocr-types", "layout-detection", "layout-types", "embeddings", "embedding-presets", "reranker", "reranker-presets", "chunking", "keywords-yake", "keywords-rake", "language-detection", "html", "tree-sitter", "office", "email", "archives", "stopwords", "auto-rotate", "auto-rotate-types", "tokio-runtime", "api", "api-types", "mcp", "liter-llm", "quality", "svg", "transcription", "transcription-types", "classification", "captioning", "ner-onnx", "ner-llm", "diff", "markdown-footnotes", "redaction", "enrichment", "heuristics", "presets", "structured", "url-ingestion"] }
 
 [target.'cfg(target_os = "windows")'.dependencies]
-xberg = { version = "1.0.0-rc.7", features = ["windows-target"] }
+xberg = { version = "1.0.0-rc.9", features = ["windows-target"] }
 
 [target.'cfg(all(target_os = "macos", target_arch = "x86_64"))'.dependencies]
-xberg = { version = "1.0.0-rc.7", features = ["macos-intel-target"] }
+xberg = { version = "1.0.0-rc.9", features = ["macos-intel-target"] }

data/ext/xberg_rb/src/lib.rs

@@ -1,5 +1,5 @@
 // This file is auto-generated by alef. DO NOT EDIT.
-// alef:hash:0ce3ea7bbabccafc622b4dd87ac0f00356dd0426ec66977c856525dabafdc585
+// alef:hash:28933e85bc9dc440d3d48a0fa997a78b13df863b6249812908ba7914441b9232
 // Re-generate with: alef generate
 #![allow(dead_code, unused_imports, unused_variables)]
 #![allow(
@@ -24634,6 +24634,16 @@ fn list_reranker_backends() -> Result<Vec<String>, Error> {
     Ok(result)
 }
 
+fn list_tokenizer_backends() -> Result<Vec<String>, Error> {
+    let result = xberg::list_tokenizer_backends().map_err(|e| {
+        magnus::Error::new(
+            unsafe { Ruby::get_unchecked() }.exception_runtime_error(),
+            e.to_string(),
+        )
+    })?;
+    Ok(result)
+}
+
 fn list_validators() -> Result<Vec<String>, Error> {
     let result = xberg::list_validators().map_err(|e| {
         magnus::Error::new(
@@ -24732,11 +24742,17 @@ impl xberg::plugins::Plugin for RbOcrBackendBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("version", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbOcrBackendBridge] host 'version' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <String as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <String as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!("[RbOcrBackendBridge] host 'version' returned an unconvertible value; returning default: {e}");
+            Default::default()
+        })
     }
 
     fn initialize(&self) -> std::result::Result<(), xberg::XbergError> {
@@ -24941,11 +24957,19 @@ impl xberg::OcrBackend for RbOcrBackendBridge {
         ) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbOcrBackendBridge] host 'supports_language' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!(
+                "[RbOcrBackendBridge] host 'supports_language' returned an unconvertible value; returning default: {e}"
+            );
+            Default::default()
+        })
     }
 
     fn backend_type(&self) -> xberg::OcrBackendType {
@@ -24955,7 +24979,10 @@ impl xberg::OcrBackend for RbOcrBackendBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("backend_type", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbOcrBackendBridge] host 'backend_type' raised; returning default: {e}");
                 return Default::default();
             }
         };
@@ -24965,12 +24992,17 @@ impl xberg::OcrBackend for RbOcrBackendBridge {
             match val.funcall::<_, _, String>("to_json", ()) {
                 Ok(s) => s,
                 Err(e) => {
-                    let _ = e;
+                    eprintln!(
+                        "[RbOcrBackendBridge] host 'backend_type' returned a non-JSON value; returning default: {e}"
+                    );
                     return Default::default();
                 }
             }
         };
-        serde_json::from_str::<xberg::OcrBackendType>(&json_str).unwrap_or_default()
+        serde_json::from_str::<xberg::OcrBackendType>(&json_str).unwrap_or_else(|e| {
+            eprintln!("[RbOcrBackendBridge] host 'backend_type' returned an unparseable value; returning default: {e}");
+            Default::default()
+        })
     }
 
     fn supported_languages(&self) -> Vec<String> {
@@ -24983,11 +25015,17 @@ impl xberg::OcrBackend for RbOcrBackendBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("supported_languages", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbOcrBackendBridge] host 'supported_languages' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <Vec<String> as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <Vec<String> as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!("[RbOcrBackendBridge] host 'supported_languages' returned an unconvertible value; returning default: {e}");
+            Default::default()
+        })
     }
 
     fn supports_table_detection(&self) -> bool {
@@ -25000,11 +25038,17 @@ impl xberg::OcrBackend for RbOcrBackendBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("supports_table_detection", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbOcrBackendBridge] host 'supports_table_detection' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!("[RbOcrBackendBridge] host 'supports_table_detection' returned an unconvertible value; returning default: {e}");
+            Default::default()
+        })
     }
 
     fn supports_document_processing(&self) -> bool {
@@ -25017,11 +25061,17 @@ impl xberg::OcrBackend for RbOcrBackendBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("supports_document_processing", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbOcrBackendBridge] host 'supports_document_processing' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!("[RbOcrBackendBridge] host 'supports_document_processing' returned an unconvertible value; returning default: {e}");
+            Default::default()
+        })
     }
 
     fn emits_structured_markdown(&self) -> bool {
@@ -25034,11 +25084,17 @@ impl xberg::OcrBackend for RbOcrBackendBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("emits_structured_markdown", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbOcrBackendBridge] host 'emits_structured_markdown' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!("[RbOcrBackendBridge] host 'emits_structured_markdown' returned an unconvertible value; returning default: {e}");
+            Default::default()
+        })
     }
 
     async fn process_document(
@@ -25586,11 +25642,17 @@ impl xberg::plugins::Plugin for RbPostProcessorBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("version", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbPostProcessorBridge] host 'version' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <String as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <String as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!("[RbPostProcessorBridge] host 'version' returned an unconvertible value; returning default: {e}");
+            Default::default()
+        })
     }
 
     fn initialize(&self) -> std::result::Result<(), xberg::XbergError> {
@@ -25702,7 +25764,10 @@ impl xberg::PostProcessor for RbPostProcessorBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("processing_stage", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbPostProcessorBridge] host 'processing_stage' raised; returning default: {e}");
                 return Default::default();
             }
         };
@@ -25712,12 +25777,19 @@ impl xberg::PostProcessor for RbPostProcessorBridge {
             match val.funcall::<_, _, String>("to_json", ()) {
                 Ok(s) => s,
                 Err(e) => {
-                    let _ = e;
+                    eprintln!(
+                        "[RbPostProcessorBridge] host 'processing_stage' returned a non-JSON value; returning default: {e}"
+                    );
                     return Default::default();
                 }
             }
         };
-        serde_json::from_str::<xberg::ProcessingStage>(&json_str).unwrap_or_default()
+        serde_json::from_str::<xberg::ProcessingStage>(&json_str).unwrap_or_else(|e| {
+            eprintln!(
+                "[RbPostProcessorBridge] host 'processing_stage' returned an unparseable value; returning default: {e}"
+            );
+            Default::default()
+        })
     }
 
     fn should_process(&self, _result: &xberg::ExtractedDocument, _config: &xberg::ExtractionConfig) -> bool {
@@ -25744,11 +25816,19 @@ impl xberg::PostProcessor for RbPostProcessorBridge {
         ) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbPostProcessorBridge] host 'should_process' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!(
+                "[RbPostProcessorBridge] host 'should_process' returned an unconvertible value; returning default: {e}"
+            );
+            Default::default()
+        })
     }
 
     fn estimated_duration_ms(&self, _result: &xberg::ExtractedDocument) -> u64 {
@@ -25768,11 +25848,17 @@ impl xberg::PostProcessor for RbPostProcessorBridge {
         ) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbPostProcessorBridge] host 'estimated_duration_ms' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <u64 as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <u64 as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!("[RbPostProcessorBridge] host 'estimated_duration_ms' returned an unconvertible value; returning default: {e}");
+            Default::default()
+        })
     }
 
     fn priority(&self) -> i32 {
@@ -25785,11 +25871,19 @@ impl xberg::PostProcessor for RbPostProcessorBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("priority", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbPostProcessorBridge] host 'priority' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <i32 as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <i32 as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!(
+                "[RbPostProcessorBridge] host 'priority' returned an unconvertible value; returning default: {e}"
+            );
+            Default::default()
+        })
     }
 }
 
@@ -26011,11 +26105,17 @@ impl xberg::plugins::Plugin for RbValidatorBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("version", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbValidatorBridge] host 'version' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <String as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <String as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!("[RbValidatorBridge] host 'version' returned an unconvertible value; returning default: {e}");
+            Default::default()
+        })
     }
 
     fn initialize(&self) -> std::result::Result<(), xberg::XbergError> {
@@ -26144,11 +26244,19 @@ impl xberg::Validator for RbValidatorBridge {
         ) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbValidatorBridge] host 'should_validate' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!(
+                "[RbValidatorBridge] host 'should_validate' returned an unconvertible value; returning default: {e}"
+            );
+            Default::default()
+        })
     }
 
     fn priority(&self) -> i32 {
@@ -26161,11 +26269,17 @@ impl xberg::Validator for RbValidatorBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("priority", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbValidatorBridge] host 'priority' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <i32 as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <i32 as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!("[RbValidatorBridge] host 'priority' returned an unconvertible value; returning default: {e}");
+            Default::default()
+        })
     }
 }
 
@@ -26340,11 +26454,19 @@ impl xberg::plugins::Plugin for RbDocumentExtractorBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("version", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbDocumentExtractorBridge] host 'version' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <String as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <String as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!(
+                "[RbDocumentExtractorBridge] host 'version' returned an unconvertible value; returning default: {e}"
+            );
+            Default::default()
+        })
     }
 
     fn initialize(&self) -> std::result::Result<(), xberg::XbergError> {
@@ -26465,11 +26587,17 @@ impl xberg::DocumentExtractor for RbDocumentExtractorBridge {
             let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("supported_mime_types", ()) {
                 Ok(v) => v,
                 Err(e) => {
-                    let _ = e;
+                    // This method's Rust signature is infallible, so a Ruby exception cannot
+                    // propagate. Log before substituting the default — a silent default is
+                    // indistinguishable from a real result to the caller.
+                    eprintln!("[RbDocumentExtractorBridge] host 'supported_mime_types' raised; returning default: {e}");
                     return Default::default();
                 }
             };
-            <Vec<String> as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+            <Vec<String> as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!("[RbDocumentExtractorBridge] host 'supported_mime_types' returned an unconvertible value; returning default: {e}");
+            Default::default()
+        })
         };
         let __strs: Vec<&'static str> = __types
             .into_iter()
@@ -26488,11 +26616,19 @@ impl xberg::DocumentExtractor for RbDocumentExtractorBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("priority", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbDocumentExtractorBridge] host 'priority' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <i32 as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <i32 as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!(
+                "[RbDocumentExtractorBridge] host 'priority' returned an unconvertible value; returning default: {e}"
+            );
+            Default::default()
+        })
     }
 
     fn can_handle(&self, _path: &std::path::Path, _mime_type: &str) -> bool {
@@ -26518,11 +26654,19 @@ impl xberg::DocumentExtractor for RbDocumentExtractorBridge {
         ) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbDocumentExtractorBridge] host 'can_handle' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <bool as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!(
+                "[RbDocumentExtractorBridge] host 'can_handle' returned an unconvertible value; returning default: {e}"
+            );
+            Default::default()
+        })
     }
 }
 
@@ -26699,11 +26843,19 @@ impl xberg::plugins::Plugin for RbEmbeddingBackendBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("version", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbEmbeddingBackendBridge] host 'version' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <String as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <String as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!(
+                "[RbEmbeddingBackendBridge] host 'version' returned an unconvertible value; returning default: {e}"
+            );
+            Default::default()
+        })
     }
 
     fn initialize(&self) -> std::result::Result<(), xberg::XbergError> {
@@ -26756,11 +26908,19 @@ impl xberg::EmbeddingBackend for RbEmbeddingBackendBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("dimensions", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbEmbeddingBackendBridge] host 'dimensions' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <usize as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <usize as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!(
+                "[RbEmbeddingBackendBridge] host 'dimensions' returned an unconvertible value; returning default: {e}"
+            );
+            Default::default()
+        })
     }
 
     async fn embed(&self, texts: Vec<String>) -> std::result::Result<Vec<Vec<f32>>, xberg::XbergError> {
@@ -26897,11 +27057,17 @@ impl xberg::plugins::Plugin for RbRendererBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("version", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbRendererBridge] host 'version' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <String as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <String as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!("[RbRendererBridge] host 'version' returned an unconvertible value; returning default: {e}");
+            Default::default()
+        })
     }
 
     fn initialize(&self) -> std::result::Result<(), xberg::XbergError> {
@@ -27077,11 +27243,19 @@ impl xberg::plugins::Plugin for RbRerankerBackendBridge {
         let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("version", ()) {
             Ok(v) => v,
             Err(e) => {
-                let _ = e;
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbRerankerBackendBridge] host 'version' raised; returning default: {e}");
                 return Default::default();
             }
         };
-        <String as magnus::TryConvert>::try_convert(val).unwrap_or_default()
+        <String as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!(
+                "[RbRerankerBackendBridge] host 'version' returned an unconvertible value; returning default: {e}"
+            );
+            Default::default()
+        })
     }
 
     fn initialize(&self) -> std::result::Result<(), xberg::XbergError> {
@@ -27223,6 +27397,180 @@ pub fn clear_reranker_backends() -> Result<(), magnus::Error> {
     })
 }
 
+#[allow(unused_imports)]
+use magnus::TryConvert as _;
+#[allow(unused_imports)]
+use magnus::value::InnerValue as _;
+/// Wrapper that bridges a foreign Rb object to the `TokenizerBackend` trait.
+pub struct RbTokenizerBackendBridge {
+    inner: magnus::value::Opaque<magnus::Value>,
+    cached_name: String,
+    has_initialize: bool,
+    has_shutdown: bool,
+}
+
+impl std::fmt::Debug for RbTokenizerBackendBridge {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        write!(f, "RbTokenizerBackendBridge")
+    }
+}
+
+impl RbTokenizerBackendBridge {
+    /// Create a new bridge wrapping a Ruby object.
+    /// Validates that the Ruby object responds to all required methods.
+    pub fn new(rb_obj: magnus::Value, name: String) -> Result<Self, magnus::Error> {
+        if !rb_obj.respond_to("count_tokens", false).unwrap_or(false) {
+            let ruby = unsafe { magnus::Ruby::get_unchecked() };
+            return Err(magnus::Error::new(
+                ruby.exception_runtime_error(),
+                format!("Ruby object missing required method: {}", "count_tokens"),
+            ));
+        }
+        Ok(Self {
+            inner: magnus::value::Opaque::from(rb_obj),
+            cached_name: name,
+            has_initialize: rb_obj.respond_to("initialize", false).unwrap_or(false),
+            has_shutdown: rb_obj.respond_to("shutdown", false).unwrap_or(false),
+        })
+    }
+}
+
+impl xberg::plugins::Plugin for RbTokenizerBackendBridge {
+    fn name(&self) -> &str {
+        &self.cached_name
+    }
+
+    fn version(&self) -> String {
+        // SAFETY: bridge methods are only invoked from threads holding the GVL
+        let ruby = unsafe { magnus::Ruby::get_unchecked() };
+        let value = self.inner.get_inner_with(&ruby);
+        let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("version", ()) {
+            Ok(v) => v,
+            Err(e) => {
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbTokenizerBackendBridge] host 'version' raised; returning default: {e}");
+                return Default::default();
+            }
+        };
+        <String as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!(
+                "[RbTokenizerBackendBridge] host 'version' returned an unconvertible value; returning default: {e}"
+            );
+            Default::default()
+        })
+    }
+
+    fn initialize(&self) -> std::result::Result<(), xberg::XbergError> {
+        if !(self.has_initialize) {
+            return Ok(());
+        }
+        // SAFETY: bridge methods are only invoked from threads holding the GVL
+        let ruby = unsafe { magnus::Ruby::get_unchecked() };
+        let value = self.inner.get_inner_with(&ruby);
+        let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("initialize", ()) {
+            Ok(v) => v,
+            Err(e) => {
+                return Err(xberg::XbergError::Other(format!(
+                    "Ruby method 'initialize' failed: {}",
+                    e
+                )));
+            }
+        };
+        let _ = val;
+        Ok(())
+    }
+
+    fn shutdown(&self) -> std::result::Result<(), xberg::XbergError> {
+        if !(self.has_shutdown) {
+            return Ok(());
+        }
+        // SAFETY: bridge methods are only invoked from threads holding the GVL
+        let ruby = unsafe { magnus::Ruby::get_unchecked() };
+        let value = self.inner.get_inner_with(&ruby);
+        let val: magnus::Value = match value.funcall::<_, _, magnus::Value>("shutdown", ()) {
+            Ok(v) => v,
+            Err(e) => {
+                return Err(xberg::XbergError::Other(format!(
+                    "Ruby method 'shutdown' failed: {}",
+                    e
+                )));
+            }
+        };
+        let _ = val;
+        Ok(())
+    }
+}
+
+impl xberg::TokenizerBackend for RbTokenizerBackendBridge {
+    fn count_tokens(&self, text: &str) -> usize {
+        // SAFETY: bridge methods are only invoked from threads holding the GVL
+        let ruby = unsafe { magnus::Ruby::get_unchecked() };
+        let value = self.inner.get_inner_with(&ruby);
+        let val: magnus::Value = match value.funcall::<_, _, magnus::Value>(
+            "count_tokens",
+            ({
+                let ruby = unsafe { magnus::Ruby::get_unchecked() };
+                ruby.str_new(AsRef::<str>::as_ref(&text)).as_value()
+            },),
+        ) {
+            Ok(v) => v,
+            Err(e) => {
+                // This method's Rust signature is infallible, so a Ruby exception cannot
+                // propagate. Log before substituting the default — a silent default is
+                // indistinguishable from a real result to the caller.
+                eprintln!("[RbTokenizerBackendBridge] host 'count_tokens' raised; returning default: {e}");
+                return Default::default();
+            }
+        };
+        <usize as magnus::TryConvert>::try_convert(val).unwrap_or_else(|e| {
+            eprintln!(
+                "[RbTokenizerBackendBridge] host 'count_tokens' returned an unconvertible value; returning default: {e}"
+            );
+            Default::default()
+        })
+    }
+}
+
+pub fn register_tokenizer_backend(rb_obj: magnus::Value, name: String) -> Result<(), magnus::Error> {
+    let required_methods = ["count_tokens"];
+    for method in &required_methods {
+        if !rb_obj.respond_to(*method, false).unwrap_or(false) {
+            let ruby = unsafe { magnus::Ruby::get_unchecked() };
+            return Err(magnus::Error::new(
+                ruby.exception_runtime_error(),
+                format!("Backend missing required method: {}", method),
+            ));
+        }
+    }
+    let wrapper = RbTokenizerBackendBridge::new(rb_obj, name)?;
+    let arc: Arc<dyn xberg::TokenizerBackend> = Arc::new(wrapper);
+
+    xberg::plugins::registry::get_tokenizer_backend_registry()
+        .write()
+        .register(arc)
+        .map_err(|e| {
+            let ruby = unsafe { magnus::Ruby::get_unchecked() };
+            magnus::Error::new(ruby.exception_runtime_error(), format!("register failed: {}", e))
+        })?;
+    Ok(())
+}
+
+pub fn unregister_tokenizer_backend(name: String) -> Result<(), magnus::Error> {
+    xberg::plugins::tokenizer_backend::unregister_tokenizer_backend(&name).map_err(|e| {
+        let ruby = unsafe { magnus::Ruby::get_unchecked() };
+        magnus::Error::new(ruby.exception_runtime_error(), format!("{}", e))
+    })
+}
+
+pub fn clear_tokenizer_backends() -> Result<(), magnus::Error> {
+    xberg::plugins::tokenizer_backend::clear_tokenizer_backends().map_err(|e| {
+        let ruby = unsafe { magnus::Ruby::get_unchecked() };
+        magnus::Error::new(ruby.exception_runtime_error(), format!("{}", e))
+    })
+}
+
 #[allow(clippy::redundant_closure, clippy::useless_conversion)]
 impl From<xberg::CacheStats> for CacheStats {
     fn from(val: xberg::CacheStats) -> Self {
@@ -37503,6 +37851,8 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     module.define_module_function("list_reranker_backends", function!(list_reranker_backends, 0))?;
 
+    module.define_module_function("list_tokenizer_backends", function!(list_tokenizer_backends, 0))?;
+
     module.define_module_function("list_validators", function!(list_validators, 0))?;
 
     module.define_module_function("find_unmarked_claims", function!(find_unmarked_claims, 1))?;
@@ -37557,5 +37907,14 @@ fn ruby_init(ruby: &Ruby) -> Result<(), Error> {
 
     module.define_module_function("clear_reranker_backends", function!(clear_reranker_backends, 0))?;
 
+    module.define_module_function("register_tokenizer_backend", function!(register_tokenizer_backend, 2))?;
+
+    module.define_module_function(
+        "unregister_tokenizer_backend",
+        function!(unregister_tokenizer_backend, 1),
+    )?;
+
+    module.define_module_function("clear_tokenizer_backends", function!(clear_tokenizer_backends, 0))?;
+
     Ok(())
 }

data/lib/xberg/native.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:0ce3ea7bbabccafc622b4dd87ac0f00356dd0426ec66977c856525dabafdc585
+# alef:hash:28933e85bc9dc440d3d48a0fa997a78b13df863b6249812908ba7914441b9232
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true
@@ -351,9 +351,12 @@ module Xberg
   # Defaults to `Characters` (Unicode character count). When using token-based sizing,
   # chunks are sized by token count according to the specified tokenizer.
   #
-  # Token-based sizing uses HuggingFace tokenizers loaded at runtime. Any tokenizer
-  # available on HuggingFace Hub can be used, including OpenAI-compatible tokenizers
-  # (e.g., `Xenova/gpt-4o`, `Xenova/cl100k_base`).
+  # Token-based sizing uses HuggingFace tokenizers loaded at runtime, or a tokenizer
+  # backend you register yourself. Any tokenizer available on HuggingFace Hub can be
+  # used, including OpenAI-compatible tokenizers (e.g., `Xenova/gpt-4o`,
+  # `Xenova/cl100k_base`). To size chunks with your own tokenizer instead (llama.cpp/GGUF
+  # vocabularies, SentencePiece models, custom vocabs), register a `TokenizerBackend`
+  # with `register_tokenizer_backend` and set `model` to the registered name.
   module ChunkSizing
     extend T::Helpers
     extend T::Sig
@@ -392,12 +395,15 @@ module Xberg
       new
     end
   end
-  ## Size measured in tokens from a HuggingFace tokenizer.
+  ## Size measured in tokens from a HuggingFace tokenizer or a registered
+  ## tokenizer backend.
   ChunkSizingTokenizer = Data.define(:model, :cache_dir) do
     include ChunkSizing
     extend T::Sig
 
-    # HuggingFace model ID or path, e.g. "Xenova/gpt-4o", "bert-base-uncased".
+    # Name of a tokenizer backend registered via `register_tokenizer_backend`,
+    # or a HuggingFace model ID, e.g. "Xenova/gpt-4o", "bert-base-uncased".
+    # A registered backend name takes precedence over a HuggingFace ID.
     sig { returns(String) }
     # rubocop:disable Lint/UselessMethodDefinition
     def model = super

data/lib/xberg/version.rb

@@ -1,10 +1,10 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:0ce3ea7bbabccafc622b4dd87ac0f00356dd0426ec66977c856525dabafdc585
+# alef:hash:28933e85bc9dc440d3d48a0fa997a78b13df863b6249812908ba7914441b9232
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true
 
 module Xberg
   ## The version string for this package.
-  VERSION = "1.0.0.pre.rc.7"
+  VERSION = "1.0.0.pre.rc.9"
 end

data/lib/xberg.rb

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:0ce3ea7bbabccafc622b4dd87ac0f00356dd0426ec66977c856525dabafdc585
+# alef:hash:28933e85bc9dc440d3d48a0fa997a78b13df863b6249812908ba7914441b9232
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 # frozen_string_literal: true

data/sig/types.rbs

@@ -1,5 +1,5 @@
 # This file is auto-generated by alef — DO NOT EDIT.
-# alef:hash:0ce3ea7bbabccafc622b4dd87ac0f00356dd0426ec66977c856525dabafdc585
+# alef:hash:28933e85bc9dc440d3d48a0fa997a78b13df863b6249812908ba7914441b9232
 # To regenerate: alef generate
 # To verify freshness: alef verify --exit-code
 
@@ -2483,6 +2483,10 @@ def self.clear_reranker_backends: () -> void
 
 def self.list_reranker_backends: () -> Array[String]
 
+def self.clear_tokenizer_backends: () -> void
+
+def self.list_tokenizer_backends: () -> Array[String]
+
 def self.list_validators: () -> Array[String]
 
 def self.clear_validators: () -> void
@@ -2543,6 +2547,10 @@ def embed: (Array[String] texts) -> Array[Array[Float]]
 def rerank: (String query, Array[String] documents) -> Array[Float]
     end
 
+    interface _TokenizerBackend
+    def count_tokens: (String text) -> Integer
+    end
+
 def self.register_ocr_backend: (_OcrBackend backend, String name) -> nil
 
 def self.unregister_ocr_backend: (String name) -> nil
@@ -2563,4 +2571,8 @@ def self.register_reranker_backend: (_RerankerBackend backend, String name) -> n
 
 def self.unregister_reranker_backend: (String name) -> nil
 
+def self.register_tokenizer_backend: (_TokenizerBackend backend, String name) -> nil
+
+def self.unregister_tokenizer_backend: (String name) -> nil
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: xberg
 version: !ruby/object:Gem::Version
-  version: 1.0.0.pre.rc.7
+  version: 1.0.0.pre.rc.9
 platform: ruby
 authors:
 - Na'aman Hirschfeld <naaman@xberg.io>
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2026-07-04 00:00:00.000000000 Z
+date: 2026-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys