tokenizers 0.4.4 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: fe2e8b2ec97ac4dabb4e401cb426079c065be420bf8da41c9484b3d38290589c
-  data.tar.gz: bba3b9d9d94a278e3f5189fda48b3b3a7f1ff369b66ae8b85945e92a2d8b90ed
+  metadata.gz: 258211e71ca06e96bb4ee01b15e29f6f74d3c70d04af246e95b178e10f093059
+  data.tar.gz: 6e0b01c577830afdf1c7d677b1377191420d85e0f1f8638893f72cbb7ccef322
 SHA512:
-  metadata.gz: fd5876cc95c22b917f1a5715f9b9651f714291f017c00f83316ad2c3cc48c8819d4de70288b4067570c9f729cca9d06cbfa8562996a61ac694e202268a58cdea
-  data.tar.gz: cb793a73aabafe933ef8d7259b7a36848b1f38cacdf20ef3f6f8554671b7de6ece53b3c14f46f368f540ecbab5dfca087f730e99c22caf84c7b35d539b07b633
+  metadata.gz: 4e0ea1f11dbab96b213190397ee8676d6233568f4fe013970a5a2c32105ed20ec06a5c8bc7379065799de315a0fc6d5f47807f9af47bc6f47926e4147c3eabcc
+  data.tar.gz: ccd00b103577c6cff4dded6a3bc42394eccb3e24b950674a33eedf76df7c08bc89cda8219f076fce4cf20d90580da82c03e001a4e49ceb80e56ae4055b4617cf

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.5.0 (2024-05-21)
+
+- Updated Tokenizers to 0.19.1
+- Replaced `add_prefix_space` with `prepend_scheme` and `split` options for `Metaspace` decoder and pre-tokenizer
+- Dropped support for Ruby < 3.1
+
 ## 0.4.4 (2024-02-27)
 
 - Updated Tokenizers to 0.15.2

data/Cargo.lock

@@ -40,7 +40,7 @@ dependencies = [
  "regex",
  "rustc-hash",
  "shlex",
- "syn 2.0.38",
+ "syn",
 ]
 
 [[package]]
@@ -135,9 +135,9 @@ dependencies = [
 
 [[package]]
 name = "darling"
-version = "0.14.4"
+version = "0.20.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
+checksum = "54e36fcd13ed84ffdfda6f5be89b31287cbb80c439841fe69e04841435464391"
 dependencies = [
  "darling_core",
  "darling_macro",
@@ -145,58 +145,58 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.14.4"
+version = "0.20.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
+checksum = "9c2cf1c23a687a1feeb728783b993c4e1ad83d99f351801977dd809b48d0a70f"
 dependencies = [
  "fnv",
  "ident_case",
  "proc-macro2",
  "quote",
  "strsim",
- "syn 1.0.109",
+ "syn",
 ]
 
 [[package]]
 name = "darling_macro"
-version = "0.14.4"
+version = "0.20.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
+checksum = "a668eda54683121533a393014d8692171709ff57a7d61f187b6e782719f8933f"
 dependencies = [
  "darling_core",
  "quote",
- "syn 1.0.109",
+ "syn",
 ]
 
 [[package]]
 name = "derive_builder"
-version = "0.12.0"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8d67778784b508018359cbc8696edb3db78160bab2c2a28ba7f56ef6932997f8"
+checksum = "0350b5cb0331628a5916d6c5c0b72e97393b8b6b03b47a9284f4e7f5a405ffd7"
 dependencies = [
  "derive_builder_macro",
 ]
 
 [[package]]
 name = "derive_builder_core"
-version = "0.12.0"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c11bdc11a0c47bc7d37d582b5285da6849c96681023680b906673c5707af7b0f"
+checksum = "d48cda787f839151732d396ac69e3473923d54312c070ee21e9effcaa8ca0b1d"
 dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn 1.0.109",
+ "syn",
 ]
 
 [[package]]
 name = "derive_builder_macro"
-version = "0.12.0"
+version = "0.20.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
+checksum = "206868b8242f27cecce124c19fd88157fbd0dd334df2587f36417bafbc85097b"
 dependencies = [
  "derive_builder_core",
- "syn 1.0.109",
+ "syn",
 ]
 
 [[package]]
@@ -350,9 +350,9 @@ checksum = "b8dd856d451cc0da70e2ef2ce95a18e39a93b7558bedf10201ad28503f918568"
 
 [[package]]
 name = "magnus"
-version = "0.6.2"
+version = "0.6.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4778544796676e8428e9c622460ebf284bea52d8b10db3aeb449d8b5e61b3a13"
+checksum = "b1597ef40aa8c36be098249e82c9a20cf7199278ac1c1a1a995eeead6a184479"
 dependencies = [
  "magnus-macros",
  "rb-sys",
@@ -368,7 +368,7 @@ checksum = "5968c820e2960565f647819f5928a42d6e874551cab9d88d75e3e0660d7f71e3"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn",
 ]
 
 [[package]]
@@ -394,9 +394,9 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
 [[package]]
 name = "monostate"
-version = "0.1.9"
+version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "15f370ae88093ec6b11a710dec51321a61d420fafd1bad6e30d01bd9c920e8ee"
+checksum = "a20fffcd8ca4c69d31e036a71abc400147b41f90895df4edcb36497a1f8af8bf"
 dependencies = [
  "monostate-impl",
  "serde",
@@ -404,13 +404,13 @@ dependencies = [
 
 [[package]]
 name = "monostate-impl"
-version = "0.1.9"
+version = "0.1.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "371717c0a5543d6a800cac822eac735aa7d2d2fbb41002e9856a4089532dbdce"
+checksum = "bf307cbbbd777a9c10cec88ddafee572b3484caad5cce0c9236523c3803105a6"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn",
 ]
 
 [[package]]
@@ -489,18 +489,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.68"
+version = "1.0.81"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b1106fec09662ec6dd98ccac0f81cef56984d0b49f75c92d8cbad76e20c005c"
+checksum = "3d1597b0c024618f09a9c3b8655b7e430397a36d23fdafec26d6965e9eec3eba"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.33"
+version = "1.0.36"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5267fca4496028628a95160fc423a33e8b2e6af8a5302579e322e4b520293cae"
+checksum = "0fa76aaf39101c457836aec0ce2316dbdc3ab723cdda1c6bd4e6ad4208acaca7"
 dependencies = [
  "proc-macro2",
 ]
@@ -537,9 +537,9 @@ dependencies = [
 
 [[package]]
 name = "rayon"
-version = "1.8.0"
+version = "1.10.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9c27db03db7734835b3f53954b534c91069375ce6ccaa2e065441e07d9b6cdb1"
+checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa"
 dependencies = [
  "either",
  "rayon-core",
@@ -558,9 +558,9 @@ dependencies = [
 
 [[package]]
 name = "rayon-core"
-version = "1.12.0"
+version = "1.12.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ce3fb6ad83f861aac485e76e1985cd109d9a3713802152be56c3b1f0e0658ed"
+checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2"
 dependencies = [
  "crossbeam-deque",
  "crossbeam-utils",
@@ -568,18 +568,18 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.89"
+version = "0.9.97"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0d197f2c03751ef006f29d593d22aa9068c9c358e04ca503afea0329c366147c"
+checksum = "47d30bcad206b51f2f66121190ca678dce1fdf3a2eae0ac5d838d1818b19bdf5"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.89"
+version = "0.9.97"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2b50caf8fd028f12abe00d6debe2ae2adf6202c9ca3caa59487eda710d90fa28"
+checksum = "3cbd92f281615f3c2dcb9dcb0f0576624752afbf9a7f99173b37c4b55b62dd8a"
 dependencies = [
  "bindgen",
  "lazy_static",
@@ -587,7 +587,7 @@ dependencies = [
  "quote",
  "regex",
  "shell-words",
- "syn 2.0.38",
+ "syn",
 ]
 
 [[package]]
@@ -598,33 +598,27 @@ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
 
 [[package]]
 name = "regex"
-version = "1.9.5"
+version = "1.10.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "697061221ea1b4a94a624f67d0ae2bfe4e22b8a17b6a192afb11046542cc8c47"
+checksum = "c117dbdfde9c8308975b6a18d71f3f385c89461f7b3fb054288ecf2a2058ba4c"
 dependencies = [
  "aho-corasick",
  "memchr",
  "regex-automata",
- "regex-syntax 0.7.5",
+ "regex-syntax",
 ]
 
 [[package]]
 name = "regex-automata"
-version = "0.3.8"
+version = "0.4.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2f401f4955220693b56f8ec66ee9c78abffd8d1c4f23dc41a23839eb88f0795"
+checksum = "86b83b8b9847f9bf95ef68afb0b8e6cdb80f498442f5179a29fad448fcc1eaea"
 dependencies = [
  "aho-corasick",
  "memchr",
- "regex-syntax 0.7.5",
+ "regex-syntax",
 ]
 
-[[package]]
-name = "regex-syntax"
-version = "0.7.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da"
-
 [[package]]
 name = "regex-syntax"
 version = "0.8.2"
@@ -672,7 +666,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn",
 ]
 
 [[package]]
@@ -724,20 +718,9 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
 [[package]]
 name = "syn"
-version = "1.0.109"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
-dependencies = [
- "proc-macro2",
- "quote",
- "unicode-ident",
-]
-
-[[package]]
-name = "syn"
-version = "2.0.38"
+version = "2.0.59"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e96b79aaa137db8f61e26363a0c9b47d8b4ec75da28b7d1d614c2303e232408b"
+checksum = "4a6531ffc7b071655e4ce2e04bd464c4830bb585a61cabb96cf808f05172615a"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -761,24 +744,24 @@ checksum = "10712f02019e9288794769fba95cd6847df9874d49d871d062172f9dd41bc4cc"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn 2.0.38",
+ "syn",
 ]
 
 [[package]]
 name = "tokenizers"
-version = "0.4.4"
+version = "0.5.0"
 dependencies = [
  "magnus",
  "onig",
  "serde",
- "tokenizers 0.15.2",
+ "tokenizers 0.19.1",
 ]
 
 [[package]]
 name = "tokenizers"
-version = "0.15.2"
+version = "0.19.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3dd47962b0ba36e7fd33518fbf1754d136fd1474000162bbf2a8b5fcb2d3654d"
+checksum = "e500fad1dd3af3d626327e6a3fe5050e664a6eaa4708b8ca92f1794aaf73e6fd"
 dependencies = [
  "aho-corasick",
  "derive_builder",
@@ -796,7 +779,7 @@ dependencies = [
  "rayon",
  "rayon-cond",
  "regex",
- "regex-syntax 0.8.2",
+ "regex-syntax",
  "serde",
  "serde_json",
  "spm_precompiled",
@@ -823,9 +806,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-segmentation"
-version = "1.10.1"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
+checksum = "d4c87d22b6e3f4a18d4d40ef354e97c90fcb14dd91d7dc0aa9d8a1172ebf7202"
 
 [[package]]
 name = "unicode-width"

data/ext/tokenizers/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "tokenizers"
-version = "0.4.4"
+version = "0.5.0"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
-rust-version = "1.62.0"
+rust-version = "1.63.0"
 publish = false
 
 [lib]
@@ -16,6 +16,6 @@ onig = { version = "6", default-features = false }
 serde = { version = "1", features = ["rc", "derive"] }
 
 [dependencies.tokenizers]
-version = "=0.15.2" # also update in from_pretrained.rb
+version = "=0.19.1" # also update in from_pretrained.rb
 default-features = false
 features = ["progressbar", "onig", "esaxx_fast"]

data/ext/tokenizers/src/decoders.rs

@@ -1,5 +1,6 @@
 use std::sync::{Arc, RwLock};
 
+use crate::pre_tokenizers::from_string;
 use magnus::value::Lazy;
 use magnus::{
     data_type_builder, function, method, Class, DataType, DataTypeFunctions, Module, Object, RClass, RModule,
@@ -11,7 +12,7 @@ use tk::decoders::byte_fallback::ByteFallback;
 use tk::decoders::byte_level::ByteLevel;
 use tk::decoders::ctc::CTC;
 use tk::decoders::fuse::Fuse;
-use tk::decoders::metaspace::Metaspace;
+use tk::decoders::metaspace::{Metaspace, PrependScheme};
 use tk::decoders::strip::Strip;
 use tk::decoders::wordpiece::WordPiece;
 use tk::decoders::DecoderWrapper;
@@ -126,12 +127,29 @@ impl RbDecoder {
         setter!(self, Metaspace, @set_replacement, replacement);
     }
 
-    pub fn metaspace_add_prefix_space(&self) -> bool {
-        getter!(self, Metaspace, add_prefix_space)
+    pub fn metaspace_split(&self) -> bool {
+        getter!(self, Metaspace, get_split())
     }
 
-    pub fn metaspace_set_add_prefix_space(&self, add_prefix_space: bool) {
-        setter!(self, Metaspace, add_prefix_space, add_prefix_space);
+    pub fn metaspace_set_split(&self, split: bool) {
+        setter!(self, Metaspace, @set_split, split);
+    }
+
+    pub fn metaspace_prepend_scheme(&self) -> String {
+        // Assuming Metaspace has a method to get the prepend_scheme as a string
+        let scheme: PrependScheme = getter!(self, Metaspace, get_prepend_scheme());
+        match scheme {
+            PrependScheme::First => "first",
+            PrependScheme::Never => "never",
+            PrependScheme::Always => "always",
+        }
+        .to_string()
+    }
+
+    pub fn metaspace_set_prepend_scheme(&self, prepend_scheme: String) -> RbResult<()> {
+        let scheme = from_string(prepend_scheme)?;
+        setter!(self, Metaspace, @set_prepend_scheme, scheme);
+        Ok(())
     }
 
     pub fn word_piece_cleanup(&self) -> bool {
@@ -194,8 +212,9 @@ impl RbFuse {
 pub struct RbMetaspaceDecoder {}
 
 impl RbMetaspaceDecoder {
-    pub fn new(replacement: char, add_prefix_space: bool) -> RbDecoder {
-        Metaspace::new(replacement, add_prefix_space).into()
+    pub fn new(replacement: char, prepend_scheme: String, split: bool) -> RbResult<RbDecoder> {
+        let prepend_scheme = from_string(prepend_scheme)?;
+        Ok(Metaspace::new(replacement, prepend_scheme, split).into())
     }
 }
 
@@ -364,11 +383,13 @@ pub fn init_decoders(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     class.define_singleton_method("new", function!(RbFuse::new, 0))?;
 
     let class = module.define_class("Metaspace", decoder)?;
-    class.define_singleton_method("_new", function!(RbMetaspaceDecoder::new, 2))?;
-    class.define_method("add_prefix_space", method!(RbDecoder::metaspace_add_prefix_space, 0))?;
-    class.define_method("add_prefix_space=", method!(RbDecoder::metaspace_set_add_prefix_space, 1))?;
+    class.define_singleton_method("_new", function!(RbMetaspaceDecoder::new, 3))?;
+    class.define_method("prepend_scheme", method!(RbDecoder::metaspace_prepend_scheme, 0))?;
+    class.define_method("prepend_scheme=", method!(RbDecoder::metaspace_set_prepend_scheme, 1))?;
     class.define_method("replacement", method!(RbDecoder::metaspace_replacement, 0))?;
     class.define_method("replacement=", method!(RbDecoder::metaspace_set_replacement, 1))?;
+    class.define_method("split", method!(RbDecoder::metaspace_split, 0))?;
+    class.define_method("split=", method!(RbDecoder::metaspace_set_split, 1))?;
 
     let class = module.define_class("Replace", decoder)?;
     class.define_singleton_method("new", function!(RbReplaceDecoder::new, 2))?;

data/ext/tokenizers/src/pre_tokenizers.rs

@@ -1,7 +1,7 @@
 use std::sync::{Arc, RwLock};
 
 use magnus::{
-    data_type_builder, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Module, Object,
+    data_type_builder, exception, function, method, value::Lazy, Class, DataType, DataTypeFunctions, Error, Module, Object,
     RArray, RClass, RModule, Ruby, TryConvert, TypedData,
 };
 
@@ -12,7 +12,7 @@ use tk::pre_tokenizers::bert::BertPreTokenizer;
 use tk::pre_tokenizers::byte_level::ByteLevel;
 use tk::pre_tokenizers::delimiter::CharDelimiterSplit;
 use tk::pre_tokenizers::digits::Digits;
-use tk::pre_tokenizers::metaspace::Metaspace;
+use tk::pre_tokenizers::metaspace::{Metaspace, PrependScheme};
 use tk::pre_tokenizers::punctuation::Punctuation;
 use tk::pre_tokenizers::split::Split;
 use tk::pre_tokenizers::unicode_scripts::UnicodeScripts;
@@ -118,14 +118,6 @@ impl RbPreTokenizer {
         setter!(self, Digits, individual_digits, individual_digits);
     }
 
-    fn metaspace_add_prefix_space(&self) -> bool {
-        getter!(self, Metaspace, add_prefix_space)
-    }
-
-    fn metaspace_set_add_prefix_space(&self, add_prefix_space: bool) {
-        setter!(self, Metaspace, add_prefix_space, add_prefix_space);
-    }
-
     fn metaspace_replacement(&self) -> String {
         getter!(self, Metaspace, get_replacement().to_string())
     }
@@ -133,6 +125,31 @@ impl RbPreTokenizer {
     fn metaspace_set_replacement(&self, replacement: char) {
         setter!(self, Metaspace, @set_replacement, replacement);
     }
+
+    fn metaspace_split(&self) -> bool {
+        getter!(self, Metaspace, get_split())
+    }
+
+    fn metaspace_set_split(&self, split: bool) {
+        setter!(self, Metaspace, @set_split, split);
+    }
+
+    fn metaspace_prepend_scheme(&self) -> String {
+        // Assuming Metaspace has a method to get the prepend_scheme as a string
+        let scheme: PrependScheme = getter!(self, Metaspace, get_prepend_scheme());
+        match scheme {
+            PrependScheme::First => "first",
+            PrependScheme::Never => "never",
+            PrependScheme::Always => "always",
+        }
+        .to_string()
+    }
+
+    fn metaspace_set_prepend_scheme(&self, prepend_scheme: String) -> RbResult<()> {
+        let scheme = from_string(prepend_scheme)?;
+        setter!(self, Metaspace, @set_prepend_scheme, scheme);
+        Ok(())
+    }
 }
 
 impl PreTokenizer for RbPreTokenizer {
@@ -180,9 +197,11 @@ pub struct RbMetaspace {}
 impl RbMetaspace {
     fn new(
         replacement: char,
-        add_prefix_space: bool,
-    ) -> RbPreTokenizer {
-        Metaspace::new(replacement, add_prefix_space).into()
+        prepend_scheme: String,
+        split: bool,
+    ) -> RbResult<RbPreTokenizer> {
+        let prepend_scheme = from_string(prepend_scheme)?;
+        Ok(Metaspace::new(replacement, prepend_scheme, split).into())
     }
 }
 
@@ -252,6 +271,21 @@ impl RbSequence {
     }
 }
 
+pub(crate) fn from_string(string: String) -> RbResult<PrependScheme> {
+    let scheme = match string.as_str() {
+        "first" => PrependScheme::First,
+        "never" => PrependScheme::Never,
+        "always" => PrependScheme::Always,
+        _ => {
+            return Err(Error::new(exception::arg_error(), format!(
+                "{} is an unknown variant, should be one of ['first', 'never', 'always']",
+                string
+            )));
+        }
+    };
+    Ok(scheme)
+}
+
 #[derive(Clone, Deserialize)]
 #[serde(untagged)]
 pub(crate) enum RbPreTokenizerWrapper {
@@ -465,11 +499,13 @@ pub fn init_pre_tokenizers(ruby: &Ruby, module: &RModule) -> RbResult<()> {
     class.define_method("individual_digits=", method!(RbPreTokenizer::digits_set_individual_digits, 1))?;
 
     let class = module.define_class("Metaspace", pre_tokenizer)?;
-    class.define_singleton_method("_new", function!(RbMetaspace::new, 2))?;
-    class.define_method("add_prefix_space", method!(RbPreTokenizer::metaspace_add_prefix_space, 0))?;
-    class.define_method("add_prefix_space=", method!(RbPreTokenizer::metaspace_set_add_prefix_space, 1))?;
+    class.define_singleton_method("_new", function!(RbMetaspace::new, 3))?;
+    class.define_method("prepend_scheme", method!(RbPreTokenizer::metaspace_prepend_scheme, 0))?;
+    class.define_method("prepend_scheme=", method!(RbPreTokenizer::metaspace_set_prepend_scheme, 1))?;
     class.define_method("replacement", method!(RbPreTokenizer::metaspace_replacement, 0))?;
     class.define_method("replacement=", method!(RbPreTokenizer::metaspace_set_replacement, 1))?;
+    class.define_method("split", method!(RbPreTokenizer::metaspace_split, 0))?;
+    class.define_method("split=", method!(RbPreTokenizer::metaspace_set_split, 1))?;
 
     let class = module.define_class("Punctuation", pre_tokenizer)?;
     class.define_singleton_method("_new", function!(RbPunctuation::new, 1))?;

data/lib/tokenizers/decoders/metaspace.rb

@@ -1,8 +1,8 @@
 module Tokenizers
   module Decoders
     class Metaspace
-      def self.new(replacement: "\u2581", add_prefix_space: true)
-        _new(replacement, add_prefix_space)
+      def self.new(replacement: "\u2581", prepend_scheme: "always", split: true)
+        _new(replacement, prepend_scheme, split)
       end
     end
   end

data/lib/tokenizers/from_pretrained.rb

@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.15.2"
+    TOKENIZERS_VERSION = "0.19.1"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux

data/lib/tokenizers/pre_tokenizers/metaspace.rb

@@ -1,8 +1,8 @@
 module Tokenizers
   module PreTokenizers
     class Metaspace
-      def self.new(replacement: "\u2581", add_prefix_space: true)
-        _new(replacement, add_prefix_space)
+      def self.new(replacement: "\u2581", prepend_scheme: "always", split: true)
+        _new(replacement, prepend_scheme, split)
       end
     end
   end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.4.4"
+  VERSION = "0.5.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.4.4
+  version: 0.5.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-02-27 00:00:00.000000000 Z
+date: 2024-05-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -93,14 +93,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.3
+rubygems_version: 3.5.9
 signing_key:
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby