tokenizers 0.3.1 → 0.3.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b298d2a912a3d7fe32b9dc60036822b39ff463ad9739fec9ca00809922c6f45c
-  data.tar.gz: 8491e0c8532d9eefe5ae1c0940f57ae3986480fb9460897d7612e8937d97e384
+  metadata.gz: e6e88ec5618e36e317434410c960603695806bb59dadb2252f2957d8dbf0525b
+  data.tar.gz: 33a04a4a5faada27e6e7246c16d836a4ff9f6793e89de3cfd4880e30c6c8ed0d
 SHA512:
-  metadata.gz: 3616d92731fe9c7e4214166392b05734b10dec0e004106c01c756cab544491523b596b5d0a44ef8a47708719fa32839b86a1038a31104fae6d3de3a2e17ccca9
-  data.tar.gz: 88a375508473b97ae0b30dfb12a2c930a20477b003e5f2977a670904dab077ab4e2771b98dd3a446fc9b9c417e886f3b7355bd7eec527eb898251c9c60a0337c
+  metadata.gz: 88e4f2ad57fd1d66cd5fcf0d8b7ff6b1ea902258296fb02d207a446032134189e3445a104658074e94f914331c94f46cfdd09eed7c745c0483cb3b32b09e6abf
+  data.tar.gz: e8a1721ecbd36874322477077331743b0d1ba2de6f90076e07ad5456c230f76625d7f28ed6e6026c11395c6bb27701a6b8c0feedf2050387d32d9b777baa51fe

data/CHANGELOG.md

@@ -1,8 +1,18 @@
-## 0.3.1 (2022-02-08)
+## 0.3.3 (2023-04-09)
+
+- Updated Tokenizers to 0.13.3
+- Added `ByteFallback`, `Fuse`, `Replace`, and `Strip` decoders
+- Added `Prepend` normalizer
+
+## 0.3.2 (2023-03-06)
+
+- Added precompiled gem for Linux x86-64 MUSL
+
+## 0.3.1 (2023-02-08)
 
 - Fixed error with Ruby 2.7
 
-## 0.3.0 (2022-02-07)
+## 0.3.0 (2023-02-07)
 
 - Added support for training tokenizers
 - Added more methods to `Tokenizer`
@@ -11,20 +21,20 @@
 - Changed `encode` method to include special tokens by default
 - Changed how offsets are calculated for strings with multibyte characters
 
-## 0.2.3 (2022-01-22)
+## 0.2.3 (2023-01-22)
 
 - Added `add_special_tokens` option to `encode` method
 - Added warning about `encode` method including special tokens by default in 0.3.0
 - Added more methods to `Encoding`
 - Fixed error with precompiled gem on Mac ARM
 
-## 0.2.2 (2022-01-15)
+## 0.2.2 (2023-01-15)
 
 - Added precompiled gem for Linux ARM
 - Added `from_file` method
 - Fixed error with precompiled gem on Linux x86-64
 
-## 0.2.1 (2022-01-12)
+## 0.2.1 (2023-01-12)
 
 - Added support for Ruby 3.2
 

data/Cargo.lock

@@ -71,9 +71,9 @@ checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
 
 [[package]]
 name = "clang-sys"
-version = "1.4.0"
+version = "1.6.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa2e27ae6ab525c3d369ded447057bca5438d86dc3a68f6faafb8269ba82ebf3"
+checksum = "c688fc74432808e3eb684cae8830a86be1d66a2bd58e1f248ed0960a590baf6f"
 dependencies = [
  "glob",
  "libc",
@@ -95,9 +95,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.6"
+version = "0.5.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c2dd04ddaf88237dc3b8d8f9a3c1004b506b54b3313403944054d23c0870c521"
+checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200"
 dependencies = [
  "cfg-if",
  "crossbeam-utils",
@@ -105,9 +105,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-deque"
-version = "0.8.2"
+version = "0.8.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "715e8152b692bba2d374b53d4875445368fdf21a94751410af607a5ac677d1fc"
+checksum = "ce6fd6f855243022dcecf8702fef0c297d4338e226845fe067f6341ad9fa0cef"
 dependencies = [
  "cfg-if",
  "crossbeam-epoch",
@@ -116,9 +116,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-epoch"
-version = "0.9.13"
+version = "0.9.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "01a9af1f4c2ef74bb8aa1f7e19706bc72d03598c8a570bb5de72243c7a9d9d5a"
+checksum = "46bd5f3f85273295a9d14aedfb86f6aadbff6d8f5295c4a9edb08e819dcf5695"
 dependencies = [
  "autocfg",
  "cfg-if",
@@ -129,18 +129,18 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-utils"
-version = "0.8.14"
+version = "0.8.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4fb766fa798726286dbbb842f174001dab8abc7b627a1dd86e0b7222a95d929f"
+checksum = "3c063cd8cc95f5c377ed0d4b49a4b21f632396ff690e8470c29b3359b346984b"
 dependencies = [
  "cfg-if",
 ]
 
 [[package]]
 name = "darling"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c0808e1bd8671fb44a113a14e13497557533369847788fa2ae912b6ebfce9fa8"
+checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
 dependencies = [
  "darling_core",
  "darling_macro",
@@ -148,27 +148,27 @@ dependencies = [
 
 [[package]]
 name = "darling_core"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "001d80444f28e193f30c2f293455da62dcf9a6b29918a4253152ae2b1de592cb"
+checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
 dependencies = [
  "fnv",
  "ident_case",
  "proc-macro2",
  "quote",
  "strsim",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
 name = "darling_macro"
-version = "0.14.3"
+version = "0.14.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b36230598a2d5de7ec1c6f51f72d8a99a9208daff41de2084d06e3fd3ea56685"
+checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
 dependencies = [
  "darling_core",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -189,7 +189,7 @@ dependencies = [
  "darling",
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -199,7 +199,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "ebcda35c7a396850a55ffeac740804b40ffec779b98fffbb1738f4033f0ee79e"
 dependencies = [
  "derive_builder_core",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -231,9 +231,9 @@ checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
 [[package]]
 name = "getrandom"
-version = "0.2.8"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31"
+checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4"
 dependencies = [
  "cfg-if",
  "libc",
@@ -293,9 +293,9 @@ dependencies = [
 
 [[package]]
 name = "itoa"
-version = "1.0.5"
+version = "1.0.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fad582f4b9e86b6caa621cabeb0963332d92eea04729ab12892c2533951e6440"
+checksum = "453ad9f582a441959e5f0d088b02ce04cfe8d51a8eaf077f12ac6d3e94164ca6"
 
 [[package]]
 name = "lazy_static"
@@ -311,9 +311,9 @@ checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
 
 [[package]]
 name = "libc"
-version = "0.2.139"
+version = "0.2.141"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "201de327520df007757c1f0adce6e827fe8562fbc28bfd9c15571c66ca1f5f79"
+checksum = "3304a64d199bb964be99741b7a14d26972741915b3649639149b2479bb46f4b5"
 
 [[package]]
 name = "libloading"
@@ -352,8 +352,9 @@ checksum = "58093314a45e00c77d5c508f76e77c3396afbbc0d01506e7fae47b018bac2b1d"
 
 [[package]]
 name = "magnus"
-version = "0.5.0"
-source = "git+https://github.com/matsadler/magnus#eda735faa7e03da2443eaf2c4058a184917d6b87"
+version = "0.5.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c8dc14463c2552e753ef562961f486ca76f17a857c121db40e9f3ade3f35ab81"
 dependencies = [
  "magnus-macros",
  "rb-sys",
@@ -362,12 +363,13 @@ dependencies = [
 
 [[package]]
 name = "magnus-macros"
-version = "0.3.0"
-source = "git+https://github.com/matsadler/magnus#eda735faa7e03da2443eaf2c4058a184917d6b87"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6cc17af1d45442c011aa579d727ec6cff8a69aea8a6bbad26736e7112d749bfb"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -378,9 +380,9 @@ checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d"
 
 [[package]]
 name = "memoffset"
-version = "0.7.1"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5de893c32cde5f383baa4c04c5d6dbdd735cfd4a794b0debdb2bb1b421da5ff4"
+checksum = "d61c719bcfbcf5d62b3a09efa6088de8c54bc0bfcd3ea7ae39fcc186108b8de1"
 dependencies = [
  "autocfg",
 ]
@@ -391,6 +393,27 @@ version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
 
+[[package]]
+name = "monostate"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0230b703f1ac35df1e24f6d0d2255472bcccaf657ecdfa4f1fcbcad1ad5bb98a"
+dependencies = [
+ "monostate-impl",
+ "serde",
+]
+
+[[package]]
+name = "monostate-impl"
+version = "0.1.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8795add3e14028f11f8e848bd3294898a8294767b3776b6f733560d33bd2530b"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn 2.0.13",
+]
+
 [[package]]
 name = "nom"
 version = "7.1.3"
@@ -419,9 +442,9 @@ checksum = "17b02fc0ff9a9e4b35b3342880f48e896ebf69f2967921fe8646bf5b7125956a"
 
 [[package]]
 name = "once_cell"
-version = "1.17.0"
+version = "1.17.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6f61fba1741ea2b3d6a1e3178721804bb716a68a6aeba1149b5d52e3d464ea66"
+checksum = "b7e5500299e16ebb147ae15a00a942af264cf3688f47923b8fc2cd5858f23ad3"
 
 [[package]]
 name = "onig"
@@ -447,9 +470,9 @@ dependencies = [
 
 [[package]]
 name = "paste"
-version = "1.0.11"
+version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d01a5bd0424d00070b0098dd17ebca6f961a959dead1dbcbbbc1d1cd8d3deeba"
+checksum = "9f746c4065a8fa3fe23974dd82f15431cc8d40779821001404d10d2e79ca7d79"
 
 [[package]]
 name = "peeking_take_while"
@@ -471,18 +494,18 @@ checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de"
 
 [[package]]
 name = "proc-macro2"
-version = "1.0.51"
+version = "1.0.56"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5d727cae5b39d21da60fa540906919ad737832fe0b1c165da3a34d6548c849d6"
+checksum = "2b63bdb0cd06f1f4dedf69b254734f9b45af66e4a031e42a7480257d9898b435"
 dependencies = [
  "unicode-ident",
 ]
 
 [[package]]
 name = "quote"
-version = "1.0.23"
+version = "1.0.26"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8856d8364d252a14d474036ea1358d63c9e6965c8e5c1885c18f73d70bff9c7b"
+checksum = "4424af4bf778aae2051a77b60283332f386554255d722233d09fbfc7e30da2fc"
 dependencies = [
  "proc-macro2",
 ]
@@ -519,9 +542,9 @@ dependencies = [
 
 [[package]]
 name = "rayon"
-version = "1.6.1"
+version = "1.7.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6db3a213adf02b3bcfd2d3846bb41cb22857d131789e01df434fb7e7bc0759b7"
+checksum = "1d2df5196e37bcc87abebc0053e20787d73847bb33134a69841207dd0a47f03b"
 dependencies = [
  "either",
  "rayon-core",
@@ -540,9 +563,9 @@ dependencies = [
 
 [[package]]
 name = "rayon-core"
-version = "1.10.2"
+version = "1.11.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "356a0625f1954f730c0201cdab48611198dc6ce21f4acff55089b5a78e6e835b"
+checksum = "4b8f95bd6966f5c87776639160a66bd8ab9895d9d4ab01ddba9fc60661aebe8d"
 dependencies = [
  "crossbeam-channel",
  "crossbeam-deque",
@@ -552,22 +575,26 @@ dependencies = [
 
 [[package]]
 name = "rb-sys"
-version = "0.9.64"
+version = "0.9.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cc8945662df8083245deda89e236647173cc7ad750f481ddcd7bbfd3afe3fa5e"
+checksum = "156bfedced1e236600bcaad538477097ff2ed5c6b474e411d15b791e1d24c0f1"
 dependencies = [
  "rb-sys-build",
 ]
 
 [[package]]
 name = "rb-sys-build"
-version = "0.9.64"
+version = "0.9.71"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ae8c3cdf9edc3908ee1555b7a1bca58ee1b499439b32cd1c1ec3e66736a8df48"
+checksum = "5cb2e4a32cbc290b543a74567072ad24b708aff7bb5dde5a68d5690379cd7938"
 dependencies = [
  "bindgen",
+ "lazy_static",
+ "proc-macro2",
+ "quote",
  "regex",
  "shell-words",
+ "syn 1.0.109",
 ]
 
 [[package]]
@@ -578,9 +605,9 @@ checksum = "a35802679f07360454b418a5d1735c89716bde01d35b1560fc953c1415a0b3bb"
 
 [[package]]
 name = "regex"
-version = "1.7.1"
+version = "1.7.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "48aaa5748ba571fb95cd2c85c09f629215d3a6ece942baa100950af03a34f733"
+checksum = "8b1f693b24f6ac912f4893ef08244d70b6067480d2f1a46e950c9691e6749d1d"
 dependencies = [
  "aho-corasick",
  "memchr",
@@ -589,9 +616,9 @@ dependencies = [
 
 [[package]]
 name = "regex-syntax"
-version = "0.6.28"
+version = "0.6.29"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "456c603be3e8d448b072f410900c09faf164fbce2d480456f50eea6e25f9c848"
+checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1"
 
 [[package]]
 name = "rustc-hash"
@@ -601,9 +628,9 @@ checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
 
 [[package]]
 name = "ryu"
-version = "1.0.12"
+version = "1.0.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b4b9743ed687d4b4bcedf9ff5eaa7398495ae14e61cba0a295704edbc7decde"
+checksum = "f91339c0467de62360649f8d3e185ca8de4224ff281f66000de5eb2a77a79041"
 
 [[package]]
 name = "scopeguard"
@@ -613,29 +640,29 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd"
 
 [[package]]
 name = "serde"
-version = "1.0.152"
+version = "1.0.159"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb"
+checksum = "3c04e8343c3daeec41f58990b9d77068df31209f2af111e059e9fe9646693065"
 dependencies = [
  "serde_derive",
 ]
 
 [[package]]
 name = "serde_derive"
-version = "1.0.152"
+version = "1.0.159"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e"
+checksum = "4c614d17805b093df4b147b51339e7e44bf05ef59fba1e45d83500bcfb4d8585"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.13",
 ]
 
 [[package]]
 name = "serde_json"
-version = "1.0.92"
+version = "1.0.95"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7434af0dc1cbd59268aa98b4c22c131c0584d2232f6fb166efb993e2832e896a"
+checksum = "d721eca97ac802aa7777b701877c8004d950fc142651367300d21c1cc0194744"
 dependencies = [
  "itoa",
  "ryu",
@@ -680,9 +707,20 @@ checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
 
 [[package]]
 name = "syn"
-version = "1.0.107"
+version = "1.0.109"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1f4064b5b16e03ae50984a5a8ed5d4f8803e6bc1fd170a3cda91a1be4b18e3f5"
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-ident",
+]
+
+[[package]]
+name = "syn"
+version = "2.0.13"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4c9da457c5285ac1f936ebd076af6dac17a61cfe7826f2076b4d015cf47bc8ec"
 dependencies = [
  "proc-macro2",
  "quote",
@@ -691,38 +729,39 @@ dependencies = [
 
 [[package]]
 name = "thiserror"
-version = "1.0.38"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6a9cd18aa97d5c45c6603caea1da6628790b37f7a34b6ca89522331c5180fed0"
+checksum = "978c9a314bd8dc99be594bc3c175faaa9794be04a5a5e153caba6915336cebac"
 dependencies = [
  "thiserror-impl",
 ]
 
 [[package]]
 name = "thiserror-impl"
-version = "1.0.38"
+version = "1.0.40"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1fb327af4685e4d03fa8cbcf1716380da910eeb2bb8be417e7f9fd3fb164f36f"
+checksum = "f9456a42c5b0d803c8cd86e73dd7cc9edd429499f37a3550d286d5e86720569f"
 dependencies = [
  "proc-macro2",
  "quote",
- "syn",
+ "syn 2.0.13",
 ]
 
 [[package]]
 name = "tokenizers"
-version = "0.3.1"
+version = "0.3.3"
 dependencies = [
  "magnus",
  "onig",
  "serde",
- "tokenizers 0.13.2",
+ "tokenizers 0.13.3",
 ]
 
 [[package]]
 name = "tokenizers"
-version = "0.13.2"
-source = "git+https://github.com/huggingface/tokenizers#fa66caf0abff16bae2213658ffa3e969c5445750"
+version = "0.13.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5cf49017523bf0bc01c9966f172c5f120bbb7b96cccd1708772dd42e767fb9f5"
 dependencies = [
  "aho-corasick",
  "derive_builder",
@@ -733,6 +772,7 @@ dependencies = [
  "lazy_static",
  "log",
  "macro_rules_attribute",
+ "monostate",
  "onig",
  "paste",
  "rand",
@@ -751,9 +791,9 @@ dependencies = [
 
 [[package]]
 name = "unicode-ident"
-version = "1.0.6"
+version = "1.0.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "84a22b9f218b40614adcb3f4ff08b703773ad44fa9423e4e0d346d5db86e4ebc"
+checksum = "e5464a87b239f13a63a501f2701565754bae92d243d4bb7eb12f6d57d2269bf4"
 
 [[package]]
 name = "unicode-normalization-alignments"
@@ -827,42 +867,42 @@ dependencies = [
 
 [[package]]
 name = "windows_aarch64_gnullvm"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608"
+checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8"
 
 [[package]]
 name = "windows_aarch64_msvc"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7"
+checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43"
 
 [[package]]
 name = "windows_i686_gnu"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640"
+checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f"
 
 [[package]]
 name = "windows_i686_msvc"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605"
+checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060"
 
 [[package]]
 name = "windows_x86_64_gnu"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45"
+checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36"
 
 [[package]]
 name = "windows_x86_64_gnullvm"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463"
+checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3"
 
 [[package]]
 name = "windows_x86_64_msvc"
-version = "0.42.1"
+version = "0.42.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd"
+checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0"

data/ext/tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers"
-version = "0.3.1"
+version = "0.3.3"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -10,12 +10,11 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-magnus = { git = "https://github.com/matsadler/magnus" }
-onig = { version = "6.0", default-features = false }
-serde = { version = "1.0", features = ["rc", "derive"] }
+magnus = "0.5"
+onig = { version = "6", default-features = false }
+serde = { version = "1", features = ["rc", "derive"] }
 
 [dependencies.tokenizers]
-version = "0.13.2" # also update in from_pretrained.rb
-git = "https://github.com/huggingface/tokenizers"
+version = "=0.13.3" # also update in from_pretrained.rb
 default-features = false
 features = ["progressbar", "onig", "esaxx_fast"]

data/ext/tokenizers/src/decoders.rs

@@ -7,14 +7,19 @@ use magnus::{
 };
 use serde::{Deserialize, Serialize};
 use tk::decoders::bpe::BPEDecoder;
+use tk::decoders::byte_fallback::ByteFallback;
 use tk::decoders::byte_level::ByteLevel;
 use tk::decoders::ctc::CTC;
+use tk::decoders::fuse::Fuse;
 use tk::decoders::metaspace::Metaspace;
+use tk::decoders::strip::Strip;
 use tk::decoders::wordpiece::WordPiece;
 use tk::decoders::DecoderWrapper;
 use tk::Decoder;
+use tk::normalizers::replace::Replace;
 
-use super::RbResult;
+use super::utils::*;
+use super::{RbError, RbResult};
 
 #[derive(DataTypeFunctions, Clone, Deserialize, Serialize)]
 pub struct RbDecoder {
@@ -89,6 +94,30 @@ impl RbDecoder {
         setter!(self, CTC, word_delimiter_token, word_delimiter_token);
     }
 
+    fn strip_content(&self) -> char {
+        getter!(self, Strip, content)
+    }
+
+    fn strip_set_content(&self, content: char) {
+        setter!(self, Strip, content, content)
+    }
+
+    fn strip_start(&self) -> usize {
+        getter!(self, Strip, start)
+    }
+
+    fn strip_set_start(&self, start: usize) {
+        setter!(self, Strip, start, start)
+    }
+
+    fn strip_stop(&self) -> usize {
+        getter!(self, Strip, stop)
+    }
+
+    fn strip_set_stop(&self, stop: usize) {
+        setter!(self, Strip, stop, stop)
+    }
+
     pub fn metaspace_replacement(&self) -> char {
         getter!(self, Metaspace, get_replacement().clone())
     }
@@ -130,6 +159,14 @@ impl RbBPEDecoder {
     }
 }
 
+pub struct RbByteFallbackDecoder {}
+
+impl RbByteFallbackDecoder {
+    pub fn new() -> RbDecoder {
+        ByteFallback::default().into()
+    }
+}
+
 pub struct RbByteLevelDecoder {}
 
 impl RbByteLevelDecoder {
@@ -146,6 +183,14 @@ impl RbCTC {
     }
 }
 
+pub struct RbFuse {}
+
+impl RbFuse {
+    pub fn new() -> RbDecoder {
+        Fuse::default().into()
+    }
+}
+
 pub struct RbMetaspaceDecoder {}
 
 impl RbMetaspaceDecoder {
@@ -154,6 +199,22 @@ impl RbMetaspaceDecoder {
     }
 }
 
+pub struct RbReplaceDecoder {}
+
+impl RbReplaceDecoder {
+    pub fn new(pattern: RbPattern, content: String) -> RbResult<RbDecoder> {
+        Replace::new(pattern, content).map(|v| v.into()).map_err(RbError::from)
+    }
+}
+
+pub struct RbStripDecoder {}
+
+impl RbStripDecoder {
+    pub fn new(content: char, start: usize, stop: usize) -> RbDecoder {
+        Strip::new(content, start, stop).into()
+    }
+}
+
 pub struct RbWordPieceDecoder {}
 
 impl RbWordPieceDecoder {
@@ -219,6 +280,11 @@ unsafe impl TypedData for RbDecoder {
                     class.undef_alloc_func();
                     class
                 }),
+                DecoderWrapper::ByteFallback(_) => *memoize!(RClass: {
+                    let class: RClass = crate::decoders().const_get("ByteFallback").unwrap();
+                    class.undef_alloc_func();
+                    class
+                }),
                 DecoderWrapper::ByteLevel(_) => *memoize!(RClass: {
                     let class: RClass = crate::decoders().const_get("ByteLevel").unwrap();
                     class.undef_alloc_func();
@@ -229,11 +295,26 @@ unsafe impl TypedData for RbDecoder {
                     class.undef_alloc_func();
                     class
                 }),
+                DecoderWrapper::Fuse(_) => *memoize!(RClass: {
+                    let class: RClass = crate::decoders().const_get("Fuse").unwrap();
+                    class.undef_alloc_func();
+                    class
+                }),
                 DecoderWrapper::Metaspace(_) => *memoize!(RClass: {
                     let class: RClass = crate::decoders().const_get("Metaspace").unwrap();
                     class.undef_alloc_func();
                     class
                 }),
+                DecoderWrapper::Replace(_) => *memoize!(RClass: {
+                    let class: RClass = crate::decoders().const_get("Replace").unwrap();
+                    class.undef_alloc_func();
+                    class
+                }),
+                DecoderWrapper::Strip(_) => *memoize!(RClass: {
+                    let class: RClass = crate::decoders().const_get("Strip").unwrap();
+                    class.undef_alloc_func();
+                    class
+                }),
                 DecoderWrapper::WordPiece(_) => *memoize!(RClass: {
                     let class: RClass = crate::decoders().const_get("WordPiece").unwrap();
                     class.undef_alloc_func();
@@ -253,6 +334,9 @@ pub fn decoders(module: &RModule) -> RbResult<()> {
     class.define_method("suffix", method!(RbDecoder::bpe_suffix, 0))?;
     class.define_method("suffix=", method!(RbDecoder::bpe_set_suffix, 1))?;
 
+    let class = module.define_class("ByteFallback", decoder)?;
+    class.define_singleton_method("new", function!(RbByteFallbackDecoder::new, 0))?;
+
     let class = module.define_class("ByteLevel", decoder)?;
     class.define_singleton_method("new", function!(RbByteLevelDecoder::new, 0))?;
 
@@ -265,6 +349,9 @@ pub fn decoders(module: &RModule) -> RbResult<()> {
     class.define_method("word_delimiter_token", method!(RbDecoder::ctc_word_delimiter_token, 0))?;
     class.define_method("word_delimiter_token=", method!(RbDecoder::ctc_set_word_delimiter_token, 1))?;
 
+    let class = module.define_class("Fuse", decoder)?;
+    class.define_singleton_method("new", function!(RbFuse::new, 0))?;
+
     let class = module.define_class("Metaspace", decoder)?;
     class.define_singleton_method("_new", function!(RbMetaspaceDecoder::new, 2))?;
     class.define_method("add_prefix_space", method!(RbDecoder::metaspace_add_prefix_space, 0))?;
@@ -272,6 +359,18 @@ pub fn decoders(module: &RModule) -> RbResult<()> {
     class.define_method("replacement", method!(RbDecoder::metaspace_replacement, 0))?;
     class.define_method("replacement=", method!(RbDecoder::metaspace_set_replacement, 1))?;
 
+    let class = module.define_class("Replace", decoder)?;
+    class.define_singleton_method("new", function!(RbReplaceDecoder::new, 2))?;
+
+    let class = module.define_class("Strip", decoder)?;
+    class.define_singleton_method("_new", function!(RbStripDecoder::new, 3))?;
+    class.define_method("content", method!(RbDecoder::strip_content, 0))?;
+    class.define_method("content=", method!(RbDecoder::strip_set_content, 1))?;
+    class.define_method("start", method!(RbDecoder::strip_start, 0))?;
+    class.define_method("start=", method!(RbDecoder::strip_set_start, 1))?;
+    class.define_method("stop", method!(RbDecoder::strip_stop, 0))?;
+    class.define_method("stop=", method!(RbDecoder::strip_set_stop, 1))?;
+
     let class = module.define_class("WordPiece", decoder)?;
     class.define_singleton_method("_new", function!(RbWordPieceDecoder::new, 2))?;
     class.define_method("cleanup", method!(RbDecoder::word_piece_cleanup, 0))?;

data/ext/tokenizers/src/models.rs

@@ -101,6 +101,11 @@ impl RbBPE {
             builder = builder.fuse_unk(value.try_convert()?);
         }
 
+        let value: Value = kwargs.delete(Symbol::new("byte_fallback"))?;
+        if !value.is_nil() {
+            builder = builder.byte_fallback(value.try_convert()?);
+        }
+
         if !kwargs.is_empty() {
             // TODO improve message
             return Err(Error::new(exception::arg_error(), "unknown keyword"));
@@ -169,6 +174,14 @@ impl RbModel {
         setter!(self, BPE, fuse_unk, fuse_unk);
     }
 
+    pub fn bpe_byte_fallback(&self) -> bool {
+        getter!(self, BPE, byte_fallback)
+    }
+
+    pub fn bpe_set_byte_fallback(&self, byte_fallback: bool) {
+        setter!(self, BPE, byte_fallback, byte_fallback);
+    }
+
     pub fn bpe_continuing_subword_prefix(&self) -> Option<String> {
         getter!(self, BPE, continuing_subword_prefix.clone())
     }
@@ -355,6 +368,8 @@ pub fn models(module: &RModule) -> RbResult<()> {
     class.define_method("end_of_word_suffix=", method!(RbModel::bpe_set_end_of_word_suffix, 1))?;
     class.define_method("fuse_unk", method!(RbModel::bpe_fuse_unk, 0))?;
     class.define_method("fuse_unk=", method!(RbModel::bpe_set_fuse_unk, 1))?;
+    class.define_method("byte_fallback", method!(RbModel::bpe_byte_fallback, 0))?;
+    class.define_method("byte_fallback=", method!(RbModel::bpe_set_byte_fallback, 1))?;
 
     let class = module.define_class("Unigram", model)?;
     class.define_singleton_method("_new", function!(RbUnigram::new, 2))?;

data/ext/tokenizers/src/normalizers.rs

@@ -8,7 +8,7 @@ use magnus::{
 use serde::ser::SerializeStruct;
 use serde::{Deserialize, Serialize, Serializer};
 use tk::normalizers::{
-    BertNormalizer, Lowercase, Nmt, NormalizerWrapper, Replace, Strip, StripAccents,
+    BertNormalizer, Lowercase, Nmt, NormalizerWrapper, Replace, Prepend, Strip, StripAccents,
     NFC, NFD, NFKC, NFKD,
 };
 use tk::{NormalizedString, Normalizer};
@@ -44,7 +44,7 @@ macro_rules! getter {
     ($self: ident, $variant: ident, $name: ident) => {{
         if let RbNormalizerTypeWrapper::Single(ref norm) = &$self.normalizer {
             let wrapper = norm.read().unwrap();
-            if let RbNormalizerWrapper::Wrapped(NormalizerWrapper::$variant(o)) = *wrapper {
+            if let RbNormalizerWrapper::Wrapped(NormalizerWrapper::$variant(o)) = (*wrapper).clone() {
                 o.$name
             } else {
                 unreachable!()
@@ -105,6 +105,14 @@ impl RbNormalizer {
         setter!(self, BertNormalizer, lowercase, lowercase)
     }
 
+    fn prepend_prepend(&self) -> String {
+        getter!(self, Prepend, prepend)
+    }
+
+    fn prepend_set_prepend(&self, prepend: String) {
+        setter!(self, Prepend, prepend, prepend)
+    }
+
     fn strip_left(&self) -> bool {
         getter!(self, StripNormalizer, strip_left)
     }
@@ -186,6 +194,14 @@ impl RbReplace {
     }
 }
 
+pub struct RbPrepend {}
+
+impl RbPrepend {
+    pub fn new(prepend: String) -> RbNormalizer {
+        Prepend::new(prepend).into()
+    }
+}
+
 pub struct RbStrip {}
 
 impl RbStrip {
@@ -372,6 +388,11 @@ unsafe impl TypedData for RbNormalizer {
                         class.undef_alloc_func();
                         class
                     }),
+                    NormalizerWrapper::Prepend(_) => *memoize!(RClass: {
+                        let class: RClass = crate::normalizers().const_get("Prepend").unwrap();
+                        class.undef_alloc_func();
+                        class
+                    }),
                     NormalizerWrapper::StripNormalizer(_) => *memoize!(RClass: {
                         let class: RClass = crate::normalizers().const_get("Strip").unwrap();
                         class.undef_alloc_func();
@@ -428,6 +449,11 @@ pub fn normalizers(module: &RModule) -> RbResult<()> {
     let class = module.define_class("Replace", normalizer)?;
     class.define_singleton_method("new", function!(RbReplace::new, 2))?;
 
+    let class = module.define_class("Prepend", normalizer)?;
+    class.define_singleton_method("_new", function!(RbPrepend::new, 1))?;
+    class.define_method("prepend", method!(RbNormalizer::prepend_prepend, 0))?;
+    class.define_method("prepend=", method!(RbNormalizer::prepend_set_prepend, 1))?;
+
     let class = module.define_class("Strip", normalizer)?;
     class.define_singleton_method("_new", function!(RbStrip::new, 2))?;
     class.define_method("left", method!(RbNormalizer::strip_left, 0))?;

data/lib/tokenizers/decoders/strip.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Decoders
+    class Strip
+      def self.new(content: " ", start: 0, stop: 0)
+        _new(content, start, stop)
+      end
+    end
+  end
+end

data/lib/tokenizers/from_pretrained.rb

@@ -1,7 +1,7 @@
 module Tokenizers
   module FromPretrained
     # for user agent
-    TOKENIZERS_VERSION = "0.13.2"
+    TOKENIZERS_VERSION = "0.13.3"
 
     # use Ruby for downloads
     # this avoids the need to vendor OpenSSL on Linux

data/lib/tokenizers/normalizers/prepend.rb

@@ -0,0 +1,9 @@
+module Tokenizers
+  module Normalizers
+    class Prepend
+      def self.new(prepend: "▁")
+        _new(prepend)
+      end
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.3.1"
+  VERSION = "0.3.3"
 end

data/lib/tokenizers.rb

@@ -9,6 +9,7 @@ end
 require_relative "tokenizers/decoders/bpe_decoder"
 require_relative "tokenizers/decoders/ctc"
 require_relative "tokenizers/decoders/metaspace"
+require_relative "tokenizers/decoders/strip"
 require_relative "tokenizers/decoders/word_piece"
 
 # models
@@ -19,6 +20,7 @@ require_relative "tokenizers/models/unigram"
 
 # normalizers
 require_relative "tokenizers/normalizers/bert_normalizer"
+require_relative "tokenizers/normalizers/prepend"
 require_relative "tokenizers/normalizers/strip"
 
 # pre-tokenizers

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.3.1
+  version: 0.3.3
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-02-09 00:00:00.000000000 Z
+date: 2023-04-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -56,6 +56,7 @@ files:
 - lib/tokenizers/decoders/bpe_decoder.rb
 - lib/tokenizers/decoders/ctc.rb
 - lib/tokenizers/decoders/metaspace.rb
+- lib/tokenizers/decoders/strip.rb
 - lib/tokenizers/decoders/word_piece.rb
 - lib/tokenizers/encoding.rb
 - lib/tokenizers/from_pretrained.rb
@@ -64,6 +65,7 @@ files:
 - lib/tokenizers/models/word_level.rb
 - lib/tokenizers/models/word_piece.rb
 - lib/tokenizers/normalizers/bert_normalizer.rb
+- lib/tokenizers/normalizers/prepend.rb
 - lib/tokenizers/normalizers/strip.rb
 - lib/tokenizers/pre_tokenizers/byte_level.rb
 - lib/tokenizers/pre_tokenizers/digits.rb
@@ -98,7 +100,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.6
+rubygems_version: 3.4.10
 signing_key:
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby