tokenizers 0.5.2-x86_64-linux-musl → 0.5.3-x86_64-linux-musl

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 952e7aa7504f266d818ec8cff8c9bd81d890bcb57c07113650fbfcd0875fb3b9
-  data.tar.gz: 6a0a3915a1076b7675b4ffcc3e55447906ffc1e378c9a95d82132f40c76034e7
+  metadata.gz: fce129f3910c54922d726c2dfa4f501eee8e41cc35374636663219a14fae5556
+  data.tar.gz: a8c7931b5eb452264cd4c5901d194bc0ba955d8857078e5216544edd15db0d5f
 SHA512:
-  metadata.gz: cd2390e33c79d8b6528199c8ca9af7727b1bf63f58fb1a4ce074caa243ab796dcf4cd77eab26f2f76e872d68ae29f169c616128affad49a39147717b3d01b428
-  data.tar.gz: 4e8baa0e28f5218480baa55320c4afd0db653f5ebd42b6e60b975d0a23a713ca3afd6f8666fb40a52017355e385178100946f50f9e6308910c5f8e564cfa9655
+  metadata.gz: '08fa0d532d72de71ace9d071cce5c10d08a6c7f119381c6476a7eeeadd5968a0f37a30897c91e6e1dc975122d4899161ed9bff181169eed0940f1d7fa0363405'
+  data.tar.gz: 1ab0ded21ce97e9bc5de491d42c880fbb2e186a9725546db2fb86ef0fcebb64d8e92ace94ce2cbda0540cc07a6c806fae8ae359d82b71c2998c3799a59b266a1

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.5.3 (2024-09-17)
+
+- Added `AddedToken` class
+- Added precompiled gem for Windows
+
 ## 0.5.2 (2024-08-26)
 
 - Added `from_str` method to `Tokenizer`

data/Cargo.lock

@@ -724,7 +724,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.5.2"
+version = "0.5.3"
 dependencies = [
  "magnus",
  "onig",

data/lib/tokenizers/added_token.rb

@@ -0,0 +1,7 @@
+module Tokenizers
+  class AddedToken
+    def self.new(content, **kwargs)
+      _new(content, kwargs)
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.5.2"
+  VERSION = "0.5.3"
 end

data/lib/tokenizers.rb

@@ -42,6 +42,7 @@ require_relative "tokenizers/trainers/word_level_trainer"
 require_relative "tokenizers/trainers/word_piece_trainer"
 
 # other
+require_relative "tokenizers/added_token"
 require_relative "tokenizers/char_bpe_tokenizer"
 require_relative "tokenizers/encoding"
 require_relative "tokenizers/from_pretrained"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: x86_64-linux-musl
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-08-26 00:00:00.000000000 Z
+date: 2024-09-17 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -26,6 +26,7 @@ files:
 - lib/tokenizers/3.1/tokenizers.so
 - lib/tokenizers/3.2/tokenizers.so
 - lib/tokenizers/3.3/tokenizers.so
+- lib/tokenizers/added_token.rb
 - lib/tokenizers/char_bpe_tokenizer.rb
 - lib/tokenizers/decoders/bpe_decoder.rb
 - lib/tokenizers/decoders/ctc.rb