tokenizers 0.5.2-aarch64-linux-musl → 0.5.3-aarch64-linux-musl

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: af71e9e211a8a3bc3e1a05a2c5fa937d54b0a92dcc42dc784369d14b40bb6d5f
-  data.tar.gz: d2dc41d5e280fde549f6d39ce96bc639150937321ca6630f54e245690597fd22
+  metadata.gz: '08700fb9bdcad5fd8f72a201ff9fb9d4d5738d0994ed395f8870696e3d2c8420'
+  data.tar.gz: ae92e4266649e977717c38ecfa0e7435e2a4cd5439f4156ebe9d60db05251413
 SHA512:
-  metadata.gz: 6c9db5d94f60535efaaf09efeac122bd703c1bbb7328c5d9963606bdebb6e92bea4239233471ff635d0b7c4d126d195a8e9ecd54ad31a3077e16c78ffadfbf72
-  data.tar.gz: 02ab410e4350520f433c6b08059b415d80316a2e763d51c4c81db88d52d823d1850a6fc62dd4694cab7cff7bca0ee35790db3ac05302a19573abc84973dd0ff5
+  metadata.gz: a1bbba481e1d2f8c65ced5020d2d1b53526d9b31f7927a62fd39fafddf254dbcd87a525860f1734e91c4ac102453b449832995428c5626181c0d8e4506401d9d
+  data.tar.gz: 93d5e32e26ab8e9277b148e331d35f9673708fed1d3dd92cc0ae9d9651d4fe7aeda1c850c3326b91899d4ce65fd2efc2806a6e8fd21e0a1846d0594614a48cc0

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.5.3 (2024-09-17)
+
+- Added `AddedToken` class
+- Added precompiled gem for Windows
+
 ## 0.5.2 (2024-08-26)
 
 - Added `from_str` method to `Tokenizer`

data/Cargo.lock

@@ -724,7 +724,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.5.2"
+version = "0.5.3"
 dependencies = [
  "magnus",
  "onig",

data/lib/tokenizers/added_token.rb

@@ -0,0 +1,7 @@
+module Tokenizers
+  class AddedToken
+    def self.new(content, **kwargs)
+      _new(content, kwargs)
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.5.2"
+  VERSION = "0.5.3"
 end

data/lib/tokenizers.rb

@@ -42,6 +42,7 @@ require_relative "tokenizers/trainers/word_level_trainer"
 require_relative "tokenizers/trainers/word_piece_trainer"
 
 # other
+require_relative "tokenizers/added_token"
 require_relative "tokenizers/char_bpe_tokenizer"
 require_relative "tokenizers/encoding"
 require_relative "tokenizers/from_pretrained"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: aarch64-linux-musl
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-08-26 00:00:00.000000000 Z
+date: 2024-09-17 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -26,6 +26,7 @@ files:
 - lib/tokenizers/3.1/tokenizers.so
 - lib/tokenizers/3.2/tokenizers.so
 - lib/tokenizers/3.3/tokenizers.so
+- lib/tokenizers/added_token.rb
 - lib/tokenizers/char_bpe_tokenizer.rb
 - lib/tokenizers/decoders/bpe_decoder.rb
 - lib/tokenizers/decoders/ctc.rb