tokenizers 0.5.2-arm64-darwin → 0.5.3-arm64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a00c67519f2c2bb09c0fad42f396006a341be62b41d751096ef772158ed30c7d
-  data.tar.gz: 977b690303058422c76c41ebde6a8fd9981063cb88423aff535faf8a5c31d5a1
+  metadata.gz: dbf96bf34bc5b762a120c8bb0d9d20b637eccaf31b1650d45b46c722b4c7f1b4
+  data.tar.gz: 056b0d1dd8d14dbb52c311caa9aba9973ac9ea0ea3b9e26377e2fc69cc4ce598
 SHA512:
-  metadata.gz: 7c7ac23930faa7c58cb3c1f282c3fafb1937bc097facaaa45cfdd74f7dc0c59dee57896bfbbd5fd052e6ae3bc849affb4109dbc59a2a75ee4c4f9e644c441c65
-  data.tar.gz: 9bd2569f96710bc58aeeabd39d991ff56f13682dd86a271cc03dad2205e2a6732bb60c9eab06138405c2cf7d77df0196321d6cf7df6eac6b4f74caabb6d0a9e6
+  metadata.gz: c4488747d73279ac570b62a9afa3c8d7751d8b5a30684142a7879538a1450a31955b87cf498c7e671e80a92671166fb210ff992d04d3fa76e21d284884bef86e
+  data.tar.gz: 97151485dc583f4dfd9bd4cb4457dd411c1bf9a8953800fe28d532d2ac300a6214220f09a06b8b4e3fb052067f159d010b59feb5cc1e5396225f3d248206d469

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.5.3 (2024-09-17)
+
+- Added `AddedToken` class
+- Added precompiled gem for Windows
+
 ## 0.5.2 (2024-08-26)
 
 - Added `from_str` method to `Tokenizer`

data/Cargo.lock

@@ -724,7 +724,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.5.2"
+version = "0.5.3"
 dependencies = [
  "magnus",
  "onig",

data/lib/tokenizers/added_token.rb

@@ -0,0 +1,7 @@
+module Tokenizers
+  class AddedToken
+    def self.new(content, **kwargs)
+      _new(content, kwargs)
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.5.2"
+  VERSION = "0.5.3"
 end

data/lib/tokenizers.rb

@@ -42,6 +42,7 @@ require_relative "tokenizers/trainers/word_level_trainer"
 require_relative "tokenizers/trainers/word_piece_trainer"
 
 # other
+require_relative "tokenizers/added_token"
 require_relative "tokenizers/char_bpe_tokenizer"
 require_relative "tokenizers/encoding"
 require_relative "tokenizers/from_pretrained"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: arm64-darwin
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-08-26 00:00:00.000000000 Z
+date: 2024-09-17 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -26,6 +26,7 @@ files:
 - lib/tokenizers/3.1/tokenizers.bundle
 - lib/tokenizers/3.2/tokenizers.bundle
 - lib/tokenizers/3.3/tokenizers.bundle
+- lib/tokenizers/added_token.rb
 - lib/tokenizers/char_bpe_tokenizer.rb
 - lib/tokenizers/decoders/bpe_decoder.rb
 - lib/tokenizers/decoders/ctc.rb