tokenizers 0.5.2-x86_64-darwin → 0.5.3-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 17d7c28ef87243dd719120f643f68f7cfc005b712974beb9f9513a29c3706b68
-  data.tar.gz: 218ab31a2944142938aa15f31eb0967563151d171accbb842b3d1dcf42cbca5f
+  metadata.gz: 2db1df1eec1cbf155a1f3ac69fff4b38ec3409cf758de3360f04be39e0a18168
+  data.tar.gz: d098227ba4ae83f845497ff78ca8eca54eec20f79c4e5ad3425e3dd92eeb693c
 SHA512:
-  metadata.gz: c2e9f03bc0fb467a9187770c6825561c9d452ff853193fc176d5a4149fdb446f955e8aaf187a141ff98a4cbdc7d29085a291dfbd5bb9c46b684fa784b14cafb6
-  data.tar.gz: 20a3477fa09718a5f2dcf270bfb4a0343b33a3567097a24728456d73152ca19941d6eaa0c5194830d8c837d8774afb019f729f31788ec4f9f8be84c9d4e0dbc4
+  metadata.gz: 85e8b49905169f90c821b57800c49b845428c08526255ce38d7841bdb2cc96f687b47b87e9fa4d82fdcd21b0a4357535ac07ca888b91612ce4227d9a064db5ed
+  data.tar.gz: b6a3debceba0ae20a49dca53a968e6cc0efabcbfe865652a8ad60da723b1399d6060ba80ba271ed4e489a9d29836585bc15fbaf60ecb44f5905daf2c5fca33fe

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.5.3 (2024-09-17)
+
+- Added `AddedToken` class
+- Added precompiled gem for Windows
+
 ## 0.5.2 (2024-08-26)
 
 - Added `from_str` method to `Tokenizer`

data/Cargo.lock

@@ -724,7 +724,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.5.2"
+version = "0.5.3"
 dependencies = [
  "magnus",
  "onig",

data/lib/tokenizers/added_token.rb

@@ -0,0 +1,7 @@
+module Tokenizers
+  class AddedToken
+    def self.new(content, **kwargs)
+      _new(content, kwargs)
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.5.2"
+  VERSION = "0.5.3"
 end

data/lib/tokenizers.rb

@@ -42,6 +42,7 @@ require_relative "tokenizers/trainers/word_level_trainer"
 require_relative "tokenizers/trainers/word_piece_trainer"
 
 # other
+require_relative "tokenizers/added_token"
 require_relative "tokenizers/char_bpe_tokenizer"
 require_relative "tokenizers/encoding"
 require_relative "tokenizers/from_pretrained"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.5.2
+  version: 0.5.3
 platform: x86_64-darwin
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-08-26 00:00:00.000000000 Z
+date: 2024-09-17 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -26,6 +26,7 @@ files:
 - lib/tokenizers/3.1/tokenizers.bundle
 - lib/tokenizers/3.2/tokenizers.bundle
 - lib/tokenizers/3.3/tokenizers.bundle
+- lib/tokenizers/added_token.rb
 - lib/tokenizers/char_bpe_tokenizer.rb
 - lib/tokenizers/decoders/bpe_decoder.rb
 - lib/tokenizers/decoders/ctc.rb