tokenizers 0.2.2-x86_64-darwin → 0.2.3-x86_64-darwin

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: cd9f4e13c08d5a2bbc017d0a297de6be483f1f894e3089a8c2a87bcddf09eb92
-  data.tar.gz: 144af0a6041863212aa44a42729e9d721b78f09862943d339c2aef695728b88d
+  metadata.gz: 4ccf6bfafaf681a2e5f9088b32fc41df079b48a69f865dcf7f5e3e5c9990eedb
+  data.tar.gz: 7909f67176ea9015c73e7fa76407de53c64efb6abf62972fad12504a526bb1c2
 SHA512:
-  metadata.gz: 0ee668f662c8c30aea39f618745e97810ec7a2c017f6a8cf8022d595ca7edfc7a9eb03bcf5cecb23e5035c0d36f5c3d656af7e7b615827109b07917c8b7a340f
-  data.tar.gz: a3c1dbe246a323ae8ff3e6572fa610a0e1ccf025e178f49956bf54616511e776dfead2da6a36a6a064a27fdd81ad441f6601c755c415e10df3009f615274b5f3
+  metadata.gz: 7e7bfb6230c87af3af44ed59aa45cef5fc7019b09233ba297f08e660cc2791e2e0fbbb34bd4bc5c7761198895ed56baf0df8a63c67d25379ff4821568b6ffb23
+  data.tar.gz: 834a44cc9fc48405599982477c71f3bffa52580331884b0a29b87a5d4db2fb973be0f6fd0eb7f57f70890defcd8a6486cded8c7dc6473b7b954fbd15ec0069ed

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.2.3 (2022-01-22)
+
+- Added `add_special_tokens` option to `encode` method
+- Added warning about `encode` method including special tokens by default in 0.3.0
+- Added more methods to `Encoding`
+- Fixed error with precompiled gem on Mac ARM
+
 ## 0.2.2 (2022-01-15)
 
 - Added precompiled gem for Linux ARM

data/Cargo.lock

@@ -753,7 +753,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.2.2"
+version = "0.2.3"
 dependencies = [
  "magnus",
  "tokenizers 0.13.2",

data/LICENSE-THIRD-PARTY.txt

@@ -12446,6 +12446,32 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
+================================================================================
+rb-sys-env LICENSE-MIT
+================================================================================
+
+The MIT License (MIT)
+
+Copyright (c) 2021-2022 Ian Ker-Seymer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 ================================================================================
 rb-sys-env LICENSE-APACHE
 ================================================================================
@@ -12641,32 +12667,6 @@ rb-sys-env LICENSE-APACHE
    See the License for the specific language governing permissions and
    limitations under the License.
 
-================================================================================
-rb-sys-env LICENSE-MIT
-================================================================================
-
-The MIT License (MIT)
-
-Copyright (c) 2021-2022 Ian Ker-Seymer
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
 ================================================================================
 regex LICENSE-APACHE
 ================================================================================

data/lib/tokenizers/char_bpe_tokenizer.rb

@@ -8,8 +8,8 @@ module Tokenizers
       @tokenizer.decoder = BPEDecoder.new
     end
 
-    def encode(text)
-      @tokenizer.encode(text)
+    def encode(text, **options)
+      @tokenizer.encode(text, **options)
     end
 
     def decode(ids)

data/lib/tokenizers/encoding.rb

@@ -0,0 +1,19 @@
+module Tokenizers
+  class Encoding
+    def word_to_tokens(word_index, sequence_index = 0)
+      _word_to_tokens(word_index, sequence_index)
+    end
+
+    def word_to_chars(word_index, sequence_index = 0)
+      _word_to_chars(word_index, sequence_index)
+    end
+
+    def char_to_token(char_pos, sequence_index = 0)
+      _char_to_token(char_pos, sequence_index)
+    end
+
+    def char_to_word(char_pos, sequence_index = 0)
+      _char_to_word(word_index, sequence_index)
+    end
+  end
+end

data/lib/tokenizers/tokenizer.rb

@@ -0,0 +1,12 @@
+module Tokenizers
+  class Tokenizer
+    # TODO change add_special_tokens default to true in 0.3.0
+    def encode(sequence, add_special_tokens: nil)
+      if add_special_tokens.nil?
+        warn "[tokenizers] add_special_tokens will default to true in 0.3.0. Pass add_special_tokens: true/false to silence this warning."
+        add_special_tokens = false
+      end
+      _encode(sequence, add_special_tokens)
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

data/lib/tokenizers.rb

@@ -1,14 +1,16 @@
 # ext
 begin
-  require "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
+  require_relative "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
 rescue LoadError
-  require "tokenizers/tokenizers"
+  require_relative "tokenizers/tokenizers"
 end
 
 # modules
-require "tokenizers/char_bpe_tokenizer"
-require "tokenizers/from_pretrained"
-require "tokenizers/version"
+require_relative "tokenizers/char_bpe_tokenizer"
+require_relative "tokenizers/encoding"
+require_relative "tokenizers/from_pretrained"
+require_relative "tokenizers/tokenizer"
+require_relative "tokenizers/version"
 
 module Tokenizers
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: x86_64-darwin
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-15 00:00:00.000000000 Z
+date: 2023-01-22 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -28,7 +28,9 @@ files:
 - lib/tokenizers/3.1/tokenizers.bundle
 - lib/tokenizers/3.2/tokenizers.bundle
 - lib/tokenizers/char_bpe_tokenizer.rb
+- lib/tokenizers/encoding.rb
 - lib/tokenizers/from_pretrained.rb
+- lib/tokenizers/tokenizer.rb
 - lib/tokenizers/version.rb
 homepage: https://github.com/ankane/tokenizers-ruby
 licenses:
@@ -52,7 +54,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.3
+rubygems_version: 3.4.4
 signing_key: 
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby