tokenizers 0.2.2-x86_64-linux → 0.2.3-x86_64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3b490117f35a7cf069c47028ca41700a709ec0fade9a90cbd1fa5143b4670437
-  data.tar.gz: 9763a7d19bd304915a16731d30ba203dfcaa705e65e0dbde8656504a3195b82d
+  metadata.gz: 51273c1f38d9a2fcbcda6df42b1f8eff718965e84ab49233b6e54bef0825aed4
+  data.tar.gz: 3ce5e8543e7ac32c6302fcdefde06fdebb249be42db234bbbbe8671bb414a69a
 SHA512:
-  metadata.gz: 84aae3326fc7ed7e63b907581f78b07f435f80c47fe5317314ce92028016eccec652673aa96cded2615e7d5cba859fec8327d6e31e310856af22d4389c4d906e
-  data.tar.gz: a350a314a1d8e39f04fb6e2537e039da6a914012ce260face27ee375a07c55d56832c85ddf6604e65779055ee17d9eecfbf2fcea598c4cc90410cde07f403a24
+  metadata.gz: 45946d725ed104ca0001cf323c4bc0146050f051e8f9150aba22f5169640cef2227729ccbd9f705c43ada82ff53703ddaceb52a77259f0096066739c93c13a07
+  data.tar.gz: b056d1024ab43363c3ba3b0ae1cec2874fab31630de474eb0a6694484b88418bad408155121bf2adc0ed0ed4a70064765565e630d3ba648cb23e6b5c34d9aefc

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.2.3 (2022-01-22)
+
+- Added `add_special_tokens` option to `encode` method
+- Added warning about `encode` method including special tokens by default in 0.3.0
+- Added more methods to `Encoding`
+- Fixed error with precompiled gem on Mac ARM
+
 ## 0.2.2 (2022-01-15)
 
 - Added precompiled gem for Linux ARM

data/Cargo.lock

@@ -753,7 +753,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.2.2"
+version = "0.2.3"
 dependencies = [
  "magnus",
  "tokenizers 0.13.2",

data/LICENSE-THIRD-PARTY.txt

@@ -12446,6 +12446,32 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
+================================================================================
+rb-sys-env LICENSE-MIT
+================================================================================
+
+The MIT License (MIT)
+
+Copyright (c) 2021-2022 Ian Ker-Seymer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 ================================================================================
 rb-sys-env LICENSE-APACHE
 ================================================================================
@@ -12641,32 +12667,6 @@ rb-sys-env LICENSE-APACHE
    See the License for the specific language governing permissions and
    limitations under the License.
 
-================================================================================
-rb-sys-env LICENSE-MIT
-================================================================================
-
-The MIT License (MIT)
-
-Copyright (c) 2021-2022 Ian Ker-Seymer
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
 ================================================================================
 regex LICENSE-APACHE
 ================================================================================

data/lib/tokenizers/char_bpe_tokenizer.rb

@@ -8,8 +8,8 @@ module Tokenizers
       @tokenizer.decoder = BPEDecoder.new
     end
 
-    def encode(text)
-      @tokenizer.encode(text)
+    def encode(text, **options)
+      @tokenizer.encode(text, **options)
     end
 
     def decode(ids)

data/lib/tokenizers/encoding.rb

@@ -0,0 +1,19 @@
+module Tokenizers
+  class Encoding
+    def word_to_tokens(word_index, sequence_index = 0)
+      _word_to_tokens(word_index, sequence_index)
+    end
+
+    def word_to_chars(word_index, sequence_index = 0)
+      _word_to_chars(word_index, sequence_index)
+    end
+
+    def char_to_token(char_pos, sequence_index = 0)
+      _char_to_token(char_pos, sequence_index)
+    end
+
+    def char_to_word(char_pos, sequence_index = 0)
+      _char_to_word(word_index, sequence_index)
+    end
+  end
+end

data/lib/tokenizers/tokenizer.rb

@@ -0,0 +1,12 @@
+module Tokenizers
+  class Tokenizer
+    # TODO change add_special_tokens default to true in 0.3.0
+    def encode(sequence, add_special_tokens: nil)
+      if add_special_tokens.nil?
+        warn "[tokenizers] add_special_tokens will default to true in 0.3.0. Pass add_special_tokens: true/false to silence this warning."
+        add_special_tokens = false
+      end
+      _encode(sequence, add_special_tokens)
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

data/lib/tokenizers.rb

@@ -1,14 +1,16 @@
 # ext
 begin
-  require "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
+  require_relative "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
 rescue LoadError
-  require "tokenizers/tokenizers"
+  require_relative "tokenizers/tokenizers"
 end
 
 # modules
-require "tokenizers/char_bpe_tokenizer"
-require "tokenizers/from_pretrained"
-require "tokenizers/version"
+require_relative "tokenizers/char_bpe_tokenizer"
+require_relative "tokenizers/encoding"
+require_relative "tokenizers/from_pretrained"
+require_relative "tokenizers/tokenizer"
+require_relative "tokenizers/version"
 
 module Tokenizers
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: x86_64-linux
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-15 00:00:00.000000000 Z
+date: 2023-01-22 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -28,7 +28,9 @@ files:
 - lib/tokenizers/3.1/tokenizers.so
 - lib/tokenizers/3.2/tokenizers.so
 - lib/tokenizers/char_bpe_tokenizer.rb
+- lib/tokenizers/encoding.rb
 - lib/tokenizers/from_pretrained.rb
+- lib/tokenizers/tokenizer.rb
 - lib/tokenizers/version.rb
 homepage: https://github.com/ankane/tokenizers-ruby
 licenses:
@@ -52,7 +54,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.3
+rubygems_version: 3.4.4
 signing_key: 
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby