tokenizers 0.2.2-aarch64-linux → 0.2.3-aarch64-linux

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 6eb598afb972974b72da4aabbfb72ecac6470f62e620ad4ee0cefb4aaa403173
-  data.tar.gz: a469da134839a57c31326c13721cb0f1fb34408b8b3a18a6cd16dde1b331c1a4
+  metadata.gz: 8ebac1b83e37b024ec99ac2e96b1085bdbdd060207b53a0d5411f826c15e7a05
+  data.tar.gz: 1278eb2b6c661f441f65eb53a28595690c51b532a5e468b606ad75fa57aaecba
 SHA512:
-  metadata.gz: 5f78428fae46491dc2414a047ee6381cb787cffe17c59ba317e48453c62e2c809abfeae7e126271f778e893aaa7a4b9ed3b90ab9fe46bec17b0392befef5d0fe
-  data.tar.gz: 6613e888311b6630a0c6ced1ba353cd5eb2c7b33ddc00c7476896c78cfe71d44d5634adb401457995d06d86d476553a343db0ae597f18c611e180c8406b9f0de
+  metadata.gz: 6ddca76a32fdf761150981de37a876b4f3b20283f77fd3b1adb0dbfb3be52af7d95d4ed6d883c7ed98cf4fd6ce32ea50f2396781400ab7071d84b340dc8ccbd2
+  data.tar.gz: 0a53cb1b001870aae8fbfdfcf6e93fc83ec19543327338764c2d8bd4af4d2ec8620dfe1b8f5e1f83bf0a45fe123e7973664c86650622e99ecd01839b0da6a450

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.2.3 (2022-01-22)
+
+- Added `add_special_tokens` option to `encode` method
+- Added warning about `encode` method including special tokens by default in 0.3.0
+- Added more methods to `Encoding`
+- Fixed error with precompiled gem on Mac ARM
+
 ## 0.2.2 (2022-01-15)
 
 - Added precompiled gem for Linux ARM

data/Cargo.lock

@@ -753,7 +753,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.2.2"
+version = "0.2.3"
 dependencies = [
  "magnus",
  "tokenizers 0.13.2",

data/LICENSE-THIRD-PARTY.txt

@@ -12446,6 +12446,32 @@ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
 
+================================================================================
+rb-sys-env LICENSE-MIT
+================================================================================
+
+The MIT License (MIT)
+
+Copyright (c) 2021-2022 Ian Ker-Seymer
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
 ================================================================================
 rb-sys-env LICENSE-APACHE
 ================================================================================
@@ -12641,32 +12667,6 @@ rb-sys-env LICENSE-APACHE
    See the License for the specific language governing permissions and
    limitations under the License.
 
-================================================================================
-rb-sys-env LICENSE-MIT
-================================================================================
-
-The MIT License (MIT)
-
-Copyright (c) 2021-2022 Ian Ker-Seymer
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-
 ================================================================================
 regex LICENSE-APACHE
 ================================================================================

data/lib/tokenizers/char_bpe_tokenizer.rb

@@ -8,8 +8,8 @@ module Tokenizers
       @tokenizer.decoder = BPEDecoder.new
     end
 
-    def encode(text)
-      @tokenizer.encode(text)
+    def encode(text, **options)
+      @tokenizer.encode(text, **options)
     end
 
     def decode(ids)

data/lib/tokenizers/encoding.rb

@@ -0,0 +1,19 @@
+module Tokenizers
+  class Encoding
+    def word_to_tokens(word_index, sequence_index = 0)
+      _word_to_tokens(word_index, sequence_index)
+    end
+
+    def word_to_chars(word_index, sequence_index = 0)
+      _word_to_chars(word_index, sequence_index)
+    end
+
+    def char_to_token(char_pos, sequence_index = 0)
+      _char_to_token(char_pos, sequence_index)
+    end
+
+    def char_to_word(char_pos, sequence_index = 0)
+      _char_to_word(word_index, sequence_index)
+    end
+  end
+end

data/lib/tokenizers/tokenizer.rb

@@ -0,0 +1,12 @@
+module Tokenizers
+  class Tokenizer
+    # TODO change add_special_tokens default to true in 0.3.0
+    def encode(sequence, add_special_tokens: nil)
+      if add_special_tokens.nil?
+        warn "[tokenizers] add_special_tokens will default to true in 0.3.0. Pass add_special_tokens: true/false to silence this warning."
+        add_special_tokens = false
+      end
+      _encode(sequence, add_special_tokens)
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

data/lib/tokenizers.rb

@@ -1,14 +1,16 @@
 # ext
 begin
-  require "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
+  require_relative "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
 rescue LoadError
-  require "tokenizers/tokenizers"
+  require_relative "tokenizers/tokenizers"
 end
 
 # modules
-require "tokenizers/char_bpe_tokenizer"
-require "tokenizers/from_pretrained"
-require "tokenizers/version"
+require_relative "tokenizers/char_bpe_tokenizer"
+require_relative "tokenizers/encoding"
+require_relative "tokenizers/from_pretrained"
+require_relative "tokenizers/tokenizer"
+require_relative "tokenizers/version"
 
 module Tokenizers
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: aarch64-linux
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2023-01-15 00:00:00.000000000 Z
+date: 2023-01-22 00:00:00.000000000 Z
 dependencies: []
 description: 
 email: andrew@ankane.org
@@ -28,7 +28,9 @@ files:
 - lib/tokenizers/3.1/tokenizers.so
 - lib/tokenizers/3.2/tokenizers.so
 - lib/tokenizers/char_bpe_tokenizer.rb
+- lib/tokenizers/encoding.rb
 - lib/tokenizers/from_pretrained.rb
+- lib/tokenizers/tokenizer.rb
 - lib/tokenizers/version.rb
 homepage: https://github.com/ankane/tokenizers-ruby
 licenses:
@@ -52,7 +54,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.3
+rubygems_version: 3.4.4
 signing_key: 
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby