tokenizers 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 197131371ec438d82623bc0aacb8fe82ba255904e847eeb9259358f38a7063f0
-  data.tar.gz: 42ef490120e56fbb79d847ec1eb2b0a6b0ca7aa8f2ad90c09d2053d167491350
+  metadata.gz: e4f3cb98cb867df67a1c8a00b56f9ec5f4c6fafa178d760655dafb6735160773
+  data.tar.gz: 88c420f7a42f56330ce091df7f131878efd552488232282388e69d7a3c4b4aa2
 SHA512:
-  metadata.gz: 0a21b4811cc9e31565209eb514e55d6b22302c350371a76205aeb3b67cf94ea6dabf85074cebd48c65f9eca56e8e750b83a1df841807e53afb1275961bca50ce
-  data.tar.gz: 222bb9d759e3a2cc00ad7a4950c821fdbad1bbf6d4413f237bcf9cdc0698c2011022890b3f306be6df3d70b05abd446ad43066851ffa6c27387ddf3191f7557d
+  metadata.gz: 8e4746ccdf33dce78dc2b86d847f47f83576ca0d637671f825ad006a53b7ac3374654f7724f1e889618f322f9cfa5081e30083997ee9810eab282b9a8b99f807
+  data.tar.gz: 5dfe7b502d908f85ae16cfb28ebe1bd2ff51348c31151c7ee531504c00a0315dc22ea76fea963690de8c7390c7adb50d392e39de6db4a22101e91d31de1fa4e8

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.2.3 (2022-01-22)
+
+- Added `add_special_tokens` option to `encode` method
+- Added warning about `encode` method including special tokens by default in 0.3.0
+- Added more methods to `Encoding`
+- Fixed error with precompiled gem on Mac ARM
+
 ## 0.2.2 (2022-01-15)
 
 - Added precompiled gem for Linux ARM

data/Cargo.lock

@@ -753,7 +753,7 @@ dependencies = [
 
 [[package]]
 name = "tokenizers"
-version = "0.2.2"
+version = "0.2.3"
 dependencies = [
  "magnus",
  "tokenizers 0.13.2",

data/ext/tokenizers/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "tokenizers"
-version = "0.2.2"
+version = "0.2.3"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"

data/ext/tokenizers/src/encoding.rs

@@ -1,16 +1,90 @@
-use tk::Encoding;
+use tk::{Encoding, Offsets};
 
 #[magnus::wrap(class = "Tokenizers::Encoding")]
+#[repr(transparent)]
 pub struct RbEncoding {
     pub encoding: Encoding,
 }
 
+impl From<Encoding> for RbEncoding {
+    fn from(v: Encoding) -> Self {
+        Self { encoding: v }
+    }
+}
+
 impl RbEncoding {
+    pub fn n_sequences(&self) -> usize {
+        self.encoding.n_sequences()
+    }
+
     pub fn ids(&self) -> Vec<u32> {
-        self.encoding.get_ids().into()
+        self.encoding.get_ids().to_vec()
     }
 
     pub fn tokens(&self) -> Vec<String> {
-        self.encoding.get_tokens().into()
+        self.encoding.get_tokens().to_vec()
+    }
+
+    pub fn word_ids(&self) -> Vec<Option<u32>> {
+        self.encoding.get_word_ids().to_vec()
+    }
+
+    pub fn sequence_ids(&self) -> Vec<Option<usize>> {
+        self.encoding.get_sequence_ids()
+    }
+
+    pub fn type_ids(&self) -> Vec<u32> {
+        self.encoding.get_type_ids().to_vec()
+    }
+
+    pub fn offsets(&self) -> Vec<(usize, usize)> {
+        self.encoding.get_offsets().to_vec()
+    }
+
+    pub fn special_tokens_mask(&self) -> Vec<u32> {
+        self.encoding.get_special_tokens_mask().to_vec()
+    }
+
+    pub fn attention_mask(&self) -> Vec<u32> {
+        self.encoding.get_attention_mask().to_vec()
+    }
+
+    pub fn overflowing(&self) -> Vec<Self> {
+        self.encoding
+            .get_overflowing()
+            .clone()
+            .into_iter()
+            .map(|e| e.into())
+            .collect()
+    }
+
+    pub fn word_to_tokens(&self, word_index: u32, sequence_index: usize) -> Option<(usize, usize)> {
+        self.encoding.word_to_tokens(word_index, sequence_index)
+    }
+
+    pub fn word_to_chars(&self, word_index: u32, sequence_index: usize) -> Option<Offsets> {
+        self.encoding.word_to_chars(word_index, sequence_index)
+    }
+
+    pub fn token_to_sequence(&self, token_index: usize) -> Option<usize> {
+        self.encoding.token_to_sequence(token_index)
+    }
+
+    pub fn token_to_chars(&self, token_index: usize) -> Option<Offsets> {
+        let (_, offsets) = self.encoding.token_to_chars(token_index)?;
+        Some(offsets)
+    }
+
+    pub fn token_to_word(&self, token_index: usize) -> Option<u32> {
+        let (_, word_idx) = self.encoding.token_to_word(token_index)?;
+        Some(word_idx)
+    }
+
+    pub fn char_to_token(&self, char_pos: usize, sequence_index: usize) -> Option<usize> {
+        self.encoding.char_to_token(char_pos, sequence_index)
+    }
+
+    pub fn char_to_word(&self, char_pos: usize, sequence_index: usize) -> Option<u32> {
+        self.encoding.char_to_word(char_pos, sequence_index)
     }
 }

data/ext/tokenizers/src/lib.rs

@@ -38,15 +38,39 @@ fn init() -> RbResult<()> {
         "add_special_tokens",
         method!(RbTokenizer::add_special_tokens, 1),
     )?;
-    class.define_method("encode", method!(RbTokenizer::encode, 1))?;
+    class.define_method("add_tokens", method!(RbTokenizer::add_tokens, 1))?;
+    class.define_method("_encode", method!(RbTokenizer::encode, 2))?;
     class.define_method("decode", method!(RbTokenizer::decode, 1))?;
     class.define_method("decoder=", method!(RbTokenizer::set_decoder, 1))?;
     class.define_method("pre_tokenizer=", method!(RbTokenizer::set_pre_tokenizer, 1))?;
     class.define_method("normalizer=", method!(RbTokenizer::set_normalizer, 1))?;
+    class.define_method("token_to_id", method!(RbTokenizer::token_to_id, 1))?;
+    class.define_method("id_to_token", method!(RbTokenizer::id_to_token, 1))?;
 
     let class = module.define_class("Encoding", Default::default())?;
+    class.define_method("n_sequences", method!(RbEncoding::n_sequences, 0))?;
     class.define_method("ids", method!(RbEncoding::ids, 0))?;
     class.define_method("tokens", method!(RbEncoding::tokens, 0))?;
+    class.define_method("word_ids", method!(RbEncoding::word_ids, 0))?;
+    class.define_method("sequence_ids", method!(RbEncoding::sequence_ids, 0))?;
+    class.define_method("type_ids", method!(RbEncoding::type_ids, 0))?;
+    class.define_method("offsets", method!(RbEncoding::offsets, 0))?;
+    class.define_method(
+        "special_tokens_mask",
+        method!(RbEncoding::special_tokens_mask, 0),
+    )?;
+    class.define_method("attention_mask", method!(RbEncoding::attention_mask, 0))?;
+    class.define_method("overflowing", method!(RbEncoding::overflowing, 0))?;
+    class.define_method("_word_to_tokens", method!(RbEncoding::word_to_tokens, 2))?;
+    class.define_method("_word_to_chars", method!(RbEncoding::word_to_chars, 2))?;
+    class.define_method(
+        "token_to_sequence",
+        method!(RbEncoding::token_to_sequence, 1),
+    )?;
+    class.define_method("token_to_chars", method!(RbEncoding::token_to_chars, 1))?;
+    class.define_method("token_to_word", method!(RbEncoding::token_to_word, 1))?;
+    class.define_method("_char_to_token", method!(RbEncoding::char_to_token, 2))?;
+    class.define_method("_char_to_word", method!(RbEncoding::char_to_word, 2))?;
 
     let class = module.define_class("BPEDecoder", Default::default())?;
     class.define_singleton_method("new", function!(RbBPEDecoder::new, 0))?;

data/ext/tokenizers/src/tokenizer.rs

@@ -36,10 +36,16 @@ impl RbTokenizer {
         // TODO return self
     }
 
-    pub fn encode(&self, text: String) -> RbResult<RbEncoding> {
+    pub fn add_tokens(&self, tokens: Vec<String>) {
+        let tokens: Vec<AddedToken> = tokens.iter().map(|t| AddedToken::from(t, true)).collect();
+        self.tokenizer.borrow_mut().add_tokens(&tokens);
+        // TODO return self
+    }
+
+    pub fn encode(&self, sequence: String, add_special_tokens: bool) -> RbResult<RbEncoding> {
         self.tokenizer
             .borrow()
-            .encode(text, false)
+            .encode(sequence, add_special_tokens)
             .map(|v| RbEncoding { encoding: v })
             .map_err(RbError::from)
     }
@@ -68,4 +74,12 @@ impl RbTokenizer {
             .borrow_mut()
             .with_normalizer(normalizer.normalizer);
     }
+
+    pub fn token_to_id(&self, token: String) -> Option<u32> {
+        self.tokenizer.borrow().token_to_id(&token)
+    }
+
+    pub fn id_to_token(&self, id: u32) -> Option<String> {
+        self.tokenizer.borrow().id_to_token(id)
+    }
 }

data/lib/tokenizers/char_bpe_tokenizer.rb

@@ -8,8 +8,8 @@ module Tokenizers
       @tokenizer.decoder = BPEDecoder.new
     end
 
-    def encode(text)
-      @tokenizer.encode(text)
+    def encode(text, **options)
+      @tokenizer.encode(text, **options)
     end
 
     def decode(ids)

data/lib/tokenizers/encoding.rb

@@ -0,0 +1,19 @@
+module Tokenizers
+  class Encoding
+    def word_to_tokens(word_index, sequence_index = 0)
+      _word_to_tokens(word_index, sequence_index)
+    end
+
+    def word_to_chars(word_index, sequence_index = 0)
+      _word_to_chars(word_index, sequence_index)
+    end
+
+    def char_to_token(char_pos, sequence_index = 0)
+      _char_to_token(char_pos, sequence_index)
+    end
+
+    def char_to_word(char_pos, sequence_index = 0)
+      _char_to_word(word_index, sequence_index)
+    end
+  end
+end

data/lib/tokenizers/tokenizer.rb

@@ -0,0 +1,12 @@
+module Tokenizers
+  class Tokenizer
+    # TODO change add_special_tokens default to true in 0.3.0
+    def encode(sequence, add_special_tokens: nil)
+      if add_special_tokens.nil?
+        warn "[tokenizers] add_special_tokens will default to true in 0.3.0. Pass add_special_tokens: true/false to silence this warning."
+        add_special_tokens = false
+      end
+      _encode(sequence, add_special_tokens)
+    end
+  end
+end

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.2.2"
+  VERSION = "0.2.3"
 end

data/lib/tokenizers.rb

@@ -1,14 +1,16 @@
 # ext
 begin
-  require "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
+  require_relative "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
 rescue LoadError
-  require "tokenizers/tokenizers"
+  require_relative "tokenizers/tokenizers"
 end
 
 # modules
-require "tokenizers/char_bpe_tokenizer"
-require "tokenizers/from_pretrained"
-require "tokenizers/version"
+require_relative "tokenizers/char_bpe_tokenizer"
+require_relative "tokenizers/encoding"
+require_relative "tokenizers/from_pretrained"
+require_relative "tokenizers/tokenizer"
+require_relative "tokenizers/version"
 
 module Tokenizers
   class Error < StandardError; end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-01-15 00:00:00.000000000 Z
+date: 2023-01-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys
@@ -48,7 +48,9 @@ files:
 - ext/tokenizers/src/tokenizer.rs
 - lib/tokenizers.rb
 - lib/tokenizers/char_bpe_tokenizer.rb
+- lib/tokenizers/encoding.rb
 - lib/tokenizers/from_pretrained.rb
+- lib/tokenizers/tokenizer.rb
 - lib/tokenizers/version.rb
 homepage: https://github.com/ankane/tokenizers-ruby
 licenses: