tokenizers 0.1.3 → 0.2.0

data/Cargo.toml

@@ -1,14 +1,11 @@
-[package]
-name = "tokenizers-ruby"
-version = "0.1.0"
-authors = ["Andrew Kane <andrew@ankane.org>"]
-edition = "2018"
+[workspace]
+members = ["ext/tokenizers"]
 
-[lib]
-name = "tokenizers"
-crate-type = ["cdylib"]
+[profile.release]
+strip = true
 
-[dependencies]
-lazy_static = "1"
-rutie = "0.8.4"
-tokenizers = "0.13.1"
+[patch.crates-io]
+magnus-macros = { git = "https://github.com/matsadler/magnus" }
+number_prefix = { git = "https://github.com/ankane/rust-number-prefix", branch = "license-file" }
+rb-sys-env = { git = "https://github.com/oxidize-rb/rb-sys" }
+tokenizers = { git = "https://github.com/huggingface/tokenizers" }

data/README.md

@@ -12,8 +12,6 @@ Add this line to your application’s Gemfile:
 gem "tokenizers"
 ```
 
-Note: Rust and pkg-config are currently required for installation, and it can take 5-10 minutes to compile the extension.
-
 ## Getting Started
 
 Load a pretrained tokenizer
@@ -61,7 +59,7 @@ To get started with development:
 git clone https://github.com/ankane/tokenizers-ruby.git
 cd tokenizers-ruby
 bundle install
-bundle exec ruby ext/tokenizers/extconf.rb && make && make install
+bundle exec rake compile
 bundle exec rake download:files
 bundle exec rake test
 ```

data/ext/tokenizers/Cargo.toml

@@ -0,0 +1,17 @@
+[package]
+name = "tokenizers"
+version = "0.2.0"
+authors = ["Andrew Kane <andrew@ankane.org>"]
+edition = "2021"
+publish = false
+
+[lib]
+crate-type = ["cdylib"]
+
+[dependencies]
+magnus = "0.4"
+
+[dependencies.tokenizers]
+version = "0.13.2"
+default-features = false
+features = ["progressbar", "http", "onig", "esaxx_fast"]

data/ext/tokenizers/extconf.rb

@@ -1,11 +1,4 @@
-require "pathname"
+require "mkmf"
+require "rb_sys/mkmf"
 
-dest = Pathname.new(__dir__).relative_path_from(Pathname.pwd).join("../../lib/tokenizers/ext.#{RbConfig::CONFIG["DLEXT"]}")
-
-File.write "Makefile", <<~EOS
-  all:
-  \tcargo build --release --target-dir target
-  install:
-  \tmv target/release/libtokenizers.#{RbConfig::CONFIG["SOEXT"]} #{dest}
-  clean:
-EOS
+create_rust_makefile("tokenizers/tokenizers")

data/ext/tokenizers/src/decoders.rs

@@ -0,0 +1,14 @@
+use tk::decoders::bpe::BPEDecoder;
+
+#[magnus::wrap(class = "Tokenizers::BPEDecoder")]
+pub struct RbBPEDecoder {
+    pub decoder: BPEDecoder,
+}
+
+impl RbBPEDecoder {
+    pub fn new() -> Self {
+        RbBPEDecoder {
+            decoder: BPEDecoder::default(),
+        }
+    }
+}

data/ext/tokenizers/src/encoding.rs

@@ -0,0 +1,16 @@
+use tk::Encoding;
+
+#[magnus::wrap(class = "Tokenizers::Encoding")]
+pub struct RbEncoding {
+    pub encoding: Encoding,
+}
+
+impl RbEncoding {
+    pub fn ids(&self) -> Vec<u32> {
+        self.encoding.get_ids().into()
+    }
+
+    pub fn tokens(&self) -> Vec<String> {
+        self.encoding.get_tokens().into()
+    }
+}

data/ext/tokenizers/src/error.rs

@@ -0,0 +1,16 @@
+use magnus::{exception, memoize, Error, ExceptionClass, Module};
+
+use super::module;
+
+pub struct RbError {}
+
+impl RbError {
+    // convert to Error instead of Self
+    pub fn from(e: Box<dyn std::error::Error + Send + Sync>) -> Error {
+        Error::new(error(), e.to_string())
+    }
+}
+
+fn error() -> ExceptionClass {
+    *memoize!(ExceptionClass: module().define_error("Error", exception::standard_error()).unwrap())
+}

data/ext/tokenizers/src/lib.rs

@@ -0,0 +1,64 @@
+extern crate tokenizers as tk;
+
+mod decoders;
+mod encoding;
+mod error;
+mod models;
+mod normalizers;
+mod pre_tokenizers;
+mod tokenizer;
+
+use decoders::RbBPEDecoder;
+use encoding::RbEncoding;
+use error::RbError;
+use models::RbBPE;
+use normalizers::RbBertNormalizer;
+use pre_tokenizers::RbBertPreTokenizer;
+use tokenizer::RbTokenizer;
+
+use magnus::{define_module, function, memoize, method, prelude::*, Error, RModule};
+
+type RbResult<T> = Result<T, Error>;
+
+fn module() -> RModule {
+    *memoize!(RModule: define_module("Tokenizers").unwrap())
+}
+
+#[magnus::init]
+fn init() -> RbResult<()> {
+    let module = module();
+    module.define_singleton_method(
+        "_from_pretrained",
+        function!(RbTokenizer::from_pretrained, 3),
+    )?;
+
+    let class = module.define_class("BPE", Default::default())?;
+    class.define_singleton_method("new", function!(RbBPE::new, 2))?;
+
+    let class = module.define_class("Tokenizer", Default::default())?;
+    class.define_singleton_method("new", function!(RbTokenizer::new, 1))?;
+    class.define_method(
+        "add_special_tokens",
+        method!(RbTokenizer::add_special_tokens, 1),
+    )?;
+    class.define_method("encode", method!(RbTokenizer::encode, 1))?;
+    class.define_method("decode", method!(RbTokenizer::decode, 1))?;
+    class.define_method("decoder=", method!(RbTokenizer::set_decoder, 1))?;
+    class.define_method("pre_tokenizer=", method!(RbTokenizer::set_pre_tokenizer, 1))?;
+    class.define_method("normalizer=", method!(RbTokenizer::set_normalizer, 1))?;
+
+    let class = module.define_class("Encoding", Default::default())?;
+    class.define_method("ids", method!(RbEncoding::ids, 0))?;
+    class.define_method("tokens", method!(RbEncoding::tokens, 0))?;
+
+    let class = module.define_class("BPEDecoder", Default::default())?;
+    class.define_singleton_method("new", function!(RbBPEDecoder::new, 0))?;
+
+    let class = module.define_class("BertPreTokenizer", Default::default())?;
+    class.define_singleton_method("new", function!(RbBertPreTokenizer::new, 0))?;
+
+    let class = module.define_class("BertNormalizer", Default::default())?;
+    class.define_singleton_method("new", function!(RbBertNormalizer::new, 0))?;
+
+    Ok(())
+}

data/ext/tokenizers/src/models.rs

@@ -0,0 +1,19 @@
+use tk::models::bpe::BPE;
+
+use super::{RbError, RbResult};
+
+#[magnus::wrap(class = "Tokenizers::BPE")]
+pub struct RbBPE {
+    pub model: BPE,
+}
+
+impl RbBPE {
+    pub fn new(vocab: String, merges: String) -> RbResult<Self> {
+        BPE::from_file(&vocab, &merges)
+            .unk_token("<unk>".into())
+            .end_of_word_suffix("</w>".into())
+            .build()
+            .map(|v| RbBPE { model: v })
+            .map_err(RbError::from)
+    }
+}

data/ext/tokenizers/src/normalizers.rs

@@ -0,0 +1,14 @@
+use tk::normalizers::BertNormalizer;
+
+#[magnus::wrap(class = "Tokenizers::BertNormalizer")]
+pub struct RbBertNormalizer {
+    pub normalizer: BertNormalizer,
+}
+
+impl RbBertNormalizer {
+    pub fn new() -> Self {
+        RbBertNormalizer {
+            normalizer: BertNormalizer::default(),
+        }
+    }
+}

data/ext/tokenizers/src/pre_tokenizers.rs

@@ -0,0 +1,14 @@
+use tk::pre_tokenizers::bert::BertPreTokenizer;
+
+#[magnus::wrap(class = "Tokenizers::BertPreTokenizer")]
+pub struct RbBertPreTokenizer {
+    pub pretok: BertPreTokenizer,
+}
+
+impl RbBertPreTokenizer {
+    pub fn new() -> Self {
+        RbBertPreTokenizer {
+            pretok: BertPreTokenizer,
+        }
+    }
+}

data/ext/tokenizers/src/tokenizer.rs

@@ -0,0 +1,85 @@
+use magnus::Module;
+use std::cell::RefCell;
+use tk::tokenizer::Tokenizer;
+use tk::AddedToken;
+
+use super::decoders::RbBPEDecoder;
+use super::encoding::RbEncoding;
+use super::models::RbBPE;
+use super::normalizers::RbBertNormalizer;
+use super::pre_tokenizers::RbBertPreTokenizer;
+use super::{module, RbError, RbResult};
+
+#[magnus::wrap(class = "Tokenizers::Tokenizer")]
+pub struct RbTokenizer {
+    tokenizer: RefCell<Tokenizer>,
+}
+
+impl RbTokenizer {
+    pub fn new(model: &RbBPE) -> Self {
+        Self {
+            tokenizer: RefCell::new(Tokenizer::new(model.model.clone())),
+        }
+    }
+
+    pub fn from_pretrained(
+        identifier: String,
+        revision: String,
+        auth_token: Option<String>,
+    ) -> RbResult<Self> {
+        let version = module().const_get("VERSION").unwrap();
+        let params = tk::FromPretrainedParameters {
+            revision,
+            auth_token,
+            user_agent: [("bindings", "Ruby".to_string()), ("version", version)]
+                .iter()
+                .map(|(k, v)| (k.to_string(), v.to_string()))
+                .collect(),
+        };
+
+        Tokenizer::from_pretrained(identifier, Some(params))
+            .map(|v| RbTokenizer {
+                tokenizer: RefCell::new(v),
+            })
+            .map_err(RbError::from)
+    }
+
+    pub fn add_special_tokens(&self, tokens: Vec<String>) {
+        let tokens: Vec<AddedToken> = tokens.iter().map(|t| AddedToken::from(t, true)).collect();
+        self.tokenizer.borrow_mut().add_special_tokens(&tokens);
+        // TODO return self
+    }
+
+    pub fn encode(&self, text: String) -> RbResult<RbEncoding> {
+        self.tokenizer
+            .borrow()
+            .encode(text, false)
+            .map(|v| RbEncoding { encoding: v })
+            .map_err(RbError::from)
+    }
+
+    pub fn decode(&self, ids: Vec<u32>) -> RbResult<String> {
+        self.tokenizer
+            .borrow()
+            .decode(ids, true)
+            .map_err(RbError::from)
+    }
+
+    pub fn set_decoder(&self, decoder: &RbBPEDecoder) {
+        self.tokenizer
+            .borrow_mut()
+            .with_decoder(decoder.decoder.clone());
+    }
+
+    pub fn set_pre_tokenizer(&self, pre_tokenizer: &RbBertPreTokenizer) {
+        self.tokenizer
+            .borrow_mut()
+            .with_pre_tokenizer(pre_tokenizer.pretok);
+    }
+
+    pub fn set_normalizer(&self, normalizer: &RbBertNormalizer) {
+        self.tokenizer
+            .borrow_mut()
+            .with_normalizer(normalizer.normalizer);
+    }
+}

data/lib/tokenizers/version.rb

@@ -1,3 +1,3 @@
 module Tokenizers
-  VERSION = "0.1.3"
+  VERSION = "0.2.0"
 end

data/lib/tokenizers.rb

@@ -1,5 +1,9 @@
-# extlib
-require "tokenizers/ext"
+# ext
+begin
+  require "tokenizers/#{RUBY_VERSION.to_f}/tokenizers"
+rescue LoadError
+  require "tokenizers/tokenizers"
+end
 
 # modules
 require "tokenizers/char_bpe_tokenizer"

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: tokenizers
 version: !ruby/object:Gem::Version
-  version: 0.1.3
+  version: 0.2.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-07 00:00:00.000000000 Z
-dependencies: []
+date: 2022-12-11 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rb_sys
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 description:
 email: andrew@ankane.org
 executables: []
@@ -22,11 +36,19 @@ files:
 - Cargo.toml
 - LICENSE.txt
 - README.md
+- ext/tokenizers/Cargo.toml
 - ext/tokenizers/extconf.rb
+- ext/tokenizers/src/decoders.rs
+- ext/tokenizers/src/encoding.rs
+- ext/tokenizers/src/error.rs
+- ext/tokenizers/src/lib.rs
+- ext/tokenizers/src/models.rs
+- ext/tokenizers/src/normalizers.rs
+- ext/tokenizers/src/pre_tokenizers.rs
+- ext/tokenizers/src/tokenizer.rs
 - lib/tokenizers.rb
 - lib/tokenizers/char_bpe_tokenizer.rb
 - lib/tokenizers/version.rb
-- src/lib.rs
 homepage: https://github.com/ankane/tokenizers-ruby
 licenses:
 - Apache-2.0
@@ -46,7 +68,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.7
+rubygems_version: 3.3.26
 signing_key:
 specification_version: 4
 summary: Fast state-of-the-art tokenizers for Ruby