youtokentome 0.2.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9fd1ff636b8d8ede5a0dbae880595216656fe26fbd54004994d322e51d870d6a
-  data.tar.gz: b633849e81472eefcf36688547cf5e562e6a0dd6a3df8c554805f2289f0ceb07
+  metadata.gz: dc70dec1b03c48e827b1904a17bcb7edfaf7d97b9d21a83789ca8e40e75cbede
+  data.tar.gz: 00c7f498eda14f3ac274c9c6207a697ecfe54e88adc834ebe9c256318ae42c00
 SHA512:
-  metadata.gz: b619f8b2cc8fa0c9cf92da1b12d66227d87dc931a6cb89307cbe3d3a54800328fcab4d9591a5968f2c4da83ed184aa67d78cd6a78d01d1faa21e09447a0609bc
-  data.tar.gz: 4b38a4db6d2adf2e24f6044cae75fb34eb6505a8e6ce8a48fe5a4ddfed3596f130431b8d59473885029ba3f4c71ace4f11fd22139a6ae8476d77d79cd0727b90
+  metadata.gz: f604358334bf73b7ce0b7a8fd3cc13988d5af01a7c04ba7a96ddd22d97eb5b17c79f8060969e9ef3dd53a7bd59b249f338488ef297559d5d2e3045541615fb4b
+  data.tar.gz: 208dedfe2bc7ff170fc6a8abf730dd19123e890e795ac62b4b0645716799392baf99e8fc0e50cf225f44770a7f41a8d73f87a0da151fe265345ac6764d92fd87

data/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.2.1 (2021-05-23)
+
+- Improved performance
+
 ## 0.2.0 (2021-05-17)
 
 - Updated to Rice 4

data/ext/youtokentome/ext.cpp

@@ -6,12 +6,7 @@
 #include <rice/rice.hpp>
 #include <rice/stl.hpp>
 
-using Rice::define_class_under;
-using Rice::define_module;
-using Rice::define_module_under;
 using Rice::Array;
-using Rice::Module;
-using Rice::Object;
 
 void check_status(vkcom::Status& status) {
   if (!status.ok()) {
@@ -41,10 +36,11 @@ namespace Rice::detail
   public:
     std::vector<int> convert(VALUE x)
     {
-      Array a = Array(x);
+      auto a = Array(x);
       std::vector<int> ret;
-      for (size_t i = 0; i < a.size(); i++) {
-        ret.push_back(Rice::detail::From_Ruby<int>().convert(a[i].value()));
+      ret.reserve(a.size());
+      for (const auto& v : a) {
+        ret.push_back(From_Ruby<int>().convert(v.value()));
       }
       return ret;
     }
@@ -56,22 +52,25 @@ namespace Rice::detail
   public:
     std::vector<std::string> convert(VALUE x)
     {
-      Array a = Array(x);
+      auto a = Array(x);
       std::vector<std::string> ret;
-      for (size_t i = 0; i < a.size(); i++) {
-        ret.push_back(Rice::detail::From_Ruby<std::string>().convert(a[i].value()));
+      ret.reserve(a.size());
+      for (const auto& v : a) {
+        ret.push_back(From_Ruby<std::string>().convert(v.value()));
       }
       return ret;
     }
   };
 }
 
-extern "C" void Init_ext() {
-  Module rb_mYouTokenToMe = define_module("YouTokenToMe");
-  Module rb_mExt = define_module_under(rb_mYouTokenToMe, "Ext")
+extern "C"
+void Init_ext()
+{
+  auto rb_mYouTokenToMe = Rice::define_module("YouTokenToMe");
+  auto rb_mExt = Rice::define_module_under(rb_mYouTokenToMe, "Ext")
     .define_singleton_function(
       "train_bpe",
-      [](const std::string &input_path, const std::string &model_path, int vocab_size, double coverage,
+      [](const std::string& input_path, const std::string& model_path, int vocab_size, double coverage,
           int n_threads, int pad_id, int unk_id, int bos_id, int eos_id) {
 
         vkcom::SpecialTokens special_tokens(pad_id, unk_id, bos_id, eos_id);
@@ -80,7 +79,7 @@ extern "C" void Init_ext() {
         check_status(status);
       });
 
-  define_class_under<vkcom::BaseEncoder>(rb_mExt, "BaseEncoder")
+  Rice::define_class_under<vkcom::BaseEncoder>(rb_mExt, "BaseEncoder")
     .define_method("vocab_size", &vkcom::BaseEncoder::vocab_size)
     .define_method("subword_to_id", &vkcom::BaseEncoder::subword_to_id)
     .define_method(
@@ -140,7 +139,7 @@ extern "C" void Init_ext() {
     .define_method("vocab", &vkcom::BaseEncoder::vocabulary)
     .define_singleton_function(
       "new",
-      [](std::string &model_path, int n_threads) {
+      [](const std::string& model_path, int n_threads) {
         auto status = vkcom::Status();
         vkcom::BaseEncoder encoder(model_path, n_threads, &status);
         check_status(status);

data/ext/youtokentome/extconf.rb

@@ -1,6 +1,6 @@
 require "mkmf-rice"
 
-$CXXFLAGS << " -std=c++17"
+$CXXFLAGS << " -std=c++17 $(optflags)"
 
 ext = File.expand_path(".", __dir__)
 youtokentome = File.expand_path("../../vendor/YouTokenToMe/youtokentome/cpp", __dir__)

data/lib/youtokentome/version.rb

@@ -1,3 +1,3 @@
 module YouTokenToMe
-  VERSION = "0.2.0"
+  VERSION = "0.2.1"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: youtokentome
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.1
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-05-17 00:00:00.000000000 Z
+date: 2021-05-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rice