fasttext 0.1.0 → 0.2.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e0df128ff3b10090bc104614fbdaafe7db5be0077acab3da63c53c442451115a
-  data.tar.gz: ff998e44fcf39523929aff86a17c91153ea284a460388107ac7849c0566d6367
+  metadata.gz: 597a9c73a6720397b11fb9bd0492c33ed458783747d904deef74618a6f319133
+  data.tar.gz: b03ab1e051cda71fafc499dbf0847d3da1c27050646e949eac148a5548460054
 SHA512:
-  metadata.gz: 56447b44cf08ba76deaec721f66ef16d122ddae9b2ccf732fb6c364b8df283e788331c267447bcb0aab4d012d3bf94861ed30375050b6d024be6e030d0773a0a
-  data.tar.gz: 9c2be8700058fb1c01cf0df61970b48670ac75688c88c9d1ec502d5ee690cc5b328d4b903b52314937b5477b043fb0d83d3d7a1b8cfeda4026d35e4300cb8756
+  metadata.gz: 844ae824ad06aa270a5c58b6dc109701660b2bb0e5bfe63e285df57bc5db854028703c1c0476a845d5b0e8c32dd02ac42e1c8b1c5c0684713928c895c776b1da
+  data.tar.gz: 7f7a07f0041981eb046e84f6b44f4207192eb9e8a786edf41b53ae15b370eb875f5b4d578248c90a371805697ad3f0fa076532d1f6789c44d336d72402a54333

data/CHANGELOG.md

@@ -1,3 +1,28 @@
-## 0.1.0
+## 0.2.1 (2021-05-23)
+
+- Improved performance
+
+## 0.2.0 (2021-05-17)
+
+- Updated to Rice 4
+- Dropped support for Ruby < 2.6
+
+## 0.1.3 (2020-04-28)
+
+- Updated fastText to 0.9.2
+- Added support for autotune
+- Added `--with-optflags` option
+
+## 0.1.2 (2020-01-10)
+
+- Fixed installation error with Ruby 2.7
+
+## 0.1.1 (2019-10-26)
+
+- Fixed installation
+- Reduced gem size
+- Added support for multiple documents to `predict` method
+
+## 0.1.0 (2019-10-26)
 
 - First release

data/LICENSE.txt

@@ -1,22 +1,22 @@
-Copyright (c) 2019 Andrew Kane
-
 MIT License
 
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
+Copyright (c) 2016-present, Facebook, Inc.
+Copyright (c) 2019-2021 Andrew Kane
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
 
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
 
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/README.md

@@ -2,6 +2,8 @@
 
 [fastText](https://fasttext.cc) - efficient text classification and representation learning - for Ruby
 
+[![Build Status](https://github.com/ankane/fastText/workflows/build/badge.svg?branch=master)](https://github.com/ankane/fastText/actions)
+
 ## Installation
 
 Add this line to your application’s Gemfile:
@@ -10,6 +12,13 @@ Add this line to your application’s Gemfile:
 gem 'fasttext'
 ```
 
+## Getting Started
+
+fastText has two primary use cases:
+
+- [text classification](#text-classification)
+- [word representations](#word-representations)
+
 ## Text Classification
 
 Prep your data
@@ -68,6 +77,12 @@ model.labels
 
 > Use `include_freq: true` to get their frequency
 
+Search for the best hyperparameters
+
+```ruby
+model.fit(x, y, autotune_set: [x_valid, y_valid])
+```
+
 Compress the model - significantly reduces size but sacrifices a little performance
 
 ```ruby
@@ -112,6 +127,12 @@ Get a word vector
 model.word_vector("carrot")
 ```
 
+Get a sentence vector
+
+```ruby
+model.sentence_vector("sentence text")
+```
+
 Get words
 
 ```ruby
@@ -157,9 +178,13 @@ FastText::Classifier.new(
   thread: 3,                  # number of threads
   lr_update_rate: 100,        # change the rate of updates for the learning rate
   t: 0.0001,                  # sampling threshold
-  label_prefix: "__label__"   # label prefix
+  label_prefix: "__label__",  # label prefix
   verbose: 2,                 # verbose
-  pretrained_vectors: nil     # pretrained word vectors (.vec file)
+  pretrained_vectors: nil,    # pretrained word vectors (.vec file)
+  autotune_metric: "f1",      # autotune optimization metric
+  autotune_predictions: 1,    # autotune predictions
+  autotune_duration: 300,     # autotune search time in seconds
+  autotune_model_size: nil    # autotune model size, like 2M
 )
 ```
 
@@ -191,7 +216,7 @@ FastText::Vectorizer.new(
 Input can be read directly from files
 
 ```ruby
-model.fit("train.txt")
+model.fit("train.txt", autotune_set: "valid.txt")
 model.test("test.txt")
 ```
 
@@ -213,7 +238,7 @@ __label__spam text from document three
 
 ## Pretrained Models
 
-There are a number of [pretrained models](https://fasttext.cc/docs/en/english-vectors.html) you can download
+There are a number of [pretrained models](https://fasttext.cc/docs/en/supervised-models.html) you can download
 
 ### Language Identification
 
@@ -229,14 +254,6 @@ Get language predictions
 model.predict("bon appétit")
 ```
 
-## rbenv
-
-This library uses [Rice](https://github.com/jasonroelofs/rice) to interface with the fastText C++ library. Unfortunately, Rice and rbenv don’t play nicely together. This is actively [being addressed](https://github.com/rbenv/ruby-build/pull/1368), but in the meantime, if you encounter an error during installation, reinstall your Ruby version with the `--enabled-shared` flag.
-
-```sh
-CONFIGURE_OPTS="--enable-shared" rbenv install 2.6.5
-```
-
 ## History
 
 View the [changelog](https://github.com/ankane/fasttext/blob/master/CHANGELOG.md)
@@ -249,3 +266,13 @@ Everyone is encouraged to help improve this project. Here are a few ways you can
 - Fix bugs and [submit pull requests](https://github.com/ankane/fasttext/pulls)
 - Write, clarify, or fix documentation
 - Suggest or add new features
+
+To get started with development:
+
+```sh
+git clone --recursive https://github.com/ankane/fastText.git
+cd fastText
+bundle install
+bundle exec rake compile
+bundle exec rake test
+```

data/ext/fasttext/ext.cpp

@@ -1,97 +1,84 @@
+// stdlib
+#include <cmath>
+#include <iterator>
+#include <sstream>
+#include <stdexcept>
+
+// fasttext
 #include <args.h>
+#include <autotune.h>
 #include <densematrix.h>
 #include <fasttext.h>
-#include <rice/Data_Type.hpp>
-#include <rice/Constructor.hpp>
-#include <rice/Array.hpp>
-#include <rice/Hash.hpp>
 #include <real.h>
 #include <vector.h>
-#include <cmath>
-#include <iterator>
-#include <sstream>
-#include <stdexcept>
 
-using namespace Rice;
+// rice
+#include <rice/rice.hpp>
+#include <rice/stl.hpp>
 
-template<>
-inline
-long long from_ruby<long long>(Object x)
-{
-  return NUM2LL(x);
-}
+using fasttext::FastText;
 
-template<>
-inline
-Object to_ruby<long long>(long long const & x)
-{
-  return LL2NUM(x);
-}
+using Rice::Array;
+using Rice::Constructor;
+using Rice::Hash;
+using Rice::Module;
+using Rice::Object;
+using Rice::define_class_under;
+using Rice::define_module;
+using Rice::define_module_under;
 
-template<>
-inline
-unsigned long long from_ruby<unsigned long long>(Object x)
+namespace Rice::detail
 {
-  return NUM2ULL(x);
-}
-
-template<>
-inline
-Object to_ruby<unsigned long long>(unsigned long long const & x)
-{
-  return ULL2NUM(x);
-}
-
-template<>
-inline
-Object to_ruby<std::vector<std::pair<fasttext::real, std::string>>>(std::vector<std::pair<fasttext::real, std::string>> const & x)
-{
-  Array ret;
-  for (const auto& v : x) {
-    Array a;
-    a.push(v.first);
-    a.push(v.second);
-    ret.push(a);
-  }
-  return ret;
+  template<>
+  class To_Ruby<std::vector<std::pair<fasttext::real, std::string>>>
+  {
+  public:
+    VALUE convert(std::vector<std::pair<fasttext::real, std::string>> const & x)
+    {
+      Array ret;
+      for (const auto& v : x) {
+        Array a;
+        a.push(v.first);
+        a.push(v.second);
+        ret.push(a);
+      }
+      return ret;
+    }
+  };
 }
 
 fasttext::Args buildArgs(Hash h) {
   fasttext::Args a;
 
-  std::vector<Hash::Entry> v;
-  Hash::iterator it = h.begin();
-  Hash::iterator end = h.end();
-
-  for(; it != end; ++it)
+  for (const auto& it : h)
   {
-    std::string name = from_ruby<std::string>(it->key.to_s());
-    Object value = it->value;
+    auto name = it.key.to_s().str();
+    auto value = (it.value).value();
 
     if (name == "input") {
-      a.input = from_ruby<std::string>(value);
+      a.input = Rice::detail::From_Ruby<std::string>().convert(value);
     } else if (name == "output") {
-      a.output = from_ruby<std::string>(value);
+      a.output = Rice::detail::From_Ruby<std::string>().convert(value);
     } else if (name == "lr") {
-      a.lr = from_ruby<double>(value);
+      a.lr = Rice::detail::From_Ruby<double>().convert(value);
     } else if (name == "lr_update_rate") {
-      a.lrUpdateRate = from_ruby<int>(value);
+      a.lrUpdateRate = Rice::detail::From_Ruby<int>().convert(value);
     } else if (name == "dim") {
-      a.dim = from_ruby<int>(value);
+      a.dim = Rice::detail::From_Ruby<int>().convert(value);
     } else if (name == "ws") {
-      a.ws = from_ruby<int>(value);
+      a.ws = Rice::detail::From_Ruby<int>().convert(value);
     } else if (name == "epoch") {
-      a.epoch = from_ruby<int>(value);
+      a.epoch = Rice::detail::From_Ruby<int>().convert(value);
     } else if (name == "min_count") {
-      a.minCount = from_ruby<int>(value);
+      a.minCount = Rice::detail::From_Ruby<int>().convert(value);
     } else if (name == "min_count_label") {
-      a.minCountLabel = from_ruby<int>(value);
+      a.minCountLabel = Rice::detail::From_Ruby<int>().convert(value);
     } else if (name == "neg") {
-      a.neg = from_ruby<int>(value);
+      a.neg = Rice::detail::From_Ruby<int>().convert(value);
     } else if (name == "word_ngrams") {
-      a.wordNgrams = from_ruby<int>(value);
+      a.wordNgrams = Rice::detail::From_Ruby<int>().convert(value);
     } else if (name == "loss") {
-      std::string str = from_ruby<std::string>(value);
+      std::string str = Rice::detail::From_Ruby<std::string>().convert(value);
       if (str == "softmax") {
         a.loss = fasttext::loss_name::softmax;
       } else if (str == "ns") {
@@ -104,7 +91,7 @@ fasttext::Args buildArgs(Hash h) {
         throw std::invalid_argument("Unknown loss: " + str);
       }
     } else if (name == "model") {
-      std::string str = from_ruby<std::string>(value);
+      std::string str = Rice::detail::From_Ruby<std::string>().convert(value);
       if (str == "supervised") {
         a.model = fasttext::model_name::sup;
       } else if (str == "skipgram") {
@@ -115,25 +102,35 @@ fasttext::Args buildArgs(Hash h) {
         throw std::invalid_argument("Unknown model: " + str);
       }
     } else if (name == "bucket") {
-      a.bucket = from_ruby<int>(value);
+      a.bucket = Rice::detail::From_Ruby<int>().convert(value);
     } else if (name == "minn") {
-      a.minn = from_ruby<int>(value);
+      a.minn = Rice::detail::From_Ruby<int>().convert(value);
     } else if (name == "maxn") {
-      a.maxn = from_ruby<int>(value);
+      a.maxn = Rice::detail::From_Ruby<int>().convert(value);
     } else if (name == "thread") {
-      a.thread = from_ruby<int>(value);
+      a.thread = Rice::detail::From_Ruby<int>().convert(value);
     } else if (name == "t") {
-      a.t = from_ruby<double>(value);
+      a.t = Rice::detail::From_Ruby<double>().convert(value);
     } else if (name == "label_prefix") {
-      a.label = from_ruby<std::string>(value);
+      a.label = Rice::detail::From_Ruby<std::string>().convert(value);
     } else if (name == "verbose") {
-      a.verbose = from_ruby<int>(value);
+      a.verbose = Rice::detail::From_Ruby<int>().convert(value);
     } else if (name == "pretrained_vectors") {
-      a.pretrainedVectors = from_ruby<std::string>(value);
+      a.pretrainedVectors = Rice::detail::From_Ruby<std::string>().convert(value);
     } else if (name == "save_output") {
-      a.saveOutput = from_ruby<bool>(value);
-    // } else if (name == "seed") {
-    //   a.seed = from_ruby<int>(value);
+      a.saveOutput = Rice::detail::From_Ruby<bool>().convert(value);
+    } else if (name == "seed") {
+      a.seed = Rice::detail::From_Ruby<int>().convert(value);
+    } else if (name == "autotune_validation_file") {
+      a.autotuneValidationFile = Rice::detail::From_Ruby<std::string>().convert(value);
+    } else if (name == "autotune_metric") {
+      a.autotuneMetric = Rice::detail::From_Ruby<std::string>().convert(value);
+    } else if (name == "autotune_predictions") {
+      a.autotunePredictions = Rice::detail::From_Ruby<int>().convert(value);
+    } else if (name == "autotune_duration") {
+      a.autotuneDuration = Rice::detail::From_Ruby<int>().convert(value);
+    } else if (name == "autotune_model_size") {
+      a.autotuneModelSize = Rice::detail::From_Ruby<std::string>().convert(value);
     } else {
       throw std::invalid_argument("Unknown argument: " + name);
     }
@@ -147,11 +144,11 @@ void Init_ext()
   Module rb_mFastText = define_module("FastText");
   Module rb_mExt = define_module_under(rb_mFastText, "Ext");
 
-  define_class_under<fasttext::FastText>(rb_mExt, "Model")
-    .define_constructor(Constructor<fasttext::FastText>())
+  define_class_under<FastText>(rb_mExt, "Model")
+    .define_constructor(Constructor<FastText>())
     .define_method(
       "words",
-      *[](fasttext::FastText& m) {
+      [](FastText& m) {
         std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
         std::vector<int64_t> freq = d->getCounts(fasttext::entry_type::word);
 
@@ -169,7 +166,7 @@ void Init_ext()
       })
     .define_method(
       "labels",
-      *[](fasttext::FastText& m) {
+      [](FastText& m) {
         std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
         std::vector<int64_t> freq = d->getCounts(fasttext::entry_type::label);
 
@@ -187,12 +184,12 @@ void Init_ext()
       })
     .define_method(
       "test",
-      *[](fasttext::FastText& m, const std::string filename, int32_t k) {
+      [](FastText& m, const std::string& filename, int32_t k) {
         std::ifstream ifs(filename);
         if (!ifs.is_open()) {
           throw std::invalid_argument("Test file cannot be opened!");
         }
-        fasttext::Meter meter;
+        fasttext::Meter meter(false);
         m.test(ifs, k, 0.0, meter);
         ifs.close();
 
@@ -204,17 +201,21 @@ void Init_ext()
       })
     .define_method(
       "load_model",
-      *[](fasttext::FastText& m, std::string s) { m.loadModel(s); })
+      [](FastText& m, const std::string& s) {
+        m.loadModel(s);
+      })
     .define_method(
       "save_model",
-      *[](fasttext::FastText& m, std::string s) { m.saveModel(s); })
-    .define_method("dimension", &fasttext::FastText::getDimension)
-    .define_method("quantized?", &fasttext::FastText::isQuant)
-    .define_method("word_id", &fasttext::FastText::getWordId)
-    .define_method("subword_id", &fasttext::FastText::getSubwordId)
+      [](FastText& m, const std::string& s) {
+        m.saveModel(s);
+      })
+    .define_method("dimension", &FastText::getDimension)
+    .define_method("quantized?", &FastText::isQuant)
+    .define_method("word_id", &FastText::getWordId)
+    .define_method("subword_id", &FastText::getSubwordId)
     .define_method(
       "predict",
-      *[](fasttext::FastText& m, const std::string text, int32_t k, float threshold) {
+      [](FastText& m, const std::string& text, int32_t k, float threshold) {
         std::stringstream ioss(text);
         std::vector<std::pair<fasttext::real, std::string>> predictions;
         m.predictLine(ioss, predictions, k, threshold);
@@ -222,14 +223,14 @@ void Init_ext()
       })
     .define_method(
       "nearest_neighbors",
-      *[](fasttext::FastText& m, const std::string& word, int32_t k) {
+      [](FastText& m, const std::string& word, int32_t k) {
         return m.getNN(word, k);
       })
-    .define_method("analogies", &fasttext::FastText::getAnalogies)
-    .define_method("ngram_vectors", &fasttext::FastText::getNgramVectors)
+    .define_method("analogies", &FastText::getAnalogies)
+    // .define_method("ngram_vectors", &FastText::getNgramVectors)
     .define_method(
       "word_vector",
-      *[](fasttext::FastText& m, const std::string word) {
+      [](FastText& m, const std::string& word) {
         int dimension = m.getDimension();
         fasttext::Vector vec = fasttext::Vector(dimension);
         m.getWordVector(vec, word);
@@ -242,7 +243,7 @@ void Init_ext()
       })
     .define_method(
       "subwords",
-      *[](fasttext::FastText& m, const std::string word) {
+      [](FastText& m, const std::string& word) {
         std::vector<std::string> subwords;
         std::vector<int32_t> ngrams;
         std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
@@ -256,7 +257,7 @@ void Init_ext()
       })
     .define_method(
       "sentence_vector",
-      *[](fasttext::FastText& m, const std::string text) {
+      [](FastText& m, const std::string& text) {
         std::istringstream in(text);
         int dimension = m.getDimension();
         fasttext::Vector vec = fasttext::Vector(dimension);
@@ -270,22 +271,28 @@ void Init_ext()
       })
     .define_method(
       "train",
-      *[](fasttext::FastText& m, Hash h) {
-        m.train(buildArgs(h));
+      [](FastText& m, Hash h) {
+        auto a = buildArgs(h);
+        if (a.hasAutotune()) {
+          fasttext::Autotune autotune(std::shared_ptr<fasttext::FastText>(&m, [](fasttext::FastText*) {}));
+          autotune.train(a);
+        } else {
+          m.train(a);
+        }
       })
     .define_method(
       "quantize",
-      *[](fasttext::FastText& m, Hash h) {
+      [](FastText& m, Hash h) {
         m.quantize(buildArgs(h));
       })
     .define_method(
       "supervised?",
-      *[](fasttext::FastText& m) {
+      [](FastText& m) {
         return m.getArgs().model == fasttext::model_name::sup;
       })
     .define_method(
       "label_prefix",
-      *[](fasttext::FastText& m) {
+      [](FastText& m) {
         return m.getArgs().label;
       });
 }