fasttext 0.1.0

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA256:
+  metadata.gz: e0df128ff3b10090bc104614fbdaafe7db5be0077acab3da63c53c442451115a
+  data.tar.gz: ff998e44fcf39523929aff86a17c91153ea284a460388107ac7849c0566d6367
+SHA512:
+  metadata.gz: 56447b44cf08ba76deaec721f66ef16d122ddae9b2ccf732fb6c364b8df283e788331c267447bcb0aab4d012d3bf94861ed30375050b6d024be6e030d0773a0a
+  data.tar.gz: 9c2be8700058fb1c01cf0df61970b48670ac75688c88c9d1ec502d5ee690cc5b328d4b903b52314937b5477b043fb0d83d3d7a1b8cfeda4026d35e4300cb8756

data/CHANGELOG.md

@@ -0,0 +1,3 @@
+## 0.1.0
+
+- First release

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2019 Andrew Kane
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,251 @@
+# fastText
+
+[fastText](https://fasttext.cc) - efficient text classification and representation learning - for Ruby
+
+## Installation
+
+Add this line to your application’s Gemfile:
+
+```ruby
+gem 'fasttext'
+```
+
+## Text Classification
+
+Prep your data
+
+```ruby
+# documents
+x = [
+  "text from document one",
+  "text from document two",
+  "text from document three"
+]
+
+# labels
+y = ["ham", "ham", "spam"]
+```
+
+> Use an array if a document has multiple labels
+
+Train a model
+
+```ruby
+model = FastText::Classifier.new
+model.fit(x, y)
+```
+
+Get predictions
+
+```ruby
+model.predict(x)
+```
+
+Save the model to a file
+
+```ruby
+model.save_model("model.bin")
+```
+
+Load the model from a file
+
+```ruby
+model = FastText.load_model("model.bin")
+```
+
+Evaluate the model
+
+```ruby
+model.test(x_test, y_test)
+```
+
+Get words and labels
+
+```ruby
+model.words
+model.labels
+```
+
+> Use `include_freq: true` to get their frequency
+
+Compress the model - significantly reduces size but sacrifices a little performance
+
+```ruby
+model.quantize
+model.save_model("model.ftz")
+```
+
+## Word Representations
+
+Prep your data
+
+```ruby
+x = [
+  "text from document one",
+  "text from document two",
+  "text from document three"
+]
+```
+
+Train a model
+
+```ruby
+model = FastText::Vectorizer.new
+model.fit(x)
+```
+
+Get nearest neighbors
+
+```ruby
+model.nearest_neighbors("asparagus")
+```
+
+Get analogies
+
+```ruby
+model.analogies("berlin", "germany", "france")
+```
+
+Get a word vector
+
+```ruby
+model.word_vector("carrot")
+```
+
+Get words
+
+```ruby
+model.words
+```
+
+Save the model to a file
+
+```ruby
+model.save_model("model.bin")
+```
+
+Load the model from a file
+
+```ruby
+model = FastText.load_model("model.bin")
+```
+
+Use continuous bag-of-words
+
+```ruby
+model = FastText::Vectorizer.new(model: "cbow")
+```
+
+## Parameters
+
+Text classification
+
+```ruby
+FastText::Classifier.new(
+  lr: 0.1,                    # learning rate
+  dim: 100,                   # size of word vectors
+  ws: 5,                      # size of the context window
+  epoch: 5,                   # number of epochs
+  min_count: 1,               # minimal number of word occurences
+  min_count_label: 1,         # minimal number of label occurences
+  minn: 0,                    # min length of char ngram
+  maxn: 0,                    # max length of char ngram
+  neg: 5,                     # number of negatives sampled
+  word_ngrams: 1,             # max length of word ngram
+  loss: "softmax",            # loss function {ns, hs, softmax, ova}
+  bucket: 2000000,            # number of buckets
+  thread: 3,                  # number of threads
+  lr_update_rate: 100,        # change the rate of updates for the learning rate
+  t: 0.0001,                  # sampling threshold
+  label_prefix: "__label__"   # label prefix
+  verbose: 2,                 # verbose
+  pretrained_vectors: nil     # pretrained word vectors (.vec file)
+)
+```
+
+Word representations
+
+```ruby
+FastText::Vectorizer.new(
+  model: "skipgram",          # unsupervised fasttext model {cbow, skipgram}
+  lr: 0.05,                   # learning rate
+  dim: 100,                   # size of word vectors
+  ws: 5,                      # size of the context window
+  epoch: 5,                   # number of epochs
+  min_count: 5,               # minimal number of word occurences
+  minn: 3,                    # min length of char ngram
+  maxn: 6,                    # max length of char ngram
+  neg: 5,                     # number of negatives sampled
+  word_ngrams: 1,             # max length of word ngram
+  loss: "ns",                 # loss function {ns, hs, softmax, ova}
+  bucket: 2000000,            # number of buckets
+  thread: 3,                  # number of threads
+  lr_update_rate: 100,        # change the rate of updates for the learning rate
+  t: 0.0001,                  # sampling threshold
+  verbose: 2                  # verbose
+)
+```
+
+## Input Files
+
+Input can be read directly from files
+
+```ruby
+model.fit("train.txt")
+model.test("test.txt")
+```
+
+Each line should be a document
+
+```txt
+text from document one
+text from document two
+text from document three
+```
+
+For text classification, lines should start with a list of labels prefixed with `__label__`
+
+```txt
+__label__ham text from document one
+__label__ham text from document two
+__label__spam text from document three
+```
+
+## Pretrained Models
+
+There are a number of [pretrained models](https://fasttext.cc/docs/en/english-vectors.html) you can download
+
+### Language Identification
+
+Download one of the [pretrained models](https://fasttext.cc/docs/en/language-identification.html) and load it
+
+```ruby
+model = FastText.load_model("lid.176.ftz")
+```
+
+Get language predictions
+
+```ruby
+model.predict("bon appétit")
+```
+
+## rbenv
+
+This library uses [Rice](https://github.com/jasonroelofs/rice) to interface with the fastText C++ library. Unfortunately, Rice and rbenv don’t play nicely together. This is actively [being addressed](https://github.com/rbenv/ruby-build/pull/1368), but in the meantime, if you encounter an error during installation, reinstall your Ruby version with the `--enabled-shared` flag.
+
+```sh
+CONFIGURE_OPTS="--enable-shared" rbenv install 2.6.5
+```
+
+## History
+
+View the [changelog](https://github.com/ankane/fasttext/blob/master/CHANGELOG.md)
+
+## Contributing
+
+Everyone is encouraged to help improve this project. Here are a few ways you can help:
+
+- [Report bugs](https://github.com/ankane/fasttext/issues)
+- Fix bugs and [submit pull requests](https://github.com/ankane/fasttext/pulls)
+- Write, clarify, or fix documentation
+- Suggest or add new features

data/ext/fasttext/ext.cpp

@@ -0,0 +1,291 @@
+#include <args.h>
+#include <densematrix.h>
+#include <fasttext.h>
+#include <rice/Data_Type.hpp>
+#include <rice/Constructor.hpp>
+#include <rice/Array.hpp>
+#include <rice/Hash.hpp>
+#include <real.h>
+#include <vector.h>
+#include <cmath>
+#include <iterator>
+#include <sstream>
+#include <stdexcept>
+
+using namespace Rice;
+
+template<>
+inline
+long long from_ruby<long long>(Object x)
+{
+  return NUM2LL(x);
+}
+
+template<>
+inline
+Object to_ruby<long long>(long long const & x)
+{
+  return LL2NUM(x);
+}
+
+template<>
+inline
+unsigned long long from_ruby<unsigned long long>(Object x)
+{
+  return NUM2ULL(x);
+}
+
+template<>
+inline
+Object to_ruby<unsigned long long>(unsigned long long const & x)
+{
+  return ULL2NUM(x);
+}
+
+template<>
+inline
+Object to_ruby<std::vector<std::pair<fasttext::real, std::string>>>(std::vector<std::pair<fasttext::real, std::string>> const & x)
+{
+  Array ret;
+  for (const auto& v : x) {
+    Array a;
+    a.push(v.first);
+    a.push(v.second);
+    ret.push(a);
+  }
+  return ret;
+}
+
+fasttext::Args buildArgs(Hash h) {
+  fasttext::Args a;
+
+  std::vector<Hash::Entry> v;
+  Hash::iterator it = h.begin();
+  Hash::iterator end = h.end();
+
+  for(; it != end; ++it)
+  {
+    std::string name = from_ruby<std::string>(it->key.to_s());
+    Object value = it->value;
+
+    if (name == "input") {
+      a.input = from_ruby<std::string>(value);
+    } else if (name == "output") {
+      a.output = from_ruby<std::string>(value);
+    } else if (name == "lr") {
+      a.lr = from_ruby<double>(value);
+    } else if (name == "lr_update_rate") {
+      a.lrUpdateRate = from_ruby<int>(value);
+    } else if (name == "dim") {
+      a.dim = from_ruby<int>(value);
+    } else if (name == "ws") {
+      a.ws = from_ruby<int>(value);
+    } else if (name == "epoch") {
+      a.epoch = from_ruby<int>(value);
+    } else if (name == "min_count") {
+      a.minCount = from_ruby<int>(value);
+    } else if (name == "min_count_label") {
+      a.minCountLabel = from_ruby<int>(value);
+    } else if (name == "neg") {
+      a.neg = from_ruby<int>(value);
+    } else if (name == "word_ngrams") {
+      a.wordNgrams = from_ruby<int>(value);
+    } else if (name == "loss") {
+      std::string str = from_ruby<std::string>(value);
+      if (str == "softmax") {
+        a.loss = fasttext::loss_name::softmax;
+      } else if (str == "ns") {
+        a.loss = fasttext::loss_name::ns;
+      } else if (str == "hs") {
+        a.loss = fasttext::loss_name::hs;
+      } else if (str == "ova") {
+        a.loss = fasttext::loss_name::ova;
+      } else {
+        throw std::invalid_argument("Unknown loss: " + str);
+      }
+    } else if (name == "model") {
+      std::string str = from_ruby<std::string>(value);
+      if (str == "supervised") {
+        a.model = fasttext::model_name::sup;
+      } else if (str == "skipgram") {
+        a.model = fasttext::model_name::sg;
+      } else if (str == "cbow") {
+        a.model = fasttext::model_name::cbow;
+      } else {
+        throw std::invalid_argument("Unknown model: " + str);
+      }
+    } else if (name == "bucket") {
+      a.bucket = from_ruby<int>(value);
+    } else if (name == "minn") {
+      a.minn = from_ruby<int>(value);
+    } else if (name == "maxn") {
+      a.maxn = from_ruby<int>(value);
+    } else if (name == "thread") {
+      a.thread = from_ruby<int>(value);
+    } else if (name == "t") {
+      a.t = from_ruby<double>(value);
+    } else if (name == "label_prefix") {
+      a.label = from_ruby<std::string>(value);
+    } else if (name == "verbose") {
+      a.verbose = from_ruby<int>(value);
+    } else if (name == "pretrained_vectors") {
+      a.pretrainedVectors = from_ruby<std::string>(value);
+    } else if (name == "save_output") {
+      a.saveOutput = from_ruby<bool>(value);
+    // } else if (name == "seed") {
+    //   a.seed = from_ruby<int>(value);
+    } else {
+      throw std::invalid_argument("Unknown argument: " + name);
+    }
+  }
+  return a;
+}
+
+extern "C"
+void Init_ext()
+{
+  Module rb_mFastText = define_module("FastText");
+  Module rb_mExt = define_module_under(rb_mFastText, "Ext");
+
+  define_class_under<fasttext::FastText>(rb_mExt, "Model")
+    .define_constructor(Constructor<fasttext::FastText>())
+    .define_method(
+      "words",
+      *[](fasttext::FastText& m) {
+        std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
+        std::vector<int64_t> freq = d->getCounts(fasttext::entry_type::word);
+
+        Array vocab_list;
+        Array vocab_freq;
+        for (int32_t i = 0; i < d->nwords(); i++) {
+          vocab_list.push(d->getWord(i));
+          vocab_freq.push(freq[i]);
+        }
+
+        Array ret;
+        ret.push(vocab_list);
+        ret.push(vocab_freq);
+        return ret;
+      })
+    .define_method(
+      "labels",
+      *[](fasttext::FastText& m) {
+        std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
+        std::vector<int64_t> freq = d->getCounts(fasttext::entry_type::label);
+
+        Array vocab_list;
+        Array vocab_freq;
+        for (int32_t i = 0; i < d->nlabels(); i++) {
+          vocab_list.push(d->getLabel(i));
+          vocab_freq.push(freq[i]);
+        }
+
+        Array ret;
+        ret.push(vocab_list);
+        ret.push(vocab_freq);
+        return ret;
+      })
+    .define_method(
+      "test",
+      *[](fasttext::FastText& m, const std::string filename, int32_t k) {
+        std::ifstream ifs(filename);
+        if (!ifs.is_open()) {
+          throw std::invalid_argument("Test file cannot be opened!");
+        }
+        fasttext::Meter meter;
+        m.test(ifs, k, 0.0, meter);
+        ifs.close();
+
+        Array ret;
+        ret.push(meter.nexamples());
+        ret.push(meter.precision());
+        ret.push(meter.recall());
+        return ret;
+      })
+    .define_method(
+      "load_model",
+      *[](fasttext::FastText& m, std::string s) { m.loadModel(s); })
+    .define_method(
+      "save_model",
+      *[](fasttext::FastText& m, std::string s) { m.saveModel(s); })
+    .define_method("dimension", &fasttext::FastText::getDimension)
+    .define_method("quantized?", &fasttext::FastText::isQuant)
+    .define_method("word_id", &fasttext::FastText::getWordId)
+    .define_method("subword_id", &fasttext::FastText::getSubwordId)
+    .define_method(
+      "predict",
+      *[](fasttext::FastText& m, const std::string text, int32_t k, float threshold) {
+        std::stringstream ioss(text);
+        std::vector<std::pair<fasttext::real, std::string>> predictions;
+        m.predictLine(ioss, predictions, k, threshold);
+        return predictions;
+      })
+    .define_method(
+      "nearest_neighbors",
+      *[](fasttext::FastText& m, const std::string& word, int32_t k) {
+        return m.getNN(word, k);
+      })
+    .define_method("analogies", &fasttext::FastText::getAnalogies)
+    .define_method("ngram_vectors", &fasttext::FastText::getNgramVectors)
+    .define_method(
+      "word_vector",
+      *[](fasttext::FastText& m, const std::string word) {
+        int dimension = m.getDimension();
+        fasttext::Vector vec = fasttext::Vector(dimension);
+        m.getWordVector(vec, word);
+        float* data = vec.data();
+        Array ret;
+        for (int i = 0; i < dimension; i++) {
+          ret.push(data[i]);
+        }
+        return ret;
+      })
+    .define_method(
+      "subwords",
+      *[](fasttext::FastText& m, const std::string word) {
+        std::vector<std::string> subwords;
+        std::vector<int32_t> ngrams;
+        std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
+        d->getSubwords(word, ngrams, subwords);
+
+        Array ret;
+        for (const auto& subword : subwords) {
+          ret.push(subword);
+        }
+        return ret;
+      })
+    .define_method(
+      "sentence_vector",
+      *[](fasttext::FastText& m, const std::string text) {
+        std::istringstream in(text);
+        int dimension = m.getDimension();
+        fasttext::Vector vec = fasttext::Vector(dimension);
+        m.getSentenceVector(in, vec);
+        float* data = vec.data();
+        Array ret;
+        for (int i = 0; i < dimension; i++) {
+          ret.push(data[i]);
+        }
+        return ret;
+      })
+    .define_method(
+      "train",
+      *[](fasttext::FastText& m, Hash h) {
+        m.train(buildArgs(h));
+      })
+    .define_method(
+      "quantize",
+      *[](fasttext::FastText& m, Hash h) {
+        m.quantize(buildArgs(h));
+      })
+    .define_method(
+      "supervised?",
+      *[](fasttext::FastText& m) {
+        return m.getArgs().model == fasttext::model_name::sup;
+      })
+    .define_method(
+      "label_prefix",
+      *[](fasttext::FastText& m) {
+        return m.getArgs().label;
+      });
+}