fasttext 0.1.0 → 0.2.1

data/vendor/fastText/src/fasttext.h

@@ -12,6 +12,7 @@
 
 #include <atomic>
 #include <chrono>
+#include <functional>
 #include <iostream>
 #include <memory>
 #include <queue>
@@ -31,24 +32,29 @@
 namespace fasttext {
 
 class FastText {
+ public:
+  using TrainCallback =
+      std::function<void(float, float, double, double, int64_t)>;
+
  protected:
   std::shared_ptr<Args> args_;
   std::shared_ptr<Dictionary> dict_;
-
   std::shared_ptr<Matrix> input_;
   std::shared_ptr<Matrix> output_;
-
   std::shared_ptr<Model> model_;
-
   std::atomic<int64_t> tokenCount_{};
   std::atomic<real> loss_{};
-
   std::chrono::steady_clock::time_point start_;
+  bool quant_;
+  int32_t version;
+  std::unique_ptr<DenseMatrix> wordVectors_;
+  std::exception_ptr trainException_;
+
   void signModel(std::ostream&);
   bool checkModel(std::istream&);
-  void startThreads();
+  void startThreads(const TrainCallback& callback = {});
   void addInputVector(Vector&, int32_t) const;
-  void trainThread(int32_t);
+  void trainThread(int32_t, const TrainCallback& callback);
   std::vector<std::pair<real, std::string>> getNN(
       const DenseMatrix& wordVectors,
       const Vector& queryVec,
@@ -68,10 +74,11 @@ class FastText {
       const std::vector<int32_t>& labels);
   void cbow(Model::State& state, real lr, const std::vector<int32_t>& line);
   void skipgram(Model::State& state, real lr, const std::vector<int32_t>& line);
-
-  bool quant_;
-  int32_t version;
-  std::unique_ptr<DenseMatrix> wordVectors_;
+  std::vector<int32_t> selectEmbeddings(int32_t cutoff) const;
+  void precomputeWordVectors(DenseMatrix& wordVectors);
+  bool keepTraining(const int64_t ntokens) const;
+  void buildModel();
+  std::tuple<int64_t, double, double> progressInfo(real progress);
 
  public:
   FastText();
@@ -80,6 +87,8 @@ class FastText {
 
   int32_t getSubwordId(const std::string& subword) const;
 
+  int32_t getLabelId(const std::string& label) const;
+
   void getWordVector(Vector& vec, const std::string& word) const;
 
   void getSubwordVector(Vector& vec, const std::string& subword) const;
@@ -95,6 +104,10 @@ class FastText {
 
   std::shared_ptr<const DenseMatrix> getInputMatrix() const;
 
+  void setMatrices(
+      const std::shared_ptr<DenseMatrix>& inputMatrix,
+      const std::shared_ptr<DenseMatrix>& outputMatrix);
+
   std::shared_ptr<const DenseMatrix> getOutputMatrix() const;
 
   void saveVectors(const std::string& filename);
@@ -109,7 +122,7 @@ class FastText {
 
   void getSentenceVector(std::istream& in, Vector& vec);
 
-  void quantize(const Args& qargs);
+  void quantize(const Args& qargs, const TrainCallback& callback = {});
 
   std::tuple<int64_t, double, double>
   test(std::istream& in, int32_t k, real threshold = 0.0);
@@ -141,51 +154,17 @@ class FastText {
       const std::string& wordB,
       const std::string& wordC);
 
-  void train(const Args& args);
+  void train(const Args& args, const TrainCallback& callback = {});
+
+  void abort();
 
   int getDimension() const;
 
   bool isQuant() const;
 
-  FASTTEXT_DEPRECATED("loadVectors is being deprecated.")
-  void loadVectors(const std::string& filename);
-
-  FASTTEXT_DEPRECATED(
-      "getVector is being deprecated and replaced by getWordVector.")
-  void getVector(Vector& vec, const std::string& word) const;
-
-  FASTTEXT_DEPRECATED(
-      "ngramVectors is being deprecated and replaced by getNgramVectors.")
-  void ngramVectors(std::string word);
-
-  FASTTEXT_DEPRECATED(
-      "analogies is being deprecated and replaced by getAnalogies.")
-  void analogies(int32_t k);
-
-  FASTTEXT_DEPRECATED("selectEmbeddings is being deprecated.")
-  std::vector<int32_t> selectEmbeddings(int32_t cutoff) const;
-
-  FASTTEXT_DEPRECATED(
-      "saveVectors is being deprecated, please use the other signature.")
-  void saveVectors();
-
-  FASTTEXT_DEPRECATED(
-      "saveOutput is being deprecated, please use the other signature.")
-  void saveOutput();
-
-  FASTTEXT_DEPRECATED(
-      "saveModel is being deprecated, please use the other signature.")
-  void saveModel();
-
-  FASTTEXT_DEPRECATED("precomputeWordVectors is being deprecated.")
-  void precomputeWordVectors(DenseMatrix& wordVectors);
-
-  FASTTEXT_DEPRECATED("findNN is being deprecated and replaced by getNN.")
-  void findNN(
-      const DenseMatrix& wordVectors,
-      const Vector& query,
-      int32_t k,
-      const std::set<std::string>& banSet,
-      std::vector<std::pair<real, std::string>>& results);
+  class AbortError : public std::runtime_error {
+   public:
+    AbortError() : std::runtime_error("Aborted.") {}
+  };
 };
 } // namespace fasttext

data/vendor/fastText/src/main.cc

@@ -11,6 +11,7 @@
 #include <queue>
 #include <stdexcept>
 #include "args.h"
+#include "autotune.h"
 #include "fasttext.h"
 
 using namespace fasttext;
@@ -20,19 +21,25 @@ void printUsage() {
       << "usage: fasttext <command> <args>\n\n"
       << "The commands supported by fasttext are:\n\n"
       << "  supervised              train a supervised classifier\n"
-      << "  quantize                quantize a model to reduce the memory usage\n"
+      << "  quantize                quantize a model to reduce the memory "
+         "usage\n"
       << "  test                    evaluate a supervised classifier\n"
-      << "  test-label              print labels with precision and recall scores\n"
+      << "  test-label              print labels with precision and recall "
+         "scores\n"
       << "  predict                 predict most likely labels\n"
-      << "  predict-prob            predict most likely labels with probabilities\n"
+      << "  predict-prob            predict most likely labels with "
+         "probabilities\n"
       << "  skipgram                train a skipgram model\n"
       << "  cbow                    train a cbow model\n"
       << "  print-word-vectors      print word vectors given a trained model\n"
-      << "  print-sentence-vectors  print sentence vectors given a trained model\n"
-      << "  print-ngrams            print ngrams given a trained model and word\n"
+      << "  print-sentence-vectors  print sentence vectors given a trained "
+         "model\n"
+      << "  print-ngrams            print ngrams given a trained model and "
+         "word\n"
       << "  nn                      query for nearest neighbors\n"
       << "  analogies               query for analogies\n"
-      << "  dump                    dump arguments,dictionary,input/output vectors\n"
+      << "  dump                    dump arguments,dictionary,input/output "
+         "vectors\n"
       << std::endl;
 }
 
@@ -141,7 +148,7 @@ void test(const std::vector<std::string>& args) {
   FastText fasttext;
   fasttext.loadModel(model);
 
-  Meter meter;
+  Meter meter(false);
 
   if (input == "-") {
     fasttext.test(std::cin, k, threshold, meter);
@@ -351,19 +358,31 @@ void analogies(const std::vector<std::string> args) {
 void train(const std::vector<std::string> args) {
   Args a = Args();
   a.parseArgs(args);
-  FastText fasttext;
-  std::string outputFileName(a.output + ".bin");
+  std::shared_ptr<FastText> fasttext = std::make_shared<FastText>();
+  std::string outputFileName;
+
+  if (a.hasAutotune() &&
+      a.getAutotuneModelSize() != Args::kUnlimitedModelSize) {
+    outputFileName = a.output + ".ftz";
+  } else {
+    outputFileName = a.output + ".bin";
+  }
   std::ofstream ofs(outputFileName);
   if (!ofs.is_open()) {
     throw std::invalid_argument(
         outputFileName + " cannot be opened for saving.");
   }
   ofs.close();
-  fasttext.train(a);
-  fasttext.saveModel(outputFileName);
-  fasttext.saveVectors(a.output + ".vec");
+  if (a.hasAutotune()) {
+    Autotune autotune(fasttext);
+    autotune.train(a);
+  } else {
+    fasttext->train(a);
+  }
+  fasttext->saveModel(outputFileName);
+  fasttext->saveVectors(a.output + ".vec");
   if (a.saveOutput) {
-    fasttext.saveOutput(a.output + ".output");
+    fasttext->saveOutput(a.output + ".output");
   }
 }
 

data/vendor/fastText/src/meter.cc

@@ -16,6 +16,9 @@
 
 namespace fasttext {
 
+constexpr int32_t kAllLabels = -1;
+constexpr real falseNegativeScore = -1.0;
+
 void Meter::log(
     const std::vector<int32_t>& labels,
     const Predictions& predictions) {
@@ -26,14 +29,23 @@ void Meter::log(
   for (const auto& prediction : predictions) {
     labelMetrics_[prediction.second].predicted++;
 
+    real score = std::min(std::exp(prediction.first), 1.0f);
+    real gold = 0.0;
     if (utils::contains(labels, prediction.second)) {
       labelMetrics_[prediction.second].predictedGold++;
       metrics_.predictedGold++;
+      gold = 1.0;
     }
+    labelMetrics_[prediction.second].scoreVsTrue.emplace_back(score, gold);
   }
 
-  for (const auto& label : labels) {
-    labelMetrics_[label].gold++;
+  if (falseNegativeLabels_) {
+    for (const auto& label : labels) {
+      labelMetrics_[label].gold++;
+      if (!utils::containsSecond(predictions, label)) {
+        labelMetrics_[label].scoreVsTrue.emplace_back(falseNegativeScore, 1.0);
+      }
+    }
   }
 }
 
@@ -57,6 +69,15 @@ double Meter::recall() const {
   return metrics_.recall();
 }
 
+double Meter::f1Score() const {
+  const double precision = this->precision();
+  const double recall = this->recall();
+  if (precision + recall != 0) {
+    return 2 * precision * recall / (precision + recall);
+  }
+  return std::numeric_limits<double>::quiet_NaN();
+}
+
 void Meter::writeGeneralMetrics(std::ostream& out, int32_t k) const {
   out << "N"
       << "\t" << nexamples_ << std::endl;
@@ -65,4 +86,129 @@ void Meter::writeGeneralMetrics(std::ostream& out, int32_t k) const {
   out << "R@" << k << "\t" << metrics_.recall() << std::endl;
 }
 
+std::vector<std::pair<uint64_t, uint64_t>> Meter::getPositiveCounts(
+    int32_t labelId) const {
+  std::vector<std::pair<uint64_t, uint64_t>> positiveCounts;
+
+  const auto& v = scoreVsTrue(labelId);
+  uint64_t truePositives = 0;
+  uint64_t falsePositives = 0;
+  double lastScore = falseNegativeScore - 1.0;
+
+  for (auto it = v.rbegin(); it != v.rend(); ++it) {
+    double score = it->first;
+    double gold = it->second;
+    if (score < 0) { // only reachable recall
+      break;
+    }
+    if (gold == 1.0) {
+      truePositives++;
+    } else {
+      falsePositives++;
+    }
+    if (score == lastScore && positiveCounts.size()) { // squeeze tied scores
+      positiveCounts.back() = {truePositives, falsePositives};
+    } else {
+      positiveCounts.emplace_back(truePositives, falsePositives);
+    }
+    lastScore = score;
+  }
+
+  return positiveCounts;
+}
+
+double Meter::precisionAtRecall(double recallQuery) const {
+  return precisionAtRecall(kAllLabels, recallQuery);
+}
+
+double Meter::precisionAtRecall(int32_t labelId, double recallQuery) const {
+  const auto& precisionRecall = precisionRecallCurve(labelId);
+  double bestPrecision = 0.0;
+  std::for_each(
+      precisionRecall.begin(),
+      precisionRecall.end(),
+      [&bestPrecision, recallQuery](const std::pair<double, double>& element) {
+        if (element.second >= recallQuery) {
+          bestPrecision = std::max(bestPrecision, element.first);
+        };
+      });
+  return bestPrecision;
+}
+
+double Meter::recallAtPrecision(double precisionQuery) const {
+  return recallAtPrecision(kAllLabels, precisionQuery);
+}
+
+double Meter::recallAtPrecision(int32_t labelId, double precisionQuery) const {
+  const auto& precisionRecall = precisionRecallCurve(labelId);
+  double bestRecall = 0.0;
+  std::for_each(
+      precisionRecall.begin(),
+      precisionRecall.end(),
+      [&bestRecall, precisionQuery](const std::pair<double, double>& element) {
+        if (element.first >= precisionQuery) {
+          bestRecall = std::max(bestRecall, element.second);
+        };
+      });
+  return bestRecall;
+}
+
+std::vector<std::pair<double, double>> Meter::precisionRecallCurve() const {
+  return precisionRecallCurve(kAllLabels);
+}
+
+std::vector<std::pair<double, double>> Meter::precisionRecallCurve(
+    int32_t labelId) const {
+  std::vector<std::pair<double, double>> precisionRecallCurve;
+  const auto& positiveCounts = getPositiveCounts(labelId);
+  if (positiveCounts.empty()) {
+    return precisionRecallCurve;
+  }
+
+  uint64_t golds =
+      (labelId == kAllLabels) ? metrics_.gold : labelMetrics_.at(labelId).gold;
+
+  auto fullRecall = std::lower_bound(
+      positiveCounts.begin(),
+      positiveCounts.end(),
+      golds,
+      utils::compareFirstLess);
+
+  if (fullRecall != positiveCounts.end()) {
+    fullRecall = std::next(fullRecall);
+  }
+
+  for (auto it = positiveCounts.begin(); it != fullRecall; it++) {
+    double precision = 0.0;
+    double truePositives = it->first;
+    double falsePositives = it->second;
+    if (truePositives + falsePositives != 0.0) {
+      precision = truePositives / (truePositives + falsePositives);
+    }
+    double recall = golds != 0 ? (truePositives / double(golds))
+                               : std::numeric_limits<double>::quiet_NaN();
+    precisionRecallCurve.emplace_back(precision, recall);
+  }
+  precisionRecallCurve.emplace_back(1.0, 0.0);
+
+  return precisionRecallCurve;
+}
+
+std::vector<std::pair<real, real>> Meter::scoreVsTrue(int32_t labelId) const {
+  std::vector<std::pair<real, real>> ret;
+  if (labelId == kAllLabels) {
+    for (const auto& k : labelMetrics_) {
+      auto& labelScoreVsTrue = labelMetrics_.at(k.first).scoreVsTrue;
+      ret.insert(ret.end(), labelScoreVsTrue.begin(), labelScoreVsTrue.end());
+    }
+  } else {
+    if (labelMetrics_.count(labelId)) {
+      ret = labelMetrics_.at(labelId).scoreVsTrue;
+    }
+  }
+  sort(ret.begin(), ret.end());
+
+  return ret;
+}
+
 } // namespace fasttext

data/vendor/fastText/src/meter.h

@@ -22,8 +22,9 @@ class Meter {
     uint64_t gold;
     uint64_t predicted;
     uint64_t predictedGold;
+    mutable std::vector<std::pair<real, real>> scoreVsTrue;
 
-    Metrics() : gold(0), predicted(0), predictedGold(0) {}
+    Metrics() : gold(0), predicted(0), predictedGold(0), scoreVsTrue() {}
 
     double precision() const {
       if (predicted == 0) {
@@ -43,18 +44,38 @@ class Meter {
       }
       return 2 * predictedGold / double(predicted + gold);
     }
+
+    std::vector<std::pair<real, real>> getScoreVsTrue() {
+      return scoreVsTrue;
+    }
   };
+  std::vector<std::pair<uint64_t, uint64_t>> getPositiveCounts(
+      int32_t labelId) const;
 
  public:
-  Meter() : metrics_(), nexamples_(0), labelMetrics_() {}
+  Meter() = delete;
+  explicit Meter(bool falseNegativeLabels)
+      : metrics_(),
+        nexamples_(0),
+        labelMetrics_(),
+        falseNegativeLabels_(falseNegativeLabels) {}
 
   void log(const std::vector<int32_t>& labels, const Predictions& predictions);
 
   double precision(int32_t);
   double recall(int32_t);
   double f1Score(int32_t);
+  std::vector<std::pair<real, real>> scoreVsTrue(int32_t labelId) const;
+  double precisionAtRecall(int32_t labelId, double recall) const;
+  double precisionAtRecall(double recall) const;
+  double recallAtPrecision(int32_t labelId, double recall) const;
+  double recallAtPrecision(double recall) const;
+  std::vector<std::pair<double, double>> precisionRecallCurve(
+      int32_t labelId) const;
+  std::vector<std::pair<double, double>> precisionRecallCurve() const;
   double precision() const;
   double recall() const;
+  double f1Score() const;
   uint64_t nexamples() const {
     return nexamples_;
   }
@@ -64,6 +85,7 @@ class Meter {
   Metrics metrics_{};
   uint64_t nexamples_;
   std::unordered_map<int32_t, Metrics> labelMetrics_;
+  bool falseNegativeLabels_;
 };
 
 } // namespace fasttext