ffi-fasttext 0.1.0

data/vendor/fasttext/PATENTS

@@ -0,0 +1,33 @@
+Additional Grant of Patent Rights Version 2
+
+"Software" means the fastText software distributed by Facebook, Inc.
+
+Facebook, Inc. ("Facebook") hereby grants to each recipient of the Software
+("you") a perpetual, worldwide, royalty-free, non-exclusive, irrevocable
+(subject to the termination provision below) license under any Necessary
+Claims, to make, have made, use, sell, offer to sell, import, and otherwise
+transfer the Software. For avoidance of doubt, no license is granted under
+Facebook’s rights in any patent claims that are infringed by (i) modifications
+to the Software made by you or any third party or (ii) the Software in
+combination with any software or other technology.
+
+The license granted hereunder will terminate, automatically and without notice,
+if you (or any of your subsidiaries, corporate affiliates or agents) initiate
+directly or indirectly, or take a direct financial interest in, any Patent
+Assertion: (i) against Facebook or any of its subsidiaries or corporate
+affiliates, (ii) against any party if such Patent Assertion arises in whole or
+in part from any software, technology, product or service of Facebook or any of
+its subsidiaries or corporate affiliates, or (iii) against any party relating
+to the Software. Notwithstanding the foregoing, if Facebook or any of its
+subsidiaries or corporate affiliates files a lawsuit alleging patent
+infringement against you in the first instance, and you respond by filing a
+patent infringement counterclaim in that lawsuit against that party that is
+unrelated to the Software, the license granted hereunder will not terminate
+under section (i) of this paragraph due to such counterclaim.
+
+A "Necessary Claim" is a claim of a patent owned by Facebook that is
+necessarily infringed by the Software standing alone.
+
+A "Patent Assertion" is any lawsuit or other action alleging direct, indirect,
+or contributory infringement or inducement to infringe any patent, including a
+cross-claim or counterclaim.

data/vendor/fasttext/args.cc

@@ -0,0 +1,250 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#include "args.h"
+
+#include <stdlib.h>
+
+#include <iostream>
+
+namespace fasttext {
+
+Args::Args() {
+  lr = 0.05;
+  dim = 100;
+  ws = 5;
+  epoch = 5;
+  minCount = 5;
+  minCountLabel = 0;
+  neg = 5;
+  wordNgrams = 1;
+  loss = loss_name::ns;
+  model = model_name::sg;
+  bucket = 2000000;
+  minn = 3;
+  maxn = 6;
+  thread = 12;
+  lrUpdateRate = 100;
+  t = 1e-4;
+  label = "__label__";
+  verbose = 2;
+  pretrainedVectors = "";
+  saveOutput = 0;
+
+  qout = false;
+  retrain = false;
+  qnorm = false;
+  cutoff = 0;
+  dsub = 2;
+}
+
+std::string Args::lossToString(loss_name ln) {
+  switch (ln) {
+    case loss_name::hs:
+      return "hs";
+    case loss_name::ns:
+      return "ns";
+    case loss_name::softmax:
+      return "softmax";
+  }
+  return "Unknown loss!"; // should never happen
+}
+
+void Args::parseArgs(const std::vector<std::string>& args) {
+  std::string command(args[1]);
+  if (command == "supervised") {
+    model = model_name::sup;
+    loss = loss_name::softmax;
+    minCount = 1;
+    minn = 0;
+    maxn = 0;
+    lr = 0.1;
+  } else if (command == "cbow") {
+    model = model_name::cbow;
+  }
+  int ai = 2;
+  while (ai < args.size()) {
+    if (args[ai][0] != '-') {
+      std::cerr << "Provided argument without a dash! Usage:" << std::endl;
+      printHelp();
+      exit(EXIT_FAILURE);
+    }
+    if (args[ai] == "-h") {
+      std::cerr << "Here is the help! Usage:" << std::endl;
+      printHelp();
+      exit(EXIT_FAILURE);
+    } else if (args[ai] == "-input") {
+      input = std::string(args[ai + 1]);
+    } else if (args[ai] == "-test") {
+      test = std::string(args[ai + 1]);
+    } else if (args[ai] == "-output") {
+      output = std::string(args[ai + 1]);
+    } else if (args[ai] == "-lr") {
+      lr = std::stof(args[ai + 1]);
+    } else if (args[ai] == "-lrUpdateRate") {
+      lrUpdateRate = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-dim") {
+      dim = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-ws") {
+      ws = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-epoch") {
+      epoch = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-minCount") {
+      minCount = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-minCountLabel") {
+      minCountLabel = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-neg") {
+      neg = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-wordNgrams") {
+      wordNgrams = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-loss") {
+      if (args[ai + 1] == "hs") {
+        loss = loss_name::hs;
+      } else if (args[ai + 1] == "ns") {
+        loss = loss_name::ns;
+      } else if (args[ai + 1] == "softmax") {
+        loss = loss_name::softmax;
+      } else {
+        std::cerr << "Unknown loss: " << args[ai + 1] << std::endl;
+        printHelp();
+        exit(EXIT_FAILURE);
+      }
+    } else if (args[ai] == "-bucket") {
+      bucket = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-minn") {
+      minn = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-maxn") {
+      maxn = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-thread") {
+      thread = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-t") {
+      t = std::stof(args[ai + 1]);
+    } else if (args[ai] == "-label") {
+      label = std::string(args[ai + 1]);
+    } else if (args[ai] == "-verbose") {
+      verbose = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-pretrainedVectors") {
+      pretrainedVectors = std::string(args[ai + 1]);
+    } else if (args[ai] == "-saveOutput") {
+      saveOutput = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-qnorm") {
+      qnorm = true; ai--;
+    } else if (args[ai] == "-retrain") {
+      retrain = true; ai--;
+    } else if (args[ai] == "-qout") {
+      qout = true; ai--;
+    } else if (args[ai] == "-cutoff") {
+    cutoff = std::stoi(args[ai + 1]);
+    } else if (args[ai] == "-dsub") {
+      dsub = std::stoi(args[ai + 1]);
+    } else {
+      std::cerr << "Unknown argument: " << args[ai] << std::endl;
+      printHelp();
+      exit(EXIT_FAILURE);
+    }
+    ai += 2;
+  }
+  if (input.empty() || output.empty()) {
+    std::cerr << "Empty input or output path." << std::endl;
+    printHelp();
+    exit(EXIT_FAILURE);
+  }
+  if (wordNgrams <= 1 && maxn == 0) {
+    bucket = 0;
+  }
+}
+
+void Args::printHelp() {
+  printBasicHelp();
+  printDictionaryHelp();
+  printTrainingHelp();
+  printQuantizationHelp();
+}
+
+
+void Args::printBasicHelp() {
+  std::cerr
+    << "\nThe following arguments are mandatory:\n"
+    << "  -input              training file path\n"
+    << "  -output             output file path\n"
+    << "\nThe following arguments are optional:\n"
+    << "  -verbose            verbosity level [" << verbose << "]\n";
+}
+
+void Args::printDictionaryHelp() {
+  std::cerr
+    << "\nThe following arguments for the dictionary are optional:\n"
+    << "  -minCount           minimal number of word occurences [" << minCount << "]\n"
+    << "  -minCountLabel      minimal number of label occurences [" << minCountLabel << "]\n"
+    << "  -wordNgrams         max length of word ngram [" << wordNgrams << "]\n"
+    << "  -bucket             number of buckets [" << bucket << "]\n"
+    << "  -minn               min length of char ngram [" << minn << "]\n"
+    << "  -maxn               max length of char ngram [" << maxn << "]\n"
+    << "  -t                  sampling threshold [" << t << "]\n"
+    << "  -label              labels prefix [" << label << "]\n";
+}
+
+void Args::printTrainingHelp() {
+  std::cerr
+    << "\nThe following arguments for training are optional:\n"
+    << "  -lr                 learning rate [" << lr << "]\n"
+    << "  -lrUpdateRate       change the rate of updates for the learning rate [" << lrUpdateRate << "]\n"
+    << "  -dim                size of word vectors [" << dim << "]\n"
+    << "  -ws                 size of the context window [" << ws << "]\n"
+    << "  -epoch              number of epochs [" << epoch << "]\n"
+    << "  -neg                number of negatives sampled [" << neg << "]\n"
+    << "  -loss               loss function {ns, hs, softmax} [" << lossToString(loss) << "]\n"
+    << "  -thread             number of threads [" << thread << "]\n"
+    << "  -pretrainedVectors  pretrained word vectors for supervised learning ["<< pretrainedVectors <<"]\n"
+    << "  -saveOutput         whether output params should be saved [" << saveOutput << "]\n";
+}
+
+void Args::printQuantizationHelp() {
+  std::cerr
+    << "\nThe following arguments for quantization are optional:\n"
+    << "  -cutoff             number of words and ngrams to retain [" << cutoff << "]\n"
+    << "  -retrain            finetune embeddings if a cutoff is applied [" << retrain << "]\n"
+    << "  -qnorm              quantizing the norm separately [" << qnorm << "]\n"
+    << "  -qout               quantizing the classifier [" << qout << "]\n"
+    << "  -dsub               size of each sub-vector [" << dsub << "]\n";
+}
+
+void Args::save(std::ostream& out) {
+  out.write((char*) &(dim), sizeof(int));
+  out.write((char*) &(ws), sizeof(int));
+  out.write((char*) &(epoch), sizeof(int));
+  out.write((char*) &(minCount), sizeof(int));
+  out.write((char*) &(neg), sizeof(int));
+  out.write((char*) &(wordNgrams), sizeof(int));
+  out.write((char*) &(loss), sizeof(loss_name));
+  out.write((char*) &(model), sizeof(model_name));
+  out.write((char*) &(bucket), sizeof(int));
+  out.write((char*) &(minn), sizeof(int));
+  out.write((char*) &(maxn), sizeof(int));
+  out.write((char*) &(lrUpdateRate), sizeof(int));
+  out.write((char*) &(t), sizeof(double));
+}
+
+void Args::load(std::istream& in) {
+  in.read((char*) &(dim), sizeof(int));
+  in.read((char*) &(ws), sizeof(int));
+  in.read((char*) &(epoch), sizeof(int));
+  in.read((char*) &(minCount), sizeof(int));
+  in.read((char*) &(neg), sizeof(int));
+  in.read((char*) &(wordNgrams), sizeof(int));
+  in.read((char*) &(loss), sizeof(loss_name));
+  in.read((char*) &(model), sizeof(model_name));
+  in.read((char*) &(bucket), sizeof(int));
+  in.read((char*) &(minn), sizeof(int));
+  in.read((char*) &(maxn), sizeof(int));
+  in.read((char*) &(lrUpdateRate), sizeof(int));
+  in.read((char*) &(t), sizeof(double));
+}
+
+}

data/vendor/fasttext/args.h

@@ -0,0 +1,71 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#ifndef FASTTEXT_ARGS_H
+#define FASTTEXT_ARGS_H
+
+#include <istream>
+#include <ostream>
+#include <string>
+#include <vector>
+
+namespace fasttext {
+
+enum class model_name : int {cbow=1, sg, sup};
+enum class loss_name : int {hs=1, ns, softmax};
+
+class Args {
+  private:
+    std::string lossToString(loss_name);
+
+  public:
+    Args();
+    std::string input;
+    std::string test;
+    std::string output;
+    double lr;
+    int lrUpdateRate;
+    int dim;
+    int ws;
+    int epoch;
+    int minCount;
+    int minCountLabel;
+    int neg;
+    int wordNgrams;
+    loss_name loss;
+    model_name model;
+    int bucket;
+    int minn;
+    int maxn;
+    int thread;
+    double t;
+    std::string label;
+    int verbose;
+    std::string pretrainedVectors;
+    int saveOutput;
+
+    bool qout;
+    bool retrain;
+    bool qnorm;
+    size_t cutoff;
+    size_t dsub;
+
+    void parseArgs(const std::vector<std::string>& args);
+    void printHelp();
+    void printBasicHelp();
+    void printDictionaryHelp();
+    void printTrainingHelp();
+    void printQuantizationHelp();
+    void save(std::ostream&);
+    void load(std::istream&);
+};
+
+}
+
+#endif

data/vendor/fasttext/dictionary.cc

@@ -0,0 +1,475 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the BSD-style license found in the
+ * LICENSE file in the root directory of this source tree. An additional grant
+ * of patent rights can be found in the PATENTS file in the same directory.
+ */
+
+#include "dictionary.h"
+
+#include <assert.h>
+
+#include <iostream>
+#include <fstream>
+#include <algorithm>
+#include <iterator>
+#include <cmath>
+
+namespace fasttext {
+
+const std::string Dictionary::EOS = "</s>";
+const std::string Dictionary::BOW = "<";
+const std::string Dictionary::EOW = ">";
+
+Dictionary::Dictionary(std::shared_ptr<Args> args) : args_(args),
+  word2int_(MAX_VOCAB_SIZE, -1), size_(0), nwords_(0), nlabels_(0),
+  ntokens_(0), pruneidx_size_(-1) {}
+
+int32_t Dictionary::find(const std::string& w) const {
+  return find(w, hash(w));
+}
+
+int32_t Dictionary::find(const std::string& w, uint32_t h) const {
+  int32_t id = h % MAX_VOCAB_SIZE;
+  while (word2int_[id] != -1 && words_[word2int_[id]].word != w) {
+    id = (id + 1) % MAX_VOCAB_SIZE;
+  }
+  return id;
+}
+
+void Dictionary::add(const std::string& w) {
+  int32_t h = find(w);
+  ntokens_++;
+  if (word2int_[h] == -1) {
+    entry e;
+    e.word = w;
+    e.count = 1;
+    e.type = getType(w);
+    words_.push_back(e);
+    word2int_[h] = size_++;
+  } else {
+    words_[word2int_[h]].count++;
+  }
+}
+
+int32_t Dictionary::nwords() const {
+  return nwords_;
+}
+
+int32_t Dictionary::nlabels() const {
+  return nlabels_;
+}
+
+int64_t Dictionary::ntokens() const {
+  return ntokens_;
+}
+
+const std::vector<int32_t>& Dictionary::getSubwords(int32_t i) const {
+  assert(i >= 0);
+  assert(i < nwords_);
+  return words_[i].subwords;
+}
+
+const std::vector<int32_t> Dictionary::getSubwords(
+    const std::string& word) const {
+  int32_t i = getId(word);
+  if (i >= 0) {
+    return getSubwords(i);
+  }
+  std::vector<int32_t> ngrams;
+  computeSubwords(BOW + word + EOW, ngrams);
+  return ngrams;
+}
+
+void Dictionary::getSubwords(const std::string& word,
+                           std::vector<int32_t>& ngrams,
+                           std::vector<std::string>& substrings) const {
+  int32_t i = getId(word);
+  ngrams.clear();
+  substrings.clear();
+  if (i >= 0) {
+    ngrams.push_back(i);
+    substrings.push_back(words_[i].word);
+  } else {
+    ngrams.push_back(-1);
+    substrings.push_back(word);
+  }
+  computeSubwords(BOW + word + EOW, ngrams, substrings);
+}
+
+bool Dictionary::discard(int32_t id, real rand) const {
+  assert(id >= 0);
+  assert(id < nwords_);
+  if (args_->model == model_name::sup) return false;
+  return rand > pdiscard_[id];
+}
+
+int32_t Dictionary::getId(const std::string& w, uint32_t h) const {
+  int32_t id = find(w, h);
+  return word2int_[id];
+}
+
+int32_t Dictionary::getId(const std::string& w) const {
+  int32_t h = find(w);
+  return word2int_[h];
+}
+
+entry_type Dictionary::getType(int32_t id) const {
+  assert(id >= 0);
+  assert(id < size_);
+  return words_[id].type;
+}
+
+entry_type Dictionary::getType(const std::string& w) const {
+  return (w.find(args_->label) == 0) ? entry_type::label : entry_type::word;
+}
+
+std::string Dictionary::getWord(int32_t id) const {
+  assert(id >= 0);
+  assert(id < size_);
+  return words_[id].word;
+}
+
+uint32_t Dictionary::hash(const std::string& str) const {
+  uint32_t h = 2166136261;
+  for (size_t i = 0; i < str.size(); i++) {
+    h = h ^ uint32_t(str[i]);
+    h = h * 16777619;
+  }
+  return h;
+}
+
+void Dictionary::computeSubwords(const std::string& word,
+                               std::vector<int32_t>& ngrams,
+                               std::vector<std::string>& substrings) const {
+  for (size_t i = 0; i < word.size(); i++) {
+    std::string ngram;
+    if ((word[i] & 0xC0) == 0x80) continue;
+    for (size_t j = i, n = 1; j < word.size() && n <= args_->maxn; n++) {
+      ngram.push_back(word[j++]);
+      while (j < word.size() && (word[j] & 0xC0) == 0x80) {
+        ngram.push_back(word[j++]);
+      }
+      if (n >= args_->minn && !(n == 1 && (i == 0 || j == word.size()))) {
+        int32_t h = hash(ngram) % args_->bucket;
+        ngrams.push_back(nwords_ + h);
+        substrings.push_back(ngram);
+      }
+    }
+  }
+}
+
+void Dictionary::computeSubwords(const std::string& word,
+                               std::vector<int32_t>& ngrams) const {
+  for (size_t i = 0; i < word.size(); i++) {
+    std::string ngram;
+    if ((word[i] & 0xC0) == 0x80) continue;
+    for (size_t j = i, n = 1; j < word.size() && n <= args_->maxn; n++) {
+      ngram.push_back(word[j++]);
+      while (j < word.size() && (word[j] & 0xC0) == 0x80) {
+        ngram.push_back(word[j++]);
+      }
+      if (n >= args_->minn && !(n == 1 && (i == 0 || j == word.size()))) {
+        int32_t h = hash(ngram) % args_->bucket;
+        pushHash(ngrams, h);
+      }
+    }
+  }
+}
+
+void Dictionary::initNgrams() {
+  for (size_t i = 0; i < size_; i++) {
+    std::string word = BOW + words_[i].word + EOW;
+    words_[i].subwords.clear();
+    words_[i].subwords.push_back(i);
+    if (words_[i].word != EOS) {
+      computeSubwords(word, words_[i].subwords);
+    }
+  }
+}
+
+bool Dictionary::readWord(std::istream& in, std::string& word) const
+{
+  char c;
+  std::streambuf& sb = *in.rdbuf();
+  word.clear();
+  while ((c = sb.sbumpc()) != EOF) {
+    if (c == ' ' || c == '\n' || c == '\r' || c == '\t' || c == '\v' ||
+        c == '\f' || c == '\0') {
+      if (word.empty()) {
+        if (c == '\n') {
+          word += EOS;
+          return true;
+        }
+        continue;
+      } else {
+        if (c == '\n')
+          sb.sungetc();
+        return true;
+      }
+    }
+    word.push_back(c);
+  }
+  // trigger eofbit
+  in.get();
+  return !word.empty();
+}
+
+void Dictionary::readFromFile(std::istream& in) {
+  std::string word;
+  int64_t minThreshold = 1;
+  while (readWord(in, word)) {
+    add(word);
+    if (ntokens_ % 1000000 == 0 && args_->verbose > 1) {
+      std::cerr << "\rRead " << ntokens_  / 1000000 << "M words" << std::flush;
+    }
+    if (size_ > 0.75 * MAX_VOCAB_SIZE) {
+      minThreshold++;
+      threshold(minThreshold, minThreshold);
+    }
+  }
+  threshold(args_->minCount, args_->minCountLabel);
+  initTableDiscard();
+  initNgrams();
+  if (args_->verbose > 0) {
+    std::cerr << "\rRead " << ntokens_  / 1000000 << "M words" << std::endl;
+    std::cerr << "Number of words:  " << nwords_ << std::endl;
+    std::cerr << "Number of labels: " << nlabels_ << std::endl;
+  }
+  if (size_ == 0) {
+    std::cerr << "Empty vocabulary. Try a smaller -minCount value."
+              << std::endl;
+    exit(EXIT_FAILURE);
+  }
+}
+
+void Dictionary::threshold(int64_t t, int64_t tl) {
+  sort(words_.begin(), words_.end(), [](const entry& e1, const entry& e2) {
+      if (e1.type != e2.type) return e1.type < e2.type;
+      return e1.count > e2.count;
+    });
+  words_.erase(remove_if(words_.begin(), words_.end(), [&](const entry& e) {
+        return (e.type == entry_type::word && e.count < t) ||
+               (e.type == entry_type::label && e.count < tl);
+      }), words_.end());
+  words_.shrink_to_fit();
+  size_ = 0;
+  nwords_ = 0;
+  nlabels_ = 0;
+  std::fill(word2int_.begin(), word2int_.end(), -1);
+  for (auto it = words_.begin(); it != words_.end(); ++it) {
+    int32_t h = find(it->word);
+    word2int_[h] = size_++;
+    if (it->type == entry_type::word) nwords_++;
+    if (it->type == entry_type::label) nlabels_++;
+  }
+}
+
+void Dictionary::initTableDiscard() {
+  pdiscard_.resize(size_);
+  for (size_t i = 0; i < size_; i++) {
+    real f = real(words_[i].count) / real(ntokens_);
+    pdiscard_[i] = std::sqrt(args_->t / f) + args_->t / f;
+  }
+}
+
+std::vector<int64_t> Dictionary::getCounts(entry_type type) const {
+  std::vector<int64_t> counts;
+  for (auto& w : words_) {
+    if (w.type == type) counts.push_back(w.count);
+  }
+  return counts;
+}
+
+void Dictionary::addWordNgrams(std::vector<int32_t>& line,
+                               const std::vector<int32_t>& hashes,
+                               int32_t n) const {
+  for (int32_t i = 0; i < hashes.size(); i++) {
+    uint64_t h = hashes[i];
+    for (int32_t j = i + 1; j < hashes.size() && j < i + n; j++) {
+      h = h * 116049371 + hashes[j];
+      pushHash(line, h % args_->bucket);
+    }
+  }
+}
+
+void Dictionary::addSubwords(std::vector<int32_t>& line,
+                             const std::string& token,
+                             int32_t wid) const {
+  if (wid < 0) { // out of vocab
+    computeSubwords(BOW + token + EOW, line);
+  } else {
+    if (args_->maxn <= 0) { // in vocab w/o subwords
+      line.push_back(wid);
+    } else { // in vocab w/ subwords
+      const std::vector<int32_t>& ngrams = getSubwords(wid);
+      line.insert(line.end(), ngrams.cbegin(), ngrams.cend());
+    }
+  }
+}
+
+void Dictionary::reset(std::istream& in) const {
+  if (in.eof()) {
+    in.clear();
+    in.seekg(std::streampos(0));
+  }
+}
+
+int32_t Dictionary::getLine(std::istream& in,
+                            std::vector<int32_t>& words,
+                            std::minstd_rand& rng) const {
+  std::uniform_real_distribution<> uniform(0, 1);
+  std::string token;
+  int32_t ntokens = 0;
+
+  reset(in);
+  words.clear();
+  while (readWord(in, token)) {
+    int32_t h = find(token);
+    int32_t wid = word2int_[h];
+    if (wid < 0) continue;
+
+    ntokens++;
+    if (getType(wid) == entry_type::word && !discard(wid, uniform(rng))) {
+      words.push_back(wid);
+    }
+    if (ntokens > MAX_LINE_SIZE || token == EOS) break;
+  }
+  return ntokens;
+}
+
+int32_t Dictionary::getLine(std::istream& in,
+                            std::vector<int32_t>& words,
+                            std::vector<int32_t>& labels,
+                            std::minstd_rand& rng) const {
+  std::vector<int32_t> word_hashes;
+  std::string token;
+  int32_t ntokens = 0;
+
+  reset(in);
+  words.clear();
+  labels.clear();
+  while (readWord(in, token)) {
+    uint32_t h = hash(token);
+    int32_t wid = getId(token, h);
+    entry_type type = wid < 0 ? getType(token) : getType(wid);
+
+    ntokens++;
+    if (type == entry_type::word) {
+      addSubwords(words, token, wid);
+      word_hashes.push_back(h);
+    } else if (type == entry_type::label && wid >= 0) {
+      labels.push_back(wid - nwords_);
+    }
+    if (token == EOS) break;
+  }
+  addWordNgrams(words, word_hashes, args_->wordNgrams);
+  return ntokens;
+}
+
+void Dictionary::pushHash(std::vector<int32_t>& hashes, int32_t id) const {
+  if (pruneidx_size_ == 0 || id < 0) return;
+  if (pruneidx_size_ > 0) {
+    if (pruneidx_.count(id)) {
+      id = pruneidx_.at(id);
+    } else {
+      return;
+    }
+  }
+  hashes.push_back(nwords_ + id);
+}
+
+std::string Dictionary::getLabel(int32_t lid) const {
+  assert(lid >= 0);
+  assert(lid < nlabels_);
+  return words_[lid + nwords_].word;
+}
+
+void Dictionary::save(std::ostream& out) const {
+  out.write((char*) &size_, sizeof(int32_t));
+  out.write((char*) &nwords_, sizeof(int32_t));
+  out.write((char*) &nlabels_, sizeof(int32_t));
+  out.write((char*) &ntokens_, sizeof(int64_t));
+  out.write((char*) &pruneidx_size_, sizeof(int64_t));
+  for (int32_t i = 0; i < size_; i++) {
+    entry e = words_[i];
+    out.write(e.word.data(), e.word.size() * sizeof(char));
+    out.put(0);
+    out.write((char*) &(e.count), sizeof(int64_t));
+    out.write((char*) &(e.type), sizeof(entry_type));
+  }
+  for (const auto pair : pruneidx_) {
+    out.write((char*) &(pair.first), sizeof(int32_t));
+    out.write((char*) &(pair.second), sizeof(int32_t));
+  }
+}
+
+void Dictionary::load(std::istream& in) {
+  words_.clear();
+  std::fill(word2int_.begin(), word2int_.end(), -1);
+  in.read((char*) &size_, sizeof(int32_t));
+  in.read((char*) &nwords_, sizeof(int32_t));
+  in.read((char*) &nlabels_, sizeof(int32_t));
+  in.read((char*) &ntokens_, sizeof(int64_t));
+  in.read((char*) &pruneidx_size_, sizeof(int64_t));
+  for (int32_t i = 0; i < size_; i++) {
+    char c;
+    entry e;
+    while ((c = in.get()) != 0) {
+      e.word.push_back(c);
+    }
+    in.read((char*) &e.count, sizeof(int64_t));
+    in.read((char*) &e.type, sizeof(entry_type));
+    words_.push_back(e);
+    word2int_[find(e.word)] = i;
+  }
+  pruneidx_.clear();
+  for (int32_t i = 0; i < pruneidx_size_; i++) {
+    int32_t first;
+    int32_t second;
+    in.read((char*) &first, sizeof(int32_t));
+    in.read((char*) &second, sizeof(int32_t));
+    pruneidx_[first] = second;
+  }
+  initTableDiscard();
+  initNgrams();
+}
+
+void Dictionary::prune(std::vector<int32_t>& idx) {
+  std::vector<int32_t> words, ngrams;
+  for (auto it = idx.cbegin(); it != idx.cend(); ++it) {
+    if (*it < nwords_) {words.push_back(*it);}
+    else {ngrams.push_back(*it);}
+  }
+  std::sort(words.begin(), words.end());
+  idx = words;
+
+  if (ngrams.size() != 0) {
+    int32_t j = 0;
+    for (const auto ngram : ngrams) {
+      pruneidx_[ngram - nwords_] = j;
+      j++;
+    }
+    idx.insert(idx.end(), ngrams.begin(), ngrams.end());
+  }
+  pruneidx_size_ = pruneidx_.size();
+
+  std::fill(word2int_.begin(), word2int_.end(), -1);
+
+  int32_t j = 0;
+  for (int32_t i = 0; i < words_.size(); i++) {
+    if (getType(i) == entry_type::label || (j < words.size() && words[j] == i)) {
+      words_[j] = words_[i];
+      word2int_[find(words_[j].word)] = j;
+      j++;
+    }
+  }
+  nwords_ = words.size();
+  size_ = nwords_ +  nlabels_;
+  words_.erase(words_.begin() + size_, words_.end());
+  initNgrams();
+}
+
+}