fasttext 0.1.0

data/vendor/fastText/python/fasttext_module/fasttext/__init__.py

@@ -0,0 +1,22 @@
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from .FastText import train_supervised
+from .FastText import train_unsupervised
+from .FastText import load_model
+from .FastText import tokenize
+from .FastText import EOS
+from .FastText import BOW
+from .FastText import EOW
+
+from .FastText import cbow
+from .FastText import skipgram
+from .FastText import supervised

data/vendor/fastText/python/fasttext_module/fasttext/pybind/fasttext_pybind.cc

@@ -0,0 +1,388 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <args.h>
+#include <densematrix.h>
+#include <fasttext.h>
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#include <real.h>
+#include <vector.h>
+#include <cmath>
+#include <iterator>
+#include <sstream>
+#include <stdexcept>
+
+using namespace pybind11::literals;
+namespace py = pybind11;
+
+py::str castToPythonString(const std::string& s, const char* onUnicodeError) {
+  PyObject* handle = PyUnicode_DecodeUTF8(s.data(), s.length(), onUnicodeError);
+  if (!handle) {
+    throw py::error_already_set();
+  }
+
+  // py::str's constructor from a PyObject assumes the string has been encoded
+  // for python 2 and not encoded for python 3 :
+  // https://github.com/pybind/pybind11/blob/ccbe68b084806dece5863437a7dc93de20bd9b15/include/pybind11/pytypes.h#L930
+#if PY_MAJOR_VERSION < 3
+  handle = PyUnicode_AsEncodedString(handle, "utf-8", onUnicodeError);
+#endif
+
+  return py::str(handle);
+}
+
+std::pair<std::vector<py::str>, std::vector<py::str>> getLineText(
+    fasttext::FastText& m,
+    const std::string text,
+    const char* onUnicodeError) {
+  std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
+  std::stringstream ioss(text);
+  std::string token;
+  std::vector<py::str> words;
+  std::vector<py::str> labels;
+  while (d->readWord(ioss, token)) {
+    uint32_t h = d->hash(token);
+    int32_t wid = d->getId(token, h);
+    fasttext::entry_type type = wid < 0 ? d->getType(token) : d->getType(wid);
+
+    if (type == fasttext::entry_type::word) {
+      words.push_back(castToPythonString(token, onUnicodeError));
+      // Labels must not be OOV!
+    } else if (type == fasttext::entry_type::label && wid >= 0) {
+      labels.push_back(castToPythonString(token, onUnicodeError));
+    }
+    if (token == fasttext::Dictionary::EOS)
+      break;
+  }
+  return std::pair<std::vector<py::str>, std::vector<py::str>>(words, labels);
+}
+
+PYBIND11_MODULE(fasttext_pybind, m) {
+  py::class_<fasttext::Args>(m, "args")
+      .def(py::init<>())
+      .def_readwrite("input", &fasttext::Args::input)
+      .def_readwrite("output", &fasttext::Args::output)
+      .def_readwrite("lr", &fasttext::Args::lr)
+      .def_readwrite("lrUpdateRate", &fasttext::Args::lrUpdateRate)
+      .def_readwrite("dim", &fasttext::Args::dim)
+      .def_readwrite("ws", &fasttext::Args::ws)
+      .def_readwrite("epoch", &fasttext::Args::epoch)
+      .def_readwrite("minCount", &fasttext::Args::minCount)
+      .def_readwrite("minCountLabel", &fasttext::Args::minCountLabel)
+      .def_readwrite("neg", &fasttext::Args::neg)
+      .def_readwrite("wordNgrams", &fasttext::Args::wordNgrams)
+      .def_readwrite("loss", &fasttext::Args::loss)
+      .def_readwrite("model", &fasttext::Args::model)
+      .def_readwrite("bucket", &fasttext::Args::bucket)
+      .def_readwrite("minn", &fasttext::Args::minn)
+      .def_readwrite("maxn", &fasttext::Args::maxn)
+      .def_readwrite("thread", &fasttext::Args::thread)
+      .def_readwrite("t", &fasttext::Args::t)
+      .def_readwrite("label", &fasttext::Args::label)
+      .def_readwrite("verbose", &fasttext::Args::verbose)
+      .def_readwrite("pretrainedVectors", &fasttext::Args::pretrainedVectors)
+      .def_readwrite("saveOutput", &fasttext::Args::saveOutput)
+
+      .def_readwrite("qout", &fasttext::Args::qout)
+      .def_readwrite("retrain", &fasttext::Args::retrain)
+      .def_readwrite("qnorm", &fasttext::Args::qnorm)
+      .def_readwrite("cutoff", &fasttext::Args::cutoff)
+      .def_readwrite("dsub", &fasttext::Args::dsub);
+
+  py::enum_<fasttext::model_name>(m, "model_name")
+      .value("cbow", fasttext::model_name::cbow)
+      .value("skipgram", fasttext::model_name::sg)
+      .value("supervised", fasttext::model_name::sup)
+      .export_values();
+
+  py::enum_<fasttext::loss_name>(m, "loss_name")
+      .value("hs", fasttext::loss_name::hs)
+      .value("ns", fasttext::loss_name::ns)
+      .value("softmax", fasttext::loss_name::softmax)
+      .value("ova", fasttext::loss_name::ova)
+      .export_values();
+
+  m.def(
+      "train",
+      [](fasttext::FastText& ft, fasttext::Args& a) { ft.train(a); },
+      py::call_guard<py::gil_scoped_release>());
+
+  py::class_<fasttext::Vector>(m, "Vector", py::buffer_protocol())
+      .def(py::init<ssize_t>())
+      .def_buffer([](fasttext::Vector& m) -> py::buffer_info {
+        return py::buffer_info(
+            m.data(),
+            sizeof(fasttext::real),
+            py::format_descriptor<fasttext::real>::format(),
+            1,
+            {m.size()},
+            {sizeof(fasttext::real)});
+      });
+
+  py::class_<fasttext::DenseMatrix>(
+      m, "DenseMatrix", py::buffer_protocol(), py::module_local())
+      .def(py::init<>())
+      .def(py::init<ssize_t, ssize_t>())
+      .def_buffer([](fasttext::DenseMatrix& m) -> py::buffer_info {
+        return py::buffer_info(
+            m.data(),
+            sizeof(fasttext::real),
+            py::format_descriptor<fasttext::real>::format(),
+            2,
+            {m.size(0), m.size(1)},
+            {sizeof(fasttext::real) * m.size(1),
+             sizeof(fasttext::real) * (int64_t)1});
+      });
+
+  py::class_<fasttext::FastText>(m, "fasttext")
+      .def(py::init<>())
+      .def("getArgs", &fasttext::FastText::getArgs)
+      .def(
+          "getInputMatrix",
+          [](fasttext::FastText& m) {
+            std::shared_ptr<const fasttext::DenseMatrix> mm =
+                m.getInputMatrix();
+            return *mm.get();
+          })
+      .def(
+          "getOutputMatrix",
+          [](fasttext::FastText& m) {
+            std::shared_ptr<const fasttext::DenseMatrix> mm =
+                m.getOutputMatrix();
+            return *mm.get();
+          })
+      .def(
+          "loadModel",
+          [](fasttext::FastText& m, std::string s) { m.loadModel(s); })
+      .def(
+          "saveModel",
+          [](fasttext::FastText& m, std::string s) { m.saveModel(s); })
+      .def(
+          "test",
+          [](fasttext::FastText& m, const std::string filename, int32_t k) {
+            std::ifstream ifs(filename);
+            if (!ifs.is_open()) {
+              throw std::invalid_argument("Test file cannot be opened!");
+            }
+            fasttext::Meter meter;
+            m.test(ifs, k, 0.0, meter);
+            ifs.close();
+            return std::tuple<int64_t, double, double>(
+                meter.nexamples(), meter.precision(), meter.recall());
+          })
+      .def(
+          "getSentenceVector",
+          [](fasttext::FastText& m,
+             fasttext::Vector& v,
+             const std::string text) {
+            std::stringstream ioss(text);
+            m.getSentenceVector(ioss, v);
+          })
+      .def(
+          "tokenize",
+          [](fasttext::FastText& m, const std::string text) {
+            std::vector<std::string> text_split;
+            std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
+            std::stringstream ioss(text);
+            std::string token;
+            while (!ioss.eof()) {
+              while (d->readWord(ioss, token)) {
+                text_split.push_back(token);
+              }
+            }
+            return text_split;
+          })
+      .def("getLine", &getLineText)
+      .def(
+          "multilineGetLine",
+          [](fasttext::FastText& m,
+             const std::vector<std::string> lines,
+             const char* onUnicodeError) {
+            std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
+            std::vector<std::vector<py::str>> all_words;
+            std::vector<std::vector<py::str>> all_labels;
+            for (const auto& text : lines) {
+              auto pair = getLineText(m, text, onUnicodeError);
+              all_words.push_back(pair.first);
+              all_labels.push_back(pair.second);
+            }
+            return std::pair<
+                std::vector<std::vector<py::str>>,
+                std::vector<std::vector<py::str>>>(all_words, all_labels);
+          })
+      .def(
+          "getVocab",
+          [](fasttext::FastText& m, const char* onUnicodeError) {
+            py::str s;
+            std::vector<py::str> vocab_list;
+            std::vector<int64_t> vocab_freq;
+            std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
+            vocab_freq = d->getCounts(fasttext::entry_type::word);
+            for (int32_t i = 0; i < vocab_freq.size(); i++) {
+              vocab_list.push_back(
+                  castToPythonString(d->getWord(i), onUnicodeError));
+            }
+            return std::pair<std::vector<py::str>, std::vector<int64_t>>(
+                vocab_list, vocab_freq);
+          })
+      .def(
+          "getLabels",
+          [](fasttext::FastText& m, const char* onUnicodeError) {
+            std::vector<py::str> labels_list;
+            std::vector<int64_t> labels_freq;
+            std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
+            labels_freq = d->getCounts(fasttext::entry_type::label);
+            for (int32_t i = 0; i < labels_freq.size(); i++) {
+              labels_list.push_back(
+                  castToPythonString(d->getLabel(i), onUnicodeError));
+            }
+            return std::pair<std::vector<py::str>, std::vector<int64_t>>(
+                labels_list, labels_freq);
+          })
+      .def(
+          "quantize",
+          [](fasttext::FastText& m,
+             const std::string input,
+             bool qout,
+             int32_t cutoff,
+             bool retrain,
+             int epoch,
+             double lr,
+             int thread,
+             int verbose,
+             int32_t dsub,
+             bool qnorm) {
+            fasttext::Args qa = fasttext::Args();
+            qa.input = input;
+            qa.qout = qout;
+            qa.cutoff = cutoff;
+            qa.retrain = retrain;
+            qa.epoch = epoch;
+            qa.lr = lr;
+            qa.thread = thread;
+            qa.verbose = verbose;
+            qa.dsub = dsub;
+            qa.qnorm = qnorm;
+            m.quantize(qa);
+          })
+      .def(
+          "predict",
+          // NOTE: text needs to end in a newline
+          // to exactly mimic the behavior of the cli
+          [](fasttext::FastText& m,
+             const std::string text,
+             int32_t k,
+             fasttext::real threshold,
+             const char* onUnicodeError) {
+            std::stringstream ioss(text);
+            std::vector<std::pair<fasttext::real, std::string>> predictions;
+            m.predictLine(ioss, predictions, k, threshold);
+
+            std::vector<std::pair<fasttext::real, py::str>>
+                transformedPredictions;
+
+            for (const auto& prediction : predictions) {
+              transformedPredictions.push_back(std::make_pair(
+                  prediction.first,
+                  castToPythonString(prediction.second, onUnicodeError)));
+            }
+
+            return transformedPredictions;
+          })
+      .def(
+          "multilinePredict",
+          // NOTE: text needs to end in a newline
+          // to exactly mimic the behavior of the cli
+          [](fasttext::FastText& m,
+             const std::vector<std::string>& lines,
+             int32_t k,
+             fasttext::real threshold,
+             const char* onUnicodeError) {
+            std::vector<std::vector<std::pair<fasttext::real, py::str>>>
+                allPredictions;
+            std::vector<std::pair<fasttext::real, std::string>> predictions;
+
+            for (const std::string& text : lines) {
+              std::stringstream ioss(text);
+              m.predictLine(ioss, predictions, k, threshold);
+              std::vector<std::pair<fasttext::real, py::str>>
+                  transformedPredictions;
+              for (const auto& prediction : predictions) {
+                transformedPredictions.push_back(std::make_pair(
+                    prediction.first,
+                    castToPythonString(prediction.second, onUnicodeError)));
+              }
+              allPredictions.push_back(transformedPredictions);
+            }
+            return allPredictions;
+          })
+      .def(
+          "testLabel",
+          [](fasttext::FastText& m,
+             const std::string filename,
+             int32_t k,
+             fasttext::real threshold) {
+            std::ifstream ifs(filename);
+            if (!ifs.is_open()) {
+              throw std::invalid_argument("Test file cannot be opened!");
+            }
+            fasttext::Meter meter;
+            m.test(ifs, k, threshold, meter);
+            std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
+            std::unordered_map<std::string, py::dict> returnedValue;
+            for (int32_t i = 0; i < d->nlabels(); i++) {
+              returnedValue[d->getLabel(i)] = py::dict(
+                  "precision"_a = meter.precision(i),
+                  "recall"_a = meter.recall(i),
+                  "f1score"_a = meter.f1Score(i));
+            }
+
+            return returnedValue;
+          })
+      .def(
+          "getWordId",
+          [](fasttext::FastText& m, const std::string word) {
+            return m.getWordId(word);
+          })
+      .def(
+          "getSubwordId",
+          [](fasttext::FastText& m, const std::string word) {
+            return m.getSubwordId(word);
+          })
+      .def(
+          "getInputVector",
+          [](fasttext::FastText& m, fasttext::Vector& vec, int32_t ind) {
+            m.getInputVector(vec, ind);
+          })
+      .def(
+          "getWordVector",
+          [](fasttext::FastText& m,
+             fasttext::Vector& vec,
+             const std::string word) { m.getWordVector(vec, word); })
+      .def(
+          "getSubwords",
+          [](fasttext::FastText& m,
+             const std::string word,
+             const char* onUnicodeError) {
+            std::vector<std::string> subwords;
+            std::vector<int32_t> ngrams;
+            std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
+            d->getSubwords(word, ngrams, subwords);
+            std::vector<py::str> transformedSubwords;
+
+            for (const auto& subword : subwords) {
+              transformedSubwords.push_back(
+                  castToPythonString(subword, onUnicodeError));
+            }
+
+            return std::pair<std::vector<py::str>, std::vector<int32_t>>(
+                transformedSubwords, ngrams);
+          })
+      .def("isQuant", [](fasttext::FastText& m) { return m.isQuant(); });
+}

data/vendor/fastText/python/fasttext_module/fasttext/tests/__init__.py

@@ -0,0 +1,14 @@
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from .test_configurations import get_supervised_models
+from .test_script import gen_tests
+from .test_script import gen_unit_tests

data/vendor/fastText/python/fasttext_module/fasttext/tests/test_configurations.py

@@ -0,0 +1,239 @@
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import multiprocessing
+
+# This script represents a collection of integration tests
+# Each integration test comes with a full set of parameters,
+# a dataset, and expected metrics.
+# These configurations can be used by various fastText APIs
+# to confirm some level of correctness.
+
+
+def max_thread():
+    return multiprocessing.cpu_count() - 1
+
+
+def check_supervised_configuration(configuration, verbose=1):
+    configuration["args"]["verbose"] = verbose
+    configuration["quant_args"]["verbose"] = verbose
+    return configuration
+
+
+def check_supervised_configurations(configurations, verbose=1):
+    for i in range(len(configurations)):
+        configurations[i] = check_supervised_configuration(
+            configurations[i], verbose=verbose
+        )
+    return configurations
+
+
+def flickr_job(thread=None):
+    if thread is None:
+        thread = max_thread()
+    config = {}
+    config["dataset"] = "YFCC100M"
+    config["args"] = {
+        "dim": 256,
+        "wordNgrams": 2,
+        "minCount": 10,
+        "bucket": 10000000,
+        "epoch": 20,
+        "loss": "hs",
+        "minCountLabel": 100,
+        "thread": thread
+    }
+    config["args"]["input"] = "YFCC100M/train"
+    config["quant_args"] = {
+        "dsub": 2,
+        "lr": 0.1,
+        "epoch": 5,
+        "cutoff": 100000,
+        "qnorm": True,
+        "retrain": True,
+        "qout": True
+    }
+    config["quant_args"]["input"] = config["args"]["input"]
+    config["test"] = {
+        "n": 647224,
+        "p1": 0.470,
+        "r1": 0.071,
+        "size": 12060039727,
+        "data": "YFCC100M/test",
+    }
+    # One quant example (to illustrate slack): 0.344, 0.0528, 64506972
+    config["quant_test"] = {
+        "n": 647224,
+        "p1": 0.300,
+        "r1": 0.0450,
+        "size": 70000000,
+        "data": "YFCC100M/test",
+    }
+    return config
+
+
+def langid_job1(thread=None):
+    if thread is None:
+        thread = max_thread()
+    config = {}
+    config["dataset"] = "langid"
+    config["args"] = {"dim": 16, "minn": 2, "maxn": 4, "thread": thread}
+    config["args"]["input"] = "langid.train"
+    config["quant_args"] = {"qnorm": True, "cutoff": 50000, "retrain": True}
+    config["quant_args"]["input"] = config["args"]["input"]
+    config["test"] = {
+        "n": 10000,
+        "p1": 0.985,
+        "r1": 0.985,
+        "size": 368132610,
+        "data": "langid.valid",
+    }
+    # One quant example (to illustrate slack): 0.984 0.984 932793
+    config["quant_test"] = {
+        "p1": 0.97,
+        "r1": 0.97,
+        "size": 1000000,
+    }
+    config["quant_test"]["n"] = config["test"]["n"]
+    config["quant_test"]["data"] = config["test"]["data"]
+    return config
+
+
+def langid_job2(thread=None):
+    if thread is None:
+        thread = max_thread()
+    config = langid_job1(thread).copy()
+    config["args"]["loss"] = "hs"
+    return config
+
+
+def cooking_job1(thread=None):
+    if thread is None:
+        thread = max_thread()
+    config = {}
+    config["dataset"] = "cooking"
+    config["args"] = {
+        "epoch": 25,
+        "lr": 1.0,
+        "wordNgrams": 2,
+        "minCount": 1,
+        "thread": thread,
+    }
+    config["args"]["input"] = "cooking.train"
+    config["quant_args"] = {"qnorm": True, "cutoff": 50000, "retrain": True}
+    config["quant_args"]["input"] = config["args"]["input"]
+    config["test"] = {
+        "n": 3000,
+        "p1": 0.59,
+        "r1": 0.25,
+        "size": 804047585,
+        "data": "cooking.valid",
+    }
+    # One quant example (to illustrate slack): 0.602 0.26 3439172
+    config["quant_test"] = {
+        "p1": 0.55,
+        "r1": 0.20,
+        "size": 4000000,
+    }
+    config["quant_test"]["n"] = config["test"]["n"]
+    config["quant_test"]["data"] = config["test"]["data"]
+    return config
+
+
+def cooking_job2(thread=None):
+    if thread is None:
+        thread = max_thread()
+    config = cooking_job1(thread).copy()
+    config["args"]["loss"] = "hs"
+    return config
+
+
+# Supervised models
+# See https://fasttext.cc/docs/en/supervised-models.html
+def get_supervised_models(thread=None, verbose=1):
+    if thread is None:
+        thread = max_thread()
+    sup_job_dataset = [
+        "ag_news", "sogou_news", "dbpedia", "yelp_review_polarity",
+        "yelp_review_full", "yahoo_answers", "amazon_review_full",
+        "amazon_review_polarity"
+    ]
+
+    sup_params = {
+        "dim": 10,
+        "wordNgrams": 2,
+        "minCount": 1,
+        "bucket": 10000000,
+        "epoch": 5,
+        "thread": thread,
+        "verbose": 1,
+    }
+    quant_params = {
+        "retrain": True,
+        "cutoff": 100000,
+        "qnorm": True,
+        "verbose": 1,
+    }
+    sup_job_lr = [0.25, 0.5, 0.5, 0.1, 0.1, 0.1, 0.05, 0.05]
+
+    sup_job_n = [7600, 60000, 70000, 38000, 50000, 60000, 650000, 400000]
+
+    sup_job_p1 = [0.915, 0.968, 0.983, 0.956, 0.638, 0.723, 0.600, 0.940]
+    sup_job_r1 = [0.915, 0.968, 0.983, 0.956, 0.638, 0.723, 0.600, 0.940]
+    sup_job_size = [
+        405607193, 421445471, 447481878, 427867393, 431292576, 517549567,
+        483742593, 493604598
+    ]
+
+    sup_job_quant_p1 = [0.918, 0.965, 0.983, 0.950, 0.625, 0.707, 0.58, 0.920]
+    sup_job_quant_r1 = [0.918, 0.965, 0.983, 0.950, 0.625, 0.707, 0.58, 0.920]
+    sup_job_quant_size = [
+        1600000, 1500000, 1700000, 1600000, 1600000, 1700000, 1600000, 1600000
+    ]
+
+    configurations = []
+    for i in range(len(sup_job_dataset)):
+        configuration = {}
+        configuration["dataset"] = sup_job_dataset[i]
+        args = sup_params.copy()
+        quant_args = quant_params.copy()
+        args["lr"] = sup_job_lr[i]
+        args["input"] = sup_job_dataset[i] + ".train"
+        quant_args["lr"] = sup_job_lr[i]
+        quant_args["input"] = sup_job_dataset[i] + ".train"
+        configuration["args"] = args
+        configuration["quant_args"] = quant_args
+        test = {
+            "n": sup_job_n[i],
+            "p1": sup_job_p1[i],
+            "r1": sup_job_r1[i],
+            "size": sup_job_size[i],
+            "data": sup_job_dataset[i] + ".test",
+        }
+        quant_test = {
+            "n": sup_job_n[i],
+            "p1": sup_job_quant_p1[i],
+            "r1": sup_job_quant_r1[i],
+            "size": sup_job_quant_size[i],
+            "data": sup_job_dataset[i] + ".test",
+        }
+        configuration["test"] = test
+        configuration["quant_test"] = quant_test
+        configurations.append(configuration)
+    configurations.append(flickr_job())
+    configurations.append(langid_job1())
+    configurations.append(langid_job2())
+    configurations.append(cooking_job1())
+    configurations.append(cooking_job2())
+    configurations = check_supervised_configurations(
+        configurations, verbose=verbose
+    )
+    return configurations