fasttext 0.1.0 → 0.2.1

data/vendor/fastText/python/fasttext_module/fasttext/__init__.py

@@ -1,22 +0,0 @@
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from .FastText import train_supervised
-from .FastText import train_unsupervised
-from .FastText import load_model
-from .FastText import tokenize
-from .FastText import EOS
-from .FastText import BOW
-from .FastText import EOW
-
-from .FastText import cbow
-from .FastText import skipgram
-from .FastText import supervised

data/vendor/fastText/python/fasttext_module/fasttext/pybind/fasttext_pybind.cc

@@ -1,388 +0,0 @@
-/**
- * Copyright (c) 2017-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <args.h>
-#include <densematrix.h>
-#include <fasttext.h>
-#include <pybind11/pybind11.h>
-#include <pybind11/stl.h>
-#include <real.h>
-#include <vector.h>
-#include <cmath>
-#include <iterator>
-#include <sstream>
-#include <stdexcept>
-
-using namespace pybind11::literals;
-namespace py = pybind11;
-
-py::str castToPythonString(const std::string& s, const char* onUnicodeError) {
-  PyObject* handle = PyUnicode_DecodeUTF8(s.data(), s.length(), onUnicodeError);
-  if (!handle) {
-    throw py::error_already_set();
-  }
-
-  // py::str's constructor from a PyObject assumes the string has been encoded
-  // for python 2 and not encoded for python 3 :
-  // https://github.com/pybind/pybind11/blob/ccbe68b084806dece5863437a7dc93de20bd9b15/include/pybind11/pytypes.h#L930
-#if PY_MAJOR_VERSION < 3
-  handle = PyUnicode_AsEncodedString(handle, "utf-8", onUnicodeError);
-#endif
-
-  return py::str(handle);
-}
-
-std::pair<std::vector<py::str>, std::vector<py::str>> getLineText(
-    fasttext::FastText& m,
-    const std::string text,
-    const char* onUnicodeError) {
-  std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
-  std::stringstream ioss(text);
-  std::string token;
-  std::vector<py::str> words;
-  std::vector<py::str> labels;
-  while (d->readWord(ioss, token)) {
-    uint32_t h = d->hash(token);
-    int32_t wid = d->getId(token, h);
-    fasttext::entry_type type = wid < 0 ? d->getType(token) : d->getType(wid);
-
-    if (type == fasttext::entry_type::word) {
-      words.push_back(castToPythonString(token, onUnicodeError));
-      // Labels must not be OOV!
-    } else if (type == fasttext::entry_type::label && wid >= 0) {
-      labels.push_back(castToPythonString(token, onUnicodeError));
-    }
-    if (token == fasttext::Dictionary::EOS)
-      break;
-  }
-  return std::pair<std::vector<py::str>, std::vector<py::str>>(words, labels);
-}
-
-PYBIND11_MODULE(fasttext_pybind, m) {
-  py::class_<fasttext::Args>(m, "args")
-      .def(py::init<>())
-      .def_readwrite("input", &fasttext::Args::input)
-      .def_readwrite("output", &fasttext::Args::output)
-      .def_readwrite("lr", &fasttext::Args::lr)
-      .def_readwrite("lrUpdateRate", &fasttext::Args::lrUpdateRate)
-      .def_readwrite("dim", &fasttext::Args::dim)
-      .def_readwrite("ws", &fasttext::Args::ws)
-      .def_readwrite("epoch", &fasttext::Args::epoch)
-      .def_readwrite("minCount", &fasttext::Args::minCount)
-      .def_readwrite("minCountLabel", &fasttext::Args::minCountLabel)
-      .def_readwrite("neg", &fasttext::Args::neg)
-      .def_readwrite("wordNgrams", &fasttext::Args::wordNgrams)
-      .def_readwrite("loss", &fasttext::Args::loss)
-      .def_readwrite("model", &fasttext::Args::model)
-      .def_readwrite("bucket", &fasttext::Args::bucket)
-      .def_readwrite("minn", &fasttext::Args::minn)
-      .def_readwrite("maxn", &fasttext::Args::maxn)
-      .def_readwrite("thread", &fasttext::Args::thread)
-      .def_readwrite("t", &fasttext::Args::t)
-      .def_readwrite("label", &fasttext::Args::label)
-      .def_readwrite("verbose", &fasttext::Args::verbose)
-      .def_readwrite("pretrainedVectors", &fasttext::Args::pretrainedVectors)
-      .def_readwrite("saveOutput", &fasttext::Args::saveOutput)
-
-      .def_readwrite("qout", &fasttext::Args::qout)
-      .def_readwrite("retrain", &fasttext::Args::retrain)
-      .def_readwrite("qnorm", &fasttext::Args::qnorm)
-      .def_readwrite("cutoff", &fasttext::Args::cutoff)
-      .def_readwrite("dsub", &fasttext::Args::dsub);
-
-  py::enum_<fasttext::model_name>(m, "model_name")
-      .value("cbow", fasttext::model_name::cbow)
-      .value("skipgram", fasttext::model_name::sg)
-      .value("supervised", fasttext::model_name::sup)
-      .export_values();
-
-  py::enum_<fasttext::loss_name>(m, "loss_name")
-      .value("hs", fasttext::loss_name::hs)
-      .value("ns", fasttext::loss_name::ns)
-      .value("softmax", fasttext::loss_name::softmax)
-      .value("ova", fasttext::loss_name::ova)
-      .export_values();
-
-  m.def(
-      "train",
-      [](fasttext::FastText& ft, fasttext::Args& a) { ft.train(a); },
-      py::call_guard<py::gil_scoped_release>());
-
-  py::class_<fasttext::Vector>(m, "Vector", py::buffer_protocol())
-      .def(py::init<ssize_t>())
-      .def_buffer([](fasttext::Vector& m) -> py::buffer_info {
-        return py::buffer_info(
-            m.data(),
-            sizeof(fasttext::real),
-            py::format_descriptor<fasttext::real>::format(),
-            1,
-            {m.size()},
-            {sizeof(fasttext::real)});
-      });
-
-  py::class_<fasttext::DenseMatrix>(
-      m, "DenseMatrix", py::buffer_protocol(), py::module_local())
-      .def(py::init<>())
-      .def(py::init<ssize_t, ssize_t>())
-      .def_buffer([](fasttext::DenseMatrix& m) -> py::buffer_info {
-        return py::buffer_info(
-            m.data(),
-            sizeof(fasttext::real),
-            py::format_descriptor<fasttext::real>::format(),
-            2,
-            {m.size(0), m.size(1)},
-            {sizeof(fasttext::real) * m.size(1),
-             sizeof(fasttext::real) * (int64_t)1});
-      });
-
-  py::class_<fasttext::FastText>(m, "fasttext")
-      .def(py::init<>())
-      .def("getArgs", &fasttext::FastText::getArgs)
-      .def(
-          "getInputMatrix",
-          [](fasttext::FastText& m) {
-            std::shared_ptr<const fasttext::DenseMatrix> mm =
-                m.getInputMatrix();
-            return *mm.get();
-          })
-      .def(
-          "getOutputMatrix",
-          [](fasttext::FastText& m) {
-            std::shared_ptr<const fasttext::DenseMatrix> mm =
-                m.getOutputMatrix();
-            return *mm.get();
-          })
-      .def(
-          "loadModel",
-          [](fasttext::FastText& m, std::string s) { m.loadModel(s); })
-      .def(
-          "saveModel",
-          [](fasttext::FastText& m, std::string s) { m.saveModel(s); })
-      .def(
-          "test",
-          [](fasttext::FastText& m, const std::string filename, int32_t k) {
-            std::ifstream ifs(filename);
-            if (!ifs.is_open()) {
-              throw std::invalid_argument("Test file cannot be opened!");
-            }
-            fasttext::Meter meter;
-            m.test(ifs, k, 0.0, meter);
-            ifs.close();
-            return std::tuple<int64_t, double, double>(
-                meter.nexamples(), meter.precision(), meter.recall());
-          })
-      .def(
-          "getSentenceVector",
-          [](fasttext::FastText& m,
-             fasttext::Vector& v,
-             const std::string text) {
-            std::stringstream ioss(text);
-            m.getSentenceVector(ioss, v);
-          })
-      .def(
-          "tokenize",
-          [](fasttext::FastText& m, const std::string text) {
-            std::vector<std::string> text_split;
-            std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
-            std::stringstream ioss(text);
-            std::string token;
-            while (!ioss.eof()) {
-              while (d->readWord(ioss, token)) {
-                text_split.push_back(token);
-              }
-            }
-            return text_split;
-          })
-      .def("getLine", &getLineText)
-      .def(
-          "multilineGetLine",
-          [](fasttext::FastText& m,
-             const std::vector<std::string> lines,
-             const char* onUnicodeError) {
-            std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
-            std::vector<std::vector<py::str>> all_words;
-            std::vector<std::vector<py::str>> all_labels;
-            for (const auto& text : lines) {
-              auto pair = getLineText(m, text, onUnicodeError);
-              all_words.push_back(pair.first);
-              all_labels.push_back(pair.second);
-            }
-            return std::pair<
-                std::vector<std::vector<py::str>>,
-                std::vector<std::vector<py::str>>>(all_words, all_labels);
-          })
-      .def(
-          "getVocab",
-          [](fasttext::FastText& m, const char* onUnicodeError) {
-            py::str s;
-            std::vector<py::str> vocab_list;
-            std::vector<int64_t> vocab_freq;
-            std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
-            vocab_freq = d->getCounts(fasttext::entry_type::word);
-            for (int32_t i = 0; i < vocab_freq.size(); i++) {
-              vocab_list.push_back(
-                  castToPythonString(d->getWord(i), onUnicodeError));
-            }
-            return std::pair<std::vector<py::str>, std::vector<int64_t>>(
-                vocab_list, vocab_freq);
-          })
-      .def(
-          "getLabels",
-          [](fasttext::FastText& m, const char* onUnicodeError) {
-            std::vector<py::str> labels_list;
-            std::vector<int64_t> labels_freq;
-            std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
-            labels_freq = d->getCounts(fasttext::entry_type::label);
-            for (int32_t i = 0; i < labels_freq.size(); i++) {
-              labels_list.push_back(
-                  castToPythonString(d->getLabel(i), onUnicodeError));
-            }
-            return std::pair<std::vector<py::str>, std::vector<int64_t>>(
-                labels_list, labels_freq);
-          })
-      .def(
-          "quantize",
-          [](fasttext::FastText& m,
-             const std::string input,
-             bool qout,
-             int32_t cutoff,
-             bool retrain,
-             int epoch,
-             double lr,
-             int thread,
-             int verbose,
-             int32_t dsub,
-             bool qnorm) {
-            fasttext::Args qa = fasttext::Args();
-            qa.input = input;
-            qa.qout = qout;
-            qa.cutoff = cutoff;
-            qa.retrain = retrain;
-            qa.epoch = epoch;
-            qa.lr = lr;
-            qa.thread = thread;
-            qa.verbose = verbose;
-            qa.dsub = dsub;
-            qa.qnorm = qnorm;
-            m.quantize(qa);
-          })
-      .def(
-          "predict",
-          // NOTE: text needs to end in a newline
-          // to exactly mimic the behavior of the cli
-          [](fasttext::FastText& m,
-             const std::string text,
-             int32_t k,
-             fasttext::real threshold,
-             const char* onUnicodeError) {
-            std::stringstream ioss(text);
-            std::vector<std::pair<fasttext::real, std::string>> predictions;
-            m.predictLine(ioss, predictions, k, threshold);
-
-            std::vector<std::pair<fasttext::real, py::str>>
-                transformedPredictions;
-
-            for (const auto& prediction : predictions) {
-              transformedPredictions.push_back(std::make_pair(
-                  prediction.first,
-                  castToPythonString(prediction.second, onUnicodeError)));
-            }
-
-            return transformedPredictions;
-          })
-      .def(
-          "multilinePredict",
-          // NOTE: text needs to end in a newline
-          // to exactly mimic the behavior of the cli
-          [](fasttext::FastText& m,
-             const std::vector<std::string>& lines,
-             int32_t k,
-             fasttext::real threshold,
-             const char* onUnicodeError) {
-            std::vector<std::vector<std::pair<fasttext::real, py::str>>>
-                allPredictions;
-            std::vector<std::pair<fasttext::real, std::string>> predictions;
-
-            for (const std::string& text : lines) {
-              std::stringstream ioss(text);
-              m.predictLine(ioss, predictions, k, threshold);
-              std::vector<std::pair<fasttext::real, py::str>>
-                  transformedPredictions;
-              for (const auto& prediction : predictions) {
-                transformedPredictions.push_back(std::make_pair(
-                    prediction.first,
-                    castToPythonString(prediction.second, onUnicodeError)));
-              }
-              allPredictions.push_back(transformedPredictions);
-            }
-            return allPredictions;
-          })
-      .def(
-          "testLabel",
-          [](fasttext::FastText& m,
-             const std::string filename,
-             int32_t k,
-             fasttext::real threshold) {
-            std::ifstream ifs(filename);
-            if (!ifs.is_open()) {
-              throw std::invalid_argument("Test file cannot be opened!");
-            }
-            fasttext::Meter meter;
-            m.test(ifs, k, threshold, meter);
-            std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
-            std::unordered_map<std::string, py::dict> returnedValue;
-            for (int32_t i = 0; i < d->nlabels(); i++) {
-              returnedValue[d->getLabel(i)] = py::dict(
-                  "precision"_a = meter.precision(i),
-                  "recall"_a = meter.recall(i),
-                  "f1score"_a = meter.f1Score(i));
-            }
-
-            return returnedValue;
-          })
-      .def(
-          "getWordId",
-          [](fasttext::FastText& m, const std::string word) {
-            return m.getWordId(word);
-          })
-      .def(
-          "getSubwordId",
-          [](fasttext::FastText& m, const std::string word) {
-            return m.getSubwordId(word);
-          })
-      .def(
-          "getInputVector",
-          [](fasttext::FastText& m, fasttext::Vector& vec, int32_t ind) {
-            m.getInputVector(vec, ind);
-          })
-      .def(
-          "getWordVector",
-          [](fasttext::FastText& m,
-             fasttext::Vector& vec,
-             const std::string word) { m.getWordVector(vec, word); })
-      .def(
-          "getSubwords",
-          [](fasttext::FastText& m,
-             const std::string word,
-             const char* onUnicodeError) {
-            std::vector<std::string> subwords;
-            std::vector<int32_t> ngrams;
-            std::shared_ptr<const fasttext::Dictionary> d = m.getDictionary();
-            d->getSubwords(word, ngrams, subwords);
-            std::vector<py::str> transformedSubwords;
-
-            for (const auto& subword : subwords) {
-              transformedSubwords.push_back(
-                  castToPythonString(subword, onUnicodeError));
-            }
-
-            return std::pair<std::vector<py::str>, std::vector<int32_t>>(
-                transformedSubwords, ngrams);
-          })
-      .def("isQuant", [](fasttext::FastText& m) { return m.isQuant(); });
-}

data/vendor/fastText/python/fasttext_module/fasttext/tests/__init__.py

@@ -1,14 +0,0 @@
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from .test_configurations import get_supervised_models
-from .test_script import gen_tests
-from .test_script import gen_unit_tests

data/vendor/fastText/python/fasttext_module/fasttext/tests/test_configurations.py

@@ -1,239 +0,0 @@
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import multiprocessing
-
-# This script represents a collection of integration tests
-# Each integration test comes with a full set of parameters,
-# a dataset, and expected metrics.
-# These configurations can be used by various fastText APIs
-# to confirm some level of correctness.
-
-
-def max_thread():
-    return multiprocessing.cpu_count() - 1
-
-
-def check_supervised_configuration(configuration, verbose=1):
-    configuration["args"]["verbose"] = verbose
-    configuration["quant_args"]["verbose"] = verbose
-    return configuration
-
-
-def check_supervised_configurations(configurations, verbose=1):
-    for i in range(len(configurations)):
-        configurations[i] = check_supervised_configuration(
-            configurations[i], verbose=verbose
-        )
-    return configurations
-
-
-def flickr_job(thread=None):
-    if thread is None:
-        thread = max_thread()
-    config = {}
-    config["dataset"] = "YFCC100M"
-    config["args"] = {
-        "dim": 256,
-        "wordNgrams": 2,
-        "minCount": 10,
-        "bucket": 10000000,
-        "epoch": 20,
-        "loss": "hs",
-        "minCountLabel": 100,
-        "thread": thread
-    }
-    config["args"]["input"] = "YFCC100M/train"
-    config["quant_args"] = {
-        "dsub": 2,
-        "lr": 0.1,
-        "epoch": 5,
-        "cutoff": 100000,
-        "qnorm": True,
-        "retrain": True,
-        "qout": True
-    }
-    config["quant_args"]["input"] = config["args"]["input"]
-    config["test"] = {
-        "n": 647224,
-        "p1": 0.470,
-        "r1": 0.071,
-        "size": 12060039727,
-        "data": "YFCC100M/test",
-    }
-    # One quant example (to illustrate slack): 0.344, 0.0528, 64506972
-    config["quant_test"] = {
-        "n": 647224,
-        "p1": 0.300,
-        "r1": 0.0450,
-        "size": 70000000,
-        "data": "YFCC100M/test",
-    }
-    return config
-
-
-def langid_job1(thread=None):
-    if thread is None:
-        thread = max_thread()
-    config = {}
-    config["dataset"] = "langid"
-    config["args"] = {"dim": 16, "minn": 2, "maxn": 4, "thread": thread}
-    config["args"]["input"] = "langid.train"
-    config["quant_args"] = {"qnorm": True, "cutoff": 50000, "retrain": True}
-    config["quant_args"]["input"] = config["args"]["input"]
-    config["test"] = {
-        "n": 10000,
-        "p1": 0.985,
-        "r1": 0.985,
-        "size": 368132610,
-        "data": "langid.valid",
-    }
-    # One quant example (to illustrate slack): 0.984 0.984 932793
-    config["quant_test"] = {
-        "p1": 0.97,
-        "r1": 0.97,
-        "size": 1000000,
-    }
-    config["quant_test"]["n"] = config["test"]["n"]
-    config["quant_test"]["data"] = config["test"]["data"]
-    return config
-
-
-def langid_job2(thread=None):
-    if thread is None:
-        thread = max_thread()
-    config = langid_job1(thread).copy()
-    config["args"]["loss"] = "hs"
-    return config
-
-
-def cooking_job1(thread=None):
-    if thread is None:
-        thread = max_thread()
-    config = {}
-    config["dataset"] = "cooking"
-    config["args"] = {
-        "epoch": 25,
-        "lr": 1.0,
-        "wordNgrams": 2,
-        "minCount": 1,
-        "thread": thread,
-    }
-    config["args"]["input"] = "cooking.train"
-    config["quant_args"] = {"qnorm": True, "cutoff": 50000, "retrain": True}
-    config["quant_args"]["input"] = config["args"]["input"]
-    config["test"] = {
-        "n": 3000,
-        "p1": 0.59,
-        "r1": 0.25,
-        "size": 804047585,
-        "data": "cooking.valid",
-    }
-    # One quant example (to illustrate slack): 0.602 0.26 3439172
-    config["quant_test"] = {
-        "p1": 0.55,
-        "r1": 0.20,
-        "size": 4000000,
-    }
-    config["quant_test"]["n"] = config["test"]["n"]
-    config["quant_test"]["data"] = config["test"]["data"]
-    return config
-
-
-def cooking_job2(thread=None):
-    if thread is None:
-        thread = max_thread()
-    config = cooking_job1(thread).copy()
-    config["args"]["loss"] = "hs"
-    return config
-
-
-# Supervised models
-# See https://fasttext.cc/docs/en/supervised-models.html
-def get_supervised_models(thread=None, verbose=1):
-    if thread is None:
-        thread = max_thread()
-    sup_job_dataset = [
-        "ag_news", "sogou_news", "dbpedia", "yelp_review_polarity",
-        "yelp_review_full", "yahoo_answers", "amazon_review_full",
-        "amazon_review_polarity"
-    ]
-
-    sup_params = {
-        "dim": 10,
-        "wordNgrams": 2,
-        "minCount": 1,
-        "bucket": 10000000,
-        "epoch": 5,
-        "thread": thread,
-        "verbose": 1,
-    }
-    quant_params = {
-        "retrain": True,
-        "cutoff": 100000,
-        "qnorm": True,
-        "verbose": 1,
-    }
-    sup_job_lr = [0.25, 0.5, 0.5, 0.1, 0.1, 0.1, 0.05, 0.05]
-
-    sup_job_n = [7600, 60000, 70000, 38000, 50000, 60000, 650000, 400000]
-
-    sup_job_p1 = [0.915, 0.968, 0.983, 0.956, 0.638, 0.723, 0.600, 0.940]
-    sup_job_r1 = [0.915, 0.968, 0.983, 0.956, 0.638, 0.723, 0.600, 0.940]
-    sup_job_size = [
-        405607193, 421445471, 447481878, 427867393, 431292576, 517549567,
-        483742593, 493604598
-    ]
-
-    sup_job_quant_p1 = [0.918, 0.965, 0.983, 0.950, 0.625, 0.707, 0.58, 0.920]
-    sup_job_quant_r1 = [0.918, 0.965, 0.983, 0.950, 0.625, 0.707, 0.58, 0.920]
-    sup_job_quant_size = [
-        1600000, 1500000, 1700000, 1600000, 1600000, 1700000, 1600000, 1600000
-    ]
-
-    configurations = []
-    for i in range(len(sup_job_dataset)):
-        configuration = {}
-        configuration["dataset"] = sup_job_dataset[i]
-        args = sup_params.copy()
-        quant_args = quant_params.copy()
-        args["lr"] = sup_job_lr[i]
-        args["input"] = sup_job_dataset[i] + ".train"
-        quant_args["lr"] = sup_job_lr[i]
-        quant_args["input"] = sup_job_dataset[i] + ".train"
-        configuration["args"] = args
-        configuration["quant_args"] = quant_args
-        test = {
-            "n": sup_job_n[i],
-            "p1": sup_job_p1[i],
-            "r1": sup_job_r1[i],
-            "size": sup_job_size[i],
-            "data": sup_job_dataset[i] + ".test",
-        }
-        quant_test = {
-            "n": sup_job_n[i],
-            "p1": sup_job_quant_p1[i],
-            "r1": sup_job_quant_r1[i],
-            "size": sup_job_quant_size[i],
-            "data": sup_job_dataset[i] + ".test",
-        }
-        configuration["test"] = test
-        configuration["quant_test"] = quant_test
-        configurations.append(configuration)
-    configurations.append(flickr_job())
-    configurations.append(langid_job1())
-    configurations.append(langid_job2())
-    configurations.append(cooking_job1())
-    configurations.append(cooking_job2())
-    configurations = check_supervised_configurations(
-        configurations, verbose=verbose
-    )
-    return configurations