fasttext 0.1.0

data/vendor/fastText/scripts/kbcompletion/svo.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+#
+# copyright (c) 2017-present, facebook, inc.
+# all rights reserved.
+#
+# this source code is licensed under the MIT license found in the
+# license file in the root directory of this source tree.
+#
+# script for SVO
+DIR=data/SVO-tensor-dataset
+FASTTEXTDIR=../../
+
+# compile
+pushd $FASTTEXTDIR
+make opt
+popd
+ft=${FASTTEXTDIR}/fasttext
+
+## Train model and test it on validation:
+
+dim=200
+epoch=3
+model=svo
+
+echo  "---- train ----"
+time $ft supervised -input ${DIR}/ft_svo_data_train_1000000.dat  \
+  -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20
+
+echo "computing raw hit@5%..."
+$ft test ${model}.bin ${DIR}/ft_svo_data_test_250000.dat 227 2> /dev/null | awk '{if(NR==3) print "raw hit@5%="$2}'
+
+
+echo  "---- train + valid ----"
+time $ft supervised -input ${DIR}/ft_svo_data-valid+train.dat  \
+  -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20
+
+echo "computing raw hit@5%..."
+$ft test ${model}.bin ${DIR}/ft_svo_data_test_250000.dat 227 2> /dev/null | awk '{if(NR==3) print "raw hit@5%="$2}'

data/vendor/fastText/scripts/kbcompletion/wn18.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# copyright (c) 2017-present, facebook, inc.
+# all rights reserved.
+#
+# this source code is licensed under the MIT license found in the
+# license file in the root directory of this source tree.
+#
+# script for WN11
+DIR=data/wordnet-mlj12/
+FASTTEXTDIR=../../
+
+# compile
+
+pushd $FASTTEXTDIR
+make opt
+popd
+ft=${FASTTEXTDIR}/fasttext
+
+g++ -std=c++0x eval.cpp -o eval
+
+# Train model and test it:
+dim=100
+epoch=100
+neg=500
+model=data/wn
+pred=data/wnpred
+
+echo  "---- train ----"
+$ft supervised -input ${DIR}/ft_wordnet-mlj12-train.txt  \
+  -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20 -loss ns -neg $neg
+
+echo "computing raw hits@10..."
+$ft test ${model}.bin ${DIR}/ft_wordnet-mlj12-test.txt 10 2> /dev/null | awk '{if(NR==3) print "raw hit@10 = "$2}'
+
+echo "computing filtered hit@10..."
+$ft predict ${model}.bin ${DIR}/ft_wordnet-mlj12-test.txt 20000 > $pred
+./eval $pred ${DIR}/ft_wordnet-mlj12-test.txt $DIR/ft_wordnet-mlj12-full.txt 10 | awk '{if(NR==2) print "filtered hit@10 = "$2}'
+
+echo  "---- train+val ----"
+$ft supervised -input ${DIR}/ft_wordnet-mlj12-valid+train.txt \
+  -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20 -loss ns -neg $neg
+
+echo "computing raw hits@10..."
+$ft test ${model}.bin ${DIR}/ft_wordnet-mlj12-test.txt 10  2> /dev/null | awk '{if(NR==3) print "raw hit@10 = "$2}'
+
+echo "computing filtered hit@10..."
+$ft predict ${model}.bin ${DIR}/ft_wordnet-mlj12-test.txt 20000 > $pred
+./eval $pred ${DIR}/ft_wordnet-mlj12-test.txt $DIR/ft_wordnet-mlj12-full.txt 10 | awk '{if(NR==2) print "filtered hit@10 = "$2}'

data/vendor/fastText/scripts/quantization/quantization-results.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+# This script applies quantization to the models from Table 1 in:
+# Bag of Tricks for Efficient Text Classification, arXiv 1607.01759, 2016
+
+set -e
+
+DATASET=(
+  ag_news
+  sogou_news
+  dbpedia
+  yelp_review_polarity
+  yelp_review_full
+  yahoo_answers
+  amazon_review_full
+  amazon_review_polarity
+)
+
+# These learning rates were chosen by validation on a subset of the training set.
+LR=( 0.25 0.5 0.5 0.1 0.1 0.1 0.05 0.05 )
+
+RESULTDIR=result
+DATADIR=data
+
+echo 'Warning! Make sure you run the classification-results.sh script before this one'
+echo 'Otherwise you can expect the commands in this script to fail'
+
+for i in {0..7}
+do
+  echo "Working on dataset ${DATASET[i]}"
+  ../../fasttext quantize -input "${DATADIR}/${DATASET[i]}.train" \
+    -output "${RESULTDIR}/${DATASET[i]}" -lr "${LR[i]}" \
+    -thread 4 -qnorm -retrain -epoch 5 -cutoff 100000 > /dev/null
+  ../../fasttext test "${RESULTDIR}/${DATASET[i]}.ftz" \
+    "${DATADIR}/${DATASET[i]}.test"
+done

data/vendor/fastText/setup.cfg

@@ -0,0 +1,2 @@
+[metadata]
+description-file = README.md

data/vendor/fastText/setup.py

@@ -0,0 +1,203 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from setuptools import setup, Extension
+from setuptools.command.build_ext import build_ext
+import sys
+import setuptools
+import os
+import subprocess
+import platform
+import io
+
+__version__ = '0.9.1'
+FASTTEXT_SRC = "src"
+
+# Based on https://github.com/pybind/python_example
+
+class get_pybind_include(object):
+    """Helper class to determine the pybind11 include path
+
+    The purpose of this class is to postpone importing pybind11
+    until it is actually installed, so that the ``get_include()``
+    method can be invoked. """
+
+    def __init__(self, user=False):
+        try:
+            import pybind11
+        except ImportError:
+            if subprocess.call([sys.executable, '-m', 'pip', 'install', 'pybind11']):
+                raise RuntimeError('pybind11 install failed.')
+
+        self.user = user
+
+    def __str__(self):
+        import pybind11
+        return pybind11.get_include(self.user)
+
+try:
+    coverage_index = sys.argv.index('--coverage')
+except ValueError:
+    coverage = False
+else:
+    del sys.argv[coverage_index]
+    coverage = True
+
+fasttext_src_files = map(str, os.listdir(FASTTEXT_SRC))
+fasttext_src_cc = list(filter(lambda x: x.endswith('.cc'), fasttext_src_files))
+
+fasttext_src_cc = list(
+    map(lambda x: str(os.path.join(FASTTEXT_SRC, x)), fasttext_src_cc)
+)
+
+ext_modules = [
+    Extension(
+        str('fasttext_pybind'),
+        [
+            str('python/fasttext_module/fasttext/pybind/fasttext_pybind.cc'),
+        ] + fasttext_src_cc,
+        include_dirs=[
+            # Path to pybind11 headers
+            get_pybind_include(),
+            get_pybind_include(user=True),
+            # Path to fasttext source code
+            FASTTEXT_SRC,
+        ],
+        language='c++',
+        extra_compile_args=["-O0 -fno-inline -fprofile-arcs -pthread -march=native" if coverage else
+                            "-O3 -funroll-loops -pthread -march=native"],
+    ),
+]
+
+
+# As of Python 3.6, CCompiler has a `has_flag` method.
+# cf http://bugs.python.org/issue26689
+def has_flag(compiler, flags):
+    """Return a boolean indicating whether a flag name is supported on
+    the specified compiler.
+    """
+    import tempfile
+    with tempfile.NamedTemporaryFile('w', suffix='.cpp') as f:
+        f.write('int main (int argc, char **argv) { return 0; }')
+        try:
+            compiler.compile([f.name], extra_postargs=flags)
+        except setuptools.distutils.errors.CompileError:
+            return False
+    return True
+
+
+def cpp_flag(compiler):
+    """Return the -std=c++[0x/11/14] compiler flag.
+    The c++14 is preferred over c++0x/11 (when it is available).
+    """
+    standards = ['-std=c++14', '-std=c++11', '-std=c++0x']
+    for standard in standards:
+        if has_flag(compiler, [standard]):
+            return standard
+    raise RuntimeError(
+        'Unsupported compiler -- at least C++0x support '
+        'is needed!'
+    )
+
+
+class BuildExt(build_ext):
+    """A custom build extension for adding compiler-specific options."""
+    c_opts = {
+        'msvc': ['/EHsc'],
+        'unix': [],
+    }
+
+    def build_extensions(self):
+        if sys.platform == 'darwin':
+            mac_osx_version = float('.'.join(platform.mac_ver()[0].split('.')[:2]))
+            os.environ['MACOSX_DEPLOYMENT_TARGET'] = str(mac_osx_version)
+            all_flags = ['-stdlib=libc++', '-mmacosx-version-min=10.7']
+            if has_flag(self.compiler, [all_flags[0]]):
+                self.c_opts['unix'] += [all_flags[0]]
+            elif has_flag(self.compiler, all_flags):
+                self.c_opts['unix'] += all_flags
+            else:
+                raise RuntimeError(
+                    'libc++ is needed! Failed to compile with {} and {}.'.
+                    format(" ".join(all_flags), all_flags[0])
+                )
+        ct = self.compiler.compiler_type
+        opts = self.c_opts.get(ct, [])
+        extra_link_args = []
+
+        if coverage:
+            coverage_option = '--coverage'
+            opts.append(coverage_option)
+            extra_link_args.append(coverage_option)
+
+        if ct == 'unix':
+            opts.append('-DVERSION_INFO="%s"' % self.distribution.get_version())
+            opts.append(cpp_flag(self.compiler))
+            if has_flag(self.compiler, ['-fvisibility=hidden']):
+                opts.append('-fvisibility=hidden')
+        elif ct == 'msvc':
+            opts.append(
+                '/DVERSION_INFO=\\"%s\\"' % self.distribution.get_version()
+            )
+        for ext in self.extensions:
+            ext.extra_compile_args = opts
+            ext.extra_link_args = extra_link_args
+        build_ext.build_extensions(self)
+
+
+def _get_readme():
+    """
+    Use pandoc to generate rst from md.
+    pandoc --from=markdown --to=rst --output=python/README.rst python/README.md
+    """
+    with io.open("python/README.rst", encoding='utf-8') as fid:
+        return fid.read()
+
+
+setup(
+    name='fasttext',
+    version=__version__,
+    author='Onur Celebi',
+    author_email='celebio@fb.com',
+    description='fasttext Python bindings',
+    long_description=_get_readme(),
+    ext_modules=ext_modules,
+    url='https://github.com/facebookresearch/fastText',
+    license='MIT',
+    classifiers=[
+        'Development Status :: 3 - Alpha',
+        'Intended Audience :: Developers',
+        'Intended Audience :: Science/Research',
+        'License :: OSI Approved :: MIT License',
+        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3.4',
+        'Programming Language :: Python :: 3.5',
+        'Programming Language :: Python :: 3.6',
+        'Topic :: Software Development',
+        'Topic :: Scientific/Engineering',
+        'Operating System :: Microsoft :: Windows',
+        'Operating System :: POSIX',
+        'Operating System :: Unix',
+        'Operating System :: MacOS',
+    ],
+    install_requires=['pybind11>=2.2', "setuptools >= 0.7.0", "numpy"],
+    cmdclass={'build_ext': BuildExt},
+    packages=[
+        str('fasttext'),
+        str('fasttext.util'),
+        str('fasttext.tests'),
+    ],
+    package_dir={str(''): str('python/fasttext_module')},
+    zip_safe=False,
+)

data/vendor/fastText/src/args.cc

@@ -0,0 +1,320 @@
+/**
+ * Copyright (c) 2016-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include "args.h"
+
+#include <stdlib.h>
+
+#include <iostream>
+#include <stdexcept>
+
+namespace fasttext {
+
+Args::Args() {
+  lr = 0.05;
+  dim = 100;
+  ws = 5;
+  epoch = 5;
+  minCount = 5;
+  minCountLabel = 0;
+  neg = 5;
+  wordNgrams = 1;
+  loss = loss_name::ns;
+  model = model_name::sg;
+  bucket = 2000000;
+  minn = 3;
+  maxn = 6;
+  thread = 12;
+  lrUpdateRate = 100;
+  t = 1e-4;
+  label = "__label__";
+  verbose = 2;
+  pretrainedVectors = "";
+  saveOutput = false;
+
+  qout = false;
+  retrain = false;
+  qnorm = false;
+  cutoff = 0;
+  dsub = 2;
+}
+
+std::string Args::lossToString(loss_name ln) const {
+  switch (ln) {
+    case loss_name::hs:
+      return "hs";
+    case loss_name::ns:
+      return "ns";
+    case loss_name::softmax:
+      return "softmax";
+    case loss_name::ova:
+      return "one-vs-all";
+  }
+  return "Unknown loss!"; // should never happen
+}
+
+std::string Args::boolToString(bool b) const {
+  if (b) {
+    return "true";
+  } else {
+    return "false";
+  }
+}
+
+std::string Args::modelToString(model_name mn) const {
+  switch (mn) {
+    case model_name::cbow:
+      return "cbow";
+    case model_name::sg:
+      return "sg";
+    case model_name::sup:
+      return "sup";
+  }
+  return "Unknown model name!"; // should never happen
+}
+
+void Args::parseArgs(const std::vector<std::string>& args) {
+  std::string command(args[1]);
+  if (command == "supervised") {
+    model = model_name::sup;
+    loss = loss_name::softmax;
+    minCount = 1;
+    minn = 0;
+    maxn = 0;
+    lr = 0.1;
+  } else if (command == "cbow") {
+    model = model_name::cbow;
+  }
+  for (int ai = 2; ai < args.size(); ai += 2) {
+    if (args[ai][0] != '-') {
+      std::cerr << "Provided argument without a dash! Usage:" << std::endl;
+      printHelp();
+      exit(EXIT_FAILURE);
+    }
+    try {
+      if (args[ai] == "-h") {
+        std::cerr << "Here is the help! Usage:" << std::endl;
+        printHelp();
+        exit(EXIT_FAILURE);
+      } else if (args[ai] == "-input") {
+        input = std::string(args.at(ai + 1));
+      } else if (args[ai] == "-output") {
+        output = std::string(args.at(ai + 1));
+      } else if (args[ai] == "-lr") {
+        lr = std::stof(args.at(ai + 1));
+      } else if (args[ai] == "-lrUpdateRate") {
+        lrUpdateRate = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-dim") {
+        dim = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-ws") {
+        ws = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-epoch") {
+        epoch = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-minCount") {
+        minCount = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-minCountLabel") {
+        minCountLabel = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-neg") {
+        neg = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-wordNgrams") {
+        wordNgrams = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-loss") {
+        if (args.at(ai + 1) == "hs") {
+          loss = loss_name::hs;
+        } else if (args.at(ai + 1) == "ns") {
+          loss = loss_name::ns;
+        } else if (args.at(ai + 1) == "softmax") {
+          loss = loss_name::softmax;
+        } else if (
+            args.at(ai + 1) == "one-vs-all" || args.at(ai + 1) == "ova") {
+          loss = loss_name::ova;
+        } else {
+          std::cerr << "Unknown loss: " << args.at(ai + 1) << std::endl;
+          printHelp();
+          exit(EXIT_FAILURE);
+        }
+      } else if (args[ai] == "-bucket") {
+        bucket = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-minn") {
+        minn = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-maxn") {
+        maxn = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-thread") {
+        thread = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-t") {
+        t = std::stof(args.at(ai + 1));
+      } else if (args[ai] == "-label") {
+        label = std::string(args.at(ai + 1));
+      } else if (args[ai] == "-verbose") {
+        verbose = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-pretrainedVectors") {
+        pretrainedVectors = std::string(args.at(ai + 1));
+      } else if (args[ai] == "-saveOutput") {
+        saveOutput = true;
+        ai--;
+      } else if (args[ai] == "-qnorm") {
+        qnorm = true;
+        ai--;
+      } else if (args[ai] == "-retrain") {
+        retrain = true;
+        ai--;
+      } else if (args[ai] == "-qout") {
+        qout = true;
+        ai--;
+      } else if (args[ai] == "-cutoff") {
+        cutoff = std::stoi(args.at(ai + 1));
+      } else if (args[ai] == "-dsub") {
+        dsub = std::stoi(args.at(ai + 1));
+      } else {
+        std::cerr << "Unknown argument: " << args[ai] << std::endl;
+        printHelp();
+        exit(EXIT_FAILURE);
+      }
+    } catch (std::out_of_range) {
+      std::cerr << args[ai] << " is missing an argument" << std::endl;
+      printHelp();
+      exit(EXIT_FAILURE);
+    }
+  }
+  if (input.empty() || output.empty()) {
+    std::cerr << "Empty input or output path." << std::endl;
+    printHelp();
+    exit(EXIT_FAILURE);
+  }
+  if (wordNgrams <= 1 && maxn == 0) {
+    bucket = 0;
+  }
+}
+
+void Args::printHelp() {
+  printBasicHelp();
+  printDictionaryHelp();
+  printTrainingHelp();
+  printQuantizationHelp();
+}
+
+void Args::printBasicHelp() {
+  std::cerr << "\nThe following arguments are mandatory:\n"
+            << "  -input              training file path\n"
+            << "  -output             output file path\n"
+            << "\nThe following arguments are optional:\n"
+            << "  -verbose            verbosity level [" << verbose << "]\n";
+}
+
+void Args::printDictionaryHelp() {
+  std::cerr << "\nThe following arguments for the dictionary are optional:\n"
+            << "  -minCount           minimal number of word occurences ["
+            << minCount << "]\n"
+            << "  -minCountLabel      minimal number of label occurences ["
+            << minCountLabel << "]\n"
+            << "  -wordNgrams         max length of word ngram [" << wordNgrams
+            << "]\n"
+            << "  -bucket             number of buckets [" << bucket << "]\n"
+            << "  -minn               min length of char ngram [" << minn
+            << "]\n"
+            << "  -maxn               max length of char ngram [" << maxn
+            << "]\n"
+            << "  -t                  sampling threshold [" << t << "]\n"
+            << "  -label              labels prefix [" << label << "]\n";
+}
+
+void Args::printTrainingHelp() {
+  std::cerr
+      << "\nThe following arguments for training are optional:\n"
+      << "  -lr                 learning rate [" << lr << "]\n"
+      << "  -lrUpdateRate       change the rate of updates for the learning rate ["
+      << lrUpdateRate << "]\n"
+      << "  -dim                size of word vectors [" << dim << "]\n"
+      << "  -ws                 size of the context window [" << ws << "]\n"
+      << "  -epoch              number of epochs [" << epoch << "]\n"
+      << "  -neg                number of negatives sampled [" << neg << "]\n"
+      << "  -loss               loss function {ns, hs, softmax, one-vs-all} ["
+      << lossToString(loss) << "]\n"
+      << "  -thread             number of threads [" << thread << "]\n"
+      << "  -pretrainedVectors  pretrained word vectors for supervised learning ["
+      << pretrainedVectors << "]\n"
+      << "  -saveOutput         whether output params should be saved ["
+      << boolToString(saveOutput) << "]\n";
+}
+
+void Args::printQuantizationHelp() {
+  std::cerr
+      << "\nThe following arguments for quantization are optional:\n"
+      << "  -cutoff             number of words and ngrams to retain ["
+      << cutoff << "]\n"
+      << "  -retrain            whether embeddings are finetuned if a cutoff is applied ["
+      << boolToString(retrain) << "]\n"
+      << "  -qnorm              whether the norm is quantized separately ["
+      << boolToString(qnorm) << "]\n"
+      << "  -qout               whether the classifier is quantized ["
+      << boolToString(qout) << "]\n"
+      << "  -dsub               size of each sub-vector [" << dsub << "]\n";
+}
+
+void Args::save(std::ostream& out) {
+  out.write((char*)&(dim), sizeof(int));
+  out.write((char*)&(ws), sizeof(int));
+  out.write((char*)&(epoch), sizeof(int));
+  out.write((char*)&(minCount), sizeof(int));
+  out.write((char*)&(neg), sizeof(int));
+  out.write((char*)&(wordNgrams), sizeof(int));
+  out.write((char*)&(loss), sizeof(loss_name));
+  out.write((char*)&(model), sizeof(model_name));
+  out.write((char*)&(bucket), sizeof(int));
+  out.write((char*)&(minn), sizeof(int));
+  out.write((char*)&(maxn), sizeof(int));
+  out.write((char*)&(lrUpdateRate), sizeof(int));
+  out.write((char*)&(t), sizeof(double));
+}
+
+void Args::load(std::istream& in) {
+  in.read((char*)&(dim), sizeof(int));
+  in.read((char*)&(ws), sizeof(int));
+  in.read((char*)&(epoch), sizeof(int));
+  in.read((char*)&(minCount), sizeof(int));
+  in.read((char*)&(neg), sizeof(int));
+  in.read((char*)&(wordNgrams), sizeof(int));
+  in.read((char*)&(loss), sizeof(loss_name));
+  in.read((char*)&(model), sizeof(model_name));
+  in.read((char*)&(bucket), sizeof(int));
+  in.read((char*)&(minn), sizeof(int));
+  in.read((char*)&(maxn), sizeof(int));
+  in.read((char*)&(lrUpdateRate), sizeof(int));
+  in.read((char*)&(t), sizeof(double));
+}
+
+void Args::dump(std::ostream& out) const {
+  out << "dim"
+      << " " << dim << std::endl;
+  out << "ws"
+      << " " << ws << std::endl;
+  out << "epoch"
+      << " " << epoch << std::endl;
+  out << "minCount"
+      << " " << minCount << std::endl;
+  out << "neg"
+      << " " << neg << std::endl;
+  out << "wordNgrams"
+      << " " << wordNgrams << std::endl;
+  out << "loss"
+      << " " << lossToString(loss) << std::endl;
+  out << "model"
+      << " " << modelToString(model) << std::endl;
+  out << "bucket"
+      << " " << bucket << std::endl;
+  out << "minn"
+      << " " << minn << std::endl;
+  out << "maxn"
+      << " " << maxn << std::endl;
+  out << "lrUpdateRate"
+      << " " << lrUpdateRate << std::endl;
+  out << "t"
+      << " " << t << std::endl;
+}
+
+} // namespace fasttext