fasttext 0.1.0

data/vendor/fastText/python/fasttext_module/fasttext/util/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from .util import test
+from .util import find_nearest_neighbor

data/vendor/fastText/python/fasttext_module/fasttext/util/util.py

@@ -0,0 +1,60 @@
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# NOTE: The purpose of this file is not to accumulate all useful utility
+# functions. This file should contain very commonly used and requested functions
+# (such as test). If you think you have a function at that level, please create
+# an issue and we will happily review your suggestion. This file is also not supposed
+# to pull in dependencies outside of numpy/scipy without very good reasons. For
+# example, this file should not use sklearn and matplotlib to produce a t-sne
+# plot of word embeddings or such.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import numpy as np
+
+
+# TODO: Add example on reproducing model.test with util.test and model.get_line
+def test(predictions, labels, k=1):
+    """
+    Return precision and recall modeled after fasttext's test
+    """
+    precision = 0.0
+    nexamples = 0
+    nlabels = 0
+    for prediction, labels in zip(predictions, labels):
+        for p in prediction:
+            if p in labels:
+                precision += 1
+        nexamples += 1
+        nlabels += len(labels)
+    return (precision / (k * nexamples), precision / nlabels)
+
+
+def find_nearest_neighbor(query, vectors, ban_set, cossims=None):
+    """
+    query is a 1d numpy array corresponding to the vector to which you want to
+    find the closest vector
+    vectors is a 2d numpy array corresponding to the vectors you want to consider
+    ban_set is a set of indicies within vectors you want to ignore for nearest match
+    cossims is a 1d numpy array of size len(vectors), which can be passed for efficiency
+
+    returns the index of the closest match to query within vectors
+
+    """
+    if cossims is None:
+        cossims = np.matmul(vectors, query, out=cossims)
+    else:
+        np.matmul(vectors, query, out=cossims)
+    rank = len(cossims) - 1
+    result_i = np.argpartition(cossims, rank)[rank]
+    while result_i in ban_set:
+        rank -= 1
+        result_i = np.argpartition(cossims, rank)[rank]
+    return result_i

data/vendor/fastText/quantization-example.sh

@@ -0,0 +1,40 @@
+myshuf() {
+  perl -MList::Util=shuffle -e 'print shuffle(<>);' "$@";
+}
+
+normalize_text() {
+  tr '[:upper:]' '[:lower:]' | sed -e 's/^/__label__/g' | \
+    sed -e "s/'/ ' /g" -e 's/"//g' -e 's/\./ \. /g' -e 's/<br \/>/ /g' \
+        -e 's/,/ , /g' -e 's/(/ ( /g' -e 's/)/ ) /g' -e 's/\!/ \! /g' \
+        -e 's/\?/ \? /g' -e 's/\;/ /g' -e 's/\:/ /g' | tr -s " " | myshuf
+}
+
+RESULTDIR=result
+DATADIR=data
+
+mkdir -p "${RESULTDIR}"
+mkdir -p "${DATADIR}"
+
+if [ ! -f "${DATADIR}/dbpedia.train" ]
+then
+  wget -c "https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz" -O "${DATADIR}/dbpedia_csv.tar.gz"
+  tar -xzvf "${DATADIR}/dbpedia_csv.tar.gz" -C "${DATADIR}"
+  cat "${DATADIR}/dbpedia_csv/train.csv" | normalize_text > "${DATADIR}/dbpedia.train"
+  cat "${DATADIR}/dbpedia_csv/test.csv" | normalize_text > "${DATADIR}/dbpedia.test"
+fi
+
+make
+
+echo "Training..."
+./fasttext supervised -input "${DATADIR}/dbpedia.train" -output "${RESULTDIR}/dbpedia" -dim 10 -lr 0.1 -wordNgrams 2 -minCount 1 -bucket 10000000 -epoch 5 -thread 4
+
+echo "Quantizing..."
+./fasttext quantize -output "${RESULTDIR}/dbpedia" -input "${DATADIR}/dbpedia.train" -qnorm -retrain -epoch 1 -cutoff 100000
+
+echo "Testing original model..."
+./fasttext test "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test"
+echo "Testing quantized model..."
+./fasttext test "${RESULTDIR}/dbpedia.ftz" "${DATADIR}/dbpedia.test"
+
+wc -c < "${RESULTDIR}/dbpedia.bin" | awk '{print "Size of the original model:\t",$1;}'
+wc -c < "${RESULTDIR}/dbpedia.ftz" | awk '{print "Size of the quantized model:\t",$1;}'

data/vendor/fastText/runtests.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+# To run the integration tests you must first fetch all the required test data.
+# Have a look at tests/fetch_test_data.sh
+# You will then need to point this script to the corresponding folder
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import unittest
+import argparse
+from fasttext.tests import gen_tests
+from fasttext.tests import gen_unit_tests
+
+
+def run_tests(tests):
+    suite = unittest.TestLoader().loadTestsFromTestCase(tests)
+    unittest.TextTestRunner(verbosity=3).run(suite)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-u", "--unit-tests", help="run unit tests", action="store_true"
+    )
+    parser.add_argument(
+        "-i",
+        "--integration-tests",
+        help="run integration tests",
+        action="store_true"
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        default=1,
+        help="verbosity level (default 1)",
+        type=int,
+    )
+    parser.add_argument("--data-dir", help="Full path to data directory")
+    args = parser.parse_args()
+    if args.unit_tests:
+        run_tests(gen_unit_tests(verbose=args.verbose))
+    if args.integration_tests:
+        if args.data_dir is None:
+            raise ValueError(
+                "Need data directory! Consult tests/fetch_test_data.sh"
+            )
+        run_tests(gen_tests(args.data_dir, verbose=args.verbose))
+    if not args.unit_tests and not args.integration_tests:
+        print("Ran no tests")

data/vendor/fastText/scripts/kbcompletion/README.md

@@ -0,0 +1,19 @@
+# Fast Linear Model for Knowledge Graph Embeddings
+
+## Knowledge base completion
+
+These scripts require the [fastText library](https://github.com/facebookresearch/fastText).
+
+Run the data.sh script to download and format the datasets. Then run any of the scripts to train and test on a given dataset.
+
+## Reference
+
+If you use this code please cite:
+
+@article{joulin2017fast,
+  title={Fast Linear Model for Knowledge Graph Embeddings},
+  author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Nickel, Maximilian and Mikolov, Tomas},
+  journal={arXiv preprint arXiv:1710.10881},
+  year={2017}
+}
+

data/vendor/fastText/scripts/kbcompletion/data.sh

@@ -0,0 +1,69 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+set -e
+DATADIR=data/
+
+if [ ! -d "$DATADIR" ]; then
+  mkdir $DATADIR
+fi
+
+cd $DATADIR
+echo "preparing WN18"
+#wget -P . https://everest.hds.utc.fr/lib/exe/fetch.php?media=en:wordnet-mlj12.tar.gz
+#mv fetch.php\?media\=en\:wordnet-mlj12.tar.gz wordnet-mlj12.tar.gz
+wget -P . https://github.com/mana-ysh/knowledge-graph-embeddings/raw/master/dat/wordnet-mlj12.tar.gz
+tar -xzvf wordnet-mlj12.tar.gz
+DIR=wordnet-mlj12
+for f in ${DIR}/wordnet-ml*.txt;
+do
+  fn=${DIR}/ft_$(basename $f)
+  awk '{print "__label__"$1,"0_"$2, $3;print $1,"1_"$2," __label__"$3}' < ${f} > ${fn};
+done
+cat ${DIR}/ft_* > ${DIR}/ft_wordnet-mlj12-full.txt
+cat ${DIR}/ft_*train.txt ${DIR}/ft_*valid.txt > ${DIR}/ft_wordnet-mlj12-valid+train.txt
+
+echo "preparing FB15K"
+#wget https://everest.hds.utc.fr/lib/exe/fetch.php?media=en:fb15k.tgz
+#mv fetch.php\?media\=en\:fb15k.tgz fb15k.tgz
+wget https://github.com/mana-ysh/knowledge-graph-embeddings/raw/master/dat/fb15k.tgz
+tar -xzvf fb15k.tgz
+DIR=FB15k/
+for f in ${DIR}/freebase*.txt;
+do
+  fn=${DIR}/ft_$(basename $f)
+  echo $f " --> " $fn
+  awk '{print "__label__"$1,"0_"$2, $3;print $1,"1_"$2," __label__"$3}' < ${f} > ${fn};
+done
+cat ${DIR}/ft_* > ${DIR}/ft_freebase_mtr100_mte100-full.txt
+cat ${DIR}/ft_*train.txt ${DIR}/ft_*valid.txt > ${DIR}/ft_freebase_mtr100_mte100-valid+train.txt
+
+echo "preparing FB15K-237"
+wget https://download.microsoft.com/download/8/7/0/8700516A-AB3D-4850-B4BB-805C515AECE1/FB15K-237.2.zip
+unzip FB15K-237.2.zip
+DIR=Release/
+for f in train.txt test.txt valid.txt
+do
+  fn=${DIR}/ft_$(basename $f)
+  echo $f " --> " $fn
+  awk -F "\t" '{print "__label__"$1,"0_"$2, $3;print $1,"1_"$2," __label__"$3}' < ${DIR}/${f} > ${fn};
+done
+cat ${DIR}/ft_*.txt > ${DIR}/ft_full.txt
+cat ${DIR}/ft_train.txt ${DIR}/ft_valid.txt > ${DIR}/ft_valid+train.txt
+
+echo "preparing SVO"
+wget . https://everest.hds.utc.fr/lib/exe/fetch.php?media=en:svo-tensor-dataset.tar.gz
+mv fetch.php?media=en:svo-tensor-dataset.tar.gz svo-tensor-dataset.tar.gz
+tar -xzvf svo-tensor-dataset.tar.gz
+DIR=SVO-tensor-dataset
+for f in ${DIR}/svo_data*.dat;
+do
+  fn=${DIR}/ft_$(basename $f)
+  awk '{print "0_"$1,"1_"$3,"__label__"$2;}' < ${f} > ${fn};
+done
+cat ${DIR}/ft_*train*.dat ${DIR}/ft_*valid*.dat > ${DIR}/ft_svo_data-valid+train.dat

data/vendor/fastText/scripts/kbcompletion/eval.cpp

@@ -0,0 +1,108 @@
+/**
+ * Copyright (c) 2017-present, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under the MIT license found in the
+ * LICENSE file in the root directory of this source tree.
+ */
+
+#include <unordered_map>
+#include <iostream>
+#include <fstream>
+#include <string>
+#include <vector>
+
+std::string EOS = "</s>";
+
+bool readWord(std::istream& in, std::string& word)
+{
+  char c;
+  std::streambuf& sb = *in.rdbuf();
+  word.clear();
+  while ((c = sb.sbumpc()) != EOF) {
+    if (c == ' ' || c == '\n' || c == '\r' || c == '\t' || c == '\v' ||
+        c == '\f' || c == '\0') {
+      if (word.empty()) {
+        if (c == '\n') {
+          word += EOS;
+          return true;
+        }
+        continue;
+      } else {
+        if (c == '\n')
+          sb.sungetc();
+        return true;
+      }
+    }
+    word.push_back(c);
+  }
+  in.get();
+  return !word.empty();
+}
+
+int main(int argc, char** argv) {
+  int k = 10;
+  if (argc < 4) {
+    std::cerr<<"eval <pred> <gt> <kb> [<k>]"<<std::endl;
+    exit(1);
+  }
+  if (argc == 5) { k = atoi(argv[4]);}
+
+  std::string predfn(argv[1]);
+  std::ifstream predf(predfn);
+  std::string gtfn(argv[2]);
+  std::ifstream gtf(gtfn);
+  std::string kbfn(argv[3]);
+  std::ifstream kbf(kbfn);
+
+  if (!predf.is_open() || !gtf.is_open() || !kbf.is_open()) {
+    std::cerr << "Files cannot be opened!" << std::endl;
+    exit(EXIT_FAILURE);
+  }
+
+  std::unordered_map< std::string,
+    std::unordered_map< std::string, bool > > KB;
+
+  while (kbf.peek() != EOF) {
+    std::string label, key, word;
+    while (readWord(kbf, word)) {
+      if (word == EOS) {break;}
+      if (word.find("__label__") == 0) {label = word;}
+      else {key += "|" + word;}
+    }
+    KB[key][label] = true;
+  }
+  kbf.close();
+
+  double precision = 0.0;
+  int32_t nexamples = 0;
+  while (predf.peek() != EOF || gtf.peek() != EOF) {
+    if (predf.peek() == EOF || gtf.peek() == EOF) {
+      std::cerr<<"pred / gt files have diff sizes"<<std::endl;
+      exit(1);
+    }
+    std::string label, key, word;
+
+    while (readWord(gtf, word)) {
+      if (word == EOS) {break;}
+      if ( word.find("__label__") == 0) {label = word;}
+      else {key += "|" + word;}
+    }
+    if (KB.find(key) == KB.end()) {
+      std::cerr<<"empty key!"<<std::endl; exit(1);
+    }
+
+    int count = 0;bool eval = true;
+    while (readWord(predf, word)) {
+      if (word == EOS) {break;}
+      if (!eval) {continue;}
+      if (label == word) {precision += 1.0; eval = false;}
+      else if (KB[key].find(word) == KB[key].end()) {count++;}
+      if (count == k) {eval = false;}
+    }
+    nexamples++;
+  }
+  predf.close(); gtf.close();
+  std::cout << "N:\t" << nexamples << std::endl;
+  std::cout << "R@" << k << "\t" << precision / nexamples << std::endl;
+}

data/vendor/fastText/scripts/kbcompletion/fb15k.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# copyright (c) 2017-present, facebook, inc.
+# all rights reserved.
+#
+# this source code is licensed under the MIT license found in the
+# license file in the root directory of this source tree.
+#
+# script for FB15k
+DIR=data/FB15k/
+FASTTEXTDIR=../../
+
+# compile
+pushd $FASTTEXTDIR
+make opt
+popd
+ft=${FASTTEXTDIR}/fasttext
+
+g++ -std=c++0x eval.cpp -o eval
+
+## Train model and test it on validation:
+dim=100
+epoch=100
+neg=100
+model=data/fb15
+pred=data/fbpred
+
+echo "---- train ----"
+$ft supervised -input $DIR/ft_freebase_mtr100_mte100-train.txt \
+  -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20 -loss ns -neg $neg -minCount 0
+
+echo "computing raw hits@10..."
+$ft test ${model}.bin $DIR/ft_freebase_mtr100_mte100-test.txt 10 2> /dev/null | awk '{if(NR==3) print "raw hit@10="$2}'
+
+echo "computing filtered hit@10..."
+$ft predict ${model}.bin $DIR/ft_freebase_mtr100_mte100-test.txt 20000 > $pred
+./eval $pred ${DIR}/ft_freebase_mtr100_mte100-test.txt $DIR/ft_freebase_mtr100_mte100-full.txt 10 | awk '{if(NR==2) print "filtered hit@10="$2}'
+
+echo  "---- train+val ----"
+
+$ft supervised -input $DIR/ft_freebase_mtr100_mte100-valid+train.txt \
+  -dim ${dim} -epoch ${dim} -output ${model} -lr .2 -thread 20 -loss ns -neg ${neg} -minCount 0
+
+echo "computing raw hits@10..."
+$ft test ${model}.bin $DIR/ft_freebase_mtr100_mte100-test.txt 10  2> /dev/null | awk '{if(NR==3) print "raw hit@10="$2}'
+
+echo "computing filtered hit@10..."
+$ft predict ${model}.bin $DIR/ft_freebase_mtr100_mte100-test.txt 20000 > $pred
+./eval $pred ${DIR}/ft_freebase_mtr100_mte100-test.txt $DIR/ft_freebase_mtr100_mte100-full.txt 10 | awk '{if(NR==2) print "filtered hit@10="$2}'

data/vendor/fastText/scripts/kbcompletion/fb15k237.sh

@@ -0,0 +1,45 @@
+#!/usr/bin/env bash
+#
+# copyright (c) 2017-present, facebook, inc.
+# all rights reserved.
+#
+# this source code is licensed under the MIT license found in the
+# license file in the root directory of this source tree.
+#
+# script for FB15k237
+DIR=data/Release/
+FASTTEXTDIR=../../
+
+# compile
+
+pushd $FASTTEXTDIR
+make opt
+popd
+ft=${FASTTEXTDIR}/fasttext
+
+g++ -std=c++0x eval.cpp -o eval
+
+## Train model and test it on validation:
+
+pred=data/fb237pred
+model=data/fb15k237
+dim=50
+epoch=10
+neg=500
+
+echo "---- train ----"
+$ft supervised -input $DIR/ft_train.txt \
+  -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20 -loss ns -neg $neg -minCount 0
+
+echo "computing filtered hit@10..."
+$ft predict ${model}.bin $DIR/ft_test.txt 20000 > $pred
+./eval $pred ${DIR}/ft_test.txt $DIR/ft_full.txt 10 | awk '{if(NR==2) print "filtered hit@10="$2}'
+
+echo  "---- train+val ----"
+
+$ft supervised -input $DIR/ft_valid+train.txt \
+  -dim ${dim} -epoch ${dim} -output ${model} -lr .2 -thread 20 -loss ns -neg ${neg} -minCount 0
+
+echo "computing filtered hit@10..."
+$ft predict ${model}.bin $DIR/ft_test.txt 20000 > $pred
+./eval $pred ${DIR}/ft_test.txt $DIR/ft_full.txt 10 | awk '{if(NR==2) print "filtered hit@10="$2}'