fasttext 0.1.0 → 0.2.1

data/vendor/fastText/python/fasttext_module/fasttext/util/__init__.py

@@ -1,13 +0,0 @@
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from .util import test
-from .util import find_nearest_neighbor

data/vendor/fastText/python/fasttext_module/fasttext/util/util.py

@@ -1,60 +0,0 @@
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-# NOTE: The purpose of this file is not to accumulate all useful utility
-# functions. This file should contain very commonly used and requested functions
-# (such as test). If you think you have a function at that level, please create
-# an issue and we will happily review your suggestion. This file is also not supposed
-# to pull in dependencies outside of numpy/scipy without very good reasons. For
-# example, this file should not use sklearn and matplotlib to produce a t-sne
-# plot of word embeddings or such.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import numpy as np
-
-
-# TODO: Add example on reproducing model.test with util.test and model.get_line
-def test(predictions, labels, k=1):
-    """
-    Return precision and recall modeled after fasttext's test
-    """
-    precision = 0.0
-    nexamples = 0
-    nlabels = 0
-    for prediction, labels in zip(predictions, labels):
-        for p in prediction:
-            if p in labels:
-                precision += 1
-        nexamples += 1
-        nlabels += len(labels)
-    return (precision / (k * nexamples), precision / nlabels)
-
-
-def find_nearest_neighbor(query, vectors, ban_set, cossims=None):
-    """
-    query is a 1d numpy array corresponding to the vector to which you want to
-    find the closest vector
-    vectors is a 2d numpy array corresponding to the vectors you want to consider
-    ban_set is a set of indicies within vectors you want to ignore for nearest match
-    cossims is a 1d numpy array of size len(vectors), which can be passed for efficiency
-
-    returns the index of the closest match to query within vectors
-
-    """
-    if cossims is None:
-        cossims = np.matmul(vectors, query, out=cossims)
-    else:
-        np.matmul(vectors, query, out=cossims)
-    rank = len(cossims) - 1
-    result_i = np.argpartition(cossims, rank)[rank]
-    while result_i in ban_set:
-        rank -= 1
-        result_i = np.argpartition(cossims, rank)[rank]
-    return result_i

data/vendor/fastText/quantization-example.sh

@@ -1,40 +0,0 @@
-myshuf() {
-  perl -MList::Util=shuffle -e 'print shuffle(<>);' "$@";
-}
-
-normalize_text() {
-  tr '[:upper:]' '[:lower:]' | sed -e 's/^/__label__/g' | \
-    sed -e "s/'/ ' /g" -e 's/"//g' -e 's/\./ \. /g' -e 's/<br \/>/ /g' \
-        -e 's/,/ , /g' -e 's/(/ ( /g' -e 's/)/ ) /g' -e 's/\!/ \! /g' \
-        -e 's/\?/ \? /g' -e 's/\;/ /g' -e 's/\:/ /g' | tr -s " " | myshuf
-}
-
-RESULTDIR=result
-DATADIR=data
-
-mkdir -p "${RESULTDIR}"
-mkdir -p "${DATADIR}"
-
-if [ ! -f "${DATADIR}/dbpedia.train" ]
-then
-  wget -c "https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz" -O "${DATADIR}/dbpedia_csv.tar.gz"
-  tar -xzvf "${DATADIR}/dbpedia_csv.tar.gz" -C "${DATADIR}"
-  cat "${DATADIR}/dbpedia_csv/train.csv" | normalize_text > "${DATADIR}/dbpedia.train"
-  cat "${DATADIR}/dbpedia_csv/test.csv" | normalize_text > "${DATADIR}/dbpedia.test"
-fi
-
-make
-
-echo "Training..."
-./fasttext supervised -input "${DATADIR}/dbpedia.train" -output "${RESULTDIR}/dbpedia" -dim 10 -lr 0.1 -wordNgrams 2 -minCount 1 -bucket 10000000 -epoch 5 -thread 4
-
-echo "Quantizing..."
-./fasttext quantize -output "${RESULTDIR}/dbpedia" -input "${DATADIR}/dbpedia.train" -qnorm -retrain -epoch 1 -cutoff 100000
-
-echo "Testing original model..."
-./fasttext test "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test"
-echo "Testing quantized model..."
-./fasttext test "${RESULTDIR}/dbpedia.ftz" "${DATADIR}/dbpedia.test"
-
-wc -c < "${RESULTDIR}/dbpedia.bin" | awk '{print "Size of the original model:\t",$1;}'
-wc -c < "${RESULTDIR}/dbpedia.ftz" | awk '{print "Size of the quantized model:\t",$1;}'

data/vendor/fastText/runtests.py

@@ -1,60 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# Copyright (c) 2016-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-#
-
-# To run the integration tests you must first fetch all the required test data.
-# Have a look at tests/fetch_test_data.sh
-# You will then need to point this script to the corresponding folder
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import unittest
-import argparse
-from fasttext.tests import gen_tests
-from fasttext.tests import gen_unit_tests
-
-
-def run_tests(tests):
-    suite = unittest.TestLoader().loadTestsFromTestCase(tests)
-    unittest.TextTestRunner(verbosity=3).run(suite)
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "-u", "--unit-tests", help="run unit tests", action="store_true"
-    )
-    parser.add_argument(
-        "-i",
-        "--integration-tests",
-        help="run integration tests",
-        action="store_true"
-    )
-    parser.add_argument(
-        "-v",
-        "--verbose",
-        default=1,
-        help="verbosity level (default 1)",
-        type=int,
-    )
-    parser.add_argument("--data-dir", help="Full path to data directory")
-    args = parser.parse_args()
-    if args.unit_tests:
-        run_tests(gen_unit_tests(verbose=args.verbose))
-    if args.integration_tests:
-        if args.data_dir is None:
-            raise ValueError(
-                "Need data directory! Consult tests/fetch_test_data.sh"
-            )
-        run_tests(gen_tests(args.data_dir, verbose=args.verbose))
-    if not args.unit_tests and not args.integration_tests:
-        print("Ran no tests")

data/vendor/fastText/scripts/kbcompletion/README.md

@@ -1,19 +0,0 @@
-# Fast Linear Model for Knowledge Graph Embeddings
-
-## Knowledge base completion
-
-These scripts require the [fastText library](https://github.com/facebookresearch/fastText).
-
-Run the data.sh script to download and format the datasets. Then run any of the scripts to train and test on a given dataset.
-
-## Reference
-
-If you use this code please cite:
-
-@article{joulin2017fast,
-  title={Fast Linear Model for Knowledge Graph Embeddings},
-  author={Joulin, Armand and Grave, Edouard and Bojanowski, Piotr and Nickel, Maximilian and Mikolov, Tomas},
-  journal={arXiv preprint arXiv:1710.10881},
-  year={2017}
-}
-

data/vendor/fastText/scripts/kbcompletion/data.sh

@@ -1,69 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-#
-set -e
-DATADIR=data/
-
-if [ ! -d "$DATADIR" ]; then
-  mkdir $DATADIR
-fi
-
-cd $DATADIR
-echo "preparing WN18"
-#wget -P . https://everest.hds.utc.fr/lib/exe/fetch.php?media=en:wordnet-mlj12.tar.gz
-#mv fetch.php\?media\=en\:wordnet-mlj12.tar.gz wordnet-mlj12.tar.gz
-wget -P . https://github.com/mana-ysh/knowledge-graph-embeddings/raw/master/dat/wordnet-mlj12.tar.gz
-tar -xzvf wordnet-mlj12.tar.gz
-DIR=wordnet-mlj12
-for f in ${DIR}/wordnet-ml*.txt;
-do
-  fn=${DIR}/ft_$(basename $f)
-  awk '{print "__label__"$1,"0_"$2, $3;print $1,"1_"$2," __label__"$3}' < ${f} > ${fn};
-done
-cat ${DIR}/ft_* > ${DIR}/ft_wordnet-mlj12-full.txt
-cat ${DIR}/ft_*train.txt ${DIR}/ft_*valid.txt > ${DIR}/ft_wordnet-mlj12-valid+train.txt
-
-echo "preparing FB15K"
-#wget https://everest.hds.utc.fr/lib/exe/fetch.php?media=en:fb15k.tgz
-#mv fetch.php\?media\=en\:fb15k.tgz fb15k.tgz
-wget https://github.com/mana-ysh/knowledge-graph-embeddings/raw/master/dat/fb15k.tgz
-tar -xzvf fb15k.tgz
-DIR=FB15k/
-for f in ${DIR}/freebase*.txt;
-do
-  fn=${DIR}/ft_$(basename $f)
-  echo $f " --> " $fn
-  awk '{print "__label__"$1,"0_"$2, $3;print $1,"1_"$2," __label__"$3}' < ${f} > ${fn};
-done
-cat ${DIR}/ft_* > ${DIR}/ft_freebase_mtr100_mte100-full.txt
-cat ${DIR}/ft_*train.txt ${DIR}/ft_*valid.txt > ${DIR}/ft_freebase_mtr100_mte100-valid+train.txt
-
-echo "preparing FB15K-237"
-wget https://download.microsoft.com/download/8/7/0/8700516A-AB3D-4850-B4BB-805C515AECE1/FB15K-237.2.zip
-unzip FB15K-237.2.zip
-DIR=Release/
-for f in train.txt test.txt valid.txt
-do
-  fn=${DIR}/ft_$(basename $f)
-  echo $f " --> " $fn
-  awk -F "\t" '{print "__label__"$1,"0_"$2, $3;print $1,"1_"$2," __label__"$3}' < ${DIR}/${f} > ${fn};
-done
-cat ${DIR}/ft_*.txt > ${DIR}/ft_full.txt
-cat ${DIR}/ft_train.txt ${DIR}/ft_valid.txt > ${DIR}/ft_valid+train.txt
-
-echo "preparing SVO"
-wget . https://everest.hds.utc.fr/lib/exe/fetch.php?media=en:svo-tensor-dataset.tar.gz
-mv fetch.php?media=en:svo-tensor-dataset.tar.gz svo-tensor-dataset.tar.gz
-tar -xzvf svo-tensor-dataset.tar.gz
-DIR=SVO-tensor-dataset
-for f in ${DIR}/svo_data*.dat;
-do
-  fn=${DIR}/ft_$(basename $f)
-  awk '{print "0_"$1,"1_"$3,"__label__"$2;}' < ${f} > ${fn};
-done
-cat ${DIR}/ft_*train*.dat ${DIR}/ft_*valid*.dat > ${DIR}/ft_svo_data-valid+train.dat

data/vendor/fastText/scripts/kbcompletion/eval.cpp

@@ -1,108 +0,0 @@
-/**
- * Copyright (c) 2017-present, Facebook, Inc.
- * All rights reserved.
- *
- * This source code is licensed under the MIT license found in the
- * LICENSE file in the root directory of this source tree.
- */
-
-#include <unordered_map>
-#include <iostream>
-#include <fstream>
-#include <string>
-#include <vector>
-
-std::string EOS = "</s>";
-
-bool readWord(std::istream& in, std::string& word)
-{
-  char c;
-  std::streambuf& sb = *in.rdbuf();
-  word.clear();
-  while ((c = sb.sbumpc()) != EOF) {
-    if (c == ' ' || c == '\n' || c == '\r' || c == '\t' || c == '\v' ||
-        c == '\f' || c == '\0') {
-      if (word.empty()) {
-        if (c == '\n') {
-          word += EOS;
-          return true;
-        }
-        continue;
-      } else {
-        if (c == '\n')
-          sb.sungetc();
-        return true;
-      }
-    }
-    word.push_back(c);
-  }
-  in.get();
-  return !word.empty();
-}
-
-int main(int argc, char** argv) {
-  int k = 10;
-  if (argc < 4) {
-    std::cerr<<"eval <pred> <gt> <kb> [<k>]"<<std::endl;
-    exit(1);
-  }
-  if (argc == 5) { k = atoi(argv[4]);}
-
-  std::string predfn(argv[1]);
-  std::ifstream predf(predfn);
-  std::string gtfn(argv[2]);
-  std::ifstream gtf(gtfn);
-  std::string kbfn(argv[3]);
-  std::ifstream kbf(kbfn);
-
-  if (!predf.is_open() || !gtf.is_open() || !kbf.is_open()) {
-    std::cerr << "Files cannot be opened!" << std::endl;
-    exit(EXIT_FAILURE);
-  }
-
-  std::unordered_map< std::string,
-    std::unordered_map< std::string, bool > > KB;
-
-  while (kbf.peek() != EOF) {
-    std::string label, key, word;
-    while (readWord(kbf, word)) {
-      if (word == EOS) {break;}
-      if (word.find("__label__") == 0) {label = word;}
-      else {key += "|" + word;}
-    }
-    KB[key][label] = true;
-  }
-  kbf.close();
-
-  double precision = 0.0;
-  int32_t nexamples = 0;
-  while (predf.peek() != EOF || gtf.peek() != EOF) {
-    if (predf.peek() == EOF || gtf.peek() == EOF) {
-      std::cerr<<"pred / gt files have diff sizes"<<std::endl;
-      exit(1);
-    }
-    std::string label, key, word;
-
-    while (readWord(gtf, word)) {
-      if (word == EOS) {break;}
-      if ( word.find("__label__") == 0) {label = word;}
-      else {key += "|" + word;}
-    }
-    if (KB.find(key) == KB.end()) {
-      std::cerr<<"empty key!"<<std::endl; exit(1);
-    }
-
-    int count = 0;bool eval = true;
-    while (readWord(predf, word)) {
-      if (word == EOS) {break;}
-      if (!eval) {continue;}
-      if (label == word) {precision += 1.0; eval = false;}
-      else if (KB[key].find(word) == KB[key].end()) {count++;}
-      if (count == k) {eval = false;}
-    }
-    nexamples++;
-  }
-  predf.close(); gtf.close();
-  std::cout << "N:\t" << nexamples << std::endl;
-  std::cout << "R@" << k << "\t" << precision / nexamples << std::endl;
-}

data/vendor/fastText/scripts/kbcompletion/fb15k.sh

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-#
-# copyright (c) 2017-present, facebook, inc.
-# all rights reserved.
-#
-# this source code is licensed under the MIT license found in the
-# license file in the root directory of this source tree.
-#
-# script for FB15k
-DIR=data/FB15k/
-FASTTEXTDIR=../../
-
-# compile
-pushd $FASTTEXTDIR
-make opt
-popd
-ft=${FASTTEXTDIR}/fasttext
-
-g++ -std=c++0x eval.cpp -o eval
-
-## Train model and test it on validation:
-dim=100
-epoch=100
-neg=100
-model=data/fb15
-pred=data/fbpred
-
-echo "---- train ----"
-$ft supervised -input $DIR/ft_freebase_mtr100_mte100-train.txt \
-  -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20 -loss ns -neg $neg -minCount 0
-
-echo "computing raw hits@10..."
-$ft test ${model}.bin $DIR/ft_freebase_mtr100_mte100-test.txt 10 2> /dev/null | awk '{if(NR==3) print "raw hit@10="$2}'
-
-echo "computing filtered hit@10..."
-$ft predict ${model}.bin $DIR/ft_freebase_mtr100_mte100-test.txt 20000 > $pred
-./eval $pred ${DIR}/ft_freebase_mtr100_mte100-test.txt $DIR/ft_freebase_mtr100_mte100-full.txt 10 | awk '{if(NR==2) print "filtered hit@10="$2}'
-
-echo  "---- train+val ----"
-
-$ft supervised -input $DIR/ft_freebase_mtr100_mte100-valid+train.txt \
-  -dim ${dim} -epoch ${dim} -output ${model} -lr .2 -thread 20 -loss ns -neg ${neg} -minCount 0
-
-echo "computing raw hits@10..."
-$ft test ${model}.bin $DIR/ft_freebase_mtr100_mte100-test.txt 10  2> /dev/null | awk '{if(NR==3) print "raw hit@10="$2}'
-
-echo "computing filtered hit@10..."
-$ft predict ${model}.bin $DIR/ft_freebase_mtr100_mte100-test.txt 20000 > $pred
-./eval $pred ${DIR}/ft_freebase_mtr100_mte100-test.txt $DIR/ft_freebase_mtr100_mte100-full.txt 10 | awk '{if(NR==2) print "filtered hit@10="$2}'

data/vendor/fastText/scripts/kbcompletion/fb15k237.sh

@@ -1,45 +0,0 @@
-#!/usr/bin/env bash
-#
-# copyright (c) 2017-present, facebook, inc.
-# all rights reserved.
-#
-# this source code is licensed under the MIT license found in the
-# license file in the root directory of this source tree.
-#
-# script for FB15k237
-DIR=data/Release/
-FASTTEXTDIR=../../
-
-# compile
-
-pushd $FASTTEXTDIR
-make opt
-popd
-ft=${FASTTEXTDIR}/fasttext
-
-g++ -std=c++0x eval.cpp -o eval
-
-## Train model and test it on validation:
-
-pred=data/fb237pred
-model=data/fb15k237
-dim=50
-epoch=10
-neg=500
-
-echo "---- train ----"
-$ft supervised -input $DIR/ft_train.txt \
-  -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20 -loss ns -neg $neg -minCount 0
-
-echo "computing filtered hit@10..."
-$ft predict ${model}.bin $DIR/ft_test.txt 20000 > $pred
-./eval $pred ${DIR}/ft_test.txt $DIR/ft_full.txt 10 | awk '{if(NR==2) print "filtered hit@10="$2}'
-
-echo  "---- train+val ----"
-
-$ft supervised -input $DIR/ft_valid+train.txt \
-  -dim ${dim} -epoch ${dim} -output ${model} -lr .2 -thread 20 -loss ns -neg ${neg} -minCount 0
-
-echo "computing filtered hit@10..."
-$ft predict ${model}.bin $DIR/ft_test.txt 20000 > $pred
-./eval $pred ${DIR}/ft_test.txt $DIR/ft_full.txt 10 | awk '{if(NR==2) print "filtered hit@10="$2}'