fasttext 0.1.0 → 0.2.1

data/vendor/fastText/scripts/kbcompletion/svo.sh

@@ -1,38 +0,0 @@
-#!/usr/bin/env bash
-#
-# copyright (c) 2017-present, facebook, inc.
-# all rights reserved.
-#
-# this source code is licensed under the MIT license found in the
-# license file in the root directory of this source tree.
-#
-# script for SVO
-DIR=data/SVO-tensor-dataset
-FASTTEXTDIR=../../
-
-# compile
-pushd $FASTTEXTDIR
-make opt
-popd
-ft=${FASTTEXTDIR}/fasttext
-
-## Train model and test it on validation:
-
-dim=200
-epoch=3
-model=svo
-
-echo  "---- train ----"
-time $ft supervised -input ${DIR}/ft_svo_data_train_1000000.dat  \
-  -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20
-
-echo "computing raw hit@5%..."
-$ft test ${model}.bin ${DIR}/ft_svo_data_test_250000.dat 227 2> /dev/null | awk '{if(NR==3) print "raw hit@5%="$2}'
-
-
-echo  "---- train + valid ----"
-time $ft supervised -input ${DIR}/ft_svo_data-valid+train.dat  \
-  -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20
-
-echo "computing raw hit@5%..."
-$ft test ${model}.bin ${DIR}/ft_svo_data_test_250000.dat 227 2> /dev/null | awk '{if(NR==3) print "raw hit@5%="$2}'

data/vendor/fastText/scripts/kbcompletion/wn18.sh

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-#
-# copyright (c) 2017-present, facebook, inc.
-# all rights reserved.
-#
-# this source code is licensed under the MIT license found in the
-# license file in the root directory of this source tree.
-#
-# script for WN11
-DIR=data/wordnet-mlj12/
-FASTTEXTDIR=../../
-
-# compile
-
-pushd $FASTTEXTDIR
-make opt
-popd
-ft=${FASTTEXTDIR}/fasttext
-
-g++ -std=c++0x eval.cpp -o eval
-
-# Train model and test it:
-dim=100
-epoch=100
-neg=500
-model=data/wn
-pred=data/wnpred
-
-echo  "---- train ----"
-$ft supervised -input ${DIR}/ft_wordnet-mlj12-train.txt  \
-  -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20 -loss ns -neg $neg
-
-echo "computing raw hits@10..."
-$ft test ${model}.bin ${DIR}/ft_wordnet-mlj12-test.txt 10 2> /dev/null | awk '{if(NR==3) print "raw hit@10 = "$2}'
-
-echo "computing filtered hit@10..."
-$ft predict ${model}.bin ${DIR}/ft_wordnet-mlj12-test.txt 20000 > $pred
-./eval $pred ${DIR}/ft_wordnet-mlj12-test.txt $DIR/ft_wordnet-mlj12-full.txt 10 | awk '{if(NR==2) print "filtered hit@10 = "$2}'
-
-echo  "---- train+val ----"
-$ft supervised -input ${DIR}/ft_wordnet-mlj12-valid+train.txt \
-  -dim $dim -epoch $epoch -output ${model} -lr .2 -thread 20 -loss ns -neg $neg
-
-echo "computing raw hits@10..."
-$ft test ${model}.bin ${DIR}/ft_wordnet-mlj12-test.txt 10  2> /dev/null | awk '{if(NR==3) print "raw hit@10 = "$2}'
-
-echo "computing filtered hit@10..."
-$ft predict ${model}.bin ${DIR}/ft_wordnet-mlj12-test.txt 20000 > $pred
-./eval $pred ${DIR}/ft_wordnet-mlj12-test.txt $DIR/ft_wordnet-mlj12-full.txt 10 | awk '{if(NR==2) print "filtered hit@10 = "$2}'

data/vendor/fastText/scripts/quantization/quantization-results.sh

@@ -1,43 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2016-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-#
-
-# This script applies quantization to the models from Table 1 in:
-# Bag of Tricks for Efficient Text Classification, arXiv 1607.01759, 2016
-
-set -e
-
-DATASET=(
-  ag_news
-  sogou_news
-  dbpedia
-  yelp_review_polarity
-  yelp_review_full
-  yahoo_answers
-  amazon_review_full
-  amazon_review_polarity
-)
-
-# These learning rates were chosen by validation on a subset of the training set.
-LR=( 0.25 0.5 0.5 0.1 0.1 0.1 0.05 0.05 )
-
-RESULTDIR=result
-DATADIR=data
-
-echo 'Warning! Make sure you run the classification-results.sh script before this one'
-echo 'Otherwise you can expect the commands in this script to fail'
-
-for i in {0..7}
-do
-  echo "Working on dataset ${DATASET[i]}"
-  ../../fasttext quantize -input "${DATADIR}/${DATASET[i]}.train" \
-    -output "${RESULTDIR}/${DATASET[i]}" -lr "${LR[i]}" \
-    -thread 4 -qnorm -retrain -epoch 5 -cutoff 100000 > /dev/null
-  ../../fasttext test "${RESULTDIR}/${DATASET[i]}.ftz" \
-    "${DATADIR}/${DATASET[i]}.test"
-done

data/vendor/fastText/setup.cfg

@@ -1,2 +0,0 @@
-[metadata]
-description-file = README.md

data/vendor/fastText/setup.py

@@ -1,203 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-#
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from setuptools import setup, Extension
-from setuptools.command.build_ext import build_ext
-import sys
-import setuptools
-import os
-import subprocess
-import platform
-import io
-
-__version__ = '0.9.1'
-FASTTEXT_SRC = "src"
-
-# Based on https://github.com/pybind/python_example
-
-class get_pybind_include(object):
-    """Helper class to determine the pybind11 include path
-
-    The purpose of this class is to postpone importing pybind11
-    until it is actually installed, so that the ``get_include()``
-    method can be invoked. """
-
-    def __init__(self, user=False):
-        try:
-            import pybind11
-        except ImportError:
-            if subprocess.call([sys.executable, '-m', 'pip', 'install', 'pybind11']):
-                raise RuntimeError('pybind11 install failed.')
-
-        self.user = user
-
-    def __str__(self):
-        import pybind11
-        return pybind11.get_include(self.user)
-
-try:
-    coverage_index = sys.argv.index('--coverage')
-except ValueError:
-    coverage = False
-else:
-    del sys.argv[coverage_index]
-    coverage = True
-
-fasttext_src_files = map(str, os.listdir(FASTTEXT_SRC))
-fasttext_src_cc = list(filter(lambda x: x.endswith('.cc'), fasttext_src_files))
-
-fasttext_src_cc = list(
-    map(lambda x: str(os.path.join(FASTTEXT_SRC, x)), fasttext_src_cc)
-)
-
-ext_modules = [
-    Extension(
-        str('fasttext_pybind'),
-        [
-            str('python/fasttext_module/fasttext/pybind/fasttext_pybind.cc'),
-        ] + fasttext_src_cc,
-        include_dirs=[
-            # Path to pybind11 headers
-            get_pybind_include(),
-            get_pybind_include(user=True),
-            # Path to fasttext source code
-            FASTTEXT_SRC,
-        ],
-        language='c++',
-        extra_compile_args=["-O0 -fno-inline -fprofile-arcs -pthread -march=native" if coverage else
-                            "-O3 -funroll-loops -pthread -march=native"],
-    ),
-]
-
-
-# As of Python 3.6, CCompiler has a `has_flag` method.
-# cf http://bugs.python.org/issue26689
-def has_flag(compiler, flags):
-    """Return a boolean indicating whether a flag name is supported on
-    the specified compiler.
-    """
-    import tempfile
-    with tempfile.NamedTemporaryFile('w', suffix='.cpp') as f:
-        f.write('int main (int argc, char **argv) { return 0; }')
-        try:
-            compiler.compile([f.name], extra_postargs=flags)
-        except setuptools.distutils.errors.CompileError:
-            return False
-    return True
-
-
-def cpp_flag(compiler):
-    """Return the -std=c++[0x/11/14] compiler flag.
-    The c++14 is preferred over c++0x/11 (when it is available).
-    """
-    standards = ['-std=c++14', '-std=c++11', '-std=c++0x']
-    for standard in standards:
-        if has_flag(compiler, [standard]):
-            return standard
-    raise RuntimeError(
-        'Unsupported compiler -- at least C++0x support '
-        'is needed!'
-    )
-
-
-class BuildExt(build_ext):
-    """A custom build extension for adding compiler-specific options."""
-    c_opts = {
-        'msvc': ['/EHsc'],
-        'unix': [],
-    }
-
-    def build_extensions(self):
-        if sys.platform == 'darwin':
-            mac_osx_version = float('.'.join(platform.mac_ver()[0].split('.')[:2]))
-            os.environ['MACOSX_DEPLOYMENT_TARGET'] = str(mac_osx_version)
-            all_flags = ['-stdlib=libc++', '-mmacosx-version-min=10.7']
-            if has_flag(self.compiler, [all_flags[0]]):
-                self.c_opts['unix'] += [all_flags[0]]
-            elif has_flag(self.compiler, all_flags):
-                self.c_opts['unix'] += all_flags
-            else:
-                raise RuntimeError(
-                    'libc++ is needed! Failed to compile with {} and {}.'.
-                    format(" ".join(all_flags), all_flags[0])
-                )
-        ct = self.compiler.compiler_type
-        opts = self.c_opts.get(ct, [])
-        extra_link_args = []
-
-        if coverage:
-            coverage_option = '--coverage'
-            opts.append(coverage_option)
-            extra_link_args.append(coverage_option)
-
-        if ct == 'unix':
-            opts.append('-DVERSION_INFO="%s"' % self.distribution.get_version())
-            opts.append(cpp_flag(self.compiler))
-            if has_flag(self.compiler, ['-fvisibility=hidden']):
-                opts.append('-fvisibility=hidden')
-        elif ct == 'msvc':
-            opts.append(
-                '/DVERSION_INFO=\\"%s\\"' % self.distribution.get_version()
-            )
-        for ext in self.extensions:
-            ext.extra_compile_args = opts
-            ext.extra_link_args = extra_link_args
-        build_ext.build_extensions(self)
-
-
-def _get_readme():
-    """
-    Use pandoc to generate rst from md.
-    pandoc --from=markdown --to=rst --output=python/README.rst python/README.md
-    """
-    with io.open("python/README.rst", encoding='utf-8') as fid:
-        return fid.read()
-
-
-setup(
-    name='fasttext',
-    version=__version__,
-    author='Onur Celebi',
-    author_email='celebio@fb.com',
-    description='fasttext Python bindings',
-    long_description=_get_readme(),
-    ext_modules=ext_modules,
-    url='https://github.com/facebookresearch/fastText',
-    license='MIT',
-    classifiers=[
-        'Development Status :: 3 - Alpha',
-        'Intended Audience :: Developers',
-        'Intended Audience :: Science/Research',
-        'License :: OSI Approved :: MIT License',
-        'Programming Language :: Python :: 2.7',
-        'Programming Language :: Python :: 3.4',
-        'Programming Language :: Python :: 3.5',
-        'Programming Language :: Python :: 3.6',
-        'Topic :: Software Development',
-        'Topic :: Scientific/Engineering',
-        'Operating System :: Microsoft :: Windows',
-        'Operating System :: POSIX',
-        'Operating System :: Unix',
-        'Operating System :: MacOS',
-    ],
-    install_requires=['pybind11>=2.2', "setuptools >= 0.7.0", "numpy"],
-    cmdclass={'build_ext': BuildExt},
-    packages=[
-        str('fasttext'),
-        str('fasttext.util'),
-        str('fasttext.tests'),
-    ],
-    package_dir={str(''): str('python/fasttext_module')},
-    zip_safe=False,
-)

data/vendor/fastText/tests/fetch_test_data.sh

@@ -1,202 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2016-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-#
-
-DATADIR=${DATADIR:-data}
-
-report_error() {
-   echo "Error on line $1 of $0"
-}
-
-myshuf() {
-  perl -MList::Util=shuffle -e 'print shuffle(<>);' "$@";
-}
-
-normalize_text() {
-  tr '[:upper:]' '[:lower:]' | sed -e 's/^/__label__/g' | \
-    sed -e "s/'/ ' /g" -e 's/"//g' -e 's/\./ \. /g' -e 's/<br \/>/ /g' \
-        -e 's/,/ , /g' -e 's/(/ ( /g' -e 's/)/ ) /g' -e 's/\!/ \! /g' \
-        -e 's/\?/ \? /g' -e 's/\;/ /g' -e 's/\:/ /g' | tr -s " " | myshuf
-}
-
-set -e
-trap 'report_error $LINENO' ERR
-
-mkdir -p "${DATADIR}"
-
-
-# Unsupervised datasets
-
-data_result="${DATADIR}/rw_queries.txt"
-if [ ! -f "$data_result" ]
-then
-  cut -f 1,2 "${DATADIR}"/rw/rw.txt | awk '{print tolower($0)}' | tr '\t' '\n' > "$data_result" || rm -f "$data_result"
-fi
-
-data_result="${DATADIR}/enwik9.zip"
-if [ ! -f "$data_result" ] || \
-   [ $(md5sum "$data_result" | cut -f 1 -d ' ') != "3e773f8a1577fda2e27f871ca17f31fd" ]
-then
-  wget -c http://mattmahoney.net/dc/enwik9.zip -P "${DATADIR}" || rm -f "$data_result"
-  unzip "$data_result" -d "${DATADIR}" || rm -f "$data_result"
-fi
-
-data_result="${DATADIR}/fil9"
-if [ ! -f "$data_result" ]
-then
-  perl wikifil.pl "${DATADIR}/enwik9" > "$data_result" || rm -f "$data_result"
-fi
-
-data_result="${DATADIR}/rw/rw.txt"
-if [ ! -f "$data_result" ]
-then
-  wget -c https://nlp.stanford.edu/~lmthang/morphoNLM/rw.zip -P "${DATADIR}"
-  unzip "${DATADIR}/rw.zip" -d "${DATADIR}" || rm -f "$data_result"
-fi
-
-# Supervised datasets
-# Each datasets comes with a .train and a .test to measure performance
-
-echo "Downloading dataset dbpedia"
-
-data_result="${DATADIR}/dbpedia_csv.tar.gz"
-if [ ! -f "$data_result" ] || \
-   [ $(md5sum "$data_result" | cut -f 1 -d ' ') != "8139d58cf075c7f70d085358e73af9b3" ]
-then
-  wget -c "https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz" -O "$data_result"
-  tar -xzvf "$data_result" -C "${DATADIR}"
-fi
-
-data_result="${DATADIR}/dbpedia.train"
-if [ ! -f "$data_result" ]
-then
-  cat "${DATADIR}/dbpedia_csv/train.csv" | normalize_text > "$data_result" || rm -f "$data_result"
-fi
-
-data_result="${DATADIR}/dbpedia.test"
-if [ ! -f "$data_result" ]
-then
-  cat "${DATADIR}/dbpedia_csv/test.csv" | normalize_text > "$data_result" || rm -f "$data_result"
-fi
-
-echo "Downloading dataset tatoeba for langid"
-
-data_result="${DATADIR}"/langid/all.txt
-if [ ! -f "$data_result" ]
-then
-  mkdir -p "${DATADIR}"/langid
-  wget http://downloads.tatoeba.org/exports/sentences.tar.bz2 -O "${DATADIR}"/langid/sentences.tar.bz2
-  tar xvfj "${DATADIR}"/langid/sentences.tar.bz2 --directory "${DATADIR}"/langid || exit 1
-  awk -F"\t" '{print"__label__"$2" "$3}' < "${DATADIR}"/langid/sentences.csv | shuf > "$data_result"
-fi
-
-data_result="${DATADIR}/langid.train"
-if [ ! -f "$data_result" ]
-then
-  tail -n +10001 "${DATADIR}"/langid/all.txt > "$data_result"
-fi
-
-data_result="${DATADIR}/langid.valid"
-if [ ! -f "$data_result" ]
-then
-  head -n 10000 "${DATADIR}"/langid/all.txt > "$data_result"
-fi
-
-echo "Downloading cooking dataset"
-
-data_result="${DATADIR}"/cooking/cooking.stackexchange.txt
-if [ ! -f "$data_result" ]
-then
-  mkdir -p "${DATADIR}"/cooking/
-  wget https://dl.fbaipublicfiles.com/fasttext/data/cooking.stackexchange.tar.gz -O "${DATADIR}"/cooking/cooking.stackexchange.tar.gz
-  tar xvzf "${DATADIR}"/cooking/cooking.stackexchange.tar.gz --directory "${DATADIR}"/cooking || exit 1
-  cat "${DATADIR}"/cooking/cooking.stackexchange.txt | sed -e "s/\([.\!?,'/()]\)/ \1 /g" | tr "[:upper:]" "[:lower:]" > "${DATADIR}"/cooking/cooking.preprocessed.txt
-fi
-
-data_result="${DATADIR}"/cooking.train
-if [ ! -f "$data_result" ]
-then
-  head -n 12404 "${DATADIR}"/cooking/cooking.preprocessed.txt > "${DATADIR}"/cooking.train
-fi
-
-data_result="${DATADIR}"/cooking.valid
-if [ ! -f "$data_result" ]
-then
-  tail -n 3000 "${DATADIR}"/cooking/cooking.preprocessed.txt > "${DATADIR}"/cooking.valid
-fi
-
-echo "Checking for YFCC100M"
-
-data_result="${DATADIR}"/YFCC100M/train
-if [ ! -f "$data_result" ]
-then
-  echo 'Download YFCC100M, unpack it and place train into the following path: '"$data_result"
-  echo 'You can download YFCC100M at :'"https://fasttext.cc/docs/en/dataset.html"
-  echo 'After you download this, run the script again'
-  exit 1
-fi
-
-data_result="${DATADIR}"/YFCC100M/test
-if [ ! -f "$data_result" ]
-then
-  echo 'Download YFCC100M, unpack it and place test into the following path: '"$data_result"
-  echo 'You can download YFCC100M at :'"https://fasttext.cc/docs/en/dataset.html"
-  echo 'After you download this, run the script again'
-  exit 1
-fi
-
-DATASET=(
-  ag_news
-  sogou_news
-  dbpedia
-  yelp_review_polarity
-  yelp_review_full
-  yahoo_answers
-  amazon_review_full
-  amazon_review_polarity
-)
-
-ID=(
-  0Bz8a_Dbh9QhbUDNpeUdjb0wxRms # ag_news
-  0Bz8a_Dbh9QhbUkVqNEszd0pHaFE # sogou_news
-  0Bz8a_Dbh9QhbQ2Vic1kxMmZZQ1k # dbpedia
-  0Bz8a_Dbh9QhbNUpYQ2N3SGlFaDg # yelp_review_polarity
-  0Bz8a_Dbh9QhbZlU4dXhHTFhZQU0 # yelp_review_full
-  0Bz8a_Dbh9Qhbd2JNdDBsQUdocVU # yahoo_answers
-  0Bz8a_Dbh9QhbZVhsUnRWRDhETzA # amazon_review_full
-  0Bz8a_Dbh9QhbaW12WVVZS2drcnM # amazon_review_polarity
-)
-
-# Small datasets first
-
-for i in {0..0}
-do
-  echo "Downloading dataset ${DATASET[i]}"
-  if [ ! -f "${DATADIR}/${DATASET[i]}.train" ]
-  then
-    wget -c "https://drive.google.com/uc?export=download&id=${ID[i]}" -O "${DATADIR}/${DATASET[i]}_csv.tar.gz"
-    tar -xzvf "${DATADIR}/${DATASET[i]}_csv.tar.gz" -C "${DATADIR}"
-    cat "${DATADIR}/${DATASET[i]}_csv/train.csv" | normalize_text > "${DATADIR}/${DATASET[i]}.train"
-    cat "${DATADIR}/${DATASET[i]}_csv/test.csv" | normalize_text > "${DATADIR}/${DATASET[i]}.test"
-  fi
-done
-
-# Large datasets require a bit more work due to the extra request page
-
-for i in {1..7}
-do
-  echo "Downloading dataset ${DATASET[i]}"
-  if [ ! -f "${DATADIR}/${DATASET[i]}.train" ]
-  then
-    curl -c /tmp/cookies "https://drive.google.com/uc?export=download&id=${ID[i]}" > /tmp/intermezzo.html
-    curl -L -b /tmp/cookies "https://drive.google.com$(cat /tmp/intermezzo.html | grep -Po 'uc-download-link" [^>]* href="\K[^"]*' | sed 's/\&amp;/\&/g')" > "${DATADIR}/${DATASET[i]}_csv.tar.gz"
-    tar -xzvf "${DATADIR}/${DATASET[i]}_csv.tar.gz" -C "${DATADIR}"
-    cat "${DATADIR}/${DATASET[i]}_csv/train.csv" | normalize_text > "${DATADIR}/${DATASET[i]}.train"
-    cat "${DATADIR}/${DATASET[i]}_csv/test.csv" | normalize_text > "${DATADIR}/${DATASET[i]}.test"
-  fi
-done