fasttext 0.1.0 → 0.2.1

data/vendor/fastText/CMakeLists.txt

@@ -1,68 +0,0 @@
-#
-# Copyright (c) 2016-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-#
-
-cmake_minimum_required(VERSION 2.8.9)
-project(fasttext)
-
-# The version number.
-set (fasttext_VERSION_MAJOR 0)
-set (fasttext_VERSION_MINOR 1)
-
-include_directories(fasttext)
-
-set(CMAKE_CXX_FLAGS " -pthread -std=c++11 -funroll-loops -O3 -march=native")
-
-set(HEADER_FILES
-    src/args.h
-    src/densematrix.h
-    src/dictionary.h
-    src/fasttext.h
-    src/loss.h
-    src/matrix.h
-    src/meter.h
-    src/model.h
-    src/productquantizer.h
-    src/quantmatrix.h
-    src/real.h
-    src/utils.h
-    src/vector.h)
-
-set(SOURCE_FILES
-    src/args.cc
-    src/densematrix.cc
-    src/dictionary.cc
-    src/fasttext.cc
-    src/loss.cc
-    src/main.cc
-    src/matrix.cc
-    src/meter.cc
-    src/model.cc
-    src/productquantizer.cc
-    src/quantmatrix.cc
-    src/utils.cc
-    src/vector.cc)
-
-add_library(fasttext-shared SHARED ${SOURCE_FILES} ${HEADER_FILES})
-add_library(fasttext-static STATIC ${SOURCE_FILES} ${HEADER_FILES})
-add_library(fasttext-static_pic STATIC ${SOURCE_FILES} ${HEADER_FILES})
-set_target_properties(fasttext-shared PROPERTIES OUTPUT_NAME fasttext)
-set_target_properties(fasttext-static PROPERTIES OUTPUT_NAME fasttext)
-set_target_properties(fasttext-static_pic PROPERTIES OUTPUT_NAME fasttext_pic
-  POSITION_INDEPENDENT_CODE True)
-add_executable(fasttext-bin src/main.cc)
-target_link_libraries(fasttext-bin pthread fasttext-static)
-set_target_properties(fasttext-bin PROPERTIES PUBLIC_HEADER "${HEADER_FILES}" OUTPUT_NAME fasttext)
-install (TARGETS fasttext-shared
-    LIBRARY DESTINATION lib)
-install (TARGETS fasttext-static
-    ARCHIVE DESTINATION lib)
-install (TARGETS fasttext-static_pic
-    ARCHIVE DESTINATION lib)
-install (TARGETS fasttext-bin
-    RUNTIME DESTINATION bin
- PUBLIC_HEADER DESTINATION include/fasttext)

data/vendor/fastText/CODE_OF_CONDUCT.md

@@ -1,2 +0,0 @@
-# Code of Conduct
-Facebook has adopted a Code of Conduct that we expect project participants to adhere to. Please [read the full text](https://code.fb.com/codeofconduct) so that you can understand what actions will and will not be tolerated.

data/vendor/fastText/CONTRIBUTING.md

@@ -1,32 +0,0 @@
-# Contributing to fastText
-We want to make contributing to this project as easy and transparent as possible.
-
-## Issues
-We use GitHub issues to track public bugs. Please ensure your description is clear and has sufficient instructions to be able to reproduce the issue.
-
-### Reproducing issues
-Please make sure that the issue you mention is not a result of one of the existing third-party libraries. For example, please do not post an issue if you encountered an error within a third-party Python library. We can only help you with errors which can be directly reproduced either with our C++ code or the corresponding Python bindings. If you do find an error, please post detailed steps to reproduce it. If we can't reproduce your error, we can't help you fix it.
-
-## Pull Requests
-Please post an Issue before submitting a pull request. This might save you some time as it is possible we can't support your contribution, albeit we try our best to accomodate your (planned) work and highly appreciate your time. Generally, it is best to have a pull request emerge from an issue rather than the other way around.
-
-To create a pull request:
-
-1. Fork the repo and create your branch from `master`.
-2. If you've added code that should be tested, add tests.
-3. If you've changed APIs, update the documentation.
-4. Ensure the test suite passes.
-5. Make sure your code lints.
-6. If you haven't already, complete the Contributor License Agreement ("CLA").
-
-## Tests
-First, you will need to make sure you have the required data. For that, please have a look at the fetch_test_data.sh script under tests. Next run the tests using the runtests.py script passing a path to the directory containing the datasets.
-
-## Contributor License Agreement ("CLA")
-In order to accept your pull request, we need you to submit a CLA. You only need
-to do this once to work on any of Facebook's open source projects.
-
-Complete your CLA here: <https://code.facebook.com/cla>
-
-## License
-By contributing to fastText, you agree that your contributions will be licensed under its MIT license.

data/vendor/fastText/MANIFEST.in

@@ -1,5 +0,0 @@
-include LICENSE
-include PATENTS
-
-recursive-include python *.md *.rst
-recursive-include src *.h

data/vendor/fastText/Makefile

@@ -1,63 +0,0 @@
-#
-# Copyright (c) 2016-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-#
-
-CXX = c++
-CXXFLAGS = -pthread -std=c++0x -march=native
-OBJS = args.o matrix.o dictionary.o loss.o productquantizer.o densematrix.o quantmatrix.o vector.o model.o utils.o meter.o fasttext.o
-INCLUDES = -I.
-
-opt: CXXFLAGS += -O3 -funroll-loops -DNDEBUG
-opt: fasttext
-
-coverage: CXXFLAGS += -O0 -fno-inline -fprofile-arcs --coverage
-coverage: fasttext
-
-debug: CXXFLAGS += -g -O0 -fno-inline
-debug: fasttext
-
-args.o: src/args.cc src/args.h
-	$(CXX) $(CXXFLAGS) -c src/args.cc
-
-matrix.o: src/matrix.cc src/matrix.h
-	$(CXX) $(CXXFLAGS) -c src/matrix.cc
-
-dictionary.o: src/dictionary.cc src/dictionary.h src/args.h
-	$(CXX) $(CXXFLAGS) -c src/dictionary.cc
-
-loss.o: src/loss.cc src/loss.h src/matrix.h src/real.h
-	$(CXX) $(CXXFLAGS) -c src/loss.cc
-
-productquantizer.o: src/productquantizer.cc src/productquantizer.h src/utils.h
-	$(CXX) $(CXXFLAGS) -c src/productquantizer.cc
-
-densematrix.o: src/densematrix.cc src/densematrix.h src/utils.h src/matrix.h
-	$(CXX) $(CXXFLAGS) -c src/densematrix.cc
-
-quantmatrix.o: src/quantmatrix.cc src/quantmatrix.h src/utils.h src/matrix.h
-	$(CXX) $(CXXFLAGS) -c src/quantmatrix.cc
-
-vector.o: src/vector.cc src/vector.h src/utils.h
-	$(CXX) $(CXXFLAGS) -c src/vector.cc
-
-model.o: src/model.cc src/model.h src/args.h
-	$(CXX) $(CXXFLAGS) -c src/model.cc
-
-utils.o: src/utils.cc src/utils.h
-	$(CXX) $(CXXFLAGS) -c src/utils.cc
-
-meter.o: src/meter.cc src/meter.h
-	$(CXX) $(CXXFLAGS) -c src/meter.cc
-
-fasttext.o: src/fasttext.cc src/*.h
-	$(CXX) $(CXXFLAGS) -c src/fasttext.cc
-
-fasttext: $(OBJS) src/fasttext.cc
-	$(CXX) $(CXXFLAGS) $(OBJS) src/main.cc -o fasttext
-
-clean:
-	rm -rf *.o *.gcno *.gcda fasttext

data/vendor/fastText/alignment/README.md

@@ -1,53 +0,0 @@
-## Alignment of Word Embeddings
-
-This directory provides code for learning alignments between word embeddings in different languages.
-
-The code is in Python 3 and requires [NumPy](http://www.numpy.org/).
-
-The script `example.sh` shows how to use this code to learn and evaluate a bilingual alignment of word embeddings.
-
-The word embeddings used in [1] can be found on the [fastText project page](https://fasttext.cc) and the supervised bilingual lexicons on the [MUSE project page](https://github.com/facebookresearch/MUSE).
-
-### Supervised alignment
-
-The script `align.py` aligns word embeddings from two languages using a bilingual lexicon as supervision.
-The details of this approach can be found in [1].
-
-### Unsupervised alignment
-
-The script `unsup_align.py` aligns word embeddings from two languages without requiring any supervision.
-The details of this approach can be found in [2].
-
-In addition to NumPy, the unsupervised method requires the [Python Optimal Transport](https://pot.readthedocs.io/en/stable/) toolbox.
-
-### Download
-
-Wikipedia fastText embeddings aligned with our method can be found [here](https://fasttext.cc/docs/en/aligned-vectors.html).
-
-### References
-
-If you use the supervised alignment method, please cite:
-
-[1] A. Joulin, P. Bojanowski, T. Mikolov, H. Jegou, E. Grave, [*Loss in Translation: Learning Bilingual Word Mapping with a Retrieval Criterion*](https://arxiv.org/abs/1804.07745)
-
-```
-@InProceedings{joulin2018loss,
-    title={Loss in Translation: Learning Bilingual Word Mapping with a Retrieval Criterion},
-    author={Joulin, Armand and Bojanowski, Piotr and Mikolov, Tomas and J\'egou, Herv\'e and Grave, Edouard},
-    year={2018},
-    booktitle={Proceedings of the 2018 Conference on Empirical Methods in Natural Language Processing},
-}
-```
-
-If you use the unsupervised alignment method, please cite:
-
-[2] E. Grave, A. Joulin, Q. Berthet, [*Unsupervised Alignment of Embeddings with Wasserstein Procrustes*](https://arxiv.org/abs/1805.11222)
-
-```
-@article{grave2018unsupervised,
-    title={Unsupervised Alignment of Embeddings with Wasserstein Procrustes},
-    author={Grave, Edouard and Joulin, Armand and Berthet, Quentin},
-    journal={arXiv preprint arXiv:1805.11222},
-    year={2018}
-}
-```

data/vendor/fastText/alignment/align.py

@@ -1,145 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# Copyright (c) 2018-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the license found in the
-# LICENSE file in the root directory of this source tree.
-
-import numpy as np
-import argparse
-from utils import *
-import sys
-
-parser = argparse.ArgumentParser(description='RCSLS for supervised word alignment')
-
-parser.add_argument("--src_emb", type=str, default='', help="Load source embeddings")
-parser.add_argument("--tgt_emb", type=str, default='', help="Load target embeddings")
-parser.add_argument('--center', action='store_true', help='whether to center embeddings or not')
-
-parser.add_argument("--dico_train", type=str, default='', help="train dictionary")
-parser.add_argument("--dico_test", type=str, default='', help="validation dictionary")
-
-parser.add_argument("--output", type=str, default='', help="where to save aligned embeddings")
-
-parser.add_argument("--knn", type=int, default=10, help="number of nearest neighbors in RCSL/CSLS")
-parser.add_argument("--maxneg", type=int, default=200000, help="Maximum number of negatives for the Extended RCSLS")
-parser.add_argument("--maxsup", type=int, default=-1, help="Maximum number of training examples")
-parser.add_argument("--maxload", type=int, default=200000, help="Maximum number of loaded vectors")
-
-parser.add_argument("--model", type=str, default="none", help="Set of constraints: spectral or none")
-parser.add_argument("--reg", type=float, default=0.0 , help='regularization parameters')
-
-parser.add_argument("--lr", type=float, default=1.0, help='learning rate')
-parser.add_argument("--niter", type=int, default=10, help='number of iterations')
-parser.add_argument('--sgd', action='store_true', help='use sgd')
-parser.add_argument("--batchsize", type=int, default=10000, help="batch size for sgd")
-
-params = parser.parse_args()
-
-###### SPECIFIC FUNCTIONS ######
-# functions specific to RCSLS
-# the rest of the functions are in utils.py
-
-def getknn(sc, x, y, k=10):
-    sidx = np.argpartition(sc, -k, axis=1)[:, -k:]
-    ytopk = y[sidx.flatten(), :]
-    ytopk = ytopk.reshape(sidx.shape[0], sidx.shape[1], y.shape[1])
-    f = np.sum(sc[np.arange(sc.shape[0])[:, None], sidx])
-    df = np.dot(ytopk.sum(1).T, x)
-    return f / k, df / k
-
-
-def rcsls(X_src, Y_tgt, Z_src, Z_tgt, R, knn=10):
-    X_trans = np.dot(X_src, R.T)
-    f = 2 * np.sum(X_trans * Y_tgt)
-    df = 2 * np.dot(Y_tgt.T, X_src)
-    fk0, dfk0 = getknn(np.dot(X_trans, Z_tgt.T), X_src, Z_tgt, knn)
-    fk1, dfk1 = getknn(np.dot(np.dot(Z_src, R.T), Y_tgt.T).T, Y_tgt, Z_src, knn)
-    f = f - fk0 -fk1
-    df = df - dfk0 - dfk1.T
-    return -f / X_src.shape[0], -df / X_src.shape[0]
-
-
-def proj_spectral(R):
-    U, s, V = np.linalg.svd(R)
-    s[s > 1] = 1
-    s[s < 0] = 0
-    return np.dot(U, np.dot(np.diag(s), V))
-
-
-###### MAIN ######
-
-# load word embeddings
-words_tgt, x_tgt = load_vectors(params.tgt_emb, maxload=params.maxload, center=params.center)
-words_src, x_src = load_vectors(params.src_emb, maxload=params.maxload, center=params.center)
-
-# load validation bilingual lexicon
-src2tgt, lexicon_size = load_lexicon(params.dico_test, words_src, words_tgt)
-
-# word --> vector indices
-idx_src = idx(words_src)
-idx_tgt = idx(words_tgt)
-
-# load train bilingual lexicon
-pairs = load_pairs(params.dico_train, idx_src, idx_tgt)
-if params.maxsup > 0 and params.maxsup < len(pairs):
-    pairs = pairs[:params.maxsup]
-
-# selecting training vector  pairs
-X_src, Y_tgt = select_vectors_from_pairs(x_src, x_tgt, pairs)
-
-# adding negatives for RCSLS
-Z_src = x_src[:params.maxneg, :]
-Z_tgt = x_tgt[:params.maxneg, :]
-
-# initialization:
-R = procrustes(X_src, Y_tgt)
-nnacc = compute_nn_accuracy(np.dot(x_src, R.T), x_tgt, src2tgt, lexicon_size=lexicon_size)
-print("[init -- Procrustes] NN: %.4f"%(nnacc))
-sys.stdout.flush()
-
-# optimization
-fold, Rold = 0, []
-niter, lr = params.niter, params.lr
-
-for it in range(0, niter + 1):
-    if lr < 1e-4:
-        break
-
-    if params.sgd:
-        indices = np.random.choice(X_src.shape[0], size=params.batchsize, replace=False)
-        f, df = rcsls(X_src[indices, :], Y_tgt[indices, :], Z_src, Z_tgt, R, params.knn)
-    else:
-        f, df = rcsls(X_src, Y_tgt, Z_src, Z_tgt, R, params.knn)
-
-    if params.reg > 0:
-        R *= (1 - lr * params.reg)
-    R -= lr * df
-    if params.model == "spectral":
-        R = proj_spectral(R)
-
-    print("[it=%d] f = %.4f" % (it, f))
-    sys.stdout.flush()
-
-    if f > fold and it > 0 and not params.sgd:
-        lr /= 2
-        f, R = fold, Rold
-
-    fold, Rold = f, R
-
-    if (it > 0 and it % 10 == 0) or it == niter:
-        nnacc = compute_nn_accuracy(np.dot(x_src, R.T), x_tgt, src2tgt, lexicon_size=lexicon_size)
-        print("[it=%d] NN = %.4f - Coverage = %.4f" % (it, nnacc, len(src2tgt) / lexicon_size))
-
-nnacc = compute_nn_accuracy(np.dot(x_src, R.T), x_tgt, src2tgt, lexicon_size=lexicon_size)
-print("[final] NN = %.4f - Coverage = %.4f" % (nnacc, len(src2tgt) / lexicon_size))
-
-if params.output != "":
-    print("Saving all aligned vectors at %s" % params.output)
-    words_full, x_full = load_vectors(params.src_emb, maxload=-1, center=params.center, verbose=False)
-    x = np.dot(x_full, R.T)
-    x /= np.linalg.norm(x, axis=1)[:, np.newaxis] + 1e-8
-    save_vectors(params.output, x, words_full)
-    save_matrix(params.output + "-mat",  R)

data/vendor/fastText/alignment/eval.py

@@ -1,60 +0,0 @@
-#!/usr/bin/env python3
-# -*- coding: utf-8 -*-
-#
-# Copyright (c) 2018-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the license found in the
-# LICENSE file in the root directory of this source tree.
-
-import io
-import numpy as np
-import argparse
-from utils import *
-
-parser = argparse.ArgumentParser(description='Evaluation of word alignment')
-parser.add_argument("--src_emb", type=str, default='', help="Load source embeddings")
-parser.add_argument("--tgt_emb", type=str, default='', help="Load target embeddings")
-parser.add_argument('--center', action='store_true', help='whether to center embeddings or not')
-parser.add_argument("--src_mat", type=str, default='', help="Load source alignment matrix. If none given, the aligment matrix is the identity.")
-parser.add_argument("--tgt_mat", type=str, default='', help="Load target alignment matrix. If none given, the aligment matrix is the identity.")
-parser.add_argument("--dico_test", type=str, default='', help="test dictionary")
-parser.add_argument("--maxload", type=int, default=200000)
-parser.add_argument("--nomatch", action='store_true', help="no exact match in lexicon")
-params = parser.parse_args()
-
-
-###### SPECIFIC FUNCTIONS ######
-# function specific to evaluation
-# the rest of the functions are in utils.py
-
-def load_transform(fname, d1=300, d2=300):
-    fin = io.open(fname, 'r', encoding='utf-8', newline='\n', errors='ignore')
-    R = np.zeros([d1, d2])
-    for i, line in enumerate(fin):
-        tokens = line.split(' ')
-        R[i, :] = np.array(tokens[0:d2], dtype=float)
-    return R
-
-
-###### MAIN ######
-
-print("Evaluation of alignment on %s" % params.dico_test)
-if params.nomatch:
-    print("running without exact string matches")
-
-words_tgt, x_tgt = load_vectors(params.tgt_emb, maxload=params.maxload, center=params.center)
-words_src, x_src = load_vectors(params.src_emb, maxload=params.maxload, center=params.center)
-
-if params.tgt_mat != "":
-    R_tgt = load_transform(params.tgt_mat)
-    x_tgt = np.dot(x_tgt, R_tgt)
-if params.src_mat != "":
-    R_src = load_transform(params.src_mat)
-    x_src = np.dot(x_src, R_src)
-
-src2tgt, lexicon_size = load_lexicon(params.dico_test, words_src, words_tgt)
-
-nnacc = compute_nn_accuracy(x_src, x_tgt, src2tgt, lexicon_size=lexicon_size)
-cslsproc = compute_csls_accuracy(x_src, x_tgt, src2tgt, lexicon_size=lexicon_size)
-print("NN = %.4f - CSLS = %.4f - Coverage = %.4f" % (nnacc, cslsproc, len(src2tgt) / lexicon_size))

data/vendor/fastText/alignment/example.sh

@@ -1,51 +0,0 @@
-#!/bin/usr/env sh
-# Copyright (c) 2018-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the license found in the
-# LICENSE file in the root directory of this source tree.
-
-set -e
-s=${1:-en}
-t=${2:-es}
-echo "Example based on the ${s}->${t} alignment"
-
-if [ ! -d data/ ]; then
-  mkdir -p data;
-fi
-
-if [ ! -d res/ ]; then
-  mkdir -p res;
-fi
-
-dico_train=data/${s}-${t}.0-5000.txt
-if [ ! -f "${dico_train}" ]; then
-  DICO=$(basename -- "${dico_train}")
-  wget -c "https://dl.fbaipublicfiles.com/arrival/dictionaries/${DICO}" -P data/
-fi
-
-dico_test=data/${s}-${t}.5000-6500.txt
-if [ ! -f "${dico_test}" ]; then
-  DICO=$(basename -- "${dico_test}")
-  wget -c "https://dl.fbaipublicfiles.com/arrival/dictionaries/${DICO}" -P data/
-fi
-
-src_emb=data/wiki.${s}.vec
-if [ ! -f "${src_emb}" ]; then
-  EMB=$(basename -- "${src_emb}")
-  wget -c "https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/${EMB}" -P data/
-fi
-
-tgt_emb=data/wiki.${t}.vec
-if [ ! -f "${tgt_emb}" ]; then
-  EMB=$(basename -- "${tgt_emb}")
-  wget -c "https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/${EMB}" -P data/
-fi
-
-output=res/wiki.${s}-${t}.vec
-
-python3 align.py --src_emb "${src_emb}" --tgt_emb "${tgt_emb}" \
-  --dico_train "${dico_train}" --dico_test "${dico_test}" --output "${output}" \
-  --lr 25 --niter 10
-python3 eval.py --src_emb "${output}" --tgt_emb "${tgt_emb}" \
-  --dico_test "${dico_test}"