fasttext 0.1.0 → 0.1.1

data/vendor/fastText/alignment/unsup_align.py

@@ -1,109 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2018-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-import codecs, sys, time, math, argparse, ot
-import numpy as np
-from utils import *
-
-parser = argparse.ArgumentParser(description='Wasserstein Procrustes for Embedding Alignment')
-parser.add_argument('--model_src', type=str, help='Path to source word embeddings')
-parser.add_argument('--model_tgt', type=str, help='Path to target word embeddings')
-parser.add_argument('--lexicon', type=str, help='Path to the evaluation lexicon')
-parser.add_argument('--output_src', default='', type=str, help='Path to save the aligned source embeddings')
-parser.add_argument('--output_tgt', default='', type=str, help='Path to save the aligned target embeddings')
-parser.add_argument('--seed', default=1111, type=int, help='Random number generator seed')
-parser.add_argument('--nepoch', default=5, type=int, help='Number of epochs')
-parser.add_argument('--niter', default=5000, type=int, help='Initial number of iterations')
-parser.add_argument('--bsz', default=500, type=int, help='Initial batch size')
-parser.add_argument('--lr', default=500., type=float, help='Learning rate')
-parser.add_argument('--nmax', default=20000, type=int, help='Vocabulary size for learning the alignment')
-parser.add_argument('--reg', default=0.05, type=float, help='Regularization parameter for sinkhorn')
-args = parser.parse_args()
-
-
-def objective(X, Y, R, n=5000):
-    Xn, Yn = X[:n], Y[:n]
-    C = -np.dot(np.dot(Xn, R), Yn.T)
-    P = ot.sinkhorn(np.ones(n), np.ones(n), C, 0.025, stopThr=1e-3)
-    return 1000 * np.linalg.norm(np.dot(Xn, R) - np.dot(P, Yn)) / n
-
-
-def sqrt_eig(x):
-    U, s, VT = np.linalg.svd(x, full_matrices=False)
-    return np.dot(U, np.dot(np.diag(np.sqrt(s)), VT))
-
-
-def align(X, Y, R, lr=10., bsz=200, nepoch=5, niter=1000,
-          nmax=10000, reg=0.05, verbose=True):
-    for epoch in range(1, nepoch + 1):
-        for _it in range(1, niter + 1):
-            # sample mini-batch
-            xt = X[np.random.permutation(nmax)[:bsz], :]
-            yt = Y[np.random.permutation(nmax)[:bsz], :]
-            # compute OT on minibatch
-            C = -np.dot(np.dot(xt, R), yt.T)
-            P = ot.sinkhorn(np.ones(bsz), np.ones(bsz), C, reg, stopThr=1e-3)
-            # compute gradient
-            G = - np.dot(xt.T, np.dot(P, yt))
-            R -= lr / bsz * G
-            # project on orthogonal matrices
-            U, s, VT = np.linalg.svd(R)
-            R = np.dot(U, VT)
-        bsz *= 2
-        niter //= 4
-        if verbose:
-            print("epoch: %d  obj: %.3f" % (epoch, objective(X, Y, R)))
-    return R
-
-
-def convex_init(X, Y, niter=100, reg=0.05, apply_sqrt=False):
-    n, d = X.shape
-    if apply_sqrt:
-        X, Y = sqrt_eig(X), sqrt_eig(Y)
-    K_X, K_Y = np.dot(X, X.T), np.dot(Y, Y.T)
-    K_Y *= np.linalg.norm(K_X) / np.linalg.norm(K_Y)
-    K2_X, K2_Y = np.dot(K_X, K_X), np.dot(K_Y, K_Y)
-    P = np.ones([n, n]) / float(n)
-    for it in range(1, niter + 1):
-        G = np.dot(P, K2_X) + np.dot(K2_Y, P) - 2 * np.dot(K_Y, np.dot(P, K_X))
-        q = ot.sinkhorn(np.ones(n), np.ones(n), G, reg, stopThr=1e-3)
-        alpha = 2.0 / float(2.0 + it)
-        P = alpha * q + (1.0 - alpha) * P
-    obj = np.linalg.norm(np.dot(P, K_X) - np.dot(K_Y, P))
-    print(obj)
-    return procrustes(np.dot(P, X), Y).T
-
-
-print("\n*** Wasserstein Procrustes ***\n")
-
-np.random.seed(args.seed)
-
-maxload = 200000
-w_src, x_src = load_vectors(args.model_src, maxload, norm=True, center=True)
-w_tgt, x_tgt = load_vectors(args.model_tgt, maxload, norm=True, center=True)
-src2trg, _ = load_lexicon(args.lexicon, w_src, w_tgt)
-
-print("\nComputing initial mapping with convex relaxation...")
-t0 = time.time()
-R0 = convex_init(x_src[:2500], x_tgt[:2500], reg=args.reg, apply_sqrt=True)
-print("Done [%03d sec]" % math.floor(time.time() - t0))
-
-print("\nComputing mapping with Wasserstein Procrustes...")
-t0 = time.time()
-R = align(x_src, x_tgt, R0.copy(), bsz=args.bsz, lr=args.lr, niter=args.niter,
-          nepoch=args.nepoch, reg=args.reg, nmax=args.nmax)
-print("Done [%03d sec]" % math.floor(time.time() - t0))
-
-acc = compute_nn_accuracy(x_src, np.dot(x_tgt, R.T), src2trg)
-print("\nPrecision@1: %.3f\n" % acc)
-
-if args.output_src != '':
-    x_src = x_src / np.linalg.norm(x_src, 2, 1).reshape([-1, 1])
-    save_vectors(args.output_src, x_src, w_src)
-if args.output_tgt != '':
-    x_tgt = x_tgt / np.linalg.norm(x_tgt, 2, 1).reshape([-1, 1])
-    save_vectors(args.output_tgt, np.dot(x_tgt, R.T), w_tgt)

data/vendor/fastText/alignment/utils.py

@@ -1,154 +0,0 @@
-#!/usr/bin/env python3
-# Copyright (c) 2018-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the license found in the
-# LICENSE file in the root directory of this source tree.
-
-import io
-import numpy as np
-import collections
-
-
-def load_vectors(fname, maxload=200000, norm=True, center=False, verbose=True):
-    if verbose:
-        print("Loading vectors from %s" % fname)
-    fin = io.open(fname, 'r', encoding='utf-8', newline='\n', errors='ignore')
-    n, d = map(int, fin.readline().split())
-    if maxload > 0:
-        n = min(n, maxload)
-    x = np.zeros([n, d])
-    words = []
-    for i, line in enumerate(fin):
-        if i >= n:
-            break
-        tokens = line.rstrip().split(' ')
-        words.append(tokens[0])
-        v = np.array(tokens[1:], dtype=float)
-        x[i, :] = v
-    if norm:
-        x /= np.linalg.norm(x, axis=1)[:, np.newaxis] + 1e-8
-    if center:
-        x -= x.mean(axis=0)[np.newaxis, :]
-        x /= np.linalg.norm(x, axis=1)[:, np.newaxis] + 1e-8
-    if verbose:
-        print("%d word vectors loaded" % (len(words)))
-    return words, x
-
-
-def idx(words):
-    w2i = {}
-    for i, w in enumerate(words):
-        if w not in w2i:
-            w2i[w] = i
-    return w2i
-
-
-def save_vectors(fname, x, words):
-    n, d = x.shape
-    fout = io.open(fname, 'w', encoding='utf-8')
-    fout.write(u"%d %d\n" % (n, d))
-    for i in range(n):
-        fout.write(words[i] + " " + " ".join(map(lambda a: "%.4f" % a, x[i, :])) + "\n")
-    fout.close()
-
-
-def save_matrix(fname, x):
-    n, d = x.shape
-    fout = io.open(fname, 'w', encoding='utf-8')
-    fout.write(u"%d %d\n" % (n, d))
-    for i in range(n):
-        fout.write(" ".join(map(lambda a: "%.4f" % a, x[i, :])) + "\n")
-    fout.close()
-
-
-def procrustes(X_src, Y_tgt):
-    U, s, V = np.linalg.svd(np.dot(Y_tgt.T, X_src))
-    return np.dot(U, V)
-
-
-def select_vectors_from_pairs(x_src, y_tgt, pairs):
-    n = len(pairs)
-    d = x_src.shape[1]
-    x = np.zeros([n, d])
-    y = np.zeros([n, d])
-    for k, ij in enumerate(pairs):
-        i, j = ij
-        x[k, :] = x_src[i, :]
-        y[k, :] = y_tgt[j, :]
-    return x, y
-
-
-def load_lexicon(filename, words_src, words_tgt, verbose=True):
-    f = io.open(filename, 'r', encoding='utf-8')
-    lexicon = collections.defaultdict(set)
-    idx_src , idx_tgt = idx(words_src), idx(words_tgt)
-    vocab = set()
-    for line in f:
-        word_src, word_tgt = line.split()
-        if word_src in idx_src and word_tgt in idx_tgt:
-            lexicon[idx_src[word_src]].add(idx_tgt[word_tgt])
-        vocab.add(word_src)
-    if verbose:
-        coverage = len(lexicon) / float(len(vocab))
-        print("Coverage of source vocab: %.4f" % (coverage))
-    return lexicon, float(len(vocab))
-
-
-def load_pairs(filename, idx_src, idx_tgt, verbose=True):
-    f = io.open(filename, 'r', encoding='utf-8')
-    pairs = []
-    tot = 0
-    for line in f:
-        a, b = line.rstrip().split(' ')
-        tot += 1
-        if a in idx_src and b in idx_tgt:
-            pairs.append((idx_src[a], idx_tgt[b]))
-    if verbose:
-        coverage = (1.0 * len(pairs)) / tot
-        print("Found pairs for training: %d - Total pairs in file: %d - Coverage of pairs: %.4f" % (len(pairs), tot, coverage))
-    return pairs
-
-
-def compute_nn_accuracy(x_src, x_tgt, lexicon, bsz=100, lexicon_size=-1):
-    if lexicon_size < 0:
-        lexicon_size = len(lexicon)
-    idx_src = list(lexicon.keys())
-    acc = 0.0
-    x_src /= np.linalg.norm(x_src, axis=1)[:, np.newaxis] + 1e-8
-    x_tgt /= np.linalg.norm(x_tgt, axis=1)[:, np.newaxis] + 1e-8
-    for i in range(0, len(idx_src), bsz):
-        e = min(i + bsz, len(idx_src))
-        scores = np.dot(x_tgt, x_src[idx_src[i:e]].T)
-        pred = scores.argmax(axis=0)
-        for j in range(i, e):
-            if pred[j - i] in lexicon[idx_src[j]]:
-                acc += 1.0
-    return acc / lexicon_size
-
-
-def compute_csls_accuracy(x_src, x_tgt, lexicon, lexicon_size=-1, k=10, bsz=1024):
-    if lexicon_size < 0:
-        lexicon_size = len(lexicon)
-    idx_src = list(lexicon.keys())
-
-    x_src /= np.linalg.norm(x_src, axis=1)[:, np.newaxis] + 1e-8
-    x_tgt /= np.linalg.norm(x_tgt, axis=1)[:, np.newaxis] + 1e-8
-
-    sr = x_src[list(idx_src)]
-    sc = np.dot(sr, x_tgt.T)
-    similarities = 2 * sc
-    sc2 = np.zeros(x_tgt.shape[0])
-    for i in range(0, x_tgt.shape[0], bsz):
-        j = min(i + bsz, x_tgt.shape[0])
-        sc_batch = np.dot(x_tgt[i:j, :], x_src.T)
-        dotprod = np.partition(sc_batch, -k, axis=1)[:, -k:]
-        sc2[i:j] = np.mean(dotprod, axis=1)
-    similarities -= sc2[np.newaxis, :]
-
-    nn = np.argmax(similarities, axis=1).tolist()
-    correct = 0.0
-    for k in range(0, len(lexicon)):
-        if nn[k] in lexicon[idx_src[k]]:
-            correct += 1.0
-    return correct / lexicon_size

data/vendor/fastText/classification-example.sh

@@ -1,41 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2016-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-#
-
-myshuf() {
-  perl -MList::Util=shuffle -e 'print shuffle(<>);' "$@";
-}
-
-normalize_text() {
-  tr '[:upper:]' '[:lower:]' | sed -e 's/^/__label__/g' | \
-    sed -e "s/'/ ' /g" -e 's/"//g' -e 's/\./ \. /g' -e 's/<br \/>/ /g' \
-        -e 's/,/ , /g' -e 's/(/ ( /g' -e 's/)/ ) /g' -e 's/\!/ \! /g' \
-        -e 's/\?/ \? /g' -e 's/\;/ /g' -e 's/\:/ /g' | tr -s " " | myshuf
-}
-
-RESULTDIR=result
-DATADIR=data
-
-mkdir -p "${RESULTDIR}"
-mkdir -p "${DATADIR}"
-
-if [ ! -f "${DATADIR}/dbpedia.train" ]
-then
-  wget -c "https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz" -O "${DATADIR}/dbpedia_csv.tar.gz"
-  tar -xzvf "${DATADIR}/dbpedia_csv.tar.gz" -C "${DATADIR}"
-  cat "${DATADIR}/dbpedia_csv/train.csv" | normalize_text > "${DATADIR}/dbpedia.train"
-  cat "${DATADIR}/dbpedia_csv/test.csv" | normalize_text > "${DATADIR}/dbpedia.test"
-fi
-
-make
-
-./fasttext supervised -input "${DATADIR}/dbpedia.train" -output "${RESULTDIR}/dbpedia" -dim 10 -lr 0.1 -wordNgrams 2 -minCount 1 -bucket 10000000 -epoch 5 -thread 4
-
-./fasttext test "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test"
-
-./fasttext predict "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" > "${RESULTDIR}/dbpedia.test.predict"

data/vendor/fastText/classification-results.sh

@@ -1,94 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2016-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-#
-
-# This script produces the results from Table 1 in the following paper:
-# Bag of Tricks for Efficient Text Classification, arXiv 1607.01759, 2016
-
-myshuf() {
-  perl -MList::Util=shuffle -e 'print shuffle(<>);' "$@";
-}
-
-normalize_text() {
-  tr '[:upper:]' '[:lower:]' | sed -e 's/^/__label__/g' | \
-    sed -e "s/'/ ' /g" -e 's/"//g' -e 's/\./ \. /g' -e 's/<br \/>/ /g' \
-        -e 's/,/ , /g' -e 's/(/ ( /g' -e 's/)/ ) /g' -e 's/\!/ \! /g' \
-        -e 's/\?/ \? /g' -e 's/\;/ /g' -e 's/\:/ /g' | tr -s " " | myshuf
-}
-
-DATASET=(
-  ag_news
-  sogou_news
-  dbpedia
-  yelp_review_polarity
-  yelp_review_full
-  yahoo_answers
-  amazon_review_full
-  amazon_review_polarity
-)
-
-ID=(
-  0Bz8a_Dbh9QhbUDNpeUdjb0wxRms # ag_news
-  0Bz8a_Dbh9QhbUkVqNEszd0pHaFE # sogou_news
-  0Bz8a_Dbh9QhbQ2Vic1kxMmZZQ1k # dbpedia
-  0Bz8a_Dbh9QhbNUpYQ2N3SGlFaDg # yelp_review_polarity
-  0Bz8a_Dbh9QhbZlU4dXhHTFhZQU0 # yelp_review_full
-  0Bz8a_Dbh9Qhbd2JNdDBsQUdocVU # yahoo_answers
-  0Bz8a_Dbh9QhbZVhsUnRWRDhETzA # amazon_review_full
-  0Bz8a_Dbh9QhbaW12WVVZS2drcnM # amazon_review_polarity
-)
-
-# These learning rates were chosen by validation on a subset of the training set.
-LR=( 0.25 0.5 0.5 0.1 0.1 0.1 0.05 0.05 )
-
-RESULTDIR=result
-DATADIR=data
-
-mkdir -p "${RESULTDIR}"
-mkdir -p "${DATADIR}"
-
-# Small datasets first
-
-for i in {0..0}
-do
-  echo "Downloading dataset ${DATASET[i]}"
-  if [ ! -f "${DATADIR}/${DATASET[i]}.train" ]
-  then
-    wget -c "https://drive.google.com/uc?export=download&id=${ID[i]}" -O "${DATADIR}/${DATASET[i]}_csv.tar.gz"
-    tar -xzvf "${DATADIR}/${DATASET[i]}_csv.tar.gz" -C "${DATADIR}"
-    cat "${DATADIR}/${DATASET[i]}_csv/train.csv" | normalize_text > "${DATADIR}/${DATASET[i]}.train"
-    cat "${DATADIR}/${DATASET[i]}_csv/test.csv" | normalize_text > "${DATADIR}/${DATASET[i]}.test"
-  fi
-done
-
-# Large datasets require a bit more work due to the extra request page
-
-for i in {1..7}
-do
-  echo "Downloading dataset ${DATASET[i]}"
-  if [ ! -f "${DATADIR}/${DATASET[i]}.train" ]
-  then
-    curl -c /tmp/cookies "https://drive.google.com/uc?export=download&id=${ID[i]}" > /tmp/intermezzo.html
-    curl -L -b /tmp/cookies "https://drive.google.com$(cat /tmp/intermezzo.html | grep -Po 'uc-download-link" [^>]* href="\K[^"]*' | sed 's/\&amp;/\&/g')" > "${DATADIR}/${DATASET[i]}_csv.tar.gz"
-    tar -xzvf "${DATADIR}/${DATASET[i]}_csv.tar.gz" -C "${DATADIR}"
-    cat "${DATADIR}/${DATASET[i]}_csv/train.csv" | normalize_text > "${DATADIR}/${DATASET[i]}.train"
-    cat "${DATADIR}/${DATASET[i]}_csv/test.csv" | normalize_text > "${DATADIR}/${DATASET[i]}.test"
-  fi
-done
-
-make
-
-for i in {0..7}
-do
-  echo "Working on dataset ${DATASET[i]}"
-  ./fasttext supervised -input "${DATADIR}/${DATASET[i]}.train" \
-    -output "${RESULTDIR}/${DATASET[i]}" -dim 10 -lr "${LR[i]}" -wordNgrams 2 \
-    -minCount 1 -bucket 10000000 -epoch 5 -thread 4 > /dev/null
-  ./fasttext test "${RESULTDIR}/${DATASET[i]}.bin" \
-    "${DATADIR}/${DATASET[i]}.test"
-done

data/vendor/fastText/crawl/README.md

@@ -1,26 +0,0 @@
-## Preprocessing Common Crawl
-
-This code downloads, preprocesses and splits per language the data from [Common Crawl](http://commoncrawl.org/).
-
-This script uses the scripts and language identifier of [1].
-
-This code inherits its requirements form [fastText](https://github.com/facebookresearch/fastText).
-
-Set the variable WET_PATHS_URL to the crawl you want to process.
-Please also set the variables NUM_LANGID and NUM_DEDUP in `download_crawl.sh` according to the capacity of your machine.
-Langid processes are mostly limited by CPU usage, while dedup processes are likely to be limited by RAM usage (each use 2GB of RAM).
-
-### Reference
-
-If you use this code, please cite:
-
-[1] E. Grave*, P. Bojanowski*, P. Gupta, A. Joulin, T. Mikolov, [*Learning Word Vectors for 157 Languages*](https://arxiv.org/abs/1802.06893)
-
-```
-@inproceedings{grave2018learning,
-  title={Learning Word Vectors for 157 Languages},
-  author={Grave, Edouard and Bojanowski, Piotr and Gupta, Prakhar and Joulin, Armand and Mikolov, Tomas},
-  booktitle={Proceedings of the International Conference on Language Resources and Evaluation (LREC 2018)},
-  year={2018}
-}
-```

data/vendor/fastText/crawl/dedup.cc

@@ -1,51 +0,0 @@
-// Copyright (c) 2018-present, Facebook, Inc.
-// All rights reserved.
-//
-// This source code is licensed under the MIT license found in the
-// LICENSE file in the root directory of this source tree.
-
-#include <cstdint>
-#include <iostream>
-#include <fstream>
-#include <string>
-#include <vector>
-
-uint64_t fnv1a_64(uint8_t *data, size_t sz, uint64_t h=14695981039346656037ull)
-{
-  for (size_t i = 0; i < sz; i++, data++) {
-    h ^= uint64_t(*data);
-    h *= 1099511628211ull;
-  }
-  return h;
-}
-
-int main(int argc, char** argv)
-{
-  uint64_t init_values[] = {
-    14695981039346656037ull,
-    9425296925403859339ull,
-    13716263814064014149ull,
-    3525492407291847033ull,
-    8607404175481815707ull,
-    9818874561736458749ull,
-    10026508429719773353ull,
-    3560712257386009938ull
-  };
-  size_t n = 1ull<<34, num_hashes = 2;
-  std::vector<bool> seen(n);
-
-  std::ios_base::sync_with_stdio(false);
-
-  for (std::string line; std::getline(std::cin, line);) {
-    bool b = true;
-    for (size_t i = 0; i < num_hashes; i++) {
-      uint64_t h = fnv1a_64((uint8_t*) line.data(), line.length(), init_values[i]) % n;
-      b = b && seen[h];
-      seen[h] = true;
-    }
-    if (!b) {
-      std::cout << line << std::endl;
-    }
-  }
-  return 0;
-}