fasttext 0.1.0

data/vendor/fastText/alignment/eval.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+#
+# Copyright (c) 2018-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import io
+import numpy as np
+import argparse
+from utils import *
+
+parser = argparse.ArgumentParser(description='Evaluation of word alignment')
+parser.add_argument("--src_emb", type=str, default='', help="Load source embeddings")
+parser.add_argument("--tgt_emb", type=str, default='', help="Load target embeddings")
+parser.add_argument('--center', action='store_true', help='whether to center embeddings or not')
+parser.add_argument("--src_mat", type=str, default='', help="Load source alignment matrix. If none given, the aligment matrix is the identity.")
+parser.add_argument("--tgt_mat", type=str, default='', help="Load target alignment matrix. If none given, the aligment matrix is the identity.")
+parser.add_argument("--dico_test", type=str, default='', help="test dictionary")
+parser.add_argument("--maxload", type=int, default=200000)
+parser.add_argument("--nomatch", action='store_true', help="no exact match in lexicon")
+params = parser.parse_args()
+
+
+###### SPECIFIC FUNCTIONS ######
+# function specific to evaluation
+# the rest of the functions are in utils.py
+
+def load_transform(fname, d1=300, d2=300):
+    fin = io.open(fname, 'r', encoding='utf-8', newline='\n', errors='ignore')
+    R = np.zeros([d1, d2])
+    for i, line in enumerate(fin):
+        tokens = line.split(' ')
+        R[i, :] = np.array(tokens[0:d2], dtype=float)
+    return R
+
+
+###### MAIN ######
+
+print("Evaluation of alignment on %s" % params.dico_test)
+if params.nomatch:
+    print("running without exact string matches")
+
+words_tgt, x_tgt = load_vectors(params.tgt_emb, maxload=params.maxload, center=params.center)
+words_src, x_src = load_vectors(params.src_emb, maxload=params.maxload, center=params.center)
+
+if params.tgt_mat != "":
+    R_tgt = load_transform(params.tgt_mat)
+    x_tgt = np.dot(x_tgt, R_tgt)
+if params.src_mat != "":
+    R_src = load_transform(params.src_mat)
+    x_src = np.dot(x_src, R_src)
+
+src2tgt, lexicon_size = load_lexicon(params.dico_test, words_src, words_tgt)
+
+nnacc = compute_nn_accuracy(x_src, x_tgt, src2tgt, lexicon_size=lexicon_size)
+cslsproc = compute_csls_accuracy(x_src, x_tgt, src2tgt, lexicon_size=lexicon_size)
+print("NN = %.4f - CSLS = %.4f - Coverage = %.4f" % (nnacc, cslsproc, len(src2tgt) / lexicon_size))

data/vendor/fastText/alignment/example.sh

@@ -0,0 +1,51 @@
+#!/bin/usr/env sh
+# Copyright (c) 2018-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+set -e
+s=${1:-en}
+t=${2:-es}
+echo "Example based on the ${s}->${t} alignment"
+
+if [ ! -d data/ ]; then
+  mkdir -p data;
+fi
+
+if [ ! -d res/ ]; then
+  mkdir -p res;
+fi
+
+dico_train=data/${s}-${t}.0-5000.txt
+if [ ! -f "${dico_train}" ]; then
+  DICO=$(basename -- "${dico_train}")
+  wget -c "https://dl.fbaipublicfiles.com/arrival/dictionaries/${DICO}" -P data/
+fi
+
+dico_test=data/${s}-${t}.5000-6500.txt
+if [ ! -f "${dico_test}" ]; then
+  DICO=$(basename -- "${dico_test}")
+  wget -c "https://dl.fbaipublicfiles.com/arrival/dictionaries/${DICO}" -P data/
+fi
+
+src_emb=data/wiki.${s}.vec
+if [ ! -f "${src_emb}" ]; then
+  EMB=$(basename -- "${src_emb}")
+  wget -c "https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/${EMB}" -P data/
+fi
+
+tgt_emb=data/wiki.${t}.vec
+if [ ! -f "${tgt_emb}" ]; then
+  EMB=$(basename -- "${tgt_emb}")
+  wget -c "https://dl.fbaipublicfiles.com/fasttext/vectors-wiki/${EMB}" -P data/
+fi
+
+output=res/wiki.${s}-${t}.vec
+
+python3 align.py --src_emb "${src_emb}" --tgt_emb "${tgt_emb}" \
+  --dico_train "${dico_train}" --dico_test "${dico_test}" --output "${output}" \
+  --lr 25 --niter 10
+python3 eval.py --src_emb "${output}" --tgt_emb "${tgt_emb}" \
+  --dico_test "${dico_test}"

data/vendor/fastText/alignment/unsup_align.py

@@ -0,0 +1,109 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+import codecs, sys, time, math, argparse, ot
+import numpy as np
+from utils import *
+
+parser = argparse.ArgumentParser(description='Wasserstein Procrustes for Embedding Alignment')
+parser.add_argument('--model_src', type=str, help='Path to source word embeddings')
+parser.add_argument('--model_tgt', type=str, help='Path to target word embeddings')
+parser.add_argument('--lexicon', type=str, help='Path to the evaluation lexicon')
+parser.add_argument('--output_src', default='', type=str, help='Path to save the aligned source embeddings')
+parser.add_argument('--output_tgt', default='', type=str, help='Path to save the aligned target embeddings')
+parser.add_argument('--seed', default=1111, type=int, help='Random number generator seed')
+parser.add_argument('--nepoch', default=5, type=int, help='Number of epochs')
+parser.add_argument('--niter', default=5000, type=int, help='Initial number of iterations')
+parser.add_argument('--bsz', default=500, type=int, help='Initial batch size')
+parser.add_argument('--lr', default=500., type=float, help='Learning rate')
+parser.add_argument('--nmax', default=20000, type=int, help='Vocabulary size for learning the alignment')
+parser.add_argument('--reg', default=0.05, type=float, help='Regularization parameter for sinkhorn')
+args = parser.parse_args()
+
+
+def objective(X, Y, R, n=5000):
+    Xn, Yn = X[:n], Y[:n]
+    C = -np.dot(np.dot(Xn, R), Yn.T)
+    P = ot.sinkhorn(np.ones(n), np.ones(n), C, 0.025, stopThr=1e-3)
+    return 1000 * np.linalg.norm(np.dot(Xn, R) - np.dot(P, Yn)) / n
+
+
+def sqrt_eig(x):
+    U, s, VT = np.linalg.svd(x, full_matrices=False)
+    return np.dot(U, np.dot(np.diag(np.sqrt(s)), VT))
+
+
+def align(X, Y, R, lr=10., bsz=200, nepoch=5, niter=1000,
+          nmax=10000, reg=0.05, verbose=True):
+    for epoch in range(1, nepoch + 1):
+        for _it in range(1, niter + 1):
+            # sample mini-batch
+            xt = X[np.random.permutation(nmax)[:bsz], :]
+            yt = Y[np.random.permutation(nmax)[:bsz], :]
+            # compute OT on minibatch
+            C = -np.dot(np.dot(xt, R), yt.T)
+            P = ot.sinkhorn(np.ones(bsz), np.ones(bsz), C, reg, stopThr=1e-3)
+            # compute gradient
+            G = - np.dot(xt.T, np.dot(P, yt))
+            R -= lr / bsz * G
+            # project on orthogonal matrices
+            U, s, VT = np.linalg.svd(R)
+            R = np.dot(U, VT)
+        bsz *= 2
+        niter //= 4
+        if verbose:
+            print("epoch: %d  obj: %.3f" % (epoch, objective(X, Y, R)))
+    return R
+
+
+def convex_init(X, Y, niter=100, reg=0.05, apply_sqrt=False):
+    n, d = X.shape
+    if apply_sqrt:
+        X, Y = sqrt_eig(X), sqrt_eig(Y)
+    K_X, K_Y = np.dot(X, X.T), np.dot(Y, Y.T)
+    K_Y *= np.linalg.norm(K_X) / np.linalg.norm(K_Y)
+    K2_X, K2_Y = np.dot(K_X, K_X), np.dot(K_Y, K_Y)
+    P = np.ones([n, n]) / float(n)
+    for it in range(1, niter + 1):
+        G = np.dot(P, K2_X) + np.dot(K2_Y, P) - 2 * np.dot(K_Y, np.dot(P, K_X))
+        q = ot.sinkhorn(np.ones(n), np.ones(n), G, reg, stopThr=1e-3)
+        alpha = 2.0 / float(2.0 + it)
+        P = alpha * q + (1.0 - alpha) * P
+    obj = np.linalg.norm(np.dot(P, K_X) - np.dot(K_Y, P))
+    print(obj)
+    return procrustes(np.dot(P, X), Y).T
+
+
+print("\n*** Wasserstein Procrustes ***\n")
+
+np.random.seed(args.seed)
+
+maxload = 200000
+w_src, x_src = load_vectors(args.model_src, maxload, norm=True, center=True)
+w_tgt, x_tgt = load_vectors(args.model_tgt, maxload, norm=True, center=True)
+src2trg, _ = load_lexicon(args.lexicon, w_src, w_tgt)
+
+print("\nComputing initial mapping with convex relaxation...")
+t0 = time.time()
+R0 = convex_init(x_src[:2500], x_tgt[:2500], reg=args.reg, apply_sqrt=True)
+print("Done [%03d sec]" % math.floor(time.time() - t0))
+
+print("\nComputing mapping with Wasserstein Procrustes...")
+t0 = time.time()
+R = align(x_src, x_tgt, R0.copy(), bsz=args.bsz, lr=args.lr, niter=args.niter,
+          nepoch=args.nepoch, reg=args.reg, nmax=args.nmax)
+print("Done [%03d sec]" % math.floor(time.time() - t0))
+
+acc = compute_nn_accuracy(x_src, np.dot(x_tgt, R.T), src2trg)
+print("\nPrecision@1: %.3f\n" % acc)
+
+if args.output_src != '':
+    x_src = x_src / np.linalg.norm(x_src, 2, 1).reshape([-1, 1])
+    save_vectors(args.output_src, x_src, w_src)
+if args.output_tgt != '':
+    x_tgt = x_tgt / np.linalg.norm(x_tgt, 2, 1).reshape([-1, 1])
+    save_vectors(args.output_tgt, np.dot(x_tgt, R.T), w_tgt)

data/vendor/fastText/alignment/utils.py

@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+# Copyright (c) 2018-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+import io
+import numpy as np
+import collections
+
+
+def load_vectors(fname, maxload=200000, norm=True, center=False, verbose=True):
+    if verbose:
+        print("Loading vectors from %s" % fname)
+    fin = io.open(fname, 'r', encoding='utf-8', newline='\n', errors='ignore')
+    n, d = map(int, fin.readline().split())
+    if maxload > 0:
+        n = min(n, maxload)
+    x = np.zeros([n, d])
+    words = []
+    for i, line in enumerate(fin):
+        if i >= n:
+            break
+        tokens = line.rstrip().split(' ')
+        words.append(tokens[0])
+        v = np.array(tokens[1:], dtype=float)
+        x[i, :] = v
+    if norm:
+        x /= np.linalg.norm(x, axis=1)[:, np.newaxis] + 1e-8
+    if center:
+        x -= x.mean(axis=0)[np.newaxis, :]
+        x /= np.linalg.norm(x, axis=1)[:, np.newaxis] + 1e-8
+    if verbose:
+        print("%d word vectors loaded" % (len(words)))
+    return words, x
+
+
+def idx(words):
+    w2i = {}
+    for i, w in enumerate(words):
+        if w not in w2i:
+            w2i[w] = i
+    return w2i
+
+
+def save_vectors(fname, x, words):
+    n, d = x.shape
+    fout = io.open(fname, 'w', encoding='utf-8')
+    fout.write(u"%d %d\n" % (n, d))
+    for i in range(n):
+        fout.write(words[i] + " " + " ".join(map(lambda a: "%.4f" % a, x[i, :])) + "\n")
+    fout.close()
+
+
+def save_matrix(fname, x):
+    n, d = x.shape
+    fout = io.open(fname, 'w', encoding='utf-8')
+    fout.write(u"%d %d\n" % (n, d))
+    for i in range(n):
+        fout.write(" ".join(map(lambda a: "%.4f" % a, x[i, :])) + "\n")
+    fout.close()
+
+
+def procrustes(X_src, Y_tgt):
+    U, s, V = np.linalg.svd(np.dot(Y_tgt.T, X_src))
+    return np.dot(U, V)
+
+
+def select_vectors_from_pairs(x_src, y_tgt, pairs):
+    n = len(pairs)
+    d = x_src.shape[1]
+    x = np.zeros([n, d])
+    y = np.zeros([n, d])
+    for k, ij in enumerate(pairs):
+        i, j = ij
+        x[k, :] = x_src[i, :]
+        y[k, :] = y_tgt[j, :]
+    return x, y
+
+
+def load_lexicon(filename, words_src, words_tgt, verbose=True):
+    f = io.open(filename, 'r', encoding='utf-8')
+    lexicon = collections.defaultdict(set)
+    idx_src , idx_tgt = idx(words_src), idx(words_tgt)
+    vocab = set()
+    for line in f:
+        word_src, word_tgt = line.split()
+        if word_src in idx_src and word_tgt in idx_tgt:
+            lexicon[idx_src[word_src]].add(idx_tgt[word_tgt])
+        vocab.add(word_src)
+    if verbose:
+        coverage = len(lexicon) / float(len(vocab))
+        print("Coverage of source vocab: %.4f" % (coverage))
+    return lexicon, float(len(vocab))
+
+
+def load_pairs(filename, idx_src, idx_tgt, verbose=True):
+    f = io.open(filename, 'r', encoding='utf-8')
+    pairs = []
+    tot = 0
+    for line in f:
+        a, b = line.rstrip().split(' ')
+        tot += 1
+        if a in idx_src and b in idx_tgt:
+            pairs.append((idx_src[a], idx_tgt[b]))
+    if verbose:
+        coverage = (1.0 * len(pairs)) / tot
+        print("Found pairs for training: %d - Total pairs in file: %d - Coverage of pairs: %.4f" % (len(pairs), tot, coverage))
+    return pairs
+
+
+def compute_nn_accuracy(x_src, x_tgt, lexicon, bsz=100, lexicon_size=-1):
+    if lexicon_size < 0:
+        lexicon_size = len(lexicon)
+    idx_src = list(lexicon.keys())
+    acc = 0.0
+    x_src /= np.linalg.norm(x_src, axis=1)[:, np.newaxis] + 1e-8
+    x_tgt /= np.linalg.norm(x_tgt, axis=1)[:, np.newaxis] + 1e-8
+    for i in range(0, len(idx_src), bsz):
+        e = min(i + bsz, len(idx_src))
+        scores = np.dot(x_tgt, x_src[idx_src[i:e]].T)
+        pred = scores.argmax(axis=0)
+        for j in range(i, e):
+            if pred[j - i] in lexicon[idx_src[j]]:
+                acc += 1.0
+    return acc / lexicon_size
+
+
+def compute_csls_accuracy(x_src, x_tgt, lexicon, lexicon_size=-1, k=10, bsz=1024):
+    if lexicon_size < 0:
+        lexicon_size = len(lexicon)
+    idx_src = list(lexicon.keys())
+
+    x_src /= np.linalg.norm(x_src, axis=1)[:, np.newaxis] + 1e-8
+    x_tgt /= np.linalg.norm(x_tgt, axis=1)[:, np.newaxis] + 1e-8
+
+    sr = x_src[list(idx_src)]
+    sc = np.dot(sr, x_tgt.T)
+    similarities = 2 * sc
+    sc2 = np.zeros(x_tgt.shape[0])
+    for i in range(0, x_tgt.shape[0], bsz):
+        j = min(i + bsz, x_tgt.shape[0])
+        sc_batch = np.dot(x_tgt[i:j, :], x_src.T)
+        dotprod = np.partition(sc_batch, -k, axis=1)[:, -k:]
+        sc2[i:j] = np.mean(dotprod, axis=1)
+    similarities -= sc2[np.newaxis, :]
+
+    nn = np.argmax(similarities, axis=1).tolist()
+    correct = 0.0
+    for k in range(0, len(lexicon)):
+        if nn[k] in lexicon[idx_src[k]]:
+            correct += 1.0
+    return correct / lexicon_size

data/vendor/fastText/classification-example.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+myshuf() {
+  perl -MList::Util=shuffle -e 'print shuffle(<>);' "$@";
+}
+
+normalize_text() {
+  tr '[:upper:]' '[:lower:]' | sed -e 's/^/__label__/g' | \
+    sed -e "s/'/ ' /g" -e 's/"//g' -e 's/\./ \. /g' -e 's/<br \/>/ /g' \
+        -e 's/,/ , /g' -e 's/(/ ( /g' -e 's/)/ ) /g' -e 's/\!/ \! /g' \
+        -e 's/\?/ \? /g' -e 's/\;/ /g' -e 's/\:/ /g' | tr -s " " | myshuf
+}
+
+RESULTDIR=result
+DATADIR=data
+
+mkdir -p "${RESULTDIR}"
+mkdir -p "${DATADIR}"
+
+if [ ! -f "${DATADIR}/dbpedia.train" ]
+then
+  wget -c "https://github.com/le-scientifique/torchDatasets/raw/master/dbpedia_csv.tar.gz" -O "${DATADIR}/dbpedia_csv.tar.gz"
+  tar -xzvf "${DATADIR}/dbpedia_csv.tar.gz" -C "${DATADIR}"
+  cat "${DATADIR}/dbpedia_csv/train.csv" | normalize_text > "${DATADIR}/dbpedia.train"
+  cat "${DATADIR}/dbpedia_csv/test.csv" | normalize_text > "${DATADIR}/dbpedia.test"
+fi
+
+make
+
+./fasttext supervised -input "${DATADIR}/dbpedia.train" -output "${RESULTDIR}/dbpedia" -dim 10 -lr 0.1 -wordNgrams 2 -minCount 1 -bucket 10000000 -epoch 5 -thread 4
+
+./fasttext test "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test"
+
+./fasttext predict "${RESULTDIR}/dbpedia.bin" "${DATADIR}/dbpedia.test" > "${RESULTDIR}/dbpedia.test.predict"

data/vendor/fastText/classification-results.sh

@@ -0,0 +1,94 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+# This script produces the results from Table 1 in the following paper:
+# Bag of Tricks for Efficient Text Classification, arXiv 1607.01759, 2016
+
+myshuf() {
+  perl -MList::Util=shuffle -e 'print shuffle(<>);' "$@";
+}
+
+normalize_text() {
+  tr '[:upper:]' '[:lower:]' | sed -e 's/^/__label__/g' | \
+    sed -e "s/'/ ' /g" -e 's/"//g' -e 's/\./ \. /g' -e 's/<br \/>/ /g' \
+        -e 's/,/ , /g' -e 's/(/ ( /g' -e 's/)/ ) /g' -e 's/\!/ \! /g' \
+        -e 's/\?/ \? /g' -e 's/\;/ /g' -e 's/\:/ /g' | tr -s " " | myshuf
+}
+
+DATASET=(
+  ag_news
+  sogou_news
+  dbpedia
+  yelp_review_polarity
+  yelp_review_full
+  yahoo_answers
+  amazon_review_full
+  amazon_review_polarity
+)
+
+ID=(
+  0Bz8a_Dbh9QhbUDNpeUdjb0wxRms # ag_news
+  0Bz8a_Dbh9QhbUkVqNEszd0pHaFE # sogou_news
+  0Bz8a_Dbh9QhbQ2Vic1kxMmZZQ1k # dbpedia
+  0Bz8a_Dbh9QhbNUpYQ2N3SGlFaDg # yelp_review_polarity
+  0Bz8a_Dbh9QhbZlU4dXhHTFhZQU0 # yelp_review_full
+  0Bz8a_Dbh9Qhbd2JNdDBsQUdocVU # yahoo_answers
+  0Bz8a_Dbh9QhbZVhsUnRWRDhETzA # amazon_review_full
+  0Bz8a_Dbh9QhbaW12WVVZS2drcnM # amazon_review_polarity
+)
+
+# These learning rates were chosen by validation on a subset of the training set.
+LR=( 0.25 0.5 0.5 0.1 0.1 0.1 0.05 0.05 )
+
+RESULTDIR=result
+DATADIR=data
+
+mkdir -p "${RESULTDIR}"
+mkdir -p "${DATADIR}"
+
+# Small datasets first
+
+for i in {0..0}
+do
+  echo "Downloading dataset ${DATASET[i]}"
+  if [ ! -f "${DATADIR}/${DATASET[i]}.train" ]
+  then
+    wget -c "https://drive.google.com/uc?export=download&id=${ID[i]}" -O "${DATADIR}/${DATASET[i]}_csv.tar.gz"
+    tar -xzvf "${DATADIR}/${DATASET[i]}_csv.tar.gz" -C "${DATADIR}"
+    cat "${DATADIR}/${DATASET[i]}_csv/train.csv" | normalize_text > "${DATADIR}/${DATASET[i]}.train"
+    cat "${DATADIR}/${DATASET[i]}_csv/test.csv" | normalize_text > "${DATADIR}/${DATASET[i]}.test"
+  fi
+done
+
+# Large datasets require a bit more work due to the extra request page
+
+for i in {1..7}
+do
+  echo "Downloading dataset ${DATASET[i]}"
+  if [ ! -f "${DATADIR}/${DATASET[i]}.train" ]
+  then
+    curl -c /tmp/cookies "https://drive.google.com/uc?export=download&id=${ID[i]}" > /tmp/intermezzo.html
+    curl -L -b /tmp/cookies "https://drive.google.com$(cat /tmp/intermezzo.html | grep -Po 'uc-download-link" [^>]* href="\K[^"]*' | sed 's/\&amp;/\&/g')" > "${DATADIR}/${DATASET[i]}_csv.tar.gz"
+    tar -xzvf "${DATADIR}/${DATASET[i]}_csv.tar.gz" -C "${DATADIR}"
+    cat "${DATADIR}/${DATASET[i]}_csv/train.csv" | normalize_text > "${DATADIR}/${DATASET[i]}.train"
+    cat "${DATADIR}/${DATASET[i]}_csv/test.csv" | normalize_text > "${DATADIR}/${DATASET[i]}.test"
+  fi
+done
+
+make
+
+for i in {0..7}
+do
+  echo "Working on dataset ${DATASET[i]}"
+  ./fasttext supervised -input "${DATADIR}/${DATASET[i]}.train" \
+    -output "${RESULTDIR}/${DATASET[i]}" -dim 10 -lr "${LR[i]}" -wordNgrams 2 \
+    -minCount 1 -bucket 10000000 -epoch 5 -thread 4 > /dev/null
+  ./fasttext test "${RESULTDIR}/${DATASET[i]}.bin" \
+    "${DATADIR}/${DATASET[i]}.test"
+done