fasttext 0.1.0

data/vendor/fastText/python/benchmarks/README.rst

@@ -0,0 +1,3 @@
+These programs allow us to compare the performance of a few key operations when consindering changes. 
+
+It is important to run these to make sure a change doesn't introduce a regression.

data/vendor/fastText/python/benchmarks/get_word_vector.py

@@ -0,0 +1,49 @@
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from fasttext import load_model
+from fasttext import tokenize
+import sys
+import time
+import tempfile
+import argparse
+
+
+def get_word_vector(data, model):
+    t1 = time.time()
+    print("Reading")
+    with open(data, 'r') as f:
+        tokens = tokenize(f.read())
+    t2 = time.time()
+    print("Read TIME: " + str(t2 - t1))
+    print("Read NUM : " + str(len(tokens)))
+    f = load_model(model)
+    # This is not equivalent to piping the data into
+    # print-word-vector, because the data is tokenized
+    # first.
+    t3 = time.time()
+    i = 0
+    for t in tokens:
+        f.get_word_vector(t)
+        i += 1
+        if i % 10000 == 0:
+            sys.stderr.write("\ri: " + str(float(i / len(tokens))))
+            sys.stderr.flush()
+    t4 = time.time()
+    print("\nVectoring: " + str(t4 - t3))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description='Simple benchmark for get_word_vector.')
+    parser.add_argument('model', help='A model file to use for benchmarking.')
+    parser.add_argument('data', help='A data file to use for benchmarking.')
+    args = parser.parse_args()
+    get_word_vector(args.data, args.model)

data/vendor/fastText/python/doc/examples/FastTextEmbeddingBag.py

@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+# NOTE: This requires PyTorch! We do not provide installation scripts to install PyTorch.
+# It is up to you to install this dependency if you want to execute this example.
+# PyTorch's website should give you clear instructions on this: http://pytorch.org/
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from torch.nn.modules.sparse import EmbeddingBag
+import numpy as np
+import torch
+import random
+import string
+import time
+from fasttext import load_model
+from torch.autograd import Variable
+
+
+class FastTextEmbeddingBag(EmbeddingBag):
+    def __init__(self, model_path):
+        self.model = load_model(model_path)
+        input_matrix = self.model.get_input_matrix()
+        input_matrix_shape = input_matrix.shape
+        super().__init__(input_matrix_shape[0], input_matrix_shape[1])
+        self.weight.data.copy_(torch.FloatTensor(input_matrix))
+
+    def forward(self, words):
+        word_subinds = np.empty([0], dtype=np.int64)
+        word_offsets = [0]
+        for word in words:
+            _, subinds = self.model.get_subwords(word)
+            word_subinds = np.concatenate((word_subinds, subinds))
+            word_offsets.append(word_offsets[-1] + len(subinds))
+        word_offsets = word_offsets[:-1]
+        ind = Variable(torch.LongTensor(word_subinds))
+        offsets = Variable(torch.LongTensor(word_offsets))
+        return super().forward(ind, offsets)
+
+
+def random_word(N):
+    return ''.join(
+        random.choices(
+            string.ascii_uppercase + string.ascii_lowercase + string.digits,
+            k=N
+        )
+    )
+
+
+if __name__ == "__main__":
+    ft_emb = FastTextEmbeddingBag("fil9.bin")
+    model = load_model("fil9.bin")
+    num_lines = 200
+    total_seconds = 0.0
+    total_words = 0
+    for _ in range(num_lines):
+        words = [
+            random_word(random.randint(1, 10))
+            for _ in range(random.randint(15, 25))
+        ]
+        total_words += len(words)
+        words_average_length = sum([len(word) for word in words]) / len(words)
+        start = time.clock()
+        words_emb = ft_emb(words)
+        total_seconds += (time.clock() - start)
+        for i in range(len(words)):
+            word = words[i]
+            ft_word_emb = model.get_word_vector(word)
+            py_emb = np.array(words_emb[i].data)
+            assert (np.isclose(ft_word_emb, py_emb).all())
+    print(
+        "Avg. {:2.5f} seconds to build embeddings for {} lines with a total of {} words.".
+        format(total_seconds, num_lines, total_words)
+    )

data/vendor/fastText/python/doc/examples/bin_to_vec.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division, absolute_import, print_function
+
+from fasttext import load_model
+import argparse
+import errno
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description=("Print fasttext .vec file to stdout from .bin file")
+    )
+    parser.add_argument(
+        "model",
+        help="Model to use",
+    )
+    args = parser.parse_args()
+
+    f = load_model(args.model)
+    words = f.get_words()
+    print(str(len(words)) + " " + str(f.get_dimension()))
+    for w in words:
+        v = f.get_word_vector(w)
+        vstr = ""
+        for vi in v:
+            vstr += " " + str(vi)
+        try:
+            print(w + vstr)
+        except IOError as e:
+            if e.errno == errno.EPIPE:
+                pass

data/vendor/fastText/python/doc/examples/compute_accuracy.py

@@ -0,0 +1,163 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division, absolute_import, print_function
+
+from fasttext import load_model
+from fasttext import util
+import argparse
+import numpy as np
+
+
+def process_question(question, cossims, model, words, vectors):
+    correct = 0
+    num_qs = 0
+    num_lines = 0
+    for line in question:
+        num_lines += 1
+        qwords = line.split()
+        # We lowercase all words to correspond to the preprocessing
+        # we applied to our data.
+        qwords = [x.lower().strip() for x in qwords]
+        # If one of the words is not in the vocabulary we skip this question
+        found = True
+        for w in qwords:
+            if w not in words:
+                found = False
+                break
+        if not found:
+            continue
+        # The first three words form the query
+        # We retrieve their word vectors and normalize them
+        query = qwords[:3]
+        query = [model.get_word_vector(x) for x in query]
+        query = [x / np.linalg.norm(x) for x in query]
+        # Get the query vector. Example:
+        # Germany  - Berlin + France
+        query = query[1] - query[0] + query[2]
+        # We don't need to rank all the words, only until we found
+        # the first word not equal to our set of query words.
+        ban_set = list(map(lambda x: words.index(x), qwords[:3]))
+        if words[util.find_nearest_neighbor(
+            query, vectors, ban_set, cossims=cossims
+        )] == qwords[3]:
+            correct += 1
+        num_qs += 1
+    return correct, num_qs, num_lines
+
+
+# We use the same conventions as within compute-accuracy
+def print_compute_accuracy_score(
+    question, correct, num_qs, total_accuracy, semantic_accuracy,
+    syntactic_accuracy
+):
+    print(
+        (
+            "{0:>30}: ACCURACY TOP1: {3:.2f} %  ({1} / {2})\t  Total accuracy: {4:.2f} %   Semantic accuracy: {5:.2f} %   Syntactic accuracy: {6:.2f} %"
+        ).format(
+            question,
+            correct,
+            num_qs,
+            correct / float(num_qs) * 100 if num_qs > 0 else 0,
+            total_accuracy * 100,
+            semantic_accuracy * 100,
+            syntactic_accuracy * 100,
+        )
+    )
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description=(
+            "compute_accuracy equivalent in Python. "
+            "See https://github.com/tmikolov/word2vec/blob/master/demo-word-accuracy.sh"
+        )
+    )
+    parser.add_argument(
+        "model",
+        help="Model to use",
+    )
+    parser.add_argument(
+        "question_words",
+        help="word questions similar to tmikolov's file (see help for link)",
+    )
+    parser.add_argument(
+        "threshold",
+        help="threshold used to limit number of words used",
+    )
+    args = parser.parse_args()
+    args.threshold = int(args.threshold)
+
+    # Retrieve list of normalized word vectors for the first words up
+    # until the threshold count.
+    f = load_model(args.model)
+    # Gets words with associated frequeny sorted by default by descending order
+    words, freq = f.get_words(include_freq=True)
+    words = words[:args.threshold]
+    vectors = np.zeros((len(words), f.get_dimension()), dtype=float)
+    for i in range(len(words)):
+        wv = f.get_word_vector(words[i])
+        wv = wv / np.linalg.norm(wv)
+        vectors[i] = wv
+
+    total_correct = 0
+    total_qs = 0
+    total_num_lines = 0
+
+    total_se_correct = 0
+    total_se_qs = 0
+
+    total_sy_correct = 0
+    total_sy_qs = 0
+
+    qid = 0
+    questions = []
+    with open(args.question_words, 'r') as fqw:
+        questions = fqw.read().split(':')[1:]
+    # For efficiency preallocate the memory to calculate cosine similarities
+    cossims = np.zeros(len(words), dtype=float)
+    for question in questions:
+        quads = question.split('\n')
+        question = quads[0].strip()
+        quads = quads[1:-1]
+        correct, num_qs, num_lines = process_question(
+            quads, cossims, f, words, vectors
+        )
+        total_qs += num_qs
+        total_correct += correct
+        total_num_lines += num_lines
+
+        if (qid < 5):
+            total_se_correct += correct
+            total_se_qs += num_qs
+        else:
+            total_sy_correct += correct
+            total_sy_qs += num_qs
+
+        print_compute_accuracy_score(
+            question,
+            correct,
+            num_qs,
+            total_correct / float(total_qs) if total_qs > 0 else 0,
+            total_se_correct / float(total_se_qs) if total_se_qs > 0 else 0,
+            total_sy_correct / float(total_sy_qs) if total_sy_qs > 0 else 0,
+        )
+        qid += 1
+
+    print(
+        "Questions seen / total: {0} {1}   {2:.2f} %".
+        format(
+            total_qs,
+            total_num_lines,
+            total_qs / total_num_lines * 100 if total_num_lines > 0 else 0,
+        )
+    )

data/vendor/fastText/python/doc/examples/get_vocab.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division, absolute_import, print_function
+
+from fasttext import load_model
+import argparse
+import errno
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description=(
+            "Print words or labels and frequency of a model's dictionary"
+        )
+    )
+    parser.add_argument(
+        "model",
+        help="Model to use",
+    )
+    parser.add_argument(
+        "-l",
+        "--labels",
+        help="Print labels instead of words",
+        action='store_true',
+        default=False,
+    )
+    args = parser.parse_args()
+
+    f = load_model(args.model)
+    if args.labels:
+        words, freq = f.get_labels(include_freq=True)
+    else:
+        words, freq = f.get_words(include_freq=True)
+    for w, f in zip(words, freq):
+        try:
+            print(w + "\t" + str(f))
+        except IOError as e:
+            if e.errno == errno.EPIPE:
+                pass

data/vendor/fastText/python/doc/examples/train_supervised.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+from fasttext import train_supervised
+
+
+def print_results(N, p, r):
+    print("N\t" + str(N))
+    print("P@{}\t{:.3f}".format(1, p))
+    print("R@{}\t{:.3f}".format(1, r))
+
+if __name__ == "__main__":
+    train_data = os.path.join(os.getenv("DATADIR", ''), 'cooking.train')
+    valid_data = os.path.join(os.getenv("DATADIR", ''), 'cooking.valid')
+
+    # train_supervised uses the same arguments and defaults as the fastText cli
+    model = train_supervised(
+        input=train_data, epoch=25, lr=1.0, wordNgrams=2, verbose=2, minCount=1
+    )
+    print_results(*model.test(valid_data))
+
+    model = train_supervised(
+        input=train_data, epoch=25, lr=1.0, wordNgrams=2, verbose=2, minCount=1,
+        loss="hs"
+    )
+    print_results(*model.test(valid_data))
+    model.save_model("cooking.bin")
+
+    model.quantize(input=train_data, qnorm=True, retrain=True, cutoff=100000)
+    print_results(*model.test(valid_data))
+    model.save_model("cooking.ftz")

data/vendor/fastText/python/doc/examples/train_unsupervised.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python
+
+# Copyright (c) 2017-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+from __future__ import division, absolute_import, print_function
+
+from fasttext import train_unsupervised
+import numpy as np
+import os
+from scipy import stats
+
+
+# Because of fasttext we don't need to account for OOV
+def compute_similarity(data_path):
+    def similarity(v1, v2):
+        n1 = np.linalg.norm(v1)
+        n2 = np.linalg.norm(v2)
+        return np.dot(v1, v2) / n1 / n2
+
+    mysim = []
+    gold = []
+
+    with open(data_path, 'rb') as fin:
+        for line in fin:
+            tline = line.split()
+            word1 = tline[0].lower()
+            word2 = tline[1].lower()
+
+            v1 = model.get_word_vector(word1)
+            v2 = model.get_word_vector(word2)
+            d = similarity(v1, v2)
+            mysim.append(d)
+            gold.append(float(tline[2]))
+
+    corr = stats.spearmanr(mysim, gold)
+    dataset = os.path.basename(data_path)
+    correlation = corr[0] * 100
+    return dataset, correlation, 0
+
+
+if __name__ == "__main__":
+    model = train_unsupervised(
+        input=os.path.join(os.getenv("DATADIR", ''), 'fil9'),
+        model='skipgram',
+    )
+    model.save_model("fil9.bin")
+    dataset, corr, oov = compute_similarity('rw.txt')
+    print("{0:20s}: {1:2.0f}  (OOV: {2:2.0f}%)".format(dataset, corr, 0))