fasttext 0.1.0 → 0.2.1

data/vendor/fastText/python/benchmarks/README.rst

@@ -1,3 +0,0 @@
-These programs allow us to compare the performance of a few key operations when consindering changes. 
-
-It is important to run these to make sure a change doesn't introduce a regression.

data/vendor/fastText/python/benchmarks/get_word_vector.py

@@ -1,49 +0,0 @@
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-from fasttext import load_model
-from fasttext import tokenize
-import sys
-import time
-import tempfile
-import argparse
-
-
-def get_word_vector(data, model):
-    t1 = time.time()
-    print("Reading")
-    with open(data, 'r') as f:
-        tokens = tokenize(f.read())
-    t2 = time.time()
-    print("Read TIME: " + str(t2 - t1))
-    print("Read NUM : " + str(len(tokens)))
-    f = load_model(model)
-    # This is not equivalent to piping the data into
-    # print-word-vector, because the data is tokenized
-    # first.
-    t3 = time.time()
-    i = 0
-    for t in tokens:
-        f.get_word_vector(t)
-        i += 1
-        if i % 10000 == 0:
-            sys.stderr.write("\ri: " + str(float(i / len(tokens))))
-            sys.stderr.flush()
-    t4 = time.time()
-    print("\nVectoring: " + str(t4 - t3))
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(description='Simple benchmark for get_word_vector.')
-    parser.add_argument('model', help='A model file to use for benchmarking.')
-    parser.add_argument('data', help='A data file to use for benchmarking.')
-    args = parser.parse_args()
-    get_word_vector(args.data, args.model)

data/vendor/fastText/python/doc/examples/FastTextEmbeddingBag.py

@@ -1,81 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-# NOTE: This requires PyTorch! We do not provide installation scripts to install PyTorch.
-# It is up to you to install this dependency if you want to execute this example.
-# PyTorch's website should give you clear instructions on this: http://pytorch.org/
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-from torch.nn.modules.sparse import EmbeddingBag
-import numpy as np
-import torch
-import random
-import string
-import time
-from fasttext import load_model
-from torch.autograd import Variable
-
-
-class FastTextEmbeddingBag(EmbeddingBag):
-    def __init__(self, model_path):
-        self.model = load_model(model_path)
-        input_matrix = self.model.get_input_matrix()
-        input_matrix_shape = input_matrix.shape
-        super().__init__(input_matrix_shape[0], input_matrix_shape[1])
-        self.weight.data.copy_(torch.FloatTensor(input_matrix))
-
-    def forward(self, words):
-        word_subinds = np.empty([0], dtype=np.int64)
-        word_offsets = [0]
-        for word in words:
-            _, subinds = self.model.get_subwords(word)
-            word_subinds = np.concatenate((word_subinds, subinds))
-            word_offsets.append(word_offsets[-1] + len(subinds))
-        word_offsets = word_offsets[:-1]
-        ind = Variable(torch.LongTensor(word_subinds))
-        offsets = Variable(torch.LongTensor(word_offsets))
-        return super().forward(ind, offsets)
-
-
-def random_word(N):
-    return ''.join(
-        random.choices(
-            string.ascii_uppercase + string.ascii_lowercase + string.digits,
-            k=N
-        )
-    )
-
-
-if __name__ == "__main__":
-    ft_emb = FastTextEmbeddingBag("fil9.bin")
-    model = load_model("fil9.bin")
-    num_lines = 200
-    total_seconds = 0.0
-    total_words = 0
-    for _ in range(num_lines):
-        words = [
-            random_word(random.randint(1, 10))
-            for _ in range(random.randint(15, 25))
-        ]
-        total_words += len(words)
-        words_average_length = sum([len(word) for word in words]) / len(words)
-        start = time.clock()
-        words_emb = ft_emb(words)
-        total_seconds += (time.clock() - start)
-        for i in range(len(words)):
-            word = words[i]
-            ft_word_emb = model.get_word_vector(word)
-            py_emb = np.array(words_emb[i].data)
-            assert (np.isclose(ft_word_emb, py_emb).all())
-    print(
-        "Avg. {:2.5f} seconds to build embeddings for {} lines with a total of {} words.".
-        format(total_seconds, num_lines, total_words)
-    )

data/vendor/fastText/python/doc/examples/bin_to_vec.py

@@ -1,41 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-from __future__ import division, absolute_import, print_function
-
-from fasttext import load_model
-import argparse
-import errno
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description=("Print fasttext .vec file to stdout from .bin file")
-    )
-    parser.add_argument(
-        "model",
-        help="Model to use",
-    )
-    args = parser.parse_args()
-
-    f = load_model(args.model)
-    words = f.get_words()
-    print(str(len(words)) + " " + str(f.get_dimension()))
-    for w in words:
-        v = f.get_word_vector(w)
-        vstr = ""
-        for vi in v:
-            vstr += " " + str(vi)
-        try:
-            print(w + vstr)
-        except IOError as e:
-            if e.errno == errno.EPIPE:
-                pass

data/vendor/fastText/python/doc/examples/compute_accuracy.py

@@ -1,163 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-from __future__ import division, absolute_import, print_function
-
-from fasttext import load_model
-from fasttext import util
-import argparse
-import numpy as np
-
-
-def process_question(question, cossims, model, words, vectors):
-    correct = 0
-    num_qs = 0
-    num_lines = 0
-    for line in question:
-        num_lines += 1
-        qwords = line.split()
-        # We lowercase all words to correspond to the preprocessing
-        # we applied to our data.
-        qwords = [x.lower().strip() for x in qwords]
-        # If one of the words is not in the vocabulary we skip this question
-        found = True
-        for w in qwords:
-            if w not in words:
-                found = False
-                break
-        if not found:
-            continue
-        # The first three words form the query
-        # We retrieve their word vectors and normalize them
-        query = qwords[:3]
-        query = [model.get_word_vector(x) for x in query]
-        query = [x / np.linalg.norm(x) for x in query]
-        # Get the query vector. Example:
-        # Germany  - Berlin + France
-        query = query[1] - query[0] + query[2]
-        # We don't need to rank all the words, only until we found
-        # the first word not equal to our set of query words.
-        ban_set = list(map(lambda x: words.index(x), qwords[:3]))
-        if words[util.find_nearest_neighbor(
-            query, vectors, ban_set, cossims=cossims
-        )] == qwords[3]:
-            correct += 1
-        num_qs += 1
-    return correct, num_qs, num_lines
-
-
-# We use the same conventions as within compute-accuracy
-def print_compute_accuracy_score(
-    question, correct, num_qs, total_accuracy, semantic_accuracy,
-    syntactic_accuracy
-):
-    print(
-        (
-            "{0:>30}: ACCURACY TOP1: {3:.2f} %  ({1} / {2})\t  Total accuracy: {4:.2f} %   Semantic accuracy: {5:.2f} %   Syntactic accuracy: {6:.2f} %"
-        ).format(
-            question,
-            correct,
-            num_qs,
-            correct / float(num_qs) * 100 if num_qs > 0 else 0,
-            total_accuracy * 100,
-            semantic_accuracy * 100,
-            syntactic_accuracy * 100,
-        )
-    )
-
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description=(
-            "compute_accuracy equivalent in Python. "
-            "See https://github.com/tmikolov/word2vec/blob/master/demo-word-accuracy.sh"
-        )
-    )
-    parser.add_argument(
-        "model",
-        help="Model to use",
-    )
-    parser.add_argument(
-        "question_words",
-        help="word questions similar to tmikolov's file (see help for link)",
-    )
-    parser.add_argument(
-        "threshold",
-        help="threshold used to limit number of words used",
-    )
-    args = parser.parse_args()
-    args.threshold = int(args.threshold)
-
-    # Retrieve list of normalized word vectors for the first words up
-    # until the threshold count.
-    f = load_model(args.model)
-    # Gets words with associated frequeny sorted by default by descending order
-    words, freq = f.get_words(include_freq=True)
-    words = words[:args.threshold]
-    vectors = np.zeros((len(words), f.get_dimension()), dtype=float)
-    for i in range(len(words)):
-        wv = f.get_word_vector(words[i])
-        wv = wv / np.linalg.norm(wv)
-        vectors[i] = wv
-
-    total_correct = 0
-    total_qs = 0
-    total_num_lines = 0
-
-    total_se_correct = 0
-    total_se_qs = 0
-
-    total_sy_correct = 0
-    total_sy_qs = 0
-
-    qid = 0
-    questions = []
-    with open(args.question_words, 'r') as fqw:
-        questions = fqw.read().split(':')[1:]
-    # For efficiency preallocate the memory to calculate cosine similarities
-    cossims = np.zeros(len(words), dtype=float)
-    for question in questions:
-        quads = question.split('\n')
-        question = quads[0].strip()
-        quads = quads[1:-1]
-        correct, num_qs, num_lines = process_question(
-            quads, cossims, f, words, vectors
-        )
-        total_qs += num_qs
-        total_correct += correct
-        total_num_lines += num_lines
-
-        if (qid < 5):
-            total_se_correct += correct
-            total_se_qs += num_qs
-        else:
-            total_sy_correct += correct
-            total_sy_qs += num_qs
-
-        print_compute_accuracy_score(
-            question,
-            correct,
-            num_qs,
-            total_correct / float(total_qs) if total_qs > 0 else 0,
-            total_se_correct / float(total_se_qs) if total_se_qs > 0 else 0,
-            total_sy_correct / float(total_sy_qs) if total_sy_qs > 0 else 0,
-        )
-        qid += 1
-
-    print(
-        "Questions seen / total: {0} {1}   {2:.2f} %".
-        format(
-            total_qs,
-            total_num_lines,
-            total_qs / total_num_lines * 100 if total_num_lines > 0 else 0,
-        )
-    )

data/vendor/fastText/python/doc/examples/get_vocab.py

@@ -1,48 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-from __future__ import division, absolute_import, print_function
-
-from fasttext import load_model
-import argparse
-import errno
-
-if __name__ == "__main__":
-    parser = argparse.ArgumentParser(
-        description=(
-            "Print words or labels and frequency of a model's dictionary"
-        )
-    )
-    parser.add_argument(
-        "model",
-        help="Model to use",
-    )
-    parser.add_argument(
-        "-l",
-        "--labels",
-        help="Print labels instead of words",
-        action='store_true',
-        default=False,
-    )
-    args = parser.parse_args()
-
-    f = load_model(args.model)
-    if args.labels:
-        words, freq = f.get_labels(include_freq=True)
-    else:
-        words, freq = f.get_words(include_freq=True)
-    for w, f in zip(words, freq):
-        try:
-            print(w + "\t" + str(f))
-        except IOError as e:
-            if e.errno == errno.EPIPE:
-                pass

data/vendor/fastText/python/doc/examples/train_supervised.py

@@ -1,42 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import os
-from fasttext import train_supervised
-
-
-def print_results(N, p, r):
-    print("N\t" + str(N))
-    print("P@{}\t{:.3f}".format(1, p))
-    print("R@{}\t{:.3f}".format(1, r))
-
-if __name__ == "__main__":
-    train_data = os.path.join(os.getenv("DATADIR", ''), 'cooking.train')
-    valid_data = os.path.join(os.getenv("DATADIR", ''), 'cooking.valid')
-
-    # train_supervised uses the same arguments and defaults as the fastText cli
-    model = train_supervised(
-        input=train_data, epoch=25, lr=1.0, wordNgrams=2, verbose=2, minCount=1
-    )
-    print_results(*model.test(valid_data))
-
-    model = train_supervised(
-        input=train_data, epoch=25, lr=1.0, wordNgrams=2, verbose=2, minCount=1,
-        loss="hs"
-    )
-    print_results(*model.test(valid_data))
-    model.save_model("cooking.bin")
-
-    model.quantize(input=train_data, qnorm=True, retrain=True, cutoff=100000)
-    print_results(*model.test(valid_data))
-    model.save_model("cooking.ftz")

data/vendor/fastText/python/doc/examples/train_unsupervised.py

@@ -1,56 +0,0 @@
-#!/usr/bin/env python
-
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-from __future__ import division, absolute_import, print_function
-
-from fasttext import train_unsupervised
-import numpy as np
-import os
-from scipy import stats
-
-
-# Because of fasttext we don't need to account for OOV
-def compute_similarity(data_path):
-    def similarity(v1, v2):
-        n1 = np.linalg.norm(v1)
-        n2 = np.linalg.norm(v2)
-        return np.dot(v1, v2) / n1 / n2
-
-    mysim = []
-    gold = []
-
-    with open(data_path, 'rb') as fin:
-        for line in fin:
-            tline = line.split()
-            word1 = tline[0].lower()
-            word2 = tline[1].lower()
-
-            v1 = model.get_word_vector(word1)
-            v2 = model.get_word_vector(word2)
-            d = similarity(v1, v2)
-            mysim.append(d)
-            gold.append(float(tline[2]))
-
-    corr = stats.spearmanr(mysim, gold)
-    dataset = os.path.basename(data_path)
-    correlation = corr[0] * 100
-    return dataset, correlation, 0
-
-
-if __name__ == "__main__":
-    model = train_unsupervised(
-        input=os.path.join(os.getenv("DATADIR", ''), 'fil9'),
-        model='skipgram',
-    )
-    model.save_model("fil9.bin")
-    dataset, corr, oov = compute_similarity('rw.txt')
-    print("{0:20s}: {1:2.0f}  (OOV: {2:2.0f}%)".format(dataset, corr, 0))