fasttext 0.1.0 → 0.2.1

data/vendor/fastText/python/fasttext_module/fasttext/FastText.py

@@ -1,468 +0,0 @@
-# Copyright (c) 2017-present, Facebook, Inc.
-# All rights reserved.
-#
-# This source code is licensed under the MIT license found in the
-# LICENSE file in the root directory of this source tree.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-from __future__ import unicode_literals
-
-import fasttext_pybind as fasttext
-import numpy as np
-import multiprocessing
-import sys
-from itertools import chain
-
-loss_name = fasttext.loss_name
-model_name = fasttext.model_name
-EOS = "</s>"
-BOW = "<"
-EOW = ">"
-
-
-def eprint(cls, *args, **kwargs):
-    print(*args, file=sys.stderr, **kwargs)
-
-
-class _FastText(object):
-    """
-    This class defines the API to inspect models and should not be used to
-    create objects. It will be returned by functions such as load_model or
-    train.
-
-    In general this API assumes to be given only unicode for Python2 and the
-    Python3 equvalent called str for any string-like arguments. All unicode
-    strings are then encoded as UTF-8 and fed to the fastText C++ API.
-    """
-
-    def __init__(self, model_path=None, args=None):
-        self.f = fasttext.fasttext()
-        if model_path is not None:
-            self.f.loadModel(model_path)
-        self._words = None
-        self._labels = None
-
-        if args:
-            arg_names = ['lr', 'dim', 'ws', 'epoch', 'minCount',
-                         'minCountLabel', 'minn', 'maxn', 'neg', 'wordNgrams',
-                         'loss', 'bucket', 'thread', 'lrUpdateRate', 't',
-                         'label', 'verbose', 'pretrainedVectors']
-            for arg_name in arg_names:
-                setattr(self, arg_name, getattr(args, arg_name))
-
-    def is_quantized(self):
-        return self.f.isQuant()
-
-    def get_dimension(self):
-        """Get the dimension (size) of a lookup vector (hidden layer)."""
-        a = self.f.getArgs()
-        return a.dim
-
-    def get_word_vector(self, word):
-        """Get the vector representation of word."""
-        dim = self.get_dimension()
-        b = fasttext.Vector(dim)
-        self.f.getWordVector(b, word)
-        return np.array(b)
-
-    def get_sentence_vector(self, text):
-        """
-        Given a string, get a single vector represenation. This function
-        assumes to be given a single line of text. We split words on
-        whitespace (space, newline, tab, vertical tab) and the control
-        characters carriage return, formfeed and the null character.
-        """
-        if text.find('\n') != -1:
-            raise ValueError(
-                "predict processes one line at a time (remove \'\\n\')"
-            )
-        text += "\n"
-        dim = self.get_dimension()
-        b = fasttext.Vector(dim)
-        self.f.getSentenceVector(b, text)
-        return np.array(b)
-
-    def get_word_id(self, word):
-        """
-        Given a word, get the word id within the dictionary.
-        Returns -1 if word is not in the dictionary.
-        """
-        return self.f.getWordId(word)
-
-    def get_subword_id(self, subword):
-        """
-        Given a subword, return the index (within input matrix) it hashes to.
-        """
-        return self.f.getSubwordId(subword)
-
-    def get_subwords(self, word, on_unicode_error='strict'):
-        """
-        Given a word, get the subwords and their indicies.
-        """
-        pair = self.f.getSubwords(word, on_unicode_error)
-        return pair[0], np.array(pair[1])
-
-    def get_input_vector(self, ind):
-        """
-        Given an index, get the corresponding vector of the Input Matrix.
-        """
-        dim = self.get_dimension()
-        b = fasttext.Vector(dim)
-        self.f.getInputVector(b, ind)
-        return np.array(b)
-
-    def predict(self, text, k=1, threshold=0.0, on_unicode_error='strict'):
-        """
-        Given a string, get a list of labels and a list of
-        corresponding probabilities. k controls the number
-        of returned labels. A choice of 5, will return the 5
-        most probable labels. By default this returns only
-        the most likely label and probability. threshold filters
-        the returned labels by a threshold on probability. A
-        choice of 0.5 will return labels with at least 0.5
-        probability. k and threshold will be applied together to
-        determine the returned labels.
-
-        This function assumes to be given
-        a single line of text. We split words on whitespace (space,
-        newline, tab, vertical tab) and the control characters carriage
-        return, formfeed and the null character.
-
-        If the model is not supervised, this function will throw a ValueError.
-
-        If given a list of strings, it will return a list of results as usually
-        received for a single line of text.
-        """
-
-        def check(entry):
-            if entry.find('\n') != -1:
-                raise ValueError(
-                    "predict processes one line at a time (remove \'\\n\')"
-                )
-            entry += "\n"
-            return entry
-
-        if type(text) == list:
-            text = [check(entry) for entry in text]
-            predictions = self.f.multilinePredict(text, k, threshold, on_unicode_error)
-            dt = np.dtype([('probability', 'float64'), ('label', 'object')])
-            result_as_pair = np.array(predictions, dtype=dt)
-
-            return result_as_pair['label'].tolist(), result_as_pair['probability']
-        else:
-            text = check(text)
-            predictions = self.f.predict(text, k, threshold, on_unicode_error)
-            probs, labels = zip(*predictions)
-
-            return labels, np.array(probs, copy=False)
-
-    def get_input_matrix(self):
-        """
-        Get a copy of the full input matrix of a Model. This only
-        works if the model is not quantized.
-        """
-        if self.f.isQuant():
-            raise ValueError("Can't get quantized Matrix")
-        return np.array(self.f.getInputMatrix())
-
-    def get_output_matrix(self):
-        """
-        Get a copy of the full output matrix of a Model. This only
-        works if the model is not quantized.
-        """
-        if self.f.isQuant():
-            raise ValueError("Can't get quantized Matrix")
-        return np.array(self.f.getOutputMatrix())
-
-    def get_words(self, include_freq=False, on_unicode_error='strict'):
-        """
-        Get the entire list of words of the dictionary optionally
-        including the frequency of the individual words. This
-        does not include any subwords. For that please consult
-        the function get_subwords.
-        """
-        pair = self.f.getVocab(on_unicode_error)
-        if include_freq:
-            return (pair[0], np.array(pair[1]))
-        else:
-            return pair[0]
-
-    def get_labels(self, include_freq=False, on_unicode_error='strict'):
-        """
-        Get the entire list of labels of the dictionary optionally
-        including the frequency of the individual labels. Unsupervised
-        models use words as labels, which is why get_labels
-        will call and return get_words for this type of
-        model.
-        """
-        a = self.f.getArgs()
-        if a.model == model_name.supervised:
-            pair = self.f.getLabels(on_unicode_error)
-            if include_freq:
-                return (pair[0], np.array(pair[1]))
-            else:
-                return pair[0]
-        else:
-            return self.get_words(include_freq)
-
-    def get_line(self, text, on_unicode_error='strict'):
-        """
-        Split a line of text into words and labels. Labels must start with
-        the prefix used to create the model (__label__ by default).
-        """
-
-        def check(entry):
-            if entry.find('\n') != -1:
-                raise ValueError(
-                    "get_line processes one line at a time (remove \'\\n\')"
-                )
-            entry += "\n"
-            return entry
-
-        if type(text) == list:
-            text = [check(entry) for entry in text]
-            return self.f.multilineGetLine(text, on_unicode_error)
-        else:
-            text = check(text)
-            return self.f.getLine(text, on_unicode_error)
-
-    def save_model(self, path):
-        """Save the model to the given path"""
-        self.f.saveModel(path)
-
-    def test(self, path, k=1):
-        """Evaluate supervised model using file given by path"""
-        return self.f.test(path, k)
-
-    def test_label(self, path, k=1, threshold=0.0):
-        """
-        Return the precision and recall score for each label.
-
-        The returned value is a dictionary, where the key is the label.
-        For example:
-        f.test_label(...)
-        {'__label__italian-cuisine' : {'precision' : 0.7, 'recall' : 0.74}}
-        """
-        return self.f.testLabel(path, k, threshold)
-
-    def quantize(
-        self,
-        input=None,
-        qout=False,
-        cutoff=0,
-        retrain=False,
-        epoch=None,
-        lr=None,
-        thread=None,
-        verbose=None,
-        dsub=2,
-        qnorm=False
-    ):
-        """
-        Quantize the model reducing the size of the model and
-        it's memory footprint.
-        """
-        a = self.f.getArgs()
-        if not epoch:
-            epoch = a.epoch
-        if not lr:
-            lr = a.lr
-        if not thread:
-            thread = a.thread
-        if not verbose:
-            verbose = a.verbose
-        if retrain and not input:
-            raise ValueError("Need input file path if retraining")
-        if input is None:
-            input = ""
-        self.f.quantize(
-            input, qout, cutoff, retrain, epoch, lr, thread, verbose, dsub,
-            qnorm
-        )
-
-    @property
-    def words(self):
-        if self._words is None:
-            self._words = self.get_words()
-        return self._words
-
-    @property
-    def labels(self):
-        if self._labels is None:
-            self._labels = self.get_labels()
-        return self._labels
-
-    def __getitem__(self, word):
-        return self.get_word_vector(word)
-
-    def __contains__(self, word):
-        return word in self.words
-
-
-def _parse_model_string(string):
-    if string == "cbow":
-        return model_name.cbow
-    if string == "skipgram":
-        return model_name.skipgram
-    if string == "supervised":
-        return model_name.supervised
-    else:
-        raise ValueError("Unrecognized model name")
-
-
-def _parse_loss_string(string):
-    if string == "ns":
-        return loss_name.ns
-    if string == "hs":
-        return loss_name.hs
-    if string == "softmax":
-        return loss_name.softmax
-    if string == "ova":
-        return loss_name.ova
-    else:
-        raise ValueError("Unrecognized loss name")
-
-
-def _build_args(args):
-    args["model"] = _parse_model_string(args["model"])
-    args["loss"] = _parse_loss_string(args["loss"])
-    a = fasttext.args()
-    for (k, v) in args.items():
-        setattr(a, k, v)
-    a.output = ""  # User should use save_model
-    a.saveOutput = 0  # Never use this
-    if a.wordNgrams <= 1 and a.maxn == 0:
-        a.bucket = 0
-    return a
-
-
-def tokenize(text):
-    """Given a string of text, tokenize it and return a list of tokens"""
-    f = fasttext.fasttext()
-    return f.tokenize(text)
-
-
-def load_model(path):
-    """Load a model given a filepath and return a model object."""
-    eprint("Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.")
-    return _FastText(model_path=path)
-
-
-unsupervised_default = {
-    'model' : "skipgram",
-    'lr' : 0.05,
-    'dim' : 100,
-    'ws' : 5,
-    'epoch' : 5,
-    'minCount' : 5,
-    'minCountLabel' : 0,
-    'minn' : 3,
-    'maxn' : 6,
-    'neg' : 5,
-    'wordNgrams' : 1,
-    'loss' : "ns",
-    'bucket' : 2000000,
-    'thread' : multiprocessing.cpu_count() - 1,
-    'lrUpdateRate' : 100,
-    't' : 1e-4,
-    'label' : "__label__",
-    'verbose' : 2,
-    'pretrainedVectors' : "",
-}
-
-
-def read_args(arg_list, arg_dict, arg_names, default_values):
-    param_map = {
-        'min_count' : 'minCount',
-        'word_ngrams' : 'wordNgrams',
-        'lr_update_rate' : 'lrUpdateRate',
-        'label_prefix' : 'label',
-        'pretrained_vectors' : 'pretrainedVectors'
-    }
-
-    ret = {}
-    for (arg_name, arg_value) in chain(zip(arg_names, arg_list), arg_dict.items()):
-        if arg_name in param_map:
-            arg_name = param_map[arg_name]
-        if arg_name not in arg_names:
-            raise TypeError("unexpected keyword argument '%s'" % arg_name)
-        if arg_name in ret:
-            raise TypeError("multiple values for argument '%s'" % arg_name)
-        ret[arg_name] = arg_value
-
-    for (arg_name, arg_value) in default_values.items():
-        if arg_name not in ret:
-            ret[arg_name] = arg_value
-
-    return ret
-
-
-def train_supervised(*kargs, **kwargs):
-    """
-    Train a supervised model and return a model object.
-
-    input must be a filepath. The input text does not need to be tokenized
-    as per the tokenize function, but it must be preprocessed and encoded
-    as UTF-8. You might want to consult standard preprocessing scripts such
-    as tokenizer.perl mentioned here: http://www.statmt.org/wmt07/baseline.html
-
-    The input file must must contain at least one label per line. For an
-    example consult the example datasets which are part of the fastText
-    repository such as the dataset pulled by classification-example.sh.
-    """
-    supervised_default = unsupervised_default.copy()
-    supervised_default.update({
-        'lr' : 0.1,
-        'minCount' : 1,
-        'minn' : 0,
-        'maxn' : 0,
-        'loss' : "softmax",
-        'model' : "supervised"
-    })
-
-    arg_names = ['input', 'lr', 'dim', 'ws', 'epoch', 'minCount',
-        'minCountLabel', 'minn', 'maxn', 'neg', 'wordNgrams', 'loss', 'bucket',
-        'thread', 'lrUpdateRate', 't', 'label', 'verbose', 'pretrainedVectors']
-    params = read_args(kargs, kwargs, arg_names, supervised_default)
-    a = _build_args(params)
-    ft = _FastText(args=a)
-    fasttext.train(ft.f, a)
-    return ft
-
-
-def train_unsupervised(*kargs, **kwargs):
-    """
-    Train an unsupervised model and return a model object.
-
-    input must be a filepath. The input text does not need to be tokenized
-    as per the tokenize function, but it must be preprocessed and encoded
-    as UTF-8. You might want to consult standard preprocessing scripts such
-    as tokenizer.perl mentioned here: http://www.statmt.org/wmt07/baseline.html
-
-    The input field must not contain any labels or use the specified label prefix
-    unless it is ok for those words to be ignored. For an example consult the
-    dataset pulled by the example script word-vector-example.sh, which is
-    part of the fastText repository.
-    """
-    arg_names = ['input', 'model', 'lr', 'dim', 'ws', 'epoch', 'minCount',
-        'minCountLabel', 'minn', 'maxn', 'neg', 'wordNgrams', 'loss', 'bucket',
-        'thread', 'lrUpdateRate', 't', 'label', 'verbose', 'pretrainedVectors']
-    params = read_args(kargs, kwargs, arg_names, unsupervised_default)
-    a = _build_args(params)
-    ft = _FastText(args=a)
-    fasttext.train(ft.f, a)
-    return ft
-
-
-def cbow(*kargs, **kwargs):
-    raise Exception("`cbow` is not supported any more. Please use `train_unsupervised` with model=`cbow`. For more information please refer to https://fasttext.cc/blog/2019/06/25/blog-post.html#2-you-were-using-the-unofficial-fasttext-module")
-
-
-def skipgram(*kargs, **kwargs):
-    raise Exception("`skipgram` is not supported any more. Please use `train_unsupervised` with model=`skipgram`. For more information please refer to https://fasttext.cc/blog/2019/06/25/blog-post.html#2-you-were-using-the-unofficial-fasttext-module")
-
-
-def supervised(*kargs, **kwargs):
-    raise Exception("`supervised` is not supported any more. Please use `train_supervised`. For more information please refer to https://fasttext.cc/blog/2019/06/25/blog-post.html#2-you-were-using-the-unofficial-fasttext-module")