fasttext 0.1.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/CHANGELOG.md +3 -0
- data/LICENSE.txt +22 -0
- data/README.md +251 -0
- data/ext/fasttext/ext.cpp +291 -0
- data/ext/fasttext/extconf.rb +15 -0
- data/lib/fasttext.rb +41 -0
- data/lib/fasttext/classifier.rb +92 -0
- data/lib/fasttext/ext.bundle +0 -0
- data/lib/fasttext/model.rb +60 -0
- data/lib/fasttext/vectorizer.rb +58 -0
- data/lib/fasttext/version.rb +3 -0
- data/vendor/fastText/CMakeLists.txt +68 -0
- data/vendor/fastText/CODE_OF_CONDUCT.md +2 -0
- data/vendor/fastText/CONTRIBUTING.md +32 -0
- data/vendor/fastText/LICENSE +21 -0
- data/vendor/fastText/MANIFEST.in +5 -0
- data/vendor/fastText/Makefile +63 -0
- data/vendor/fastText/README.md +339 -0
- data/vendor/fastText/alignment/README.md +53 -0
- data/vendor/fastText/alignment/align.py +145 -0
- data/vendor/fastText/alignment/eval.py +60 -0
- data/vendor/fastText/alignment/example.sh +51 -0
- data/vendor/fastText/alignment/unsup_align.py +109 -0
- data/vendor/fastText/alignment/utils.py +154 -0
- data/vendor/fastText/classification-example.sh +41 -0
- data/vendor/fastText/classification-results.sh +94 -0
- data/vendor/fastText/crawl/README.md +26 -0
- data/vendor/fastText/crawl/dedup.cc +51 -0
- data/vendor/fastText/crawl/download_crawl.sh +57 -0
- data/vendor/fastText/crawl/filter_dedup.sh +13 -0
- data/vendor/fastText/crawl/filter_utf8.cc +105 -0
- data/vendor/fastText/crawl/process_wet_file.sh +30 -0
- data/vendor/fastText/docs/aligned-vectors.md +64 -0
- data/vendor/fastText/docs/api.md +6 -0
- data/vendor/fastText/docs/cheatsheet.md +66 -0
- data/vendor/fastText/docs/crawl-vectors.md +125 -0
- data/vendor/fastText/docs/dataset.md +6 -0
- data/vendor/fastText/docs/english-vectors.md +53 -0
- data/vendor/fastText/docs/faqs.md +63 -0
- data/vendor/fastText/docs/language-identification.md +47 -0
- data/vendor/fastText/docs/options.md +50 -0
- data/vendor/fastText/docs/pretrained-vectors.md +142 -0
- data/vendor/fastText/docs/python-module.md +314 -0
- data/vendor/fastText/docs/references.md +41 -0
- data/vendor/fastText/docs/supervised-models.md +54 -0
- data/vendor/fastText/docs/supervised-tutorial.md +349 -0
- data/vendor/fastText/docs/support.md +58 -0
- data/vendor/fastText/docs/unsupervised-tutorials.md +309 -0
- data/vendor/fastText/eval.py +95 -0
- data/vendor/fastText/get-wikimedia.sh +79 -0
- data/vendor/fastText/python/README.md +322 -0
- data/vendor/fastText/python/README.rst +406 -0
- data/vendor/fastText/python/benchmarks/README.rst +3 -0
- data/vendor/fastText/python/benchmarks/get_word_vector.py +49 -0
- data/vendor/fastText/python/doc/examples/FastTextEmbeddingBag.py +81 -0
- data/vendor/fastText/python/doc/examples/bin_to_vec.py +41 -0
- data/vendor/fastText/python/doc/examples/compute_accuracy.py +163 -0
- data/vendor/fastText/python/doc/examples/get_vocab.py +48 -0
- data/vendor/fastText/python/doc/examples/train_supervised.py +42 -0
- data/vendor/fastText/python/doc/examples/train_unsupervised.py +56 -0
- data/vendor/fastText/python/fasttext_module/fasttext/FastText.py +468 -0
- data/vendor/fastText/python/fasttext_module/fasttext/__init__.py +22 -0
- data/vendor/fastText/python/fasttext_module/fasttext/pybind/fasttext_pybind.cc +388 -0
- data/vendor/fastText/python/fasttext_module/fasttext/tests/__init__.py +14 -0
- data/vendor/fastText/python/fasttext_module/fasttext/tests/test_configurations.py +239 -0
- data/vendor/fastText/python/fasttext_module/fasttext/tests/test_script.py +629 -0
- data/vendor/fastText/python/fasttext_module/fasttext/util/__init__.py +13 -0
- data/vendor/fastText/python/fasttext_module/fasttext/util/util.py +60 -0
- data/vendor/fastText/quantization-example.sh +40 -0
- data/vendor/fastText/runtests.py +60 -0
- data/vendor/fastText/scripts/kbcompletion/README.md +19 -0
- data/vendor/fastText/scripts/kbcompletion/data.sh +69 -0
- data/vendor/fastText/scripts/kbcompletion/eval.cpp +108 -0
- data/vendor/fastText/scripts/kbcompletion/fb15k.sh +49 -0
- data/vendor/fastText/scripts/kbcompletion/fb15k237.sh +45 -0
- data/vendor/fastText/scripts/kbcompletion/svo.sh +38 -0
- data/vendor/fastText/scripts/kbcompletion/wn18.sh +49 -0
- data/vendor/fastText/scripts/quantization/quantization-results.sh +43 -0
- data/vendor/fastText/setup.cfg +2 -0
- data/vendor/fastText/setup.py +203 -0
- data/vendor/fastText/src/args.cc +320 -0
- data/vendor/fastText/src/args.h +68 -0
- data/vendor/fastText/src/densematrix.cc +155 -0
- data/vendor/fastText/src/densematrix.h +75 -0
- data/vendor/fastText/src/dictionary.cc +540 -0
- data/vendor/fastText/src/dictionary.h +111 -0
- data/vendor/fastText/src/fasttext.cc +821 -0
- data/vendor/fastText/src/fasttext.h +191 -0
- data/vendor/fastText/src/loss.cc +346 -0
- data/vendor/fastText/src/loss.h +163 -0
- data/vendor/fastText/src/main.cc +435 -0
- data/vendor/fastText/src/matrix.cc +25 -0
- data/vendor/fastText/src/matrix.h +44 -0
- data/vendor/fastText/src/meter.cc +68 -0
- data/vendor/fastText/src/meter.h +69 -0
- data/vendor/fastText/src/model.cc +98 -0
- data/vendor/fastText/src/model.h +79 -0
- data/vendor/fastText/src/productquantizer.cc +251 -0
- data/vendor/fastText/src/productquantizer.h +63 -0
- data/vendor/fastText/src/quantmatrix.cc +117 -0
- data/vendor/fastText/src/quantmatrix.h +60 -0
- data/vendor/fastText/src/real.h +15 -0
- data/vendor/fastText/src/utils.cc +28 -0
- data/vendor/fastText/src/utils.h +43 -0
- data/vendor/fastText/src/vector.cc +97 -0
- data/vendor/fastText/src/vector.h +61 -0
- data/vendor/fastText/tests/fetch_test_data.sh +202 -0
- data/vendor/fastText/website/README.md +6 -0
- data/vendor/fastText/website/blog/2016-08-18-blog-post.md +42 -0
- data/vendor/fastText/website/blog/2017-05-02-blog-post.md +60 -0
- data/vendor/fastText/website/blog/2017-10-02-blog-post.md +90 -0
- data/vendor/fastText/website/blog/2019-06-25-blog-post.md +168 -0
- data/vendor/fastText/website/core/Footer.js +127 -0
- data/vendor/fastText/website/package.json +12 -0
- data/vendor/fastText/website/pages/en/index.js +286 -0
- data/vendor/fastText/website/sidebars.json +18 -0
- data/vendor/fastText/website/siteConfig.js +102 -0
- data/vendor/fastText/website/static/docs/en/html/annotated.html +115 -0
- data/vendor/fastText/website/static/docs/en/html/annotated_dup.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/args_8cc.html +113 -0
- data/vendor/fastText/website/static/docs/en/html/args_8h.html +134 -0
- data/vendor/fastText/website/static/docs/en/html/args_8h.js +14 -0
- data/vendor/fastText/website/static/docs/en/html/args_8h_source.html +139 -0
- data/vendor/fastText/website/static/docs/en/html/bc_s.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/bdwn.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/classes.html +121 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Args-members.html +140 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Args.html +753 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Args.js +40 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Dictionary-members.html +148 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Dictionary.html +1266 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Dictionary.js +43 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1FastText-members.html +145 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1FastText.html +1149 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1FastText.js +45 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Matrix-members.html +123 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Matrix.html +610 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Matrix.js +23 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Model-members.html +150 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Model.html +1400 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Model.js +48 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1ProductQuantizer-members.html +131 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1ProductQuantizer.html +950 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1ProductQuantizer.js +31 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1QMatrix-members.html +122 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1QMatrix.html +565 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1QMatrix.js +22 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Vector-members.html +121 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Vector.html +542 -0
- data/vendor/fastText/website/static/docs/en/html/classfasttext_1_1Vector.js +21 -0
- data/vendor/fastText/website/static/docs/en/html/closed.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/dictionary_8cc.html +116 -0
- data/vendor/fastText/website/static/docs/en/html/dictionary_8h.html +142 -0
- data/vendor/fastText/website/static/docs/en/html/dictionary_8h.js +10 -0
- data/vendor/fastText/website/static/docs/en/html/dictionary_8h_source.html +127 -0
- data/vendor/fastText/website/static/docs/en/html/dir_68267d1309a1af8e8297ef4c3efbcdba.html +145 -0
- data/vendor/fastText/website/static/docs/en/html/dir_68267d1309a1af8e8297ef4c3efbcdba.js +29 -0
- data/vendor/fastText/website/static/docs/en/html/doc.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/doxygen.css +1596 -0
- data/vendor/fastText/website/static/docs/en/html/doxygen.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/dynsections.js +97 -0
- data/vendor/fastText/website/static/docs/en/html/fasttext_8cc.html +119 -0
- data/vendor/fastText/website/static/docs/en/html/fasttext_8h.html +168 -0
- data/vendor/fastText/website/static/docs/en/html/fasttext_8h.js +6 -0
- data/vendor/fastText/website/static/docs/en/html/fasttext_8h_source.html +155 -0
- data/vendor/fastText/website/static/docs/en/html/favicon.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/files.html +125 -0
- data/vendor/fastText/website/static/docs/en/html/files.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/folderclosed.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/folderopen.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/functions.html +139 -0
- data/vendor/fastText/website/static/docs/en/html/functions_0x7e.html +112 -0
- data/vendor/fastText/website/static/docs/en/html/functions_b.html +115 -0
- data/vendor/fastText/website/static/docs/en/html/functions_c.html +143 -0
- data/vendor/fastText/website/static/docs/en/html/functions_d.html +135 -0
- data/vendor/fastText/website/static/docs/en/html/functions_dup.js +27 -0
- data/vendor/fastText/website/static/docs/en/html/functions_e.html +115 -0
- data/vendor/fastText/website/static/docs/en/html/functions_f.html +112 -0
- data/vendor/fastText/website/static/docs/en/html/functions_func.html +563 -0
- data/vendor/fastText/website/static/docs/en/html/functions_g.html +145 -0
- data/vendor/fastText/website/static/docs/en/html/functions_h.html +112 -0
- data/vendor/fastText/website/static/docs/en/html/functions_i.html +121 -0
- data/vendor/fastText/website/static/docs/en/html/functions_k.html +106 -0
- data/vendor/fastText/website/static/docs/en/html/functions_l.html +140 -0
- data/vendor/fastText/website/static/docs/en/html/functions_m.html +153 -0
- data/vendor/fastText/website/static/docs/en/html/functions_n.html +164 -0
- data/vendor/fastText/website/static/docs/en/html/functions_o.html +116 -0
- data/vendor/fastText/website/static/docs/en/html/functions_p.html +161 -0
- data/vendor/fastText/website/static/docs/en/html/functions_q.html +135 -0
- data/vendor/fastText/website/static/docs/en/html/functions_r.html +116 -0
- data/vendor/fastText/website/static/docs/en/html/functions_s.html +159 -0
- data/vendor/fastText/website/static/docs/en/html/functions_t.html +138 -0
- data/vendor/fastText/website/static/docs/en/html/functions_u.html +106 -0
- data/vendor/fastText/website/static/docs/en/html/functions_v.html +106 -0
- data/vendor/fastText/website/static/docs/en/html/functions_vars.html +486 -0
- data/vendor/fastText/website/static/docs/en/html/functions_w.html +124 -0
- data/vendor/fastText/website/static/docs/en/html/functions_z.html +104 -0
- data/vendor/fastText/website/static/docs/en/html/globals.html +170 -0
- data/vendor/fastText/website/static/docs/en/html/globals_defs.html +113 -0
- data/vendor/fastText/website/static/docs/en/html/globals_func.html +155 -0
- data/vendor/fastText/website/static/docs/en/html/index.html +100 -0
- data/vendor/fastText/website/static/docs/en/html/jquery.js +87 -0
- data/vendor/fastText/website/static/docs/en/html/main_8cc.html +582 -0
- data/vendor/fastText/website/static/docs/en/html/main_8cc.js +22 -0
- data/vendor/fastText/website/static/docs/en/html/matrix_8cc.html +114 -0
- data/vendor/fastText/website/static/docs/en/html/matrix_8h.html +121 -0
- data/vendor/fastText/website/static/docs/en/html/matrix_8h_source.html +123 -0
- data/vendor/fastText/website/static/docs/en/html/menu.js +26 -0
- data/vendor/fastText/website/static/docs/en/html/menudata.js +90 -0
- data/vendor/fastText/website/static/docs/en/html/model_8cc.html +113 -0
- data/vendor/fastText/website/static/docs/en/html/model_8h.html +183 -0
- data/vendor/fastText/website/static/docs/en/html/model_8h.js +8 -0
- data/vendor/fastText/website/static/docs/en/html/model_8h_source.html +139 -0
- data/vendor/fastText/website/static/docs/en/html/namespacefasttext.html +343 -0
- data/vendor/fastText/website/static/docs/en/html/namespacefasttext.js +13 -0
- data/vendor/fastText/website/static/docs/en/html/namespacefasttext_1_1utils.html +158 -0
- data/vendor/fastText/website/static/docs/en/html/namespacemembers.html +125 -0
- data/vendor/fastText/website/static/docs/en/html/namespacemembers_enum.html +107 -0
- data/vendor/fastText/website/static/docs/en/html/namespacemembers_func.html +110 -0
- data/vendor/fastText/website/static/docs/en/html/namespacemembers_type.html +104 -0
- data/vendor/fastText/website/static/docs/en/html/namespaces.html +106 -0
- data/vendor/fastText/website/static/docs/en/html/namespaces.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/nav_f.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/nav_g.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/nav_h.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/navtree.css +146 -0
- data/vendor/fastText/website/static/docs/en/html/navtree.js +517 -0
- data/vendor/fastText/website/static/docs/en/html/navtreedata.js +40 -0
- data/vendor/fastText/website/static/docs/en/html/navtreeindex0.js +253 -0
- data/vendor/fastText/website/static/docs/en/html/navtreeindex1.js +139 -0
- data/vendor/fastText/website/static/docs/en/html/open.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/productquantizer_8cc.html +118 -0
- data/vendor/fastText/website/static/docs/en/html/productquantizer_8cc.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/productquantizer_8h.html +124 -0
- data/vendor/fastText/website/static/docs/en/html/productquantizer_8h_source.html +133 -0
- data/vendor/fastText/website/static/docs/en/html/qmatrix_8cc.html +112 -0
- data/vendor/fastText/website/static/docs/en/html/qmatrix_8h.html +126 -0
- data/vendor/fastText/website/static/docs/en/html/qmatrix_8h_source.html +128 -0
- data/vendor/fastText/website/static/docs/en/html/real_8h.html +117 -0
- data/vendor/fastText/website/static/docs/en/html/real_8h.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/real_8h_source.html +103 -0
- data/vendor/fastText/website/static/docs/en/html/resize.js +114 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_0.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_0.js +17 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_1.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_1.js +8 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_10.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_10.js +10 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_11.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_11.js +25 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_12.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_12.js +15 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_13.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_13.js +7 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_14.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_14.js +7 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_15.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_15.js +11 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_16.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_16.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_17.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_17.js +7 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_2.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_2.js +17 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_3.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_3.js +17 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_4.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_4.js +10 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_5.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_5.js +12 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_6.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_6.js +18 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_7.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_7.js +8 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_8.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_8.js +11 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_9.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_9.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_a.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_a.js +17 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_b.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_b.js +27 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_c.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_c.js +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_d.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_d.js +9 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_e.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_e.js +35 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_f.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/all_f.js +16 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_0.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_0.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_1.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_1.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_2.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_2.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_3.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_3.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_4.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_4.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_5.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_5.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_6.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_6.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_7.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_7.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_8.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/classes_8.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/close.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/search/defines_0.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/defines_0.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/defines_1.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/defines_1.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/defines_2.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/defines_2.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/defines_3.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/defines_3.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/enums_0.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/enums_0.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/enums_1.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/enums_1.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/enums_2.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/enums_2.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/enumvalues_0.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/enumvalues_0.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/enumvalues_1.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/enumvalues_1.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/enumvalues_2.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/enumvalues_2.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/enumvalues_3.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/enumvalues_3.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/enumvalues_4.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/enumvalues_4.js +6 -0
- data/vendor/fastText/website/static/docs/en/html/search/enumvalues_5.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/enumvalues_5.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_0.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_0.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_1.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_1.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_2.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_2.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_3.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_3.js +8 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_4.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_4.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_5.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_5.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_6.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_6.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_7.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_7.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_8.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/files_8.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_0.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_0.js +14 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_1.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_1.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_10.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_10.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_11.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_11.js +18 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_12.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_12.js +8 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_13.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_13.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_14.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_14.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_15.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_15.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_16.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_16.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_17.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_17.js +7 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_2.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_2.js +11 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_3.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_3.js +9 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_4.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_4.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_5.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_5.js +7 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_6.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_6.js +17 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_7.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_7.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_8.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_8.js +8 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_9.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_9.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_a.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_a.js +8 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_b.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_b.js +10 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_c.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_c.js +10 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_d.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_d.js +6 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_e.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_e.js +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_f.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/functions_f.js +6 -0
- data/vendor/fastText/website/static/docs/en/html/search/mag_sel.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/search/namespaces_0.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/namespaces_0.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/nomatches.html +12 -0
- data/vendor/fastText/website/static/docs/en/html/search/search.css +271 -0
- data/vendor/fastText/website/static/docs/en/html/search/search.js +791 -0
- data/vendor/fastText/website/static/docs/en/html/search/search_l.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/search/search_m.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/search/search_r.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/search/searchdata.js +42 -0
- data/vendor/fastText/website/static/docs/en/html/search/typedefs_0.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/typedefs_0.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/typedefs_1.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/typedefs_1.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_0.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_0.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_1.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_1.js +6 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_10.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_10.js +8 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_11.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_11.js +11 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_12.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_12.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_13.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_13.js +10 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_2.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_2.js +9 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_3.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_3.js +9 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_4.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_4.js +7 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_5.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_5.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_6.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_6.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_7.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_7.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_8.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_8.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_9.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_9.js +10 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_a.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_a.js +14 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_b.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_b.js +17 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_c.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_c.js +6 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_d.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_d.js +10 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_e.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_e.js +11 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_f.html +26 -0
- data/vendor/fastText/website/static/docs/en/html/search/variables_f.js +6 -0
- data/vendor/fastText/website/static/docs/en/html/splitbar.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/structfasttext_1_1Node-members.html +108 -0
- data/vendor/fastText/website/static/docs/en/html/structfasttext_1_1Node.html +194 -0
- data/vendor/fastText/website/static/docs/en/html/structfasttext_1_1Node.js +8 -0
- data/vendor/fastText/website/static/docs/en/html/structfasttext_1_1entry-members.html +107 -0
- data/vendor/fastText/website/static/docs/en/html/structfasttext_1_1entry.html +178 -0
- data/vendor/fastText/website/static/docs/en/html/structfasttext_1_1entry.js +7 -0
- data/vendor/fastText/website/static/docs/en/html/sync_off.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/sync_on.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/tab_a.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/tab_b.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/tab_h.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/tab_s.png +0 -0
- data/vendor/fastText/website/static/docs/en/html/tabs.css +1 -0
- data/vendor/fastText/website/static/docs/en/html/utils_8cc.html +121 -0
- data/vendor/fastText/website/static/docs/en/html/utils_8cc.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/utils_8h.html +122 -0
- data/vendor/fastText/website/static/docs/en/html/utils_8h.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/utils_8h_source.html +104 -0
- data/vendor/fastText/website/static/docs/en/html/vector_8cc.html +121 -0
- data/vendor/fastText/website/static/docs/en/html/vector_8cc.js +4 -0
- data/vendor/fastText/website/static/docs/en/html/vector_8h.html +126 -0
- data/vendor/fastText/website/static/docs/en/html/vector_8h.js +5 -0
- data/vendor/fastText/website/static/docs/en/html/vector_8h_source.html +120 -0
- data/vendor/fastText/website/static/fasttext.css +48 -0
- data/vendor/fastText/website/static/img/authors/armand_joulin.jpg +0 -0
- data/vendor/fastText/website/static/img/authors/christian_puhrsch.png +0 -0
- data/vendor/fastText/website/static/img/authors/edouard_grave.jpeg +0 -0
- data/vendor/fastText/website/static/img/authors/piotr_bojanowski.jpg +0 -0
- data/vendor/fastText/website/static/img/authors/tomas_mikolov.jpg +0 -0
- data/vendor/fastText/website/static/img/blog/2016-08-18-blog-post-img1.png +0 -0
- data/vendor/fastText/website/static/img/blog/2016-08-18-blog-post-img2.png +0 -0
- data/vendor/fastText/website/static/img/blog/2017-05-02-blog-post-img1.jpg +0 -0
- data/vendor/fastText/website/static/img/blog/2017-05-02-blog-post-img2.jpg +0 -0
- data/vendor/fastText/website/static/img/blog/2017-10-02-blog-post-img1.png +0 -0
- data/vendor/fastText/website/static/img/cbo_vs_skipgram.png +0 -0
- data/vendor/fastText/website/static/img/fasttext-icon-api.png +0 -0
- data/vendor/fastText/website/static/img/fasttext-icon-bg-web.png +0 -0
- data/vendor/fastText/website/static/img/fasttext-icon-color-square.png +0 -0
- data/vendor/fastText/website/static/img/fasttext-icon-color-web.png +0 -0
- data/vendor/fastText/website/static/img/fasttext-icon-faq.png +0 -0
- data/vendor/fastText/website/static/img/fasttext-icon-tutorial.png +0 -0
- data/vendor/fastText/website/static/img/fasttext-icon-white-web.png +0 -0
- data/vendor/fastText/website/static/img/fasttext-logo-color-web.png +0 -0
- data/vendor/fastText/website/static/img/fasttext-logo-white-web.png +0 -0
- data/vendor/fastText/website/static/img/logo-color.png +0 -0
- data/vendor/fastText/website/static/img/model-black.png +0 -0
- data/vendor/fastText/website/static/img/model-blue.png +0 -0
- data/vendor/fastText/website/static/img/model-red.png +0 -0
- data/vendor/fastText/website/static/img/ogimage.png +0 -0
- data/vendor/fastText/website/static/img/oss_logo.png +0 -0
- data/vendor/fastText/wikifil.pl +57 -0
- data/vendor/fastText/word-vector-example.sh +39 -0
- metadata +621 -0
@@ -0,0 +1,25 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2016-present, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under the MIT license found in the
|
6
|
+
* LICENSE file in the root directory of this source tree.
|
7
|
+
*/
|
8
|
+
|
9
|
+
#include "matrix.h"
|
10
|
+
|
11
|
+
namespace fasttext {
|
12
|
+
|
13
|
+
Matrix::Matrix() : m_(0), n_(0) {}
|
14
|
+
|
15
|
+
Matrix::Matrix(int64_t m, int64_t n) : m_(m), n_(n) {}
|
16
|
+
|
17
|
+
int64_t Matrix::size(int64_t dim) const {
|
18
|
+
assert(dim == 0 || dim == 1);
|
19
|
+
if (dim == 0) {
|
20
|
+
return m_;
|
21
|
+
}
|
22
|
+
return n_;
|
23
|
+
}
|
24
|
+
|
25
|
+
} // namespace fasttext
|
@@ -0,0 +1,44 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2016-present, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under the MIT license found in the
|
6
|
+
* LICENSE file in the root directory of this source tree.
|
7
|
+
*/
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include <cstdint>
|
12
|
+
#include <istream>
|
13
|
+
#include <ostream>
|
14
|
+
#include <vector>
|
15
|
+
|
16
|
+
#include <assert.h>
|
17
|
+
#include "real.h"
|
18
|
+
|
19
|
+
namespace fasttext {
|
20
|
+
|
21
|
+
class Vector;
|
22
|
+
|
23
|
+
class Matrix {
|
24
|
+
protected:
|
25
|
+
int64_t m_;
|
26
|
+
int64_t n_;
|
27
|
+
|
28
|
+
public:
|
29
|
+
Matrix();
|
30
|
+
explicit Matrix(int64_t, int64_t);
|
31
|
+
virtual ~Matrix() = default;
|
32
|
+
|
33
|
+
int64_t size(int64_t dim) const;
|
34
|
+
|
35
|
+
virtual real dotRow(const Vector&, int64_t) const = 0;
|
36
|
+
virtual void addVectorToRow(const Vector&, int64_t, real) = 0;
|
37
|
+
virtual void addRowToVector(Vector& x, int32_t i) const = 0;
|
38
|
+
virtual void addRowToVector(Vector& x, int32_t i, real a) const = 0;
|
39
|
+
virtual void save(std::ostream&) const = 0;
|
40
|
+
virtual void load(std::istream&) = 0;
|
41
|
+
virtual void dump(std::ostream&) const = 0;
|
42
|
+
};
|
43
|
+
|
44
|
+
} // namespace fasttext
|
@@ -0,0 +1,68 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2016-present, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under the MIT license found in the
|
6
|
+
* LICENSE file in the root directory of this source tree.
|
7
|
+
*/
|
8
|
+
|
9
|
+
#include "meter.h"
|
10
|
+
#include "utils.h"
|
11
|
+
|
12
|
+
#include <algorithm>
|
13
|
+
#include <cmath>
|
14
|
+
#include <iomanip>
|
15
|
+
#include <limits>
|
16
|
+
|
17
|
+
namespace fasttext {
|
18
|
+
|
19
|
+
void Meter::log(
|
20
|
+
const std::vector<int32_t>& labels,
|
21
|
+
const Predictions& predictions) {
|
22
|
+
nexamples_++;
|
23
|
+
metrics_.gold += labels.size();
|
24
|
+
metrics_.predicted += predictions.size();
|
25
|
+
|
26
|
+
for (const auto& prediction : predictions) {
|
27
|
+
labelMetrics_[prediction.second].predicted++;
|
28
|
+
|
29
|
+
if (utils::contains(labels, prediction.second)) {
|
30
|
+
labelMetrics_[prediction.second].predictedGold++;
|
31
|
+
metrics_.predictedGold++;
|
32
|
+
}
|
33
|
+
}
|
34
|
+
|
35
|
+
for (const auto& label : labels) {
|
36
|
+
labelMetrics_[label].gold++;
|
37
|
+
}
|
38
|
+
}
|
39
|
+
|
40
|
+
double Meter::precision(int32_t i) {
|
41
|
+
return labelMetrics_[i].precision();
|
42
|
+
}
|
43
|
+
|
44
|
+
double Meter::recall(int32_t i) {
|
45
|
+
return labelMetrics_[i].recall();
|
46
|
+
}
|
47
|
+
|
48
|
+
double Meter::f1Score(int32_t i) {
|
49
|
+
return labelMetrics_[i].f1Score();
|
50
|
+
}
|
51
|
+
|
52
|
+
double Meter::precision() const {
|
53
|
+
return metrics_.precision();
|
54
|
+
}
|
55
|
+
|
56
|
+
double Meter::recall() const {
|
57
|
+
return metrics_.recall();
|
58
|
+
}
|
59
|
+
|
60
|
+
void Meter::writeGeneralMetrics(std::ostream& out, int32_t k) const {
|
61
|
+
out << "N"
|
62
|
+
<< "\t" << nexamples_ << std::endl;
|
63
|
+
out << std::setprecision(3);
|
64
|
+
out << "P@" << k << "\t" << metrics_.precision() << std::endl;
|
65
|
+
out << "R@" << k << "\t" << metrics_.recall() << std::endl;
|
66
|
+
}
|
67
|
+
|
68
|
+
} // namespace fasttext
|
@@ -0,0 +1,69 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2016-present, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under the MIT license found in the
|
6
|
+
* LICENSE file in the root directory of this source tree.
|
7
|
+
*/
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include <unordered_map>
|
12
|
+
#include <vector>
|
13
|
+
|
14
|
+
#include "dictionary.h"
|
15
|
+
#include "real.h"
|
16
|
+
#include "utils.h"
|
17
|
+
|
18
|
+
namespace fasttext {
|
19
|
+
|
20
|
+
class Meter {
|
21
|
+
struct Metrics {
|
22
|
+
uint64_t gold;
|
23
|
+
uint64_t predicted;
|
24
|
+
uint64_t predictedGold;
|
25
|
+
|
26
|
+
Metrics() : gold(0), predicted(0), predictedGold(0) {}
|
27
|
+
|
28
|
+
double precision() const {
|
29
|
+
if (predicted == 0) {
|
30
|
+
return std::numeric_limits<double>::quiet_NaN();
|
31
|
+
}
|
32
|
+
return predictedGold / double(predicted);
|
33
|
+
}
|
34
|
+
double recall() const {
|
35
|
+
if (gold == 0) {
|
36
|
+
return std::numeric_limits<double>::quiet_NaN();
|
37
|
+
}
|
38
|
+
return predictedGold / double(gold);
|
39
|
+
}
|
40
|
+
double f1Score() const {
|
41
|
+
if (predicted + gold == 0) {
|
42
|
+
return std::numeric_limits<double>::quiet_NaN();
|
43
|
+
}
|
44
|
+
return 2 * predictedGold / double(predicted + gold);
|
45
|
+
}
|
46
|
+
};
|
47
|
+
|
48
|
+
public:
|
49
|
+
Meter() : metrics_(), nexamples_(0), labelMetrics_() {}
|
50
|
+
|
51
|
+
void log(const std::vector<int32_t>& labels, const Predictions& predictions);
|
52
|
+
|
53
|
+
double precision(int32_t);
|
54
|
+
double recall(int32_t);
|
55
|
+
double f1Score(int32_t);
|
56
|
+
double precision() const;
|
57
|
+
double recall() const;
|
58
|
+
uint64_t nexamples() const {
|
59
|
+
return nexamples_;
|
60
|
+
}
|
61
|
+
void writeGeneralMetrics(std::ostream& out, int32_t k) const;
|
62
|
+
|
63
|
+
private:
|
64
|
+
Metrics metrics_{};
|
65
|
+
uint64_t nexamples_;
|
66
|
+
std::unordered_map<int32_t, Metrics> labelMetrics_;
|
67
|
+
};
|
68
|
+
|
69
|
+
} // namespace fasttext
|
@@ -0,0 +1,98 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2016-present, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under the MIT license found in the
|
6
|
+
* LICENSE file in the root directory of this source tree.
|
7
|
+
*/
|
8
|
+
|
9
|
+
#include "model.h"
|
10
|
+
#include "loss.h"
|
11
|
+
#include "utils.h"
|
12
|
+
|
13
|
+
#include <assert.h>
|
14
|
+
#include <algorithm>
|
15
|
+
#include <stdexcept>
|
16
|
+
|
17
|
+
namespace fasttext {
|
18
|
+
|
19
|
+
Model::State::State(int32_t hiddenSize, int32_t outputSize, int32_t seed)
|
20
|
+
: lossValue_(0.0),
|
21
|
+
nexamples_(0),
|
22
|
+
hidden(hiddenSize),
|
23
|
+
output(outputSize),
|
24
|
+
grad(hiddenSize),
|
25
|
+
rng(seed) {}
|
26
|
+
|
27
|
+
real Model::State::getLoss() const {
|
28
|
+
return lossValue_ / nexamples_;
|
29
|
+
}
|
30
|
+
|
31
|
+
void Model::State::incrementNExamples(real loss) {
|
32
|
+
lossValue_ += loss;
|
33
|
+
nexamples_++;
|
34
|
+
}
|
35
|
+
|
36
|
+
Model::Model(
|
37
|
+
std::shared_ptr<Matrix> wi,
|
38
|
+
std::shared_ptr<Matrix> wo,
|
39
|
+
std::shared_ptr<Loss> loss,
|
40
|
+
bool normalizeGradient)
|
41
|
+
: wi_(wi), wo_(wo), loss_(loss), normalizeGradient_(normalizeGradient) {}
|
42
|
+
|
43
|
+
void Model::computeHidden(const std::vector<int32_t>& input, State& state)
|
44
|
+
const {
|
45
|
+
Vector& hidden = state.hidden;
|
46
|
+
hidden.zero();
|
47
|
+
for (auto it = input.cbegin(); it != input.cend(); ++it) {
|
48
|
+
hidden.addRow(*wi_, *it);
|
49
|
+
}
|
50
|
+
hidden.mul(1.0 / input.size());
|
51
|
+
}
|
52
|
+
|
53
|
+
void Model::predict(
|
54
|
+
const std::vector<int32_t>& input,
|
55
|
+
int32_t k,
|
56
|
+
real threshold,
|
57
|
+
Predictions& heap,
|
58
|
+
State& state) const {
|
59
|
+
if (k == Model::kUnlimitedPredictions) {
|
60
|
+
k = wo_->size(0); // output size
|
61
|
+
} else if (k <= 0) {
|
62
|
+
throw std::invalid_argument("k needs to be 1 or higher!");
|
63
|
+
}
|
64
|
+
heap.reserve(k + 1);
|
65
|
+
computeHidden(input, state);
|
66
|
+
|
67
|
+
loss_->predict(k, threshold, heap, state);
|
68
|
+
}
|
69
|
+
|
70
|
+
void Model::update(
|
71
|
+
const std::vector<int32_t>& input,
|
72
|
+
const std::vector<int32_t>& targets,
|
73
|
+
int32_t targetIndex,
|
74
|
+
real lr,
|
75
|
+
State& state) {
|
76
|
+
if (input.size() == 0) {
|
77
|
+
return;
|
78
|
+
}
|
79
|
+
computeHidden(input, state);
|
80
|
+
|
81
|
+
Vector& grad = state.grad;
|
82
|
+
grad.zero();
|
83
|
+
real lossValue = loss_->forward(targets, targetIndex, state, lr, true);
|
84
|
+
state.incrementNExamples(lossValue);
|
85
|
+
|
86
|
+
if (normalizeGradient_) {
|
87
|
+
grad.mul(1.0 / input.size());
|
88
|
+
}
|
89
|
+
for (auto it = input.cbegin(); it != input.cend(); ++it) {
|
90
|
+
wi_->addVectorToRow(grad, *it, 1.0);
|
91
|
+
}
|
92
|
+
}
|
93
|
+
|
94
|
+
real Model::std_log(real x) const {
|
95
|
+
return std::log(x + 1e-5);
|
96
|
+
}
|
97
|
+
|
98
|
+
} // namespace fasttext
|
@@ -0,0 +1,79 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2016-present, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under the MIT license found in the
|
6
|
+
* LICENSE file in the root directory of this source tree.
|
7
|
+
*/
|
8
|
+
|
9
|
+
#pragma once
|
10
|
+
|
11
|
+
#include <memory>
|
12
|
+
#include <random>
|
13
|
+
#include <utility>
|
14
|
+
#include <vector>
|
15
|
+
|
16
|
+
#include "matrix.h"
|
17
|
+
#include "real.h"
|
18
|
+
#include "utils.h"
|
19
|
+
#include "vector.h"
|
20
|
+
|
21
|
+
namespace fasttext {
|
22
|
+
|
23
|
+
class Loss;
|
24
|
+
|
25
|
+
class Model {
|
26
|
+
protected:
|
27
|
+
std::shared_ptr<Matrix> wi_;
|
28
|
+
std::shared_ptr<Matrix> wo_;
|
29
|
+
std::shared_ptr<Loss> loss_;
|
30
|
+
bool normalizeGradient_;
|
31
|
+
|
32
|
+
public:
|
33
|
+
Model(
|
34
|
+
std::shared_ptr<Matrix> wi,
|
35
|
+
std::shared_ptr<Matrix> wo,
|
36
|
+
std::shared_ptr<Loss> loss,
|
37
|
+
bool normalizeGradient);
|
38
|
+
Model(const Model& model) = delete;
|
39
|
+
Model(Model&& model) = delete;
|
40
|
+
Model& operator=(const Model& other) = delete;
|
41
|
+
Model& operator=(Model&& other) = delete;
|
42
|
+
|
43
|
+
class State {
|
44
|
+
private:
|
45
|
+
real lossValue_;
|
46
|
+
int64_t nexamples_;
|
47
|
+
|
48
|
+
public:
|
49
|
+
Vector hidden;
|
50
|
+
Vector output;
|
51
|
+
Vector grad;
|
52
|
+
std::minstd_rand rng;
|
53
|
+
|
54
|
+
State(int32_t hiddenSize, int32_t outputSize, int32_t seed);
|
55
|
+
real getLoss() const;
|
56
|
+
void incrementNExamples(real loss);
|
57
|
+
};
|
58
|
+
|
59
|
+
void predict(
|
60
|
+
const std::vector<int32_t>& input,
|
61
|
+
int32_t k,
|
62
|
+
real threshold,
|
63
|
+
Predictions& heap,
|
64
|
+
State& state) const;
|
65
|
+
void update(
|
66
|
+
const std::vector<int32_t>& input,
|
67
|
+
const std::vector<int32_t>& targets,
|
68
|
+
int32_t targetIndex,
|
69
|
+
real lr,
|
70
|
+
State& state);
|
71
|
+
void computeHidden(const std::vector<int32_t>& input, State& state) const;
|
72
|
+
|
73
|
+
real std_log(real) const;
|
74
|
+
|
75
|
+
static const int32_t kUnlimitedPredictions = -1;
|
76
|
+
static const int32_t kAllLabelsAsTarget = -1;
|
77
|
+
};
|
78
|
+
|
79
|
+
} // namespace fasttext
|
@@ -0,0 +1,251 @@
|
|
1
|
+
/**
|
2
|
+
* Copyright (c) 2016-present, Facebook, Inc.
|
3
|
+
* All rights reserved.
|
4
|
+
*
|
5
|
+
* This source code is licensed under the MIT license found in the
|
6
|
+
* LICENSE file in the root directory of this source tree.
|
7
|
+
*/
|
8
|
+
|
9
|
+
#include "productquantizer.h"
|
10
|
+
|
11
|
+
#include <algorithm>
|
12
|
+
#include <iostream>
|
13
|
+
#include <numeric>
|
14
|
+
#include <stdexcept>
|
15
|
+
#include <string>
|
16
|
+
|
17
|
+
namespace fasttext {
|
18
|
+
|
19
|
+
real distL2(const real* x, const real* y, int32_t d) {
|
20
|
+
real dist = 0;
|
21
|
+
for (auto i = 0; i < d; i++) {
|
22
|
+
auto tmp = x[i] - y[i];
|
23
|
+
dist += tmp * tmp;
|
24
|
+
}
|
25
|
+
return dist;
|
26
|
+
}
|
27
|
+
|
28
|
+
ProductQuantizer::ProductQuantizer(int32_t dim, int32_t dsub)
|
29
|
+
: dim_(dim),
|
30
|
+
nsubq_(dim / dsub),
|
31
|
+
dsub_(dsub),
|
32
|
+
centroids_(dim * ksub_),
|
33
|
+
rng(seed_) {
|
34
|
+
lastdsub_ = dim_ % dsub;
|
35
|
+
if (lastdsub_ == 0) {
|
36
|
+
lastdsub_ = dsub_;
|
37
|
+
} else {
|
38
|
+
nsubq_++;
|
39
|
+
}
|
40
|
+
}
|
41
|
+
|
42
|
+
const real* ProductQuantizer::get_centroids(int32_t m, uint8_t i) const {
|
43
|
+
if (m == nsubq_ - 1) {
|
44
|
+
return ¢roids_[m * ksub_ * dsub_ + i * lastdsub_];
|
45
|
+
}
|
46
|
+
return ¢roids_[(m * ksub_ + i) * dsub_];
|
47
|
+
}
|
48
|
+
|
49
|
+
real* ProductQuantizer::get_centroids(int32_t m, uint8_t i) {
|
50
|
+
if (m == nsubq_ - 1) {
|
51
|
+
return ¢roids_[m * ksub_ * dsub_ + i * lastdsub_];
|
52
|
+
}
|
53
|
+
return ¢roids_[(m * ksub_ + i) * dsub_];
|
54
|
+
}
|
55
|
+
|
56
|
+
real ProductQuantizer::assign_centroid(
|
57
|
+
const real* x,
|
58
|
+
const real* c0,
|
59
|
+
uint8_t* code,
|
60
|
+
int32_t d) const {
|
61
|
+
const real* c = c0;
|
62
|
+
real dis = distL2(x, c, d);
|
63
|
+
code[0] = 0;
|
64
|
+
for (auto j = 1; j < ksub_; j++) {
|
65
|
+
c += d;
|
66
|
+
real disij = distL2(x, c, d);
|
67
|
+
if (disij < dis) {
|
68
|
+
code[0] = (uint8_t)j;
|
69
|
+
dis = disij;
|
70
|
+
}
|
71
|
+
}
|
72
|
+
return dis;
|
73
|
+
}
|
74
|
+
|
75
|
+
void ProductQuantizer::Estep(
|
76
|
+
const real* x,
|
77
|
+
const real* centroids,
|
78
|
+
uint8_t* codes,
|
79
|
+
int32_t d,
|
80
|
+
int32_t n) const {
|
81
|
+
for (auto i = 0; i < n; i++) {
|
82
|
+
assign_centroid(x + i * d, centroids, codes + i, d);
|
83
|
+
}
|
84
|
+
}
|
85
|
+
|
86
|
+
void ProductQuantizer::MStep(
|
87
|
+
const real* x0,
|
88
|
+
real* centroids,
|
89
|
+
const uint8_t* codes,
|
90
|
+
int32_t d,
|
91
|
+
int32_t n) {
|
92
|
+
std::vector<int32_t> nelts(ksub_, 0);
|
93
|
+
memset(centroids, 0, sizeof(real) * d * ksub_);
|
94
|
+
const real* x = x0;
|
95
|
+
for (auto i = 0; i < n; i++) {
|
96
|
+
auto k = codes[i];
|
97
|
+
real* c = centroids + k * d;
|
98
|
+
for (auto j = 0; j < d; j++) {
|
99
|
+
c[j] += x[j];
|
100
|
+
}
|
101
|
+
nelts[k]++;
|
102
|
+
x += d;
|
103
|
+
}
|
104
|
+
|
105
|
+
real* c = centroids;
|
106
|
+
for (auto k = 0; k < ksub_; k++) {
|
107
|
+
real z = (real)nelts[k];
|
108
|
+
if (z != 0) {
|
109
|
+
for (auto j = 0; j < d; j++) {
|
110
|
+
c[j] /= z;
|
111
|
+
}
|
112
|
+
}
|
113
|
+
c += d;
|
114
|
+
}
|
115
|
+
|
116
|
+
std::uniform_real_distribution<> runiform(0, 1);
|
117
|
+
for (auto k = 0; k < ksub_; k++) {
|
118
|
+
if (nelts[k] == 0) {
|
119
|
+
int32_t m = 0;
|
120
|
+
while (runiform(rng) * (n - ksub_) >= nelts[m] - 1) {
|
121
|
+
m = (m + 1) % ksub_;
|
122
|
+
}
|
123
|
+
memcpy(centroids + k * d, centroids + m * d, sizeof(real) * d);
|
124
|
+
for (auto j = 0; j < d; j++) {
|
125
|
+
int32_t sign = (j % 2) * 2 - 1;
|
126
|
+
centroids[k * d + j] += sign * eps_;
|
127
|
+
centroids[m * d + j] -= sign * eps_;
|
128
|
+
}
|
129
|
+
nelts[k] = nelts[m] / 2;
|
130
|
+
nelts[m] -= nelts[k];
|
131
|
+
}
|
132
|
+
}
|
133
|
+
}
|
134
|
+
|
135
|
+
void ProductQuantizer::kmeans(const real* x, real* c, int32_t n, int32_t d) {
|
136
|
+
std::vector<int32_t> perm(n, 0);
|
137
|
+
std::iota(perm.begin(), perm.end(), 0);
|
138
|
+
std::shuffle(perm.begin(), perm.end(), rng);
|
139
|
+
for (auto i = 0; i < ksub_; i++) {
|
140
|
+
memcpy(&c[i * d], x + perm[i] * d, d * sizeof(real));
|
141
|
+
}
|
142
|
+
auto codes = std::vector<uint8_t>(n);
|
143
|
+
for (auto i = 0; i < niter_; i++) {
|
144
|
+
Estep(x, c, codes.data(), d, n);
|
145
|
+
MStep(x, c, codes.data(), d, n);
|
146
|
+
}
|
147
|
+
}
|
148
|
+
|
149
|
+
void ProductQuantizer::train(int32_t n, const real* x) {
|
150
|
+
if (n < ksub_) {
|
151
|
+
throw std::invalid_argument(
|
152
|
+
"Matrix too small for quantization, must have at least " +
|
153
|
+
std::to_string(ksub_) + " rows");
|
154
|
+
}
|
155
|
+
std::vector<int32_t> perm(n, 0);
|
156
|
+
std::iota(perm.begin(), perm.end(), 0);
|
157
|
+
auto d = dsub_;
|
158
|
+
auto np = std::min(n, max_points_);
|
159
|
+
auto xslice = std::vector<real>(np * dsub_);
|
160
|
+
for (auto m = 0; m < nsubq_; m++) {
|
161
|
+
if (m == nsubq_ - 1) {
|
162
|
+
d = lastdsub_;
|
163
|
+
}
|
164
|
+
if (np != n) {
|
165
|
+
std::shuffle(perm.begin(), perm.end(), rng);
|
166
|
+
}
|
167
|
+
for (auto j = 0; j < np; j++) {
|
168
|
+
memcpy(
|
169
|
+
xslice.data() + j * d,
|
170
|
+
x + perm[j] * dim_ + m * dsub_,
|
171
|
+
d * sizeof(real));
|
172
|
+
}
|
173
|
+
kmeans(xslice.data(), get_centroids(m, 0), np, d);
|
174
|
+
}
|
175
|
+
}
|
176
|
+
|
177
|
+
real ProductQuantizer::mulcode(
|
178
|
+
const Vector& x,
|
179
|
+
const uint8_t* codes,
|
180
|
+
int32_t t,
|
181
|
+
real alpha) const {
|
182
|
+
real res = 0.0;
|
183
|
+
auto d = dsub_;
|
184
|
+
const uint8_t* code = codes + nsubq_ * t;
|
185
|
+
for (auto m = 0; m < nsubq_; m++) {
|
186
|
+
const real* c = get_centroids(m, code[m]);
|
187
|
+
if (m == nsubq_ - 1) {
|
188
|
+
d = lastdsub_;
|
189
|
+
}
|
190
|
+
for (auto n = 0; n < d; n++) {
|
191
|
+
res += x[m * dsub_ + n] * c[n];
|
192
|
+
}
|
193
|
+
}
|
194
|
+
return res * alpha;
|
195
|
+
}
|
196
|
+
|
197
|
+
void ProductQuantizer::addcode(
|
198
|
+
Vector& x,
|
199
|
+
const uint8_t* codes,
|
200
|
+
int32_t t,
|
201
|
+
real alpha) const {
|
202
|
+
auto d = dsub_;
|
203
|
+
const uint8_t* code = codes + nsubq_ * t;
|
204
|
+
for (auto m = 0; m < nsubq_; m++) {
|
205
|
+
const real* c = get_centroids(m, code[m]);
|
206
|
+
if (m == nsubq_ - 1) {
|
207
|
+
d = lastdsub_;
|
208
|
+
}
|
209
|
+
for (auto n = 0; n < d; n++) {
|
210
|
+
x[m * dsub_ + n] += alpha * c[n];
|
211
|
+
}
|
212
|
+
}
|
213
|
+
}
|
214
|
+
|
215
|
+
void ProductQuantizer::compute_code(const real* x, uint8_t* code) const {
|
216
|
+
auto d = dsub_;
|
217
|
+
for (auto m = 0; m < nsubq_; m++) {
|
218
|
+
if (m == nsubq_ - 1) {
|
219
|
+
d = lastdsub_;
|
220
|
+
}
|
221
|
+
assign_centroid(x + m * dsub_, get_centroids(m, 0), code + m, d);
|
222
|
+
}
|
223
|
+
}
|
224
|
+
|
225
|
+
void ProductQuantizer::compute_codes(const real* x, uint8_t* codes, int32_t n)
|
226
|
+
const {
|
227
|
+
for (auto i = 0; i < n; i++) {
|
228
|
+
compute_code(x + i * dim_, codes + i * nsubq_);
|
229
|
+
}
|
230
|
+
}
|
231
|
+
|
232
|
+
void ProductQuantizer::save(std::ostream& out) const {
|
233
|
+
out.write((char*)&dim_, sizeof(dim_));
|
234
|
+
out.write((char*)&nsubq_, sizeof(nsubq_));
|
235
|
+
out.write((char*)&dsub_, sizeof(dsub_));
|
236
|
+
out.write((char*)&lastdsub_, sizeof(lastdsub_));
|
237
|
+
out.write((char*)centroids_.data(), centroids_.size() * sizeof(real));
|
238
|
+
}
|
239
|
+
|
240
|
+
void ProductQuantizer::load(std::istream& in) {
|
241
|
+
in.read((char*)&dim_, sizeof(dim_));
|
242
|
+
in.read((char*)&nsubq_, sizeof(nsubq_));
|
243
|
+
in.read((char*)&dsub_, sizeof(dsub_));
|
244
|
+
in.read((char*)&lastdsub_, sizeof(lastdsub_));
|
245
|
+
centroids_.resize(dim_ * ksub_);
|
246
|
+
for (auto i = 0; i < centroids_.size(); i++) {
|
247
|
+
in.read((char*)¢roids_[i], sizeof(real));
|
248
|
+
}
|
249
|
+
}
|
250
|
+
|
251
|
+
} // namespace fasttext
|