fasttext 0.1.0

data/ext/fasttext/extconf.rb

@@ -0,0 +1,15 @@
+require "mkmf-rice"
+
+abort "Missing stdc++" unless have_library("stdc++")
+
+# TODO use -std=c++14 when available
+$CXXFLAGS << " -pthread -std=c++11 -funroll-loops -O3 -march=native"
+
+src1 = File.expand_path(".", __dir__)
+src2 = File.expand_path("../../vendor/fastText/src", __dir__)
+
+$srcs = Dir["{#{src1},#{src2}}/*.{cc,cpp}"]
+$INCFLAGS << " -I$(srcdir)/../../vendor/fastText/src"
+$VPATH << "$(srcdir)/../../vendor/fastText/src"
+
+create_makefile("fasttext/ext")

data/lib/fasttext.rb

@@ -0,0 +1,41 @@
+# ext
+require "fasttext/ext"
+
+# modules
+require "fasttext/model"
+require "fasttext/classifier"
+require "fasttext/vectorizer"
+require "fasttext/version"
+
+module FastText
+  class Error < StandardError; end
+
+  class << self
+    def load_model(path)
+      m = Ext::Model.new
+      m.load_model(path)
+      model =
+        if m.supervised?
+          FastText::Classifier.new
+        else
+          FastText::Vectorizer.new
+        end
+      model.instance_variable_set("@m", m)
+      model
+    end
+
+    def train_supervised(**options)
+      input = options.delete(:input)
+      model = FastText::Classifier.new(**options)
+      model.fit(input)
+      model
+    end
+
+    def train_unsupervised(**options)
+      input = options.delete(:input)
+      model = FastText::Vectorizer.new(**options)
+      model.fit(input)
+      model
+    end
+  end
+end

data/lib/fasttext/classifier.rb

@@ -0,0 +1,92 @@
+module FastText
+  class Classifier < Model
+    DEFAULT_OPTIONS = {
+      lr: 0.1,
+      lr_update_rate: 100,
+      dim: 100,
+      ws: 5,
+      epoch: 5,
+      min_count: 1,
+      min_count_label: 0,
+      neg: 5,
+      word_ngrams: 1,
+      loss: "softmax",
+      model: "supervised",
+      bucket: 2000000,
+      minn: 0,
+      maxn: 0,
+      thread: 3,
+      t: 0.0001,
+      label_prefix: "__label__",
+      verbose: 2,
+      pretrained_vectors: "",
+      save_output: false,
+      # seed: 0
+    }
+
+    def fit(x, y = nil)
+      input = input_path(x, y)
+      @m ||= Ext::Model.new
+      m.train(DEFAULT_OPTIONS.merge(@options).merge(input: input, model: "supervised"))
+    end
+
+    # TODO support array of text
+    def predict(text, k: 1, threshold: 0.0)
+      m.predict(prep_text(text), k, threshold).map do |v|
+        [remove_prefix(v[1]), v[0]]
+      end.to_h
+    end
+
+    def test(x, y = nil, k: 1)
+      input = input_path(x, y)
+      res = m.test(input, k)
+      {
+        examples: res[0],
+        precision: res[1],
+        recall: res[2]
+      }
+    end
+
+    # TODO support options
+    def quantize
+      m.quantize({})
+    end
+
+    def labels(include_freq: false)
+      labels, freqs = m.labels
+      labels.map! { |v| remove_prefix(v) }
+      if include_freq
+        labels.zip(freqs).to_h
+      else
+        labels
+      end
+    end
+
+    private
+
+    def input_path(x, y)
+      if x.is_a?(String)
+        raise ArgumentError, "Cannot pass y with file" if y
+        x
+      else
+        tempfile = Tempfile.new("fasttext")
+        x.zip(y) do |xi, yi|
+          parts = Array(yi).map { |label| "__label__" + label }
+          parts << xi.gsub("\n", " ") # replace newlines in document
+          tempfile.write(parts.join(" "))
+          tempfile.write("\n")
+        end
+        tempfile.close
+        tempfile.path
+      end
+    end
+
+    def remove_prefix(label)
+      label.sub(label_prefix, "")
+    end
+
+    def label_prefix
+      m.label_prefix
+    end
+  end
+end

data/lib/fasttext/model.rb

@@ -0,0 +1,60 @@
+module FastText
+  class Model
+    def initialize(**options)
+      @options = options
+    end
+
+    def dimension
+      m.dimension
+    end
+
+    def quantized?
+      m.quantized?
+    end
+
+    def save_model(path)
+      m.save_model(path)
+    end
+
+    def words(include_freq: false)
+      words, freqs = m.words
+      if include_freq
+        words.zip(freqs).to_h
+      else
+        words
+      end
+    end
+
+    def word_vector(word)
+      m.word_vector(word)
+    end
+
+    def sentence_vector(text)
+      m.sentence_vector(prep_text(text))
+    end
+
+    def word_id(word)
+      m.word_id(word)
+    end
+
+    def subword_id(subword)
+      m.subword_id(subword)
+    end
+
+    def subwords(word)
+      m.subwords(word)
+    end
+
+    private
+
+    # text must end in newline for prediction to match Python and CLI
+    def prep_text(text)
+      text = "#{text}\n" unless text.end_with?("\n")
+      text
+    end
+
+    def m
+      @m || (raise Error, "Not fit")
+    end
+  end
+end

data/lib/fasttext/vectorizer.rb

@@ -0,0 +1,58 @@
+module FastText
+  class Vectorizer < Model
+    DEFAULT_OPTIONS = {
+      lr: 0.5,
+      lr_update_rate: 100,
+      dim: 100,
+      ws: 5,
+      epoch: 5,
+      min_count: 1,
+      min_count_label: 0,
+      neg: 5,
+      word_ngrams: 1,
+      loss: "ns",
+      model: "skipgram",
+      bucket: 2000000,
+      minn: 3,
+      maxn: 6,
+      thread: 3,
+      t: 0.0001,
+      verbose: 2,
+      pretrained_vectors: "",
+      save_output: false,
+      # seed: 0
+    }
+
+    def fit(x)
+      input = input_path(x)
+      @m ||= Ext::Model.new
+      m.train(DEFAULT_OPTIONS.merge(@options).merge(input: input))
+    end
+
+    def nearest_neighbors(word, k: 10)
+      m.nearest_neighbors(word, k).map(&:reverse).to_h
+    end
+
+    def analogies(word_a, word_b, word_c, k: 10)
+      m.analogies(k, word_a, word_b, word_c).map(&:reverse).to_h
+    end
+
+    private
+
+    # separate example by newlines
+    # https://github.com/facebookresearch/fastText/issues/518
+    def input_path(x)
+      if x.is_a?(String)
+        x
+      else
+        tempfile = Tempfile.new("fasttext")
+        x.each do |xi|
+          tempfile.write(xi)
+          tempfile.write("\n")
+        end
+        tempfile.close
+        tempfile.path
+      end
+    end
+  end
+end

data/lib/fasttext/version.rb

@@ -0,0 +1,3 @@
+module FastText
+  VERSION = "0.1.0"
+end

data/vendor/fastText/CMakeLists.txt

@@ -0,0 +1,68 @@
+#
+# Copyright (c) 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+cmake_minimum_required(VERSION 2.8.9)
+project(fasttext)
+
+# The version number.
+set (fasttext_VERSION_MAJOR 0)
+set (fasttext_VERSION_MINOR 1)
+
+include_directories(fasttext)
+
+set(CMAKE_CXX_FLAGS " -pthread -std=c++11 -funroll-loops -O3 -march=native")
+
+set(HEADER_FILES
+    src/args.h
+    src/densematrix.h
+    src/dictionary.h
+    src/fasttext.h
+    src/loss.h
+    src/matrix.h
+    src/meter.h
+    src/model.h
+    src/productquantizer.h
+    src/quantmatrix.h
+    src/real.h
+    src/utils.h
+    src/vector.h)
+
+set(SOURCE_FILES
+    src/args.cc
+    src/densematrix.cc
+    src/dictionary.cc
+    src/fasttext.cc
+    src/loss.cc
+    src/main.cc
+    src/matrix.cc
+    src/meter.cc
+    src/model.cc
+    src/productquantizer.cc
+    src/quantmatrix.cc
+    src/utils.cc
+    src/vector.cc)
+
+add_library(fasttext-shared SHARED ${SOURCE_FILES} ${HEADER_FILES})
+add_library(fasttext-static STATIC ${SOURCE_FILES} ${HEADER_FILES})
+add_library(fasttext-static_pic STATIC ${SOURCE_FILES} ${HEADER_FILES})
+set_target_properties(fasttext-shared PROPERTIES OUTPUT_NAME fasttext)
+set_target_properties(fasttext-static PROPERTIES OUTPUT_NAME fasttext)
+set_target_properties(fasttext-static_pic PROPERTIES OUTPUT_NAME fasttext_pic
+  POSITION_INDEPENDENT_CODE True)
+add_executable(fasttext-bin src/main.cc)
+target_link_libraries(fasttext-bin pthread fasttext-static)
+set_target_properties(fasttext-bin PROPERTIES PUBLIC_HEADER "${HEADER_FILES}" OUTPUT_NAME fasttext)
+install (TARGETS fasttext-shared
+    LIBRARY DESTINATION lib)
+install (TARGETS fasttext-static
+    ARCHIVE DESTINATION lib)
+install (TARGETS fasttext-static_pic
+    ARCHIVE DESTINATION lib)
+install (TARGETS fasttext-bin
+    RUNTIME DESTINATION bin
+ PUBLIC_HEADER DESTINATION include/fasttext)

data/vendor/fastText/CODE_OF_CONDUCT.md

@@ -0,0 +1,2 @@
+# Code of Conduct
+Facebook has adopted a Code of Conduct that we expect project participants to adhere to. Please [read the full text](https://code.fb.com/codeofconduct) so that you can understand what actions will and will not be tolerated.

data/vendor/fastText/CONTRIBUTING.md

@@ -0,0 +1,32 @@
+# Contributing to fastText
+We want to make contributing to this project as easy and transparent as possible.
+
+## Issues
+We use GitHub issues to track public bugs. Please ensure your description is clear and has sufficient instructions to be able to reproduce the issue.
+
+### Reproducing issues
+Please make sure that the issue you mention is not a result of one of the existing third-party libraries. For example, please do not post an issue if you encountered an error within a third-party Python library. We can only help you with errors which can be directly reproduced either with our C++ code or the corresponding Python bindings. If you do find an error, please post detailed steps to reproduce it. If we can't reproduce your error, we can't help you fix it.
+
+## Pull Requests
+Please post an Issue before submitting a pull request. This might save you some time as it is possible we can't support your contribution, albeit we try our best to accomodate your (planned) work and highly appreciate your time. Generally, it is best to have a pull request emerge from an issue rather than the other way around.
+
+To create a pull request:
+
+1. Fork the repo and create your branch from `master`.
+2. If you've added code that should be tested, add tests.
+3. If you've changed APIs, update the documentation.
+4. Ensure the test suite passes.
+5. Make sure your code lints.
+6. If you haven't already, complete the Contributor License Agreement ("CLA").
+
+## Tests
+First, you will need to make sure you have the required data. For that, please have a look at the fetch_test_data.sh script under tests. Next run the tests using the runtests.py script passing a path to the directory containing the datasets.
+
+## Contributor License Agreement ("CLA")
+In order to accept your pull request, we need you to submit a CLA. You only need
+to do this once to work on any of Facebook's open source projects.
+
+Complete your CLA here: <https://code.facebook.com/cla>
+
+## License
+By contributing to fastText, you agree that your contributions will be licensed under its MIT license.

data/vendor/fastText/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016-present, Facebook, Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

data/vendor/fastText/MANIFEST.in

@@ -0,0 +1,5 @@
+include LICENSE
+include PATENTS
+
+recursive-include python *.md *.rst
+recursive-include src *.h

data/vendor/fastText/Makefile

@@ -0,0 +1,63 @@
+#
+# Copyright (c) 2016-present, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under the MIT license found in the
+# LICENSE file in the root directory of this source tree.
+#
+
+CXX = c++
+CXXFLAGS = -pthread -std=c++0x -march=native
+OBJS = args.o matrix.o dictionary.o loss.o productquantizer.o densematrix.o quantmatrix.o vector.o model.o utils.o meter.o fasttext.o
+INCLUDES = -I.
+
+opt: CXXFLAGS += -O3 -funroll-loops -DNDEBUG
+opt: fasttext
+
+coverage: CXXFLAGS += -O0 -fno-inline -fprofile-arcs --coverage
+coverage: fasttext
+
+debug: CXXFLAGS += -g -O0 -fno-inline
+debug: fasttext
+
+args.o: src/args.cc src/args.h
+	$(CXX) $(CXXFLAGS) -c src/args.cc
+
+matrix.o: src/matrix.cc src/matrix.h
+	$(CXX) $(CXXFLAGS) -c src/matrix.cc
+
+dictionary.o: src/dictionary.cc src/dictionary.h src/args.h
+	$(CXX) $(CXXFLAGS) -c src/dictionary.cc
+
+loss.o: src/loss.cc src/loss.h src/matrix.h src/real.h
+	$(CXX) $(CXXFLAGS) -c src/loss.cc
+
+productquantizer.o: src/productquantizer.cc src/productquantizer.h src/utils.h
+	$(CXX) $(CXXFLAGS) -c src/productquantizer.cc
+
+densematrix.o: src/densematrix.cc src/densematrix.h src/utils.h src/matrix.h
+	$(CXX) $(CXXFLAGS) -c src/densematrix.cc
+
+quantmatrix.o: src/quantmatrix.cc src/quantmatrix.h src/utils.h src/matrix.h
+	$(CXX) $(CXXFLAGS) -c src/quantmatrix.cc
+
+vector.o: src/vector.cc src/vector.h src/utils.h
+	$(CXX) $(CXXFLAGS) -c src/vector.cc
+
+model.o: src/model.cc src/model.h src/args.h
+	$(CXX) $(CXXFLAGS) -c src/model.cc
+
+utils.o: src/utils.cc src/utils.h
+	$(CXX) $(CXXFLAGS) -c src/utils.cc
+
+meter.o: src/meter.cc src/meter.h
+	$(CXX) $(CXXFLAGS) -c src/meter.cc
+
+fasttext.o: src/fasttext.cc src/*.h
+	$(CXX) $(CXXFLAGS) -c src/fasttext.cc
+
+fasttext: $(OBJS) src/fasttext.cc
+	$(CXX) $(CXXFLAGS) $(OBJS) src/main.cc -o fasttext
+
+clean:
+	rm -rf *.o *.gcno *.gcda fasttext