ffi-fasttext 0.1.1 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 2733c8705d35255f1f65e87fa0247da4d0f83a03
-  data.tar.gz: e7e1917a8b172641441b039d147e1a9d83f217cc
+  metadata.gz: e68696e8ae484cfe2860a7c3d8b4d6ff82fa1634
+  data.tar.gz: 8d83b24e1d3a6f4088958e247576f27ecbf92075
 SHA512:
-  metadata.gz: bc1eed1ab1a402a9b6541e1737626f01c630c08e6666a9f0d8505e02edc0e1178bda200eefc133fa0fb6201c88987ec6d860459cf51f979053ac28ef8a825c85
-  data.tar.gz: a5fcd1bd377d33c4de336299a83ca7d06c7e93b2fcd51afa08363257776ea366e886cd0ea8faaf4ebd2c9964316e18f2be0c96d083f437a75d5dfe337f1c2519
+  metadata.gz: 2452d6825201de150eb467414551e847926407bbc29a92aaa560a3e59189ddee8747343cfd9429626147365e02522be8f1d4bda1133e2c2cc94fd04a3754c66c
+  data.tar.gz: e60d98a450292187fa566e5fe9a49a7534155440f20a9450bd9ce5364c6bd720c8e54422c38250b2a94d1832399ae0e977c3a0846c84c91e9e8451a045df973a

data/ext/ffi/fasttext/Rakefile

@@ -49,18 +49,19 @@ task :compile_fasttext do
   FASTTEXT_DIR = ::File.join(CWD, "..", "..", "..", "vendor", "fasttext")
 
   ::Dir.chdir(FASTTEXT_DIR) do
-    sys("g++ -c -O3 -fPIC -pthread -std=c++0x -c args.cc") unless ::File.exist?("args.o")
-    sys("g++ -c -O3 -fPIC -pthread -std=c++0x -c dictionary.cc") unless ::File.exist?("dictionary.o")
-    sys("g++ -c -O3 -fPIC -pthread -std=c++0x -c productquantizer.cc") unless ::File.exist?("productquantizer.o")
-    sys("g++ -c -O3 -fPIC -pthread -std=c++0x -c matrix.cc") unless ::File.exist?("matrix.o")
-    sys("g++ -c -O3 -fPIC -pthread -std=c++0x -c qmatrix.cc") unless ::File.exist?("qmatrix.o")
-    sys("g++ -c -O3 -fPIC -pthread -std=c++0x -c vector.cc") unless ::File.exist?("vector.o")
-    sys("g++ -c -O3 -fPIC -pthread -std=c++0x -c model.cc") unless ::File.exist?("model.o")
-    sys("g++ -c -O3 -fPIC -pthread -std=c++0x -c utils.cc") unless ::File.exist?("utils.o")
-    sys("g++ -c -O3 -fPIC -pthread -std=c++0x -c fasttext.cc") unless ::File.exist?("fasttext.o")
-    sys("g++ -c -O3 -fPIC -pthread -std=c++0x -c ffi_fasttext.cc")
-    sys("g++ -fPIC -pthread -std=c++0x args.o dictionary.o productquantizer.o matrix.o qmatrix.o vector.o model.o utils.o fasttext.o main.cc -o fasttext") unless ::File.exist?("fasttext")
-    sys("g++ -fPIC -pthread -std=c++0x args.o dictionary.o productquantizer.o matrix.o qmatrix.o vector.o model.o utils.o fasttext.o ffi_fasttext.o -shared -o libfasttext.#{::FFI::Platform::LIBSUFFIX}")
+    sys("g++ -c -O3 -fPIC -pthread -std=c++14 -c args.cc") unless ::File.exist?("args.o")
+    sys("g++ -c -O3 -fPIC -pthread -std=c++14 -c dictionary.cc") unless ::File.exist?("dictionary.o")
+    sys("g++ -c -O3 -fPIC -pthread -std=c++14 -c productquantizer.cc") unless ::File.exist?("productquantizer.o")
+    sys("g++ -c -O3 -fPIC -pthread -std=c++14 -c matrix.cc") unless ::File.exist?("matrix.o")
+    sys("g++ -c -O3 -fPIC -pthread -std=c++14 -c qmatrix.cc") unless ::File.exist?("qmatrix.o")
+    sys("g++ -c -O3 -fPIC -pthread -std=c++14 -c vector.cc") unless ::File.exist?("vector.o")
+    sys("g++ -c -O3 -fPIC -pthread -std=c++14 -c model.cc") unless ::File.exist?("model.o")
+    sys("g++ -c -O3 -fPIC -pthread -std=c++14 -c utils.cc") unless ::File.exist?("utils.o")
+    sys("g++ -c -O3 -fPIC -pthread -std=c++14 -c fasttext.cc") unless ::File.exist?("fasttext.o")
+    sys("g++ -c -O3 -fPIC -pthread -std=c++14 -c ffi_fasttext.cc")
+    sys("g++ -c -O3 -fPIC -pthread -std=c++14 -c curlstream.cc")
+    sys("g++ -fPIC -pthread -std=c++14 args.o dictionary.o productquantizer.o matrix.o qmatrix.o vector.o model.o utils.o fasttext.o main.cc -o fasttext") unless ::File.exist?("fasttext")
+    sys("g++ -fPIC -pthread -std=c++14 args.o dictionary.o productquantizer.o matrix.o qmatrix.o vector.o model.o utils.o fasttext.o ffi_fasttext.o curlstream.o -lcurl -shared -o libfasttext.#{::FFI::Platform::LIBSUFFIX}")
   end
 
   unless ::File.exist?(::File.join(FASTTEXT_DIR, "libfasttext.#{::FFI::Platform::LIBSUFFIX}"))

data/ffi-fasttext.gemspec

@@ -34,7 +34,7 @@ Gem::Specification.new do |spec|
   spec.add_dependency "ffi"
 
   spec.add_development_dependency "pry"
-  spec.add_development_dependency "bundler", "~> 1.16"
-  spec.add_development_dependency "rake", "~> 10.0"
-  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.add_development_dependency "bundler"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "minitest"
 end

data/lib/ffi/fasttext.rb

@@ -72,14 +72,15 @@ module FFI
     end
 
     attach_function :create, [:string], :pointer
+    attach_function :create_from_url, [:string], :pointer
     attach_function :destroy, [:pointer], :void
     attach_function :predict_string_free, [:pointer], :void
     attach_function :predict, [:pointer, :string, :int32_t], :strptr
 
     class Predictor
-      def initialize(model_filename)
-        raise "File does not exist" unless ::File.exist?(model_filename)
-        @ptr = ::FFI::Fasttext.create(model_filename)
+      def initialize(model_name)
+        @ptr = ::File.exist?(model_name) ? ::FFI::Fasttext.create(model_name) : ::FFI::Fasttext.create_from_url(model_name)
+        raise "Error loading model" if @ptr.null?
       end
 
       def destroy!

data/lib/ffi/fasttext/version.rb

@@ -1,5 +1,5 @@
 module FFI
   module Fasttext
-    VERSION = "0.1.1"
+    VERSION = "0.3.0"
   end
 end

data/vendor/fasttext/curlstream.cc

@@ -0,0 +1,132 @@
+#include <memory>
+#include <chrono>
+#include <thread>
+#include <streambuf>
+#include "./curlstream.h"
+
+class curl_global_initializer {
+public:
+  curl_global_initializer() { curl_global_init(CURL_GLOBAL_DEFAULT); }
+  ~curl_global_initializer() { curl_global_cleanup(); }
+};
+
+namespace {
+  bool curl_initialized()
+  {
+    static std::unique_ptr<curl_global_initializer> initializer = std::make_unique<curl_global_initializer>();
+    return initializer != nullptr;
+  }
+}
+
+CurlStreambuff::CurlStreambuff(const std::string& url)
+{
+  if (curl_initialized()) {
+    m_http_handle = curl_easy_init();
+    curl_easy_setopt(m_http_handle, CURLOPT_URL, url.c_str());
+    curl_easy_setopt(m_http_handle, CURLOPT_FOLLOWLOCATION, 1L);
+    curl_easy_setopt(m_http_handle, CURLOPT_SSL_VERIFYPEER, 0L);
+    curl_easy_setopt(m_http_handle, CURLOPT_WRITEFUNCTION, &CurlStreambuff::writer_callback);
+    curl_easy_setopt(m_http_handle, CURLOPT_WRITEDATA, this);
+
+    m_multi_handle = curl_multi_init();
+    curl_multi_add_handle(m_multi_handle, m_http_handle);
+  }
+}
+
+CurlStreambuff::~CurlStreambuff()
+{
+  if (m_multi_handle && m_http_handle) {
+    curl_multi_remove_handle(m_multi_handle, m_http_handle);
+  } 
+  if (m_http_handle) {
+    curl_easy_cleanup(m_http_handle);
+  }
+  if (m_multi_handle) {
+    curl_multi_cleanup(m_multi_handle);
+  }
+}
+
+std::streamsize CurlStreambuff::xsgetn(char *s, std::streamsize n)
+{
+  auto remaining = n;
+  while (remaining > 0) {
+    if (m_pos >= m_size) {
+      if (fillbuffer() == 0) {
+        break;
+      }
+    }
+    auto to_copy = std::min<size_t>(remaining, m_size - m_pos);
+    memcpy(s, &m_buffer[m_pos], to_copy);
+    s += to_copy;
+    m_pos += to_copy;
+    remaining -= to_copy;    
+  }
+  return n - remaining;
+}
+
+int CurlStreambuff::underflow()
+{
+  if (m_pos >= m_size) {
+    if (fillbuffer() == 0) {
+      return traits_type::eof();
+    }
+  }
+  return traits_type::to_int_type(m_buffer[m_pos]);
+}
+
+int CurlStreambuff::uflow()
+{
+  if (m_pos >= m_size) {
+    if (fillbuffer() == 0) {
+      return traits_type::eof();
+    }
+  }
+  return traits_type::to_int_type(m_buffer[m_pos++]);
+}
+
+int CurlStreambuff::writer_callback(char *data, size_t size, size_t count, void* ptr)
+{
+  auto self = static_cast<CurlStreambuff*>(ptr);
+  auto bytes = size * count;
+  if(bytes > sizeof(m_buffer) || bytes == 0) {
+    return 0;
+  }
+  memcpy(&self->m_buffer[0], data, bytes);
+  self->m_size = bytes;
+  self->m_pos = 0;
+  return bytes;
+}
+
+size_t CurlStreambuff::fillbuffer()
+{
+  using namespace std::chrono_literals;
+  m_size = 0;
+  int still_running_count = 1, repeats = 0;
+  while (still_running_count > 0) {
+    auto mc = curl_multi_perform(m_multi_handle, &still_running_count);
+    if (mc != CURLM_OK) {
+      return 0;
+    }
+    if (m_size > 0) {
+      break;
+    }
+    /* wait for activity, timeout or "nothing" */ 
+    int numfds;
+    mc = curl_multi_wait(m_multi_handle, nullptr, 0, 1000, &numfds);
+    if(mc != CURLM_OK) {
+      return 0;
+    }
+
+    /* 'numfds' being zero means either a timeout or no file descriptors to
+       wait for. Try timeout on first occurrence, then assume no file
+       descriptors and no file descriptors to wait for means wait for 100
+       milliseconds. */ 
+    if(numfds == 0) {
+      if (++repeats > 1) {
+        std::this_thread::sleep_for(100ms);
+      }
+    }
+  }
+  return m_size;
+}
+

data/vendor/fasttext/curlstream.h

@@ -0,0 +1,39 @@
+#include <iostream>
+#include <streambuf>
+#include <curl/curl.h>
+#include <string.h>
+
+class CurlStreambuff : public std::streambuf {
+public:
+  explicit CurlStreambuff(const std::string& url);
+  ~CurlStreambuff();
+
+  // Copy and move prohibited
+  CurlStreambuff(const CurlStreambuff&) = delete;
+  CurlStreambuff& operator = (const CurlStreambuff&) = delete;
+  CurlStreambuff(CurlStreambuff && src) = delete;
+  CurlStreambuff& operator = (CurlStreambuff && rhs) = delete;
+
+protected:
+  std::streamsize xsgetn(char *s, std::streamsize n) override;
+  int underflow() override;
+  int uflow() override;
+
+private:
+  static int writer_callback(char *data, size_t size, size_t count, void* ptr);
+  size_t fillbuffer();
+  
+  CURL *m_http_handle = nullptr;
+  CURLM *m_multi_handle = nullptr;
+  char m_buffer[CURL_MAX_WRITE_SIZE];
+  size_t m_pos = 0, m_size = 0;
+};
+
+class CurlStream : private CurlStreambuff, public std::istream {
+public:
+  explicit CurlStream(const std::string& url)
+    : CurlStreambuff(url), std::istream(this)
+  {
+  }
+};
+

data/vendor/fasttext/fasttext.cc

@@ -43,8 +43,7 @@ void FastText::getVector(Vector& vec, const std::string& word) const {
 void FastText::saveVectors() {
   std::ofstream ofs(args_->output + ".vec");
   if (!ofs.is_open()) {
-    std::cerr << "Error opening file for saving vectors." << std::endl;
-    exit(EXIT_FAILURE);
+    throw std::runtime_error("Error opening file for saving vectors.");
   }
   ofs << dict_->nwords() << " " << args_->dim << std::endl;
   Vector vec(args_->dim);
@@ -59,8 +58,7 @@ void FastText::saveVectors() {
 void FastText::saveOutput() {
   std::ofstream ofs(args_->output + ".output");
   if (!ofs.is_open()) {
-    std::cerr << "Error opening file for saving vectors." << std::endl;
-    exit(EXIT_FAILURE);
+    throw std::runtime_error("Error opening file for saving vectors.");
   }
   if (quant_) {
     std::cerr << "Option -saveOutput is not supported for quantized models."
@@ -110,8 +108,7 @@ void FastText::saveModel() {
   }
   std::ofstream ofs(fn, std::ofstream::binary);
   if (!ofs.is_open()) {
-    std::cerr << "Model file cannot be opened for saving!" << std::endl;
-    exit(EXIT_FAILURE);
+    throw std::runtime_error("Model file cannot be opened for saving!");
   }
   signModel(ofs);
   args_->save(ofs);
@@ -137,18 +134,16 @@ void FastText::saveModel() {
 void FastText::loadModel(const std::string& filename) {
   std::ifstream ifs(filename, std::ifstream::binary);
   if (!ifs.is_open()) {
-    std::cerr << "Model file cannot be opened for loading!" << std::endl;
-    exit(EXIT_FAILURE);
-  }
-  if (!checkModel(ifs)) {
-    std::cerr << "Model file has wrong file format!" << std::endl;
-    exit(EXIT_FAILURE);
+    throw std::runtime_error("Model file cannot be opened for loading!");
   }
   loadModel(ifs);
   ifs.close();
 }
 
 void FastText::loadModel(std::istream& in) {
+  if (!checkModel(in)) {
+    throw std::runtime_error("Model file has wrong file format!");
+  }
   args_ = std::make_shared<Args>();
   dict_ = std::make_shared<Dictionary>(args_);
   input_ = std::make_shared<Matrix>();
@@ -172,10 +167,7 @@ void FastText::loadModel(std::istream& in) {
   }
 
   if (!quant_input && dict_->isPruned()) {
-    std::cerr << "Invalid model file.\n"
-              << "Please download the updated model from www.fasttext.cc.\n"
-              << "See issue #332 on Github for more information.\n";
-    exit(1);
+    throw std::runtime_error("Invalid model file");
   }
 
   in.read((char*) &args_->qout, sizeof(bool));
@@ -228,8 +220,7 @@ std::vector FastText::selectEmbeddings(int32_t cutoff) const {
 
 void FastText::quantize(std::shared_ptr<Args> qargs) {
   if (qargs->output.empty()) {
-    std::cerr<<"No model provided!"<<std::endl;
-    exit(1);
+    throw std::runtime_error("No model provided!");
   }
   loadModel(qargs->output + ".bin");
 
@@ -600,14 +591,11 @@ void FastText::loadVectors(std::string filename) {
   std::shared_ptr<Matrix> mat; // temp. matrix for pretrained vectors
   int64_t n, dim;
   if (!in.is_open()) {
-    std::cerr << "Pretrained vectors file cannot be opened!" << std::endl;
-    exit(EXIT_FAILURE);
+    throw std::runtime_error("Pretrained vectors file cannot be opened!");
   }
   in >> n >> dim;
   if (dim != args_->dim) {
-    std::cerr << "Dimension of pretrained vectors does not match -dim option"
-              << std::endl;
-    exit(EXIT_FAILURE);
+    throw std::runtime_error("Dimension of pretrained vectors does not match -dim option");
   }
   mat = std::make_shared<Matrix>(n, dim);
   for (size_t i = 0; i < n; i++) {
@@ -639,13 +627,11 @@ void FastText::train(std::shared_ptr args) {
   dict_ = std::make_shared<Dictionary>(args_);
   if (args_->input == "-") {
     // manage expectations
-    std::cerr << "Cannot use stdin for training!" << std::endl;
-    exit(EXIT_FAILURE);
+    throw std::runtime_error("Cannot use stdin for training!");
   }
   std::ifstream ifs(args_->input);
   if (!ifs.is_open()) {
-    std::cerr << "Input file cannot be opened!" << std::endl;
-    exit(EXIT_FAILURE);
+    throw std::runtime_error("Input file cannot be opened!");
   }
   dict_->readFromFile(ifs);
   ifs.close();

data/vendor/fasttext/ffi_fasttext.cc

@@ -8,6 +8,7 @@
 
 #include "real.h"
 #include "fasttext.h"
+#include "curlstream.h"
 
 #ifdef __cplusplus
 #define EXTERN_C       extern "C"
@@ -20,9 +21,27 @@
 #endif
 
 EXTERN_C_BEGIN
-fasttext::FastText* create(const char* model_name) {
+fasttext::FastText* create(const char* model_filename) {
   fasttext::FastText* new_fasttext = new fasttext::FastText();
-  new_fasttext->loadModel(std::string(model_name));
+  try {
+    new_fasttext->loadModel(model_filename);
+  } catch(std::exception&) {
+    delete new_fasttext;
+    return nullptr;
+  }
+
+  return new_fasttext;
+}
+
+fasttext::FastText* create_from_url(const char* model_url) {
+  fasttext::FastText* new_fasttext = new fasttext::FastText();
+  try {
+    CurlStream curlstream{model_url};
+    new_fasttext->loadModel(curlstream);
+  } catch(std::exception&) {
+    delete new_fasttext;
+    return nullptr;
+  }
 
   return new_fasttext;
 }

metadata

@@ -1,85 +1,85 @@
 --- !ruby/object:Gem::Specification
 name: ffi-fasttext
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.3.0
 platform: ruby
 authors:
 - Brandon Dewitt
-autorequire: 
+autorequire:
 bindir: exe
 cert_chain: []
-date: 2018-06-28 00:00:00.000000000 Z
+date: 2020-07-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
-  name: ffi
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :runtime
+  name: ffi
   prerelease: false
+  type: :runtime
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: pry
   requirement: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-  type: :development
+  name: pry
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
-  name: bundler
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
-  type: :development
+        version: '0'
+  name: bundler
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '1.16'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
-  type: :development
+        version: '0'
+  name: rake
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '10.0'
+        version: '0'
 - !ruby/object:Gem::Dependency
-  name: minitest
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
-  type: :development
+        version: '0'
+  name: minitest
   prerelease: false
+  type: :development
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
-        version: '5.0'
+        version: '0'
 description: " FFI bindings for Facebook's FastText text classification library "
 email:
 - brandonsdewitt+rubygems@gmail.com
@@ -104,6 +104,8 @@ files:
 - vendor/fasttext/PATENTS
 - vendor/fasttext/args.cc
 - vendor/fasttext/args.h
+- vendor/fasttext/curlstream.cc
+- vendor/fasttext/curlstream.h
 - vendor/fasttext/dictionary.cc
 - vendor/fasttext/dictionary.h
 - vendor/fasttext/fasttext.cc
@@ -128,7 +130,7 @@ licenses:
 - MIT
 metadata:
   allowed_push_host: https://rubygems.org
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -143,9 +145,9 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.6.13
-signing_key: 
+rubyforge_project:
+rubygems_version: 2.6.14.1
+signing_key:
 specification_version: 4
 summary: FFI bindings for Facebook's FastText text classification library
 test_files: []