tomoto 0.4.0-aarch64-linux

data/ext/tomoto/hpa.cpp

@@ -0,0 +1,32 @@
+#include <HPA.h>
+
+#include <rice/rice.hpp>
+
+#include "utils.h"
+
+void init_hpa(Rice::Module& m) {
+  Rice::define_class_under<tomoto::IHPAModel, tomoto::IPAModel>(m, "HPA")
+    .define_singleton_function(
+      "_new",
+      [](size_t tw, size_t k1, size_t k2, tomoto::Float alpha, tomoto::Float eta, size_t seed) {
+        tomoto::HPAArgs args;
+        args.k = k1;
+        args.k2 = k2;
+        args.alpha = {alpha};
+        args.eta = eta;
+        if (seed >= 0) {
+          args.seed = seed;
+        }
+        return tomoto::IHPAModel::create((tomoto::TermWeight)tw, false, args);
+      }, Rice::Return().takeOwnership())
+    .define_method(
+      "alpha",
+      [](tomoto::IHPAModel& self) {
+        Array res;
+        // use <= to return k+1 elements
+        for (size_t i = 0; i <= self.getK(); i++) {
+          res.push(self.getAlpha(i));
+        }
+        return res;
+      });
+}

data/ext/tomoto/lda.cpp

@@ -0,0 +1,281 @@
+#include <fstream>
+#include <iostream>
+
+#include <LDA.h>
+
+#include <rice/rice.hpp>
+
+#include "utils.h"
+
+class DocumentObject
+{
+public:
+  DocumentObject(const tomoto::DocumentBase* _doc, const tomoto::ITopicModel* _tm) : doc{ _doc }, tm{ _tm } {}
+
+  const tomoto::DocumentBase* doc;
+  const tomoto::ITopicModel* tm;
+};
+
+void init_lda(Rice::Module& m) {
+  Rice::define_class_under<DocumentObject>(m, "Document")
+    .define_method(
+      "topics",
+      [](DocumentObject& self) {
+        Rice::Hash res;
+        auto topics = self.tm->getTopicsByDoc(self.doc);
+        for (size_t i = 0; i < topics.size(); i++) {
+          res[i] = topics[i];
+        }
+        return res;
+      });
+
+  Rice::define_class_under<tomoto::ILDAModel>(m, "LDA")
+    .define_singleton_function(
+      "_new",
+      [](size_t tw, size_t k, tomoto::Float alpha, tomoto::Float eta, size_t seed) {
+        tomoto::LDAArgs args;
+        args.k = k;
+        args.alpha = {alpha};
+        args.eta = eta;
+        if (seed >= 0) {
+          args.seed = seed;
+        }
+        return tomoto::ILDAModel::create((tomoto::TermWeight)tw, args);
+      }, Rice::Return().takeOwnership())
+    .define_method(
+      "_add_doc",
+      [](tomoto::ILDAModel& self, std::vector<std::string> words) {
+        return self.addDoc(buildDoc(words));
+      })
+    .define_method(
+      "_make_doc",
+      [](tomoto::ILDAModel& self, std::vector<std::string> words) {
+        return DocumentObject(self.makeDoc(buildDoc(words)).release(), &self);
+      })
+    .define_method(
+      "_infer",
+      [](tomoto::ILDAModel& self, DocumentObject& doc_object, size_t iteration, float tolerance, size_t workers, size_t ps, size_t together) {
+        std::vector<tomoto::DocumentBase*> docs;
+        auto doc = doc_object.doc;
+        docs.emplace_back(const_cast<tomoto::DocumentBase*>(doc));
+        float ll = self.infer(docs, iteration, tolerance, workers, (tomoto::ParallelScheme)ps, !!together)[0];
+
+        auto topic_dist = self.getTopicsByDoc(doc);
+        auto topic_res = Array();
+        for (size_t i = 0; i < topic_dist.size(); i++) {
+          topic_res.push(topic_dist[i]);
+        }
+
+        auto res = Array();
+        res.push(topic_res);
+        res.push(ll);
+        return res;
+      })
+    .define_method(
+      "alpha",
+      [](tomoto::ILDAModel& self) {
+        Array res;
+        for (size_t i = 0; i < self.getK(); i++) {
+          res.push(self.getAlpha(i));
+        }
+        return res;
+      })
+    .define_method(
+      "burn_in",
+      [](tomoto::ILDAModel& self) {
+        return self.getBurnInIteration();
+      })
+    .define_method(
+      "burn_in=",
+      [](tomoto::ILDAModel& self, size_t iteration) {
+        self.setBurnInIteration(iteration);
+        return iteration;
+      })
+    .define_method(
+      "_count_by_topics",
+      [](tomoto::ILDAModel& self) {
+        Array res;
+        for (auto const& v : self.getCountByTopic()) {
+          res.push(v);
+        }
+        return res;
+      })
+    .define_method(
+      "docs",
+      [](tomoto::ILDAModel& self) {
+        Array res;
+        auto n = self.getNumDocs();
+        for (size_t i = 0; i < n; i++) {
+          auto v = DocumentObject(self.getDoc(i), &self);
+          res.push(Object(Rice::detail::To_Ruby<DocumentObject>().convert(v)));
+        }
+        return res;
+      })
+    .define_method(
+      "eta",
+      [](tomoto::ILDAModel& self) {
+        return self.getEta();
+      })
+    .define_method(
+      "global_step",
+      [](tomoto::ILDAModel& self) {
+        return self.getGlobalStep();
+      })
+    .define_method(
+      "k",
+      [](tomoto::ILDAModel& self) {
+        return self.getK();
+      })
+    .define_method(
+      "_load",
+      [](tomoto::ILDAModel& self, const char* filename) {
+        std::ifstream str{ filename, std::ios_base::binary };
+        if (!str) throw std::runtime_error{ std::string("cannot open file '") + filename + std::string("'") };
+        std::vector<uint8_t> extra_data;
+        self.loadModel(str, &extra_data);
+      })
+    .define_method(
+      "ll_per_word",
+      [](tomoto::ILDAModel& self) {
+        return self.getLLPerWord();
+      })
+    .define_method(
+      "num_docs",
+      [](tomoto::ILDAModel& self) {
+        return self.getNumDocs();
+      })
+    .define_method(
+      "num_vocabs",
+      [](tomoto::ILDAModel& self) {
+        return self.getV();
+      })
+    .define_method(
+      "num_words",
+      [](tomoto::ILDAModel& self) {
+        return self.getN();
+      })
+    .define_method(
+      "optim_interval",
+      [](tomoto::ILDAModel& self) {
+        return self.getOptimInterval();
+      })
+    .define_method(
+      "optim_interval=",
+      [](tomoto::ILDAModel& self, size_t value) {
+        self.setOptimInterval(value);
+        return value;
+      })
+    .define_method(
+      "perplexity",
+      [](tomoto::ILDAModel& self) {
+        return self.getPerplexity();
+      })
+    .define_method(
+      "_prepare",
+      [](tomoto::ILDAModel& self, size_t minCnt, size_t minDf, size_t rmTop) {
+        self.prepare(true, minCnt, minDf, rmTop);
+      })
+    .define_method(
+      "_removed_top_words",
+      [](tomoto::ILDAModel& self, size_t rmTop) {
+        Array res;
+        auto dict = self.getVocabDict();
+        size_t size = dict.size();
+        for (size_t i = rmTop; i > 0; i--) {
+          res.push(dict.toWord(size - i));
+        }
+        return res;
+      })
+    .define_method(
+      "_save",
+      [](tomoto::ILDAModel& self, const char* filename, bool full) {
+        std::ofstream str{ filename, std::ios_base::binary };
+        std::vector<uint8_t> extra_data;
+        self.saveModel(str, full, &extra_data);
+      })
+    .define_method(
+      "_topic_words",
+      [](tomoto::ILDAModel& self, size_t topicId, size_t topN) {
+        Rice::Hash res;
+        for (auto const& v : self.getWordsByTopicSorted(topicId, topN)) {
+          res[v.first] = v.second;
+        }
+        return res;
+      })
+    .define_method(
+      "_train",
+      [](tomoto::ILDAModel& self, size_t iteration, size_t workers, size_t ps) {
+        self.train(iteration, workers, (tomoto::ParallelScheme)ps);
+      })
+    .define_method(
+      "_tw",
+      [](tomoto::ILDAModel& self) {
+        return (int)self.getTermWeight();
+      })
+    .define_method(
+      "used_vocab_df",
+      [](tomoto::ILDAModel& self) {
+        auto vocab = self.getVocabDf();
+        Array res;
+        for (size_t i = 0; i < self.getV(); i++) {
+          res.push(vocab[i]);
+        }
+        return res;
+      })
+    .define_method(
+      "used_vocab_freq",
+      [](tomoto::ILDAModel& self) {
+        auto vocab = self.getVocabCf();
+        Array res;
+        for (size_t i = 0; i < self.getV(); i++) {
+          res.push(vocab[i]);
+        }
+        return res;
+      })
+    .define_method(
+      "used_vocabs",
+      [](tomoto::ILDAModel& self) {
+        auto dict = self.getVocabDict();
+        Array res;
+        auto utf8 = Rice::Class(rb_cEncoding).call("const_get", "UTF_8");
+        for (size_t i = 0; i < self.getV(); i++) {
+          VALUE value = Rice::detail::To_Ruby<std::string>().convert(dict.toWord(i));
+          Object obj(value);
+          res.push(obj.call("force_encoding", utf8));
+        }
+        return res;
+      })
+    .define_method(
+      "vocab_df",
+      [](tomoto::ILDAModel& self) {
+        auto vocab = self.getVocabDf();
+        Array res;
+        for (size_t i = 0; i < vocab.size(); i++) {
+          res.push(vocab[i]);
+        }
+        return res;
+      })
+    .define_method(
+      "vocab_freq",
+      [](tomoto::ILDAModel& self) {
+        auto vocab = self.getVocabCf();
+        Array res;
+        for (size_t i = 0; i < vocab.size(); i++) {
+          res.push(vocab[i]);
+        }
+        return res;
+      })
+    .define_method(
+      "vocabs",
+      [](tomoto::ILDAModel& self) {
+        auto dict = self.getVocabDict();
+        Array res;
+        auto utf8 = Rice::Class(rb_cEncoding).call("const_get", "UTF_8");
+        for (size_t i = 0; i < dict.size(); i++) {
+          VALUE value = Rice::detail::To_Ruby<std::string>().convert(dict.toWord(i));
+          Object obj(value);
+          res.push(obj.call("force_encoding", utf8));
+        }
+        return res;
+      });
+}

data/ext/tomoto/llda.cpp

@@ -0,0 +1,46 @@
+#include <LLDA.h>
+
+#include <rice/rice.hpp>
+
+#include "utils.h"
+
+void init_llda(Rice::Module& m) {
+  Rice::define_class_under<tomoto::ILLDAModel, tomoto::ILDAModel>(m, "LLDA")
+    .define_singleton_function(
+      "_new",
+      [](size_t tw, size_t k, tomoto::Float alpha, tomoto::Float eta, size_t seed) {
+        tomoto::LDAArgs args;
+        args.k = k;
+        args.alpha = {alpha};
+        args.eta = eta;
+        if (seed >= 0) {
+          args.seed = seed;
+        }
+        return tomoto::ILLDAModel::create((tomoto::TermWeight)tw, args);
+      }, Rice::Return().takeOwnership())
+    .define_method(
+      "_add_doc",
+      [](tomoto::ILLDAModel& self, std::vector<std::string> words, std::vector<std::string> labels) {
+        auto doc = buildDoc(words);
+        doc.misc["labels"] = labels;
+        return self.addDoc(doc);
+      })
+    .define_method(
+      "topics_per_label",
+      [](tomoto::ILLDAModel& self) {
+        return self.getNumTopicsPerLabel();
+      })
+    .define_method(
+      "topic_label_dict",
+      [](tomoto::ILLDAModel& self) {
+        auto dict = self.getTopicLabelDict();
+        Array res;
+        auto utf8 = Rice::Class(rb_cEncoding).call("const_get", "UTF_8");
+        for (size_t i = 0; i < dict.size(); i++) {
+          VALUE value = Rice::detail::To_Ruby<std::string>().convert(dict.toWord(i));
+          Object obj(value);
+          res.push(obj.call("force_encoding", utf8));
+        }
+        return res;
+      });
+}

data/ext/tomoto/mglda.cpp

@@ -0,0 +1,81 @@
+#include <MGLDA.h>
+
+#include <rice/rice.hpp>
+
+#include "utils.h"
+
+void init_mglda(Rice::Module& m) {
+  Rice::define_class_under<tomoto::IMGLDAModel, tomoto::ILDAModel>(m, "MGLDA")
+    .define_singleton_function(
+      "_new",
+      [](size_t tw, size_t k_g, size_t k_l, size_t t, tomoto::Float alpha_g, tomoto::Float alpha_l, tomoto::Float alpha_mg, tomoto::Float alpha_ml, tomoto::Float eta_g) {
+        tomoto::MGLDAArgs args;
+        args.k = k_g;
+        args.kL = k_l;
+        args.t = t;
+        args.alpha = {alpha_g};
+        args.alphaL = {alpha_l};
+        args.alphaMG = alpha_mg;
+        args.alphaML = alpha_ml;
+        args.eta = eta_g;
+        // TODO more args
+        return tomoto::IMGLDAModel::create((tomoto::TermWeight)tw, args);
+      }, Rice::Return().takeOwnership())
+    .define_method(
+      "_add_doc",
+      [](tomoto::IMGLDAModel& self, std::vector<std::string> words, std::string delimiter) {
+        auto doc = buildDoc(words);
+        doc.misc["delimiter"] = delimiter;
+        return self.addDoc(doc);
+      })
+    .define_method(
+      "alpha_g",
+      [](tomoto::IMGLDAModel& self) {
+        return self.getAlpha();
+      })
+    .define_method(
+      "alpha_l",
+      [](tomoto::IMGLDAModel& self) {
+        return self.getAlphaL();
+      })
+    .define_method(
+      "alpha_mg",
+      [](tomoto::IMGLDAModel& self) {
+        return self.getAlphaM();
+      })
+    .define_method(
+      "alpha_ml",
+      [](tomoto::IMGLDAModel& self) {
+        return self.getAlphaML();
+      })
+    .define_method(
+      "eta_g",
+      [](tomoto::IMGLDAModel& self) {
+        return self.getEta();
+      })
+    .define_method(
+      "eta_l",
+      [](tomoto::IMGLDAModel& self) {
+        return self.getEtaL();
+      })
+    .define_method(
+      "gamma",
+      [](tomoto::IMGLDAModel& self) {
+        return self.getGamma();
+      })
+    .define_method(
+      "k_g",
+      [](tomoto::IMGLDAModel& self) {
+        return self.getK();
+      })
+    .define_method(
+      "k_l",
+      [](tomoto::IMGLDAModel& self) {
+        return self.getKL();
+      })
+    .define_method(
+      "t",
+      [](tomoto::IMGLDAModel& self) {
+        return self.getT();
+      });
+}

data/ext/tomoto/pa.cpp

@@ -0,0 +1,32 @@
+#include <PA.h>
+
+#include <rice/rice.hpp>
+
+#include "utils.h"
+
+void init_pa(Rice::Module& m) {
+  Rice::define_class_under<tomoto::IPAModel, tomoto::ILDAModel>(m, "PA")
+    .define_singleton_function(
+      "_new",
+      [](size_t tw, size_t k1, size_t k2, tomoto::Float alpha, tomoto::Float eta, size_t seed) {
+        tomoto::PAArgs args;
+        args.k = k1;
+        args.k2 = k2;
+        args.alpha = {alpha};
+        args.eta = eta;
+        if (seed >= 0) {
+          args.seed = seed;
+        }
+        return tomoto::IPAModel::create((tomoto::TermWeight)tw, args);
+      }, Rice::Return().takeOwnership())
+    .define_method(
+      "k1",
+      [](tomoto::IPAModel& self) {
+        return self.getK();
+      })
+    .define_method(
+      "k2",
+      [](tomoto::IPAModel& self) {
+        return self.getK2();
+      });
+}

data/ext/tomoto/plda.cpp

@@ -0,0 +1,33 @@
+#include <PLDA.h>
+
+#include <rice/rice.hpp>
+
+#include "utils.h"
+
+void init_plda(Rice::Module& m) {
+  Rice::define_class_under<tomoto::IPLDAModel, tomoto::ILLDAModel>(m, "PLDA")
+    .define_singleton_function(
+      "_new",
+      [](size_t tw, size_t latent_topics, tomoto::Float alpha, tomoto::Float eta, size_t seed) {
+        tomoto::PLDAArgs args;
+        args.numLatentTopics = latent_topics;
+        args.alpha = {alpha};
+        args.eta = eta;
+        if (seed >= 0) {
+          args.seed = seed;
+        }
+        return tomoto::IPLDAModel::create((tomoto::TermWeight)tw, args);
+      }, Rice::Return().takeOwnership())
+    .define_method(
+      "_add_doc",
+      [](tomoto::IPLDAModel& self, std::vector<std::string> words, std::vector<std::string> labels) {
+        auto doc = buildDoc(words);
+        doc.misc["labels"] = labels;
+        return self.addDoc(doc);
+      })
+    .define_method(
+      "latent_topics",
+      [](tomoto::IPLDAModel& self) {
+        return self.getNumLatentTopics();
+      });
+}

data/ext/tomoto/slda.cpp

@@ -0,0 +1,48 @@
+#include <SLDA.h>
+
+#include <rice/rice.hpp>
+
+#include "utils.h"
+
+void init_slda(Rice::Module& m) {
+  Rice::define_class_under<tomoto::ISLDAModel, tomoto::ILDAModel>(m, "SLDA")
+    .define_singleton_function(
+      "_new",
+      [](size_t tw, size_t k, Array rb_vars, tomoto::Float alpha, tomoto::Float eta, std::vector<tomoto::Float> mu, std::vector<tomoto::Float> nu_sq, std::vector<tomoto::Float> glm_param, size_t seed) {
+        std::vector<tomoto::ISLDAModel::GLM> vars;
+        vars.reserve(rb_vars.size());
+        for (auto const& v : rb_vars) {
+          vars.push_back((tomoto::ISLDAModel::GLM) Rice::detail::From_Ruby<int>().convert(v.value()));
+        }
+        tomoto::SLDAArgs args;
+        args.k = k;
+        args.vars = vars;
+        args.alpha = {alpha};
+        args.eta = eta;
+        args.mu = mu;
+        args.nuSq = nu_sq;
+        args.glmParam = glm_param;
+        if (seed >= 0) {
+          args.seed = seed;
+        }
+        return tomoto::ISLDAModel::create((tomoto::TermWeight)tw, args);
+      }, Rice::Return().takeOwnership())
+    .define_method(
+      "_add_doc",
+      [](tomoto::ISLDAModel& self, std::vector<std::string> words, std::vector<tomoto::Float> y) {
+        auto doc = buildDoc(words);
+        doc.misc["y"] = y;
+        return self.addDoc(doc);
+      })
+    .define_method(
+      "f",
+      [](tomoto::ISLDAModel& self) {
+        return self.getF();
+      })
+    .define_method(
+      "_var_type",
+      [](tomoto::ISLDAModel& self, size_t var_id) {
+        if (var_id >= self.getF()) throw std::runtime_error{ "'var_id' must be < 'f'" };
+        return self.getTypeOfVar(var_id) == tomoto::ISLDAModel::GLM::linear ? "l" : "b";
+      });
+}

data/ext/tomoto/tomoto.cpp

@@ -0,0 +1,48 @@
+#include <rice/rice.hpp>
+
+void init_lda(Rice::Module& m);
+void init_ct(Rice::Module& m);
+void init_dmr(Rice::Module& m);
+void init_dt(Rice::Module& m);
+void init_gdmr(Rice::Module& m);
+void init_hdp(Rice::Module& m);
+void init_hlda(Rice::Module& m);
+void init_pa(Rice::Module& m);
+void init_hpa(Rice::Module& m);
+void init_mglda(Rice::Module& m);
+void init_llda(Rice::Module& m);
+void init_plda(Rice::Module& m);
+void init_slda(Rice::Module& m);
+
+extern "C"
+void Init_tomoto()
+{
+  auto m = Rice::define_module("Tomoto")
+    .define_singleton_function(
+      "isa",
+      []() {
+        #ifdef __AVX2__
+          return Rice::String("avx2");
+        #elif defined(__AVX__)
+          return Rice::String("avx");
+        #elif defined(__SSE2__) || defined(__x86_64__) || defined(_WIN64)
+          return Rice::String("sse2");
+        #else
+          return Rice::String("none");
+        #endif
+      });
+
+  init_lda(m);
+  init_ct(m);
+  init_dmr(m);
+  init_dt(m);
+  init_gdmr(m);
+  init_hdp(m);
+  init_hlda(m);
+  init_pa(m);
+  init_hpa(m);
+  init_mglda(m);
+  init_llda(m);
+  init_plda(m);
+  init_slda(m);
+}

data/ext/tomoto/utils.h

@@ -0,0 +1,30 @@
+#pragma once
+
+#include <rice/rice.hpp>
+#include <rice/stl.hpp>
+
+using Rice::Array;
+using Rice::Object;
+
+namespace Rice::detail
+{
+  template<typename T>
+  class To_Ruby<std::vector<T>>
+  {
+  public:
+    VALUE convert(std::vector<T> const & x)
+    {
+      auto a = rb_ary_new2(x.size());
+      for (const auto& v : x) {
+        detail::protect(rb_ary_push, a, To_Ruby<T>().convert(v));
+      }
+      return a;
+    }
+  };
+}
+
+inline tomoto::RawDoc buildDoc(std::vector<std::string>& words) {
+  tomoto::RawDoc doc;
+  doc.rawWords = words;
+  return doc;
+}

data/lib/tomoto/ct.rb

@@ -0,0 +1,24 @@
+module Tomoto
+  class CT
+    def self.new(tw: :one, min_cf: 0, min_df: 0, rm_top: 0, k: 1, alpha: 0.1, eta: 0.01, seed: nil)
+      model = _new(to_tw(tw), k, alpha, eta, seed || -1)
+      model.instance_variable_set(:@min_cf, min_cf)
+      model.instance_variable_set(:@min_df, min_df)
+      model.instance_variable_set(:@rm_top, rm_top)
+      init_params(model, binding)
+    end
+
+    def correlations(topic_id = nil)
+      prepare
+      if topic_id
+        _correlations(topic_id)
+      else
+        k.times.map { |i| _correlations(i) }
+      end
+    end
+
+    def prior_cov
+      _prior_cov.each_slice(k).to_a
+    end
+  end
+end

data/lib/tomoto/dmr.rb

@@ -0,0 +1,27 @@
+module Tomoto
+  class DMR
+    def self.new(tw: :one, min_cf: 0, min_df: 0, rm_top: 0, k: 1, alpha: 0.1, eta: 0.01, sigma: 1.0, alpha_epsilon: 1e-10, seed: nil)
+      model = _new(to_tw(tw), k, alpha, sigma, eta, alpha_epsilon, seed || -1)
+      model.instance_variable_set(:@min_cf, min_cf)
+      model.instance_variable_set(:@min_df, min_df)
+      model.instance_variable_set(:@rm_top, rm_top)
+      init_params(model, binding)
+    end
+
+    def add_doc(doc, metadata: "")
+      _add_doc(prepare_doc(doc), metadata)
+    end
+
+    def lambdas
+      if f == 0
+        []
+      else
+        k.times.map { |i| _lambdas(i) }
+      end
+    end
+
+    def alpha
+      lambdas.map { |v| v.map { |v2| Math.exp(v2) } }
+    end
+  end
+end