melisa 0.1.0

data/README.md

@@ -0,0 +1,11 @@
+Features
+--------
+ - fast search for exact strings and prefixes
+ - has a BytesTrie that can be used to store binary data
+ - has an IntTrie that can be used to store integer values easily
+
+TODO
+----
+ - Ruby bindings aren't yet built into the gem
+
+License: MIT

data/ext/marisa/bindings/marisa-swig.cxx

@@ -0,0 +1,253 @@
+#include <cstring>
+#include <new>
+
+#include "marisa-swig.h"
+
+namespace marisa_swig {
+
+void Key::str(const char **ptr_out, size_t *length_out) const {
+  *ptr_out = key_.ptr();
+  *length_out = key_.length();
+}
+
+size_t Key::id() const {
+  return key_.id();
+}
+
+float Key::weight() const {
+  return key_.weight();
+}
+
+void Query::str(const char **ptr_out, size_t *length_out) const {
+  *ptr_out = query_.ptr();
+  *length_out = query_.length();
+}
+
+size_t Query::id() const {
+  return query_.id();
+}
+
+Keyset::Keyset() : keyset_(new (std::nothrow) marisa::Keyset) {
+  MARISA_THROW_IF(keyset_ == NULL, ::MARISA_MEMORY_ERROR);
+}
+
+Keyset::~Keyset() {
+  delete keyset_;
+}
+
+void Keyset::push_back(const marisa::Key &key) {
+  keyset_->push_back(key);
+}
+
+void Keyset::push_back(const char *ptr, size_t length, float weight) {
+  keyset_->push_back(ptr, length, weight);
+}
+
+const Key &Keyset::key(size_t i) const {
+  return reinterpret_cast<const Key &>((*keyset_)[i]);
+}
+
+void Keyset::key_str(size_t i,
+    const char **ptr_out, size_t *length_out) const {
+  *ptr_out = (*keyset_)[i].ptr();
+  *length_out = (*keyset_)[i].length();
+}
+
+size_t Keyset::key_id(size_t i) const {
+  return (*keyset_)[i].id();
+}
+
+size_t Keyset::num_keys() const {
+  return keyset_->num_keys();
+}
+
+bool Keyset::empty() const {
+  return keyset_->empty();
+}
+
+size_t Keyset::size() const {
+  return keyset_->size();
+}
+
+size_t Keyset::total_length() const {
+  return keyset_->total_length();
+}
+
+void Keyset::reset() {
+  keyset_->reset();
+}
+
+void Keyset::clear() {
+  keyset_->clear();
+}
+
+Agent::Agent()
+    : agent_(new (std::nothrow) marisa::Agent), buf_(NULL), buf_size_(0) {
+  MARISA_THROW_IF(agent_ == NULL, ::MARISA_MEMORY_ERROR);
+}
+
+Agent::~Agent() {
+  delete agent_;
+  delete [] buf_;
+}
+
+void Agent::set_query(const char *ptr, size_t length) {
+  if (length > buf_size_) {
+    size_t new_buf_size = (buf_size_ != 0) ? buf_size_ : 1;
+    if (length >= (MARISA_SIZE_MAX / 2)) {
+      new_buf_size = MARISA_SIZE_MAX;
+    } else {
+      while (new_buf_size < length) {
+        new_buf_size *= 2;
+      }
+    }
+    char *new_buf = new (std::nothrow) char[new_buf_size];
+    MARISA_THROW_IF(new_buf == NULL, MARISA_MEMORY_ERROR);
+    delete [] buf_;
+    buf_ = new_buf;
+    buf_size_ = new_buf_size;
+  }
+  std::memcpy(buf_, ptr, length);
+  agent_->set_query(buf_, length);
+}
+
+void Agent::set_query(size_t id) {
+  agent_->set_query(id);
+}
+
+const Key &Agent::key() const {
+  return reinterpret_cast<const Key &>(agent_->key());
+}
+
+const Query &Agent::query() const {
+  return reinterpret_cast<const Query &>(agent_->query());
+}
+
+void Agent::key_str(const char **ptr_out, size_t *length_out) const {
+  *ptr_out = agent_->key().ptr();
+  *length_out = agent_->key().length();
+}
+
+size_t Agent::key_id() const {
+  return agent_->key().id();
+}
+
+void Agent::query_str(const char **ptr_out, size_t *length_out) const {
+  *ptr_out = agent_->query().ptr();
+  *length_out = agent_->query().length();
+}
+
+size_t Agent::query_id() const {
+  return agent_->query().id();
+}
+
+Trie::Trie() : trie_(new (std::nothrow) marisa::Trie) {
+  MARISA_THROW_IF(trie_ == NULL, ::MARISA_MEMORY_ERROR);
+}
+
+Trie::~Trie() {
+  delete trie_;
+}
+
+void Trie::build(Keyset &keyset, int config_flags) {
+  trie_->build(*keyset.keyset_, config_flags);
+}
+
+void Trie::mmap(const char *filename) {
+  trie_->mmap(filename);
+}
+
+void Trie::load(const char *filename) {
+  trie_->load(filename);
+}
+
+void Trie::save(const char *filename) const {
+  trie_->save(filename);
+}
+
+bool Trie::lookup(Agent &agent) const {
+  return trie_->lookup(*agent.agent_);
+}
+
+void Trie::reverse_lookup(Agent &agent) const {
+  trie_->reverse_lookup(*agent.agent_);
+}
+
+bool Trie::common_prefix_search(Agent &agent) const {
+  return trie_->common_prefix_search(*agent.agent_);
+}
+
+bool Trie::predictive_search(Agent &agent) const {
+  return trie_->predictive_search(*agent.agent_);
+}
+
+size_t Trie::lookup(const char *ptr, size_t length) const {
+  marisa::Agent agent;
+  agent.set_query(ptr, length);
+  if (!trie_->lookup(agent)) {
+    return MARISA_INVALID_KEY_ID;
+  }
+  return agent.key().id();
+}
+
+void Trie::reverse_lookup(size_t id,
+    const char **ptr_out_to_be_deleted, size_t *length_out) const {
+  marisa::Agent agent;
+  agent.set_query(id);
+  trie_->reverse_lookup(agent);
+  char * const buf = new (std::nothrow) char[agent.key().length()];
+  MARISA_THROW_IF(buf == NULL, MARISA_MEMORY_ERROR);
+  std::memcpy(buf, agent.key().ptr(), agent.key().length());
+  *ptr_out_to_be_deleted = buf;
+  *length_out = agent.key().length();
+}
+
+size_t Trie::num_tries() const {
+  return trie_->num_tries();
+}
+
+size_t Trie::num_keys() const {
+  return trie_->num_keys();
+}
+
+size_t Trie::num_nodes() const {
+  return trie_->num_nodes();
+}
+
+TailMode Trie::tail_mode() const {
+  if (trie_->tail_mode() == ::MARISA_TEXT_TAIL) {
+    return TEXT_TAIL;
+  } else {
+    return BINARY_TAIL;
+  }
+}
+
+NodeOrder Trie::node_order() const {
+  if (trie_->node_order() == ::MARISA_LABEL_ORDER) {
+    return LABEL_ORDER;
+  } else {
+    return WEIGHT_ORDER;
+  }
+}
+
+bool Trie::empty() const {
+  return trie_->empty();
+}
+
+size_t Trie::size() const {
+  return trie_->size();
+}
+
+size_t Trie::total_size() const {
+  return trie_->total_size();
+}
+
+size_t Trie::io_size() const {
+  return trie_->io_size();
+}
+
+void Trie::clear() {
+  trie_->clear();
+}
+
+}  // namespace marisa_swig

data/ext/marisa/bindings/marisa-swig.h

@@ -0,0 +1,183 @@
+#ifndef MARISA_SWIG_H_
+#define MARISA_SWIG_H_
+
+#include <marisa.h>
+
+namespace marisa_swig {
+
+#define MARISA_SWIG_ENUM_COPY(name) name = MARISA_ ## name
+
+enum ErrorCode {
+  MARISA_SWIG_ENUM_COPY(OK),
+  MARISA_SWIG_ENUM_COPY(STATE_ERROR),
+  MARISA_SWIG_ENUM_COPY(NULL_ERROR),
+  MARISA_SWIG_ENUM_COPY(BOUND_ERROR),
+  MARISA_SWIG_ENUM_COPY(RANGE_ERROR),
+  MARISA_SWIG_ENUM_COPY(CODE_ERROR),
+  MARISA_SWIG_ENUM_COPY(RESET_ERROR),
+  MARISA_SWIG_ENUM_COPY(SIZE_ERROR),
+  MARISA_SWIG_ENUM_COPY(MEMORY_ERROR),
+  MARISA_SWIG_ENUM_COPY(IO_ERROR),
+  MARISA_SWIG_ENUM_COPY(FORMAT_ERROR)
+};
+
+enum NumTries {
+  MARISA_SWIG_ENUM_COPY(MIN_NUM_TRIES),
+  MARISA_SWIG_ENUM_COPY(MAX_NUM_TRIES),
+  MARISA_SWIG_ENUM_COPY(DEFAULT_NUM_TRIES)
+};
+
+enum CacheLevel {
+  MARISA_SWIG_ENUM_COPY(HUGE_CACHE),
+  MARISA_SWIG_ENUM_COPY(LARGE_CACHE),
+  MARISA_SWIG_ENUM_COPY(NORMAL_CACHE),
+  MARISA_SWIG_ENUM_COPY(SMALL_CACHE),
+  MARISA_SWIG_ENUM_COPY(TINY_CACHE),
+  MARISA_SWIG_ENUM_COPY(DEFAULT_CACHE)
+};
+
+enum TailMode {
+  MARISA_SWIG_ENUM_COPY(TEXT_TAIL),
+  MARISA_SWIG_ENUM_COPY(BINARY_TAIL),
+  MARISA_SWIG_ENUM_COPY(DEFAULT_TAIL)
+};
+
+enum NodeOrder {
+  MARISA_SWIG_ENUM_COPY(LABEL_ORDER),
+  MARISA_SWIG_ENUM_COPY(WEIGHT_ORDER),
+  MARISA_SWIG_ENUM_COPY(DEFAULT_ORDER)
+};
+
+#undef MARISA_SWIG_ENUM_COPY
+
+class Key {
+ public:
+  void str(const char **ptr_out, std::size_t *length_out) const;
+  std::size_t id() const;
+  float weight() const;
+
+ private:
+  const marisa::Key key_;
+
+  Key();
+  Key(const Key &key);
+  Key &operator=(const Key &);
+};
+
+class Query {
+ public:
+  void str(const char **ptr_out, std::size_t *length_out) const;
+  std::size_t id() const;
+
+ private:
+  const marisa::Query query_;
+
+  Query();
+  Query(const Query &query);
+  Query &operator=(const Query &);
+};
+
+class Keyset {
+ friend class Trie;
+
+ public:
+  Keyset();
+  ~Keyset();
+
+  void push_back(const marisa::Key &key);
+  void push_back(const char *ptr, std::size_t length, float weight = 1.0);
+
+  const Key &key(std::size_t i) const;
+
+  void key_str(std::size_t i,
+      const char **ptr_out, std::size_t *length_out) const;
+  std::size_t key_id(std::size_t i) const;
+
+  std::size_t num_keys() const;
+
+  bool empty() const;
+  std::size_t size() const;
+  std::size_t total_length() const;
+
+  void reset();
+  void clear();
+
+ private:
+  marisa::Keyset *keyset_;
+
+  Keyset(const Keyset &);
+  Keyset &operator=(const Keyset &);
+};
+
+class Agent {
+ friend class Trie;
+
+ public:
+  Agent();
+  ~Agent();
+
+  void set_query(const char *ptr, std::size_t length);
+  void set_query(std::size_t id);
+
+  const Key &key() const;
+  const Query &query() const;
+
+  void key_str(const char **ptr_out, std::size_t *length_out) const;
+  std::size_t key_id() const;
+
+  void query_str(const char **ptr_out, std::size_t *length_out) const;
+  std::size_t query_id() const;
+
+ private:
+  marisa::Agent *agent_;
+  char *buf_;
+  std::size_t buf_size_;
+
+  Agent(const Agent &);
+  Agent &operator=(const Agent &);
+};
+
+class Trie {
+ public:
+  Trie();
+  ~Trie();
+
+  void build(Keyset &keyset, int config_flags = 0);
+
+  void mmap(const char *filename);
+  void load(const char *filename);
+  void save(const char *filename) const;
+
+  bool lookup(Agent &agent) const;
+  void reverse_lookup(Agent &agent) const;
+  bool common_prefix_search(Agent &agent) const;
+  bool predictive_search(Agent &agent) const;
+
+  std::size_t lookup(const char *ptr, std::size_t length) const;
+  void reverse_lookup(std::size_t id,
+      const char **ptr_out_to_be_deleted, std::size_t *length_out) const;
+
+  std::size_t num_tries() const;
+  std::size_t num_keys() const;
+  std::size_t num_nodes() const;
+
+  TailMode tail_mode() const;
+  NodeOrder node_order() const;
+
+  bool empty() const;
+  std::size_t size() const;
+  std::size_t total_size() const;
+  std::size_t io_size() const;
+
+  void clear();
+
+ private:
+  marisa::Trie *trie_;
+
+  Trie(const Trie &);
+  Trie &operator=(const Trie &);
+};
+
+}  // namespace marisa_swig
+
+#endif  // MARISA_SWIG_H_

data/ext/marisa/bindings/perl/marisa-swig.cxx

@@ -0,0 +1,253 @@
+#include <cstring>
+#include <new>
+
+#include "marisa-swig.h"
+
+namespace marisa_swig {
+
+void Key::str(const char **ptr_out, size_t *length_out) const {
+  *ptr_out = key_.ptr();
+  *length_out = key_.length();
+}
+
+size_t Key::id() const {
+  return key_.id();
+}
+
+float Key::weight() const {
+  return key_.weight();
+}
+
+void Query::str(const char **ptr_out, size_t *length_out) const {
+  *ptr_out = query_.ptr();
+  *length_out = query_.length();
+}
+
+size_t Query::id() const {
+  return query_.id();
+}
+
+Keyset::Keyset() : keyset_(new (std::nothrow) marisa::Keyset) {
+  MARISA_THROW_IF(keyset_ == NULL, ::MARISA_MEMORY_ERROR);
+}
+
+Keyset::~Keyset() {
+  delete keyset_;
+}
+
+void Keyset::push_back(const marisa::Key &key) {
+  keyset_->push_back(key);
+}
+
+void Keyset::push_back(const char *ptr, size_t length, float weight) {
+  keyset_->push_back(ptr, length, weight);
+}
+
+const Key &Keyset::key(size_t i) const {
+  return reinterpret_cast<const Key &>((*keyset_)[i]);
+}
+
+void Keyset::key_str(size_t i,
+    const char **ptr_out, size_t *length_out) const {
+  *ptr_out = (*keyset_)[i].ptr();
+  *length_out = (*keyset_)[i].length();
+}
+
+size_t Keyset::key_id(size_t i) const {
+  return (*keyset_)[i].id();
+}
+
+size_t Keyset::num_keys() const {
+  return keyset_->num_keys();
+}
+
+bool Keyset::empty() const {
+  return keyset_->empty();
+}
+
+size_t Keyset::size() const {
+  return keyset_->size();
+}
+
+size_t Keyset::total_length() const {
+  return keyset_->total_length();
+}
+
+void Keyset::reset() {
+  keyset_->reset();
+}
+
+void Keyset::clear() {
+  keyset_->clear();
+}
+
+Agent::Agent()
+    : agent_(new (std::nothrow) marisa::Agent), buf_(NULL), buf_size_(0) {
+  MARISA_THROW_IF(agent_ == NULL, ::MARISA_MEMORY_ERROR);
+}
+
+Agent::~Agent() {
+  delete agent_;
+  delete [] buf_;
+}
+
+void Agent::set_query(const char *ptr, size_t length) {
+  if (length > buf_size_) {
+    size_t new_buf_size = (buf_size_ != 0) ? buf_size_ : 1;
+    if (length >= (MARISA_SIZE_MAX / 2)) {
+      new_buf_size = MARISA_SIZE_MAX;
+    } else {
+      while (new_buf_size < length) {
+        new_buf_size *= 2;
+      }
+    }
+    char *new_buf = new (std::nothrow) char[new_buf_size];
+    MARISA_THROW_IF(new_buf == NULL, MARISA_MEMORY_ERROR);
+    delete [] buf_;
+    buf_ = new_buf;
+    buf_size_ = new_buf_size;
+  }
+  std::memcpy(buf_, ptr, length);
+  agent_->set_query(buf_, length);
+}
+
+void Agent::set_query(size_t id) {
+  agent_->set_query(id);
+}
+
+const Key &Agent::key() const {
+  return reinterpret_cast<const Key &>(agent_->key());
+}
+
+const Query &Agent::query() const {
+  return reinterpret_cast<const Query &>(agent_->query());
+}
+
+void Agent::key_str(const char **ptr_out, size_t *length_out) const {
+  *ptr_out = agent_->key().ptr();
+  *length_out = agent_->key().length();
+}
+
+size_t Agent::key_id() const {
+  return agent_->key().id();
+}
+
+void Agent::query_str(const char **ptr_out, size_t *length_out) const {
+  *ptr_out = agent_->query().ptr();
+  *length_out = agent_->query().length();
+}
+
+size_t Agent::query_id() const {
+  return agent_->query().id();
+}
+
+Trie::Trie() : trie_(new (std::nothrow) marisa::Trie) {
+  MARISA_THROW_IF(trie_ == NULL, ::MARISA_MEMORY_ERROR);
+}
+
+Trie::~Trie() {
+  delete trie_;
+}
+
+void Trie::build(Keyset &keyset, int config_flags) {
+  trie_->build(*keyset.keyset_, config_flags);
+}
+
+void Trie::mmap(const char *filename) {
+  trie_->mmap(filename);
+}
+
+void Trie::load(const char *filename) {
+  trie_->load(filename);
+}
+
+void Trie::save(const char *filename) const {
+  trie_->save(filename);
+}
+
+bool Trie::lookup(Agent &agent) const {
+  return trie_->lookup(*agent.agent_);
+}
+
+void Trie::reverse_lookup(Agent &agent) const {
+  trie_->reverse_lookup(*agent.agent_);
+}
+
+bool Trie::common_prefix_search(Agent &agent) const {
+  return trie_->common_prefix_search(*agent.agent_);
+}
+
+bool Trie::predictive_search(Agent &agent) const {
+  return trie_->predictive_search(*agent.agent_);
+}
+
+size_t Trie::lookup(const char *ptr, size_t length) const {
+  marisa::Agent agent;
+  agent.set_query(ptr, length);
+  if (!trie_->lookup(agent)) {
+    return MARISA_INVALID_KEY_ID;
+  }
+  return agent.key().id();
+}
+
+void Trie::reverse_lookup(size_t id,
+    const char **ptr_out_to_be_deleted, size_t *length_out) const {
+  marisa::Agent agent;
+  agent.set_query(id);
+  trie_->reverse_lookup(agent);
+  char * const buf = new (std::nothrow) char[agent.key().length()];
+  MARISA_THROW_IF(buf == NULL, MARISA_MEMORY_ERROR);
+  std::memcpy(buf, agent.key().ptr(), agent.key().length());
+  *ptr_out_to_be_deleted = buf;
+  *length_out = agent.key().length();
+}
+
+size_t Trie::num_tries() const {
+  return trie_->num_tries();
+}
+
+size_t Trie::num_keys() const {
+  return trie_->num_keys();
+}
+
+size_t Trie::num_nodes() const {
+  return trie_->num_nodes();
+}
+
+TailMode Trie::tail_mode() const {
+  if (trie_->tail_mode() == ::MARISA_TEXT_TAIL) {
+    return TEXT_TAIL;
+  } else {
+    return BINARY_TAIL;
+  }
+}
+
+NodeOrder Trie::node_order() const {
+  if (trie_->node_order() == ::MARISA_LABEL_ORDER) {
+    return LABEL_ORDER;
+  } else {
+    return WEIGHT_ORDER;
+  }
+}
+
+bool Trie::empty() const {
+  return trie_->empty();
+}
+
+size_t Trie::size() const {
+  return trie_->size();
+}
+
+size_t Trie::total_size() const {
+  return trie_->total_size();
+}
+
+size_t Trie::io_size() const {
+  return trie_->io_size();
+}
+
+void Trie::clear() {
+  trie_->clear();
+}
+
+}  // namespace marisa_swig