melisa 0.1.0

data/ext/marisa/lib/marisa/grimoire/trie/louds-trie.h

@@ -0,0 +1,134 @@
+#ifndef MARISA_GRIMOIRE_TRIE_LOUDS_TRIE_H_
+#define MARISA_GRIMOIRE_TRIE_LOUDS_TRIE_H_
+
+#include "marisa/keyset.h"
+#include "marisa/agent.h"
+#include "marisa/grimoire/vector.h"
+#include "marisa/grimoire/trie/config.h"
+#include "marisa/grimoire/trie/key.h"
+#include "marisa/grimoire/trie/tail.h"
+#include "marisa/grimoire/trie/cache.h"
+
+namespace marisa {
+namespace grimoire {
+namespace trie {
+
+class LoudsTrie  {
+ public:
+  LoudsTrie();
+  ~LoudsTrie();
+
+  void build(Keyset &keyset, int flags);
+
+  void map(Mapper &mapper);
+  void read(Reader &reader);
+  void write(Writer &writer) const;
+
+  bool lookup(Agent &agent) const;
+  void reverse_lookup(Agent &agent) const;
+  bool common_prefix_search(Agent &agent) const;
+  bool predictive_search(Agent &agent) const;
+
+  std::size_t num_tries() const {
+    return config_.num_tries();
+  }
+  std::size_t num_keys() const {
+    return size();
+  }
+  std::size_t num_nodes() const {
+    return (louds_.size() / 2) - 1;
+  }
+
+  CacheLevel cache_level() const {
+    return config_.cache_level();
+  }
+  TailMode tail_mode() const {
+    return config_.tail_mode();
+  }
+  NodeOrder node_order() const {
+    return config_.node_order();
+  }
+
+  bool empty() const {
+    return size() == 0;
+  }
+  std::size_t size() const {
+    return terminal_flags_.num_1s();
+  }
+  std::size_t total_size() const;
+  std::size_t io_size() const;
+
+  void clear();
+  void swap(LoudsTrie &rhs);
+
+ private:
+  BitVector louds_;
+  BitVector terminal_flags_;
+  BitVector link_flags_;
+  Vector<UInt8> bases_;
+  FlatVector extras_;
+  Tail tail_;
+  scoped_ptr<LoudsTrie> next_trie_;
+  Vector<Cache> cache_;
+  std::size_t cache_mask_;
+  std::size_t num_l1_nodes_;
+  Config config_;
+  Mapper mapper_;
+
+  void build_(Keyset &keyset, const Config &config);
+
+  template <typename T>
+  void build_trie(Vector<T> &keys,
+      Vector<UInt32> *terminals, const Config &config, std::size_t trie_id);
+  template <typename T>
+  void build_current_trie(Vector<T> &keys,
+      Vector<UInt32> *terminals, const Config &config, std::size_t trie_id);
+  template <typename T>
+  void build_next_trie(Vector<T> &keys,
+      Vector<UInt32> *terminals, const Config &config, std::size_t trie_id);
+  template <typename T>
+  void build_terminals(const Vector<T> &keys,
+      Vector<UInt32> *terminals) const;
+
+  void reserve_cache(const Config &config, std::size_t trie_id,
+      std::size_t num_keys);
+  template <typename T>
+  void cache(std::size_t parent, std::size_t child,
+      float weight, char label);
+  void fill_cache();
+
+  void map_(Mapper &mapper);
+  void read_(Reader &reader);
+  void write_(Writer &writer) const;
+
+  inline bool find_child(Agent &agent) const;
+  inline bool predictive_find_child(Agent &agent) const;
+
+  inline void restore(Agent &agent, std::size_t node_id) const;
+  inline bool match(Agent &agent, std::size_t node_id) const;
+  inline bool prefix_match(Agent &agent, std::size_t node_id) const;
+
+  void restore_(Agent &agent, std::size_t node_id) const;
+  bool match_(Agent &agent, std::size_t node_id) const;
+  bool prefix_match_(Agent &agent, std::size_t node_id) const;
+
+  inline std::size_t get_cache_id(std::size_t node_id, char label) const;
+  inline std::size_t get_cache_id(std::size_t node_id) const;
+
+  inline std::size_t get_link(std::size_t node_id) const;
+  inline std::size_t get_link(std::size_t node_id,
+      std::size_t link_id) const;
+
+  inline std::size_t update_link_id(std::size_t link_id,
+      std::size_t node_id) const;
+
+  // Disallows copy and assignment.
+  LoudsTrie(const LoudsTrie &);
+  LoudsTrie &operator=(const LoudsTrie &);
+};
+
+}  // namespace trie
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_TRIE_LOUDS_TRIE_H_

data/ext/marisa/lib/marisa/grimoire/trie/range.h

@@ -0,0 +1,115 @@
+#ifndef MARISA_GRIMOIRE_TRIE_RANGE_H_
+#define MARISA_GRIMOIRE_TRIE_RANGE_H_
+
+#include "marisa/base.h"
+
+namespace marisa {
+namespace grimoire {
+namespace trie {
+
+class Range {
+ public:
+  Range() : begin_(0), end_(0), key_pos_(0) {}
+
+  void set_begin(std::size_t begin) {
+    MARISA_DEBUG_IF(begin > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    begin_ = begin;
+  }
+  void set_end(std::size_t end) {
+    MARISA_DEBUG_IF(end > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    end_ = end;
+  }
+  void set_key_pos(std::size_t key_pos) {
+    MARISA_DEBUG_IF(key_pos > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    key_pos_ = key_pos;
+  }
+
+  std::size_t begin() const {
+    return begin_;
+  }
+  std::size_t end() const {
+    return end_;
+  }
+  std::size_t key_pos() const {
+    return key_pos_;
+  }
+
+ private:
+  UInt32 begin_;
+  UInt32 end_;
+  UInt32 key_pos_;
+};
+
+inline Range make_range(std::size_t begin, std::size_t end,
+    std::size_t key_pos) {
+  Range range;
+  range.set_begin(begin);
+  range.set_end(end);
+  range.set_key_pos(key_pos);
+  return range;
+}
+
+class WeightedRange {
+ public:
+  WeightedRange() : range_(), weight_(0.0F) {}
+
+  void set_range(const Range &range) {
+    range_ = range;
+  }
+  void set_begin(std::size_t begin) {
+    range_.set_begin(begin);
+  }
+  void set_end(std::size_t end) {
+    range_.set_end(end);
+  }
+  void set_key_pos(std::size_t key_pos) {
+    range_.set_key_pos(key_pos);
+  }
+  void set_weight(float weight) {
+    weight_ = weight;
+  }
+
+  const Range &range() const {
+    return range_;
+  }
+  std::size_t begin() const {
+    return range_.begin();
+  }
+  std::size_t end() const {
+    return range_.end();
+  }
+  std::size_t key_pos() const {
+    return range_.key_pos();
+  }
+  float weight() const {
+    return weight_;
+  }
+
+ private:
+  Range range_;
+  float weight_;
+};
+
+inline bool operator<(const WeightedRange &lhs, const WeightedRange &rhs) {
+  return lhs.weight() < rhs.weight();
+}
+
+inline bool operator>(const WeightedRange &lhs, const WeightedRange &rhs) {
+  return lhs.weight() > rhs.weight();
+}
+
+inline WeightedRange make_weighted_range(std::size_t begin, std::size_t end,
+    std::size_t key_pos, float weight) {
+  WeightedRange range;
+  range.set_begin(begin);
+  range.set_end(end);
+  range.set_key_pos(key_pos);
+  range.set_weight(weight);
+  return range;
+}
+
+}  // namespace trie
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_TRIE_RANGE_H_

data/ext/marisa/lib/marisa/grimoire/trie/state.h

@@ -0,0 +1,117 @@
+#ifndef MARISA_GRIMOIRE_TRIE_STATE_H_
+#define MARISA_GRIMOIRE_TRIE_STATE_H_
+
+#include "marisa/grimoire/vector.h"
+#include "marisa/grimoire/trie/history.h"
+
+namespace marisa {
+namespace grimoire {
+namespace trie {
+
+// A search agent has its internal state and the status codes are defined
+// below.
+typedef enum StatusCode {
+  MARISA_READY_TO_ALL,
+  MARISA_READY_TO_COMMON_PREFIX_SEARCH,
+  MARISA_READY_TO_PREDICTIVE_SEARCH,
+  MARISA_END_OF_COMMON_PREFIX_SEARCH,
+  MARISA_END_OF_PREDICTIVE_SEARCH,
+} StatusCode;
+
+class State {
+ public:
+  State()
+      : key_buf_(), history_(), node_id_(0), query_pos_(0),
+        history_pos_(0), status_code_(MARISA_READY_TO_ALL) {}
+
+  void set_node_id(std::size_t node_id) {
+    MARISA_DEBUG_IF(node_id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    node_id_ = (UInt32)node_id;
+  }
+  void set_query_pos(std::size_t query_pos) {
+    MARISA_DEBUG_IF(query_pos > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    query_pos_ = (UInt32)query_pos;
+  }
+  void set_history_pos(std::size_t history_pos) {
+    MARISA_DEBUG_IF(history_pos > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    history_pos_ = (UInt32)history_pos;
+  }
+  void set_status_code(StatusCode status_code) {
+    status_code_ = status_code;
+  }
+
+  std::size_t node_id() const {
+    return node_id_;
+  }
+  std::size_t query_pos() const {
+    return query_pos_;
+  }
+  std::size_t history_pos() const {
+    return history_pos_;
+  }
+  StatusCode status_code() const {
+    return status_code_;
+  }
+
+  const Vector<char> &key_buf() const {
+    return key_buf_;
+  }
+  const Vector<History> &history() const {
+    return history_;
+  }
+
+  Vector<char> &key_buf() {
+    return key_buf_;
+  }
+  Vector<History> &history() {
+    return history_;
+  }
+
+  void reset() {
+    status_code_ = MARISA_READY_TO_ALL;
+  }
+
+  void lookup_init() {
+    node_id_ = 0;
+    query_pos_ = 0;
+    status_code_ = MARISA_READY_TO_ALL;
+  }
+  void reverse_lookup_init() {
+    key_buf_.resize(0);
+    key_buf_.reserve(32);
+    status_code_ = MARISA_READY_TO_ALL;
+  }
+  void common_prefix_search_init() {
+    node_id_ = 0;
+    query_pos_ = 0;
+    status_code_ = MARISA_READY_TO_COMMON_PREFIX_SEARCH;
+  }
+  void predictive_search_init() {
+    key_buf_.resize(0);
+    key_buf_.reserve(64);
+    history_.resize(0);
+    history_.reserve(4);
+    node_id_ = 0;
+    query_pos_ = 0;
+    history_pos_ = 0;
+    status_code_ = MARISA_READY_TO_PREDICTIVE_SEARCH;
+  }
+
+ private:
+  Vector<char> key_buf_;
+  Vector<History> history_;
+  UInt32 node_id_;
+  UInt32 query_pos_;
+  UInt32 history_pos_;
+  StatusCode status_code_;
+
+  // Disallows copy and assignment.
+  State(const State &);
+  State &operator=(const State &);
+};
+
+}  // namespace trie
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_TRIE_STATE_H_

data/ext/marisa/lib/marisa/grimoire/trie/tail.cc

@@ -0,0 +1,218 @@
+#include "marisa/grimoire/algorithm.h"
+#include "marisa/grimoire/trie/state.h"
+#include "marisa/grimoire/trie/tail.h"
+
+namespace marisa {
+namespace grimoire {
+namespace trie {
+
+Tail::Tail() : buf_(), end_flags_() {}
+
+void Tail::build(Vector<Entry> &entries, Vector<UInt32> *offsets,
+    TailMode mode) {
+  MARISA_THROW_IF(offsets == NULL, MARISA_NULL_ERROR);
+
+  switch (mode) {
+    case MARISA_TEXT_TAIL: {
+      for (std::size_t i = 0; i < entries.size(); ++i) {
+        const char * const ptr = entries[i].ptr();
+        const std::size_t length = entries[i].length();
+        for (std::size_t j = 0; j < length; ++j) {
+          if (ptr[j] == '\0') {
+            mode = MARISA_BINARY_TAIL;
+            break;
+          }
+        }
+        if (mode == MARISA_BINARY_TAIL) {
+          break;
+        }
+      }
+      break;
+    }
+    case MARISA_BINARY_TAIL: {
+      break;
+    }
+    default: {
+      MARISA_THROW(MARISA_CODE_ERROR, "undefined tail mode");
+    }
+  }
+
+  Tail temp;
+  temp.build_(entries, offsets, mode);
+  swap(temp);
+}
+
+void Tail::map(Mapper &mapper) {
+  Tail temp;
+  temp.map_(mapper);
+  swap(temp);
+}
+
+void Tail::read(Reader &reader) {
+  Tail temp;
+  temp.read_(reader);
+  swap(temp);
+}
+
+void Tail::write(Writer &writer) const {
+  write_(writer);
+}
+
+void Tail::restore(Agent &agent, std::size_t offset) const {
+  MARISA_DEBUG_IF(buf_.empty(), MARISA_STATE_ERROR);
+
+  State &state = agent.state();
+  if (end_flags_.empty()) {
+    for (const char *ptr = &buf_[offset]; *ptr != '\0'; ++ptr) {
+      state.key_buf().push_back(*ptr);
+    }
+  } else {
+    do {
+      state.key_buf().push_back(buf_[offset]);
+    } while (!end_flags_[offset++]);
+  }
+}
+
+bool Tail::match(Agent &agent, std::size_t offset) const {
+  MARISA_DEBUG_IF(buf_.empty(), MARISA_STATE_ERROR);
+  MARISA_DEBUG_IF(agent.state().query_pos() >= agent.query().length(),
+      MARISA_BOUND_ERROR);
+
+  State &state = agent.state();
+  if (end_flags_.empty()) {
+    const char * const ptr = &buf_[offset] - state.query_pos();
+    do {
+      if (ptr[state.query_pos()] != agent.query()[state.query_pos()]) {
+        return false;
+      }
+      state.set_query_pos(state.query_pos() + 1);
+      if (ptr[state.query_pos()] == '\0') {
+        return true;
+      }
+    } while (state.query_pos() < agent.query().length());
+    return false;
+  } else {
+    do {
+      if (buf_[offset] != agent.query()[state.query_pos()]) {
+        return false;
+      }
+      state.set_query_pos(state.query_pos() + 1);
+      if (end_flags_[offset++]) {
+        return true;
+      }
+    } while (state.query_pos() < agent.query().length());
+    return false;
+  }
+}
+
+bool Tail::prefix_match(Agent &agent, std::size_t offset) const {
+  MARISA_DEBUG_IF(buf_.empty(), MARISA_STATE_ERROR);
+
+  State &state = agent.state();
+  if (end_flags_.empty()) {
+    const char *ptr = &buf_[offset] - state.query_pos();
+    do {
+      if (ptr[state.query_pos()] != agent.query()[state.query_pos()]) {
+        return false;
+      }
+      state.key_buf().push_back(ptr[state.query_pos()]);
+      state.set_query_pos(state.query_pos() + 1);
+      if (ptr[state.query_pos()] == '\0') {
+        return true;
+      }
+    } while (state.query_pos() < agent.query().length());
+    ptr += state.query_pos();
+    do {
+      state.key_buf().push_back(*ptr);
+    } while (*++ptr != '\0');
+    return true;
+  } else {
+    do {
+      if (buf_[offset] != agent.query()[state.query_pos()]) {
+        return false;
+      }
+      state.key_buf().push_back(buf_[offset]);
+      state.set_query_pos(state.query_pos() + 1);
+      if (end_flags_[offset++]) {
+        return true;
+      }
+    } while (state.query_pos() < agent.query().length());
+    do {
+      state.key_buf().push_back(buf_[offset]);
+    } while (!end_flags_[offset++]);
+    return true;
+  }
+}
+
+void Tail::clear() {
+  Tail().swap(*this);
+}
+
+void Tail::swap(Tail &rhs) {
+  buf_.swap(rhs.buf_);
+  end_flags_.swap(rhs.end_flags_);
+}
+
+void Tail::build_(Vector<Entry> &entries, Vector<UInt32> *offsets,
+    TailMode mode) {
+  for (std::size_t i = 0; i < entries.size(); ++i) {
+    entries[i].set_id(i);
+  }
+  Algorithm().sort(entries.begin(), entries.end());
+
+  Vector<UInt32> temp_offsets;
+  temp_offsets.resize(entries.size(), 0);
+
+  const Entry dummy;
+  const Entry *last = &dummy;
+  for (std::size_t i = entries.size(); i > 0; --i) {
+    const Entry &current = entries[i - 1];
+    MARISA_THROW_IF(current.length() == 0, MARISA_RANGE_ERROR);
+    std::size_t match = 0;
+    while ((match < current.length()) && (match < last->length()) &&
+        ((*last)[match] == current[match])) {
+      ++match;
+    }
+    if ((match == current.length()) && (last->length() != 0)) {
+      temp_offsets[current.id()] = (UInt32)(
+          temp_offsets[last->id()] + (last->length() - match));
+    } else {
+      temp_offsets[current.id()] = (UInt32)buf_.size();
+      for (std::size_t j = 1; j <= current.length(); ++j) {
+        buf_.push_back(current[current.length() - j]);
+      }
+      if (mode == MARISA_TEXT_TAIL) {
+        buf_.push_back('\0');
+      } else {
+        for (std::size_t j = 1; j < current.length(); ++j) {
+          end_flags_.push_back(false);
+        }
+        end_flags_.push_back(true);
+      }
+      MARISA_THROW_IF(buf_.size() > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    }
+    last = &current;
+  }
+  buf_.shrink();
+
+  offsets->swap(temp_offsets);
+}
+
+void Tail::map_(Mapper &mapper) {
+  buf_.map(mapper);
+  end_flags_.map(mapper);
+}
+
+void Tail::read_(Reader &reader) {
+  buf_.read(reader);
+  end_flags_.read(reader);
+}
+
+void Tail::write_(Writer &writer) const {
+  buf_.write(writer);
+  end_flags_.write(writer);
+}
+
+}  // namespace trie
+}  // namespace grimoire
+}  // namespace marisa