melisa 0.1.0

data/ext/marisa/lib/marisa.h

@@ -0,0 +1,14 @@
+#ifndef MARISA_H_
+#define MARISA_H_
+
+// "marisa/stdio.h" includes <cstdio> for I/O using std::FILE.
+#include "marisa/stdio.h"
+
+// "marisa/iostream.h" includes <iosfwd> for I/O using std::iostream.
+#include "marisa/iostream.h"
+
+// You can use <marisa/trie.h> instead of <marisa.h> if you don't need the
+// above I/O interfaces and don't want to include the above I/O headers.
+#include "marisa/trie.h"
+
+#endif  // MARISA_H_

data/ext/marisa/lib/marisa/agent.cc

@@ -0,0 +1,51 @@
+#include <new>
+
+#include "marisa/agent.h"
+#include "marisa/grimoire/trie.h"
+
+namespace marisa {
+
+Agent::Agent() : query_(), key_(), state_() {} 
+
+Agent::~Agent() {}
+
+void Agent::set_query(const char *str) {
+  MARISA_THROW_IF(str == NULL, MARISA_NULL_ERROR);
+  if (state_.get() != NULL) {
+    state_->reset();
+  }
+  query_.set_str(str);
+}
+
+void Agent::set_query(const char *ptr, std::size_t length) {
+  MARISA_THROW_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR);
+  if (state_.get() != NULL) {
+    state_->reset();
+  }
+  query_.set_str(ptr, length);
+}
+
+void Agent::set_query(std::size_t key_id) {
+  if (state_.get() != NULL) {
+    state_->reset();
+  }
+  query_.set_id(key_id);
+}
+
+void Agent::init_state() {
+  MARISA_THROW_IF(state_.get() != NULL, MARISA_STATE_ERROR);
+  state_.reset(new (std::nothrow) grimoire::State);
+  MARISA_THROW_IF(state_.get() == NULL, MARISA_MEMORY_ERROR);
+}
+
+void Agent::clear() {
+  Agent().swap(*this);
+}
+
+void Agent::swap(Agent &rhs) {
+  query_.swap(rhs.query_);
+  key_.swap(rhs.key_);
+  state_.swap(rhs.state_);
+}
+
+}  // namespace marisa

data/ext/marisa/lib/marisa/agent.h

@@ -0,0 +1,73 @@
+#ifndef MARISA_AGENT_H_
+#define MARISA_AGENT_H_
+
+#include "marisa/key.h"
+#include "marisa/query.h"
+
+namespace marisa {
+namespace grimoire {
+namespace trie {
+
+class State;
+
+}  // namespace trie
+}  // namespace grimoire
+
+class Agent {
+ public:
+  Agent();
+  ~Agent();
+
+  const Query &query() const {
+    return query_;
+  }
+  const Key &key() const {
+    return key_;
+  }
+
+  void set_query(const char *str);
+  void set_query(const char *ptr, std::size_t length);
+  void set_query(std::size_t key_id);
+
+  const grimoire::trie::State &state() const {
+    return *state_;
+  }
+  grimoire::trie::State &state() {
+    return *state_;
+  }
+
+  void set_key(const char *str) {
+    MARISA_DEBUG_IF(str == NULL, MARISA_NULL_ERROR);
+    key_.set_str(str);
+  }
+  void set_key(const char *ptr, std::size_t length) {
+    MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR);
+    MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    key_.set_str(ptr, length);
+  }
+  void set_key(std::size_t id) {
+    MARISA_DEBUG_IF(id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    key_.set_id(id);
+  }
+
+  bool has_state() const {
+    return state_.get() != NULL;
+  }
+  void init_state();
+
+  void clear();
+  void swap(Agent &rhs);
+
+ private:
+  Query query_;
+  Key key_;
+  scoped_ptr<grimoire::trie::State> state_;
+
+  // Disallows copy and assignment.
+  Agent(const Agent &);
+  Agent &operator=(const Agent &);
+};
+
+}  // namespace marisa
+
+#endif  // MARISA_AGENT_H_

data/ext/marisa/lib/marisa/base.h

@@ -0,0 +1,193 @@
+#ifndef MARISA_BASE_H_
+#define MARISA_BASE_H_
+
+// Old Visual C++ does not provide stdint.h.
+#ifndef _MSC_VER
+ #include <stdint.h>
+#endif  // _MSC_VER
+
+#ifdef __cplusplus
+ #include <cstddef>
+#else  // __cplusplus
+ #include <stddef.h>
+#endif  // __cplusplus
+
+#ifdef __cplusplus
+extern "C" {
+#endif  // __cplusplus
+
+#ifdef _MSC_VER
+typedef unsigned __int8  marisa_uint8;
+typedef unsigned __int16 marisa_uint16;
+typedef unsigned __int32 marisa_uint32;
+typedef unsigned __int64 marisa_uint64;
+#else  // _MSC_VER
+typedef uint8_t  marisa_uint8;
+typedef uint16_t marisa_uint16;
+typedef uint32_t marisa_uint32;
+typedef uint64_t marisa_uint64;
+#endif  // _MSC_VER
+
+#if defined(_WIN64) || defined(__amd64__) || defined(__x86_64__) || \
+    defined(__ia64__) || defined(__ppc64__) || defined(__powerpc64__) || \
+    defined(__sparc64__) || defined(__mips64__) || defined(__aarch64__)
+ #define MARISA_WORD_SIZE 64
+#else  // defined(_WIN64), etc.
+ #define MARISA_WORD_SIZE 32
+#endif  // defined(_WIN64), etc.
+
+//#define MARISA_WORD_SIZE  (sizeof(void *) * 8)
+
+#define MARISA_UINT8_MAX  ((marisa_uint8)~(marisa_uint8)0)
+#define MARISA_UINT16_MAX ((marisa_uint16)~(marisa_uint16)0)
+#define MARISA_UINT32_MAX ((marisa_uint32)~(marisa_uint32)0)
+#define MARISA_UINT64_MAX ((marisa_uint64)~(marisa_uint64)0)
+#define MARISA_SIZE_MAX   ((size_t)~(size_t)0)
+
+#define MARISA_INVALID_LINK_ID MARISA_UINT32_MAX
+#define MARISA_INVALID_KEY_ID  MARISA_UINT32_MAX
+#define MARISA_INVALID_EXTRA   (MARISA_UINT32_MAX >> 8)
+
+// Error codes are defined as members of marisa_error_code. This library throws
+// an exception with one of the error codes when an error occurs.
+typedef enum marisa_error_code_ {
+  // MARISA_OK means that a requested operation has succeeded. In practice, an
+  // exception never has MARISA_OK because it is not an error.
+  MARISA_OK           = 0,
+
+  // MARISA_STATE_ERROR means that an object was not ready for a requested
+  // operation. For example, an operation to modify a fixed vector throws an
+  // exception with MARISA_STATE_ERROR.
+  MARISA_STATE_ERROR  = 1,
+
+  // MARISA_NULL_ERROR means that an invalid NULL pointer has been given.
+  MARISA_NULL_ERROR   = 2,
+
+  // MARISA_BOUND_ERROR means that an operation has tried to access an out of
+  // range address.
+  MARISA_BOUND_ERROR  = 3,
+
+  // MARISA_RANGE_ERROR means that an out of range value has appeared in
+  // operation.
+  MARISA_RANGE_ERROR  = 4,
+
+  // MARISA_CODE_ERROR means that an undefined code has appeared in operation.
+  MARISA_CODE_ERROR   = 5,
+
+  // MARISA_RESET_ERROR means that a smart pointer has tried to reset itself.
+  MARISA_RESET_ERROR  = 6,
+
+  // MARISA_SIZE_ERROR means that a size has exceeded a library limitation.
+  MARISA_SIZE_ERROR   = 7,
+
+  // MARISA_MEMORY_ERROR means that a memory allocation has failed.
+  MARISA_MEMORY_ERROR = 8,
+
+  // MARISA_IO_ERROR means that an I/O operation has failed.
+  MARISA_IO_ERROR     = 9,
+
+  // MARISA_FORMAT_ERROR means that input was in invalid format.
+  MARISA_FORMAT_ERROR = 10,
+} marisa_error_code;
+
+// Min/max values, flags and masks for dictionary settings are defined below.
+// Please note that unspecified settings will be replaced with the default
+// settings. For example, 0 is equivalent to (MARISA_DEFAULT_NUM_TRIES |
+// MARISA_DEFAULT_TRIE | MARISA_DEFAULT_TAIL | MARISA_DEFAULT_ORDER).
+
+// A dictionary consists of 3 tries in default. Usually more tries make a
+// dictionary space-efficient but time-inefficient.
+typedef enum marisa_num_tries_ {
+  MARISA_MIN_NUM_TRIES     = 0x00001,
+  MARISA_MAX_NUM_TRIES     = 0x0007F,
+  MARISA_DEFAULT_NUM_TRIES = 0x00003,
+} marisa_num_tries;
+
+// This library uses a cache technique to accelerate search functions. The
+// following enumerated type marisa_cache_level gives a list of available cache
+// size options. A larger cache enables faster search but takes a more space.
+typedef enum marisa_cache_level_ {
+  MARISA_HUGE_CACHE        = 0x00080,
+  MARISA_LARGE_CACHE       = 0x00100,
+  MARISA_NORMAL_CACHE      = 0x00200,
+  MARISA_SMALL_CACHE       = 0x00400,
+  MARISA_TINY_CACHE        = 0x00800,
+  MARISA_DEFAULT_CACHE     = MARISA_NORMAL_CACHE
+} marisa_cache_level;
+
+// This library provides 2 kinds of TAIL implementations.
+typedef enum marisa_tail_mode_ {
+  // MARISA_TEXT_TAIL merges last labels as zero-terminated strings. So, it is
+  // available if and only if the last labels do not contain a NULL character.
+  // If MARISA_TEXT_TAIL is specified and a NULL character exists in the last
+  // labels, the setting is automatically switched to MARISA_BINARY_TAIL.
+  MARISA_TEXT_TAIL         = 0x01000,
+
+  // MARISA_BINARY_TAIL also merges last labels but as byte sequences. It uses
+  // a bit vector to detect the end of a sequence, instead of NULL characters.
+  // So, MARISA_BINARY_TAIL requires a larger space if the average length of
+  // labels is greater than 8.
+  MARISA_BINARY_TAIL       = 0x02000,
+
+  MARISA_DEFAULT_TAIL      = MARISA_TEXT_TAIL,
+} marisa_tail_mode;
+
+// The arrangement of nodes affects the time cost of matching and the order of
+// predictive search.
+typedef enum marisa_node_order_ {
+  // MARISA_LABEL_ORDER arranges nodes in ascending label order.
+  // MARISA_LABEL_ORDER is useful if an application needs to predict keys in
+  // label order.
+  MARISA_LABEL_ORDER       = 0x10000,
+
+  // MARISA_WEIGHT_ORDER arranges nodes in descending weight order.
+  // MARISA_WEIGHT_ORDER is generally a better choice because it enables faster
+  // matching.
+  MARISA_WEIGHT_ORDER      = 0x20000,
+
+  MARISA_DEFAULT_ORDER     = MARISA_WEIGHT_ORDER,
+} marisa_node_order;
+
+typedef enum marisa_config_mask_ {
+  MARISA_NUM_TRIES_MASK    = 0x0007F,
+  MARISA_CACHE_LEVEL_MASK  = 0x00F80,
+  MARISA_TAIL_MODE_MASK    = 0x0F000,
+  MARISA_NODE_ORDER_MASK   = 0xF0000,
+  MARISA_CONFIG_MASK       = 0xFFFFF
+} marisa_config_mask;
+
+#ifdef __cplusplus
+}  // extern "C"
+#endif  // __cplusplus
+
+#ifdef __cplusplus
+namespace marisa {
+
+typedef ::marisa_uint8  UInt8;
+typedef ::marisa_uint16 UInt16;
+typedef ::marisa_uint32 UInt32;
+typedef ::marisa_uint64 UInt64;
+
+typedef ::marisa_error_code ErrorCode;
+
+typedef ::marisa_cache_level CacheLevel;
+typedef ::marisa_tail_mode TailMode;
+typedef ::marisa_node_order NodeOrder;
+
+template <typename T>
+inline void swap(T &lhs, T &rhs) {
+  T temp = lhs;
+  lhs = rhs;
+  rhs = temp;
+}
+
+}  // namespace marisa
+#endif  // __cplusplus
+
+#ifdef __cplusplus
+ #include "marisa/exception.h"
+ #include "marisa/scoped-ptr.h"
+ #include "marisa/scoped-array.h"
+#endif  // __cplusplus
+
+#endif  // MARISA_BASE_H_

data/ext/marisa/lib/marisa/exception.h

@@ -0,0 +1,82 @@
+#ifndef MARISA_EXCEPTION_H_
+#define MARISA_EXCEPTION_H_
+
+#include <exception>
+
+#include "marisa/base.h"
+
+namespace marisa {
+
+// An exception object keeps a filename, a line number, an error code and an
+// error message. The message format is as follows:
+//  "__FILE__:__LINE__: error_code: error_message"
+class Exception : public std::exception {
+ public:
+  Exception(const char *filename, int line,
+      ErrorCode error_code, const char *error_message)
+      : std::exception(), filename_(filename), line_(line),
+        error_code_(error_code), error_message_(error_message) {}
+  Exception(const Exception &ex)
+      : std::exception(), filename_(ex.filename_), line_(ex.line_),
+        error_code_(ex.error_code_), error_message_(ex.error_message_) {}
+  virtual ~Exception() throw() {}
+
+  Exception &operator=(const Exception &rhs) {
+    filename_ = rhs.filename_;
+    line_ = rhs.line_;
+    error_code_ = rhs.error_code_;
+    error_message_ = rhs.error_message_;
+    return *this;
+  }
+
+  const char *filename() const {
+    return filename_;
+  }
+  int line() const {
+    return line_;
+  }
+  ErrorCode error_code() const {
+    return error_code_;
+  }
+  const char *error_message() const {
+    return error_message_;
+  }
+
+  virtual const char *what() const throw() {
+    return error_message_;
+  }
+
+ private:
+  const char *filename_;
+  int line_;
+  ErrorCode error_code_;
+  const char *error_message_;
+};
+
+// These macros are used to convert a line number to a string constant.
+#define MARISA_INT_TO_STR(value) #value
+#define MARISA_LINE_TO_STR(line) MARISA_INT_TO_STR(line)
+#define MARISA_LINE_STR MARISA_LINE_TO_STR(__LINE__)
+
+// MARISA_THROW throws an exception with a filename, a line number, an error
+// code and an error message. The message format is as follows:
+//  "__FILE__:__LINE__: error_code: error_message"
+#define MARISA_THROW(error_code, error_message) \
+  (throw marisa::Exception(__FILE__, __LINE__, error_code, \
+       __FILE__ ":" MARISA_LINE_STR ": " #error_code ": " error_message))
+
+// MARISA_THROW_IF throws an exception if `condition' is true.
+#define MARISA_THROW_IF(condition, error_code) \
+  (void)((!(condition)) || (MARISA_THROW(error_code, #condition), 0))
+
+// MARISA_DEBUG_IF is ignored if _DEBUG is undefined. So, it is useful for
+// debugging time-critical codes.
+#ifdef _DEBUG
+ #define MARISA_DEBUG_IF(cond, error_code) MARISA_THROW_IF(cond, error_code)
+#else
+ #define MARISA_DEBUG_IF(cond, error_code)
+#endif
+
+}  // namespace marisa
+
+#endif  // MARISA_EXCEPTION_H_

data/ext/marisa/lib/marisa/grimoire/algorithm.h

@@ -0,0 +1,26 @@
+#ifndef MARISA_GRIMOIRE_ALGORITHM_H_
+#define MARISA_GRIMOIRE_ALGORITHM_H_
+
+#include "marisa/grimoire/algorithm/sort.h"
+
+namespace marisa {
+namespace grimoire {
+
+class Algorithm {
+ public:
+  Algorithm() {}
+
+  template <typename Iterator>
+  std::size_t sort(Iterator begin, Iterator end) const {
+    return algorithm::sort(begin, end);
+  }
+
+ private:
+  Algorithm(const Algorithm &);
+  Algorithm &operator=(const Algorithm &);
+};
+
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_ALGORITHM_H_

data/ext/marisa/lib/marisa/grimoire/algorithm/sort.h

@@ -0,0 +1,196 @@
+#ifndef MARISA_GRIMOIRE_ALGORITHM_SORT_H_
+#define MARISA_GRIMOIRE_ALGORITHM_SORT_H_
+
+#include "marisa/base.h"
+
+namespace marisa {
+namespace grimoire {
+namespace algorithm {
+namespace details {
+
+enum {
+  MARISA_INSERTION_SORT_THRESHOLD = 10
+};
+
+template <typename T>
+int get_label(const T &unit, std::size_t depth) {
+  MARISA_DEBUG_IF(depth > unit.length(), MARISA_BOUND_ERROR);
+
+  return (depth < unit.length()) ? (int)(UInt8)unit[depth] : -1;
+}
+
+template <typename T>
+int median(const T &a, const T &b, const T &c, std::size_t depth) {
+  const int x = get_label(a, depth);
+  const int y = get_label(b, depth);
+  const int z = get_label(c, depth);
+  if (x < y) {
+    if (y < z) {
+      return y;
+    } else if (x < z) {
+      return z;
+    }
+    return x;
+  } else if (x < z) {
+    return x;
+  } else if (y < z) {
+    return z;
+  }
+  return y;
+}
+
+template <typename T>
+int compare(const T &lhs, const T &rhs, std::size_t depth) {
+  for (std::size_t i = depth; i < lhs.length(); ++i) {
+    if (i == rhs.length()) {
+      return 1;
+    }
+    if (lhs[i] != rhs[i]) {
+      return (UInt8)lhs[i] - (UInt8)rhs[i];
+    }
+  }
+  if (lhs.length() == rhs.length()) {
+    return 0;
+  }
+  return (lhs.length() < rhs.length()) ? -1 : 1;
+}
+
+template <typename Iterator>
+std::size_t insertion_sort(Iterator l, Iterator r, std::size_t depth) {
+  MARISA_DEBUG_IF(l > r, MARISA_BOUND_ERROR);
+
+  std::size_t count = 1;
+  for (Iterator i = l + 1; i < r; ++i) {
+    int result = 0;
+    for (Iterator j = i; j > l; --j) {
+      result = compare(*(j - 1), *j, depth);
+      if (result <= 0) {
+        break;
+      }
+      marisa::swap(*(j - 1), *j);
+    }
+    if (result != 0) {
+      ++count;
+    }
+  }
+  return count;
+}
+
+template <typename Iterator>
+std::size_t sort(Iterator l, Iterator r, std::size_t depth) {
+  MARISA_DEBUG_IF(l > r, MARISA_BOUND_ERROR);
+
+  std::size_t count = 0;
+  while ((r - l) > MARISA_INSERTION_SORT_THRESHOLD) {
+    Iterator pl = l;
+    Iterator pr = r;
+    Iterator pivot_l = l;
+    Iterator pivot_r = r;
+
+    const int pivot = median(*l, *(l + (r - l) / 2), *(r - 1), depth);
+    for ( ; ; ) {
+      while (pl < pr) {
+        const int label = get_label(*pl, depth);
+        if (label > pivot) {
+          break;
+        } else if (label == pivot) {
+          marisa::swap(*pl, *pivot_l);
+          ++pivot_l;
+        }
+        ++pl;
+      }
+      while (pl < pr) {
+        const int label = get_label(*--pr, depth);
+        if (label < pivot) {
+          break;
+        } else if (label == pivot) {
+          marisa::swap(*pr, *--pivot_r);
+        }
+      }
+      if (pl >= pr) {
+        break;
+      }
+      marisa::swap(*pl, *pr);
+      ++pl;
+    }
+    while (pivot_l > l) {
+      marisa::swap(*--pivot_l, *--pl);
+    }
+    while (pivot_r < r) {
+      marisa::swap(*pivot_r, *pr);
+      ++pivot_r;
+      ++pr;
+    }
+
+    if (((pl - l) > (pr - pl)) || ((r - pr) > (pr - pl))) {
+      if ((pr - pl) == 1) {
+        ++count;
+      } else if ((pr - pl) > 1) {
+        if (pivot == -1) {
+          ++count;
+        } else {
+          count += sort(pl, pr, depth + 1);
+        }
+      }
+
+      if ((pl - l) < (r - pr)) {
+        if ((pl - l) == 1) {
+          ++count;
+        } else if ((pl - l) > 1) {
+          count += sort(l, pl, depth);
+        }
+        l = pr;
+      } else {
+        if ((r - pr) == 1) {
+          ++count;
+        } else if ((r - pr) > 1) {
+          count += sort(pr, r, depth);
+        }
+        r = pl;
+      }
+    } else {
+      if ((pl - l) == 1) {
+        ++count;
+      } else if ((pl - l) > 1) {
+        count += sort(l, pl, depth);
+      }
+
+      if ((r - pr) == 1) {
+        ++count;
+      } else if ((r - pr) > 1) {
+        count += sort(pr, r, depth);
+      }
+
+      l = pl, r = pr;
+      if ((pr - pl) == 1) {
+        ++count;
+      } else if ((pr - pl) > 1) {
+        if (pivot == -1) {
+          l = r;
+          ++count;
+        } else {
+          ++depth;
+        }
+      }
+    }
+  }
+
+  if ((r - l) > 1) {
+    count += insertion_sort(l, r, depth);
+  }
+  return count;
+}
+
+}  // namespace details
+
+template <typename Iterator>
+std::size_t sort(Iterator begin, Iterator end) {
+  MARISA_DEBUG_IF(begin > end, MARISA_BOUND_ERROR);
+  return details::sort(begin, end, 0);
+};
+
+}  // namespace algorithm
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_ALGORITHM_SORT_H_