melisa 0.1.0

data/ext/marisa/lib/marisa/grimoire/vector/rank-index.h

@@ -0,0 +1,82 @@
+#ifndef MARISA_GRIMOIRE_VECTOR_RANK_INDEX_H_
+#define MARISA_GRIMOIRE_VECTOR_RANK_INDEX_H_
+
+#include "marisa/base.h"
+
+namespace marisa {
+namespace grimoire {
+namespace vector {
+
+class RankIndex {
+ public:
+  RankIndex() : abs_(0), rel_lo_(0), rel_hi_(0) {}
+
+  void set_abs(std::size_t value) {
+    MARISA_DEBUG_IF(value > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    abs_ = (UInt32)value;
+  }
+  void set_rel1(std::size_t value) {
+    MARISA_DEBUG_IF(value > 64, MARISA_RANGE_ERROR);
+    rel_lo_ = (UInt32)((rel_lo_ & ~0x7FU) | (value & 0x7FU));
+  }
+  void set_rel2(std::size_t value) {
+    MARISA_DEBUG_IF(value > 128, MARISA_RANGE_ERROR);
+    rel_lo_ = (UInt32)((rel_lo_ & ~(0xFFU << 7)) | ((value & 0xFFU) << 7));
+  }
+  void set_rel3(std::size_t value) {
+    MARISA_DEBUG_IF(value > 192, MARISA_RANGE_ERROR);
+    rel_lo_ = (UInt32)((rel_lo_ & ~(0xFFU << 15)) | ((value & 0xFFU) << 15));
+  }
+  void set_rel4(std::size_t value) {
+    MARISA_DEBUG_IF(value > 256, MARISA_RANGE_ERROR);
+    rel_lo_ = (UInt32)((rel_lo_ & ~(0x1FFU << 23)) | ((value & 0x1FFU) << 23));
+  }
+  void set_rel5(std::size_t value) {
+    MARISA_DEBUG_IF(value > 320, MARISA_RANGE_ERROR);
+    rel_hi_ = (UInt32)((rel_hi_ & ~0x1FFU) | (value & 0x1FFU));
+  }
+  void set_rel6(std::size_t value) {
+    MARISA_DEBUG_IF(value > 384, MARISA_RANGE_ERROR);
+    rel_hi_ = (UInt32)((rel_hi_ & ~(0x1FFU << 9)) | ((value & 0x1FFU) << 9));
+  }
+  void set_rel7(std::size_t value) {
+    MARISA_DEBUG_IF(value > 448, MARISA_RANGE_ERROR);
+    rel_hi_ = (UInt32)((rel_hi_ & ~(0x1FFU << 18)) | ((value & 0x1FFU) << 18));
+  }
+
+  std::size_t abs() const {
+    return abs_;
+  }
+  std::size_t rel1() const {
+    return rel_lo_ & 0x7FU;
+  }
+  std::size_t rel2() const {
+    return (rel_lo_ >> 7) & 0xFFU;
+  }
+  std::size_t rel3() const {
+    return (rel_lo_ >> 15) & 0xFFU;
+  }
+  std::size_t rel4() const {
+    return (rel_lo_ >> 23) & 0x1FFU;
+  }
+  std::size_t rel5() const {
+    return rel_hi_ & 0x1FFU;
+  }
+  std::size_t rel6() const {
+    return (rel_hi_ >> 9) & 0x1FFU;
+  }
+  std::size_t rel7() const {
+    return (rel_hi_ >> 18) & 0x1FFU;
+  }
+
+ private:
+  UInt32 abs_;
+  UInt32 rel_lo_;
+  UInt32 rel_hi_;
+};
+
+}  // namespace vector
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_VECTOR_RANK_INDEX_H_

data/ext/marisa/lib/marisa/grimoire/vector/vector.h

@@ -0,0 +1,256 @@
+#ifndef MARISA_GRIMOIRE_VECTOR_VECTOR_H_
+#define MARISA_GRIMOIRE_VECTOR_VECTOR_H_
+
+#include <new>
+
+#include "marisa/grimoire/io.h"
+
+namespace marisa {
+namespace grimoire {
+namespace vector {
+
+template <typename T>
+class Vector {
+ public:
+  Vector()
+      : buf_(), objs_(NULL), const_objs_(NULL),
+        size_(0), capacity_(0), fixed_(false) {}
+  ~Vector() {
+    if (objs_ != NULL) {
+      for (std::size_t i = 0; i < size_; ++i) {
+        objs_[i].~T();
+      }
+    }
+  }
+
+  void map(Mapper &mapper) {
+    Vector temp;
+    temp.map_(mapper);
+    swap(temp);
+  }
+
+  void read(Reader &reader) {
+    Vector temp;
+    temp.read_(reader);
+    swap(temp);
+  }
+
+  void write(Writer &writer) const {
+    write_(writer);
+  }
+
+  void push_back(const T &x) {
+    MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR);
+    MARISA_DEBUG_IF(size_ == max_size(), MARISA_SIZE_ERROR);
+    reserve(size_ + 1);
+    new (&objs_[size_]) T(x);
+    ++size_;
+  }
+
+  void pop_back() {
+    MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR);
+    MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR);
+    objs_[--size_].~T();
+  }
+
+  // resize() assumes that T's placement new does not throw an exception.
+  void resize(std::size_t size) {
+    MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR);
+    reserve(size);
+    for (std::size_t i = size_; i < size; ++i) {
+      new (&objs_[i]) T;
+    }
+    for (std::size_t i = size; i < size_; ++i) {
+      objs_[i].~T();
+    }
+    size_ = size;
+  }
+
+  // resize() assumes that T's placement new does not throw an exception.
+  void resize(std::size_t size, const T &x) {
+    MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR);
+    reserve(size);
+    for (std::size_t i = size_; i < size; ++i) {
+      new (&objs_[i]) T(x);
+    }
+    for (std::size_t i = size; i < size_; ++i) {
+      objs_[i].~T();
+    }
+    size_ = size;
+  }
+
+  void reserve(std::size_t capacity) {
+    MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR);
+    if (capacity <= capacity_) {
+      return;
+    }
+    MARISA_DEBUG_IF(capacity > max_size(), MARISA_SIZE_ERROR);
+    std::size_t new_capacity = capacity;
+    if (capacity_ > (capacity / 2)) {
+      if (capacity_ > (max_size() / 2)) {
+        new_capacity = max_size();
+      } else {
+        new_capacity = capacity_ * 2;
+      }
+    }
+    realloc(new_capacity);
+  }
+
+  void shrink() {
+    MARISA_THROW_IF(fixed_, MARISA_STATE_ERROR);
+    if (size_ != capacity_) {
+      realloc(size_);
+    }
+  }
+
+  void fix() {
+    MARISA_THROW_IF(fixed_, MARISA_STATE_ERROR);
+    fixed_ = true;
+  }
+
+  const T *begin() const {
+    return const_objs_;
+  }
+  const T *end() const {
+    return const_objs_ + size_;
+  }
+  const T &operator[](std::size_t i) const {
+    MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR);
+    return const_objs_[i];
+  }
+  const T &front() const {
+    MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR);
+    return const_objs_[0];
+  }
+  const T &back() const {
+    MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR);
+    return const_objs_[size_ - 1];
+  }
+
+  T *begin() {
+    MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR);
+    return objs_;
+  }
+  T *end() {
+    MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR);
+    return objs_ + size_;
+  }
+  T &operator[](std::size_t i) {
+    MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR);
+    MARISA_DEBUG_IF(i >= size_, MARISA_BOUND_ERROR);
+    return objs_[i];
+  }
+  T &front() {
+    MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR);
+    MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR);
+    return objs_[0];
+  }
+  T &back() {
+    MARISA_DEBUG_IF(fixed_, MARISA_STATE_ERROR);
+    MARISA_DEBUG_IF(size_ == 0, MARISA_STATE_ERROR);
+    return objs_[size_ - 1];
+  }
+
+  std::size_t size() const {
+    return size_;
+  }
+  std::size_t capacity() const {
+    return capacity_;
+  }
+  bool fixed() const {
+    return fixed_;
+  }
+
+  bool empty() const {
+    return size_ == 0;
+  }
+  std::size_t total_size() const {
+    return sizeof(T) * size_;
+  }
+  std::size_t io_size() const {
+    return sizeof(UInt64) + ((total_size() + 7) & ~(std::size_t)0x07);
+  }
+
+  void clear() {
+    Vector().swap(*this);
+  }
+  void swap(Vector &rhs) {
+    buf_.swap(rhs.buf_);
+    marisa::swap(objs_, rhs.objs_);
+    marisa::swap(const_objs_, rhs.const_objs_);
+    marisa::swap(size_, rhs.size_);
+    marisa::swap(capacity_, rhs.capacity_);
+    marisa::swap(fixed_, rhs.fixed_);
+  }
+
+  static std::size_t max_size() {
+    return MARISA_SIZE_MAX / sizeof(T);
+  }
+
+ private:
+  scoped_array<char> buf_;
+  T *objs_;
+  const T *const_objs_;
+  std::size_t size_;
+  std::size_t capacity_;
+  bool fixed_;
+
+  void map_(Mapper &mapper) {
+    UInt64 total_size;
+    mapper.map(&total_size);
+    MARISA_THROW_IF(total_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR);
+    MARISA_THROW_IF((total_size % sizeof(T)) != 0, MARISA_FORMAT_ERROR);
+    const std::size_t size = (std::size_t)(total_size / sizeof(T));
+    mapper.map(&const_objs_, size);
+    mapper.seek((std::size_t)((8 - (total_size % 8)) % 8));
+    size_ = size;
+    fix();
+  }
+  void read_(Reader &reader) {
+    UInt64 total_size;
+    reader.read(&total_size);
+    MARISA_THROW_IF(total_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR);
+    MARISA_THROW_IF((total_size % sizeof(T)) != 0, MARISA_FORMAT_ERROR);
+    const std::size_t size = (std::size_t)(total_size / sizeof(T));
+    resize(size);
+    reader.read(objs_, size);
+    reader.seek((std::size_t)((8 - (total_size % 8)) % 8));
+  }
+  void write_(Writer &writer) const {
+    writer.write((UInt64)total_size());
+    writer.write(const_objs_, size_);
+    writer.seek((8 - (total_size() % 8)) % 8);
+  }
+
+  // realloc() assumes that T's placement new does not throw an exception.
+  void realloc(std::size_t new_capacity) {
+    MARISA_DEBUG_IF(new_capacity > max_size(), MARISA_SIZE_ERROR);
+
+    scoped_array<char> new_buf(
+        new (std::nothrow) char[sizeof(T) * new_capacity]);
+    MARISA_DEBUG_IF(new_buf.get() == NULL, MARISA_MEMORY_ERROR);
+    T *new_objs = reinterpret_cast<T *>(new_buf.get());
+
+    for (std::size_t i = 0; i < size_; ++i) {
+      new (&new_objs[i]) T(objs_[i]);
+    }
+    for (std::size_t i = 0; i < size_; ++i) {
+      objs_[i].~T();
+    }
+
+    buf_.swap(new_buf);
+    objs_ = new_objs;
+    const_objs_ = new_objs;
+    capacity_ = new_capacity;
+  }
+
+  // Disallows copy and assignment.
+  Vector(const Vector &);
+  Vector &operator=(const Vector &);
+};
+
+}  // namespace vector
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_VECTOR_VECTOR_H_

data/ext/marisa/lib/marisa/iostream.h

@@ -0,0 +1,18 @@
+#ifndef MARISA_IOSTREAM_H_
+#define MARISA_IOSTREAM_H_
+
+#include <iosfwd>
+
+namespace marisa {
+
+class Trie;
+
+std::istream &read(std::istream &stream, Trie *trie);
+std::ostream &write(std::ostream &stream, const Trie &trie);
+
+std::istream &operator>>(std::istream &stream, Trie &trie);
+std::ostream &operator<<(std::ostream &stream, const Trie &trie);
+
+}  // namespace marisa
+
+#endif  // MARISA_IOSTREAM_H_

data/ext/marisa/lib/marisa/key.h

@@ -0,0 +1,85 @@
+#ifndef MARISA_KEY_H_
+#define MARISA_KEY_H_
+
+#include "marisa/base.h"
+
+namespace marisa {
+
+class Key {
+ public:
+  Key() : ptr_(NULL), length_(0), union_() {
+    union_.id = 0;
+  }
+  Key(const Key &key)
+      : ptr_(key.ptr_), length_(key.length_), union_(key.union_) {}
+
+  Key &operator=(const Key &key) {
+    ptr_ = key.ptr_;
+    length_ = key.length_;
+    union_ = key.union_;
+    return *this;
+  }
+
+  char operator[](std::size_t i) const {
+    MARISA_DEBUG_IF(i >= length_, MARISA_BOUND_ERROR);
+    return ptr_[i];
+  }
+
+  void set_str(const char *str) {
+    MARISA_DEBUG_IF(str == NULL, MARISA_NULL_ERROR);
+    std::size_t length = 0;
+    while (str[length] != '\0') {
+      ++length;
+    }
+    MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    ptr_ = str;
+    length_ = (UInt32)length;
+  }
+  void set_str(const char *ptr, std::size_t length) {
+    MARISA_DEBUG_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR);
+    MARISA_DEBUG_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    ptr_ = ptr;
+    length_ = (UInt32)length;
+  }
+  void set_id(std::size_t id) {
+    MARISA_DEBUG_IF(id > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    union_.id = (UInt32)id;
+  }
+  void set_weight(float weight) {
+    union_.weight = weight;
+  }
+
+  const char *ptr() const {
+    return ptr_;
+  }
+  std::size_t length() const {
+    return length_;
+  }
+  std::size_t id() const {
+    return union_.id;
+  }
+  float weight() const {
+    return union_.weight;
+  }
+
+  void clear() {
+    Key().swap(*this);
+  }
+  void swap(Key &rhs) {
+    marisa::swap(ptr_, rhs.ptr_);
+    marisa::swap(length_, rhs.length_);
+    marisa::swap(union_.id, rhs.union_.id);
+  }
+
+ private:
+  const char *ptr_;
+  UInt32 length_;
+  union Union {
+    UInt32 id;
+    float weight;
+  } union_;
+};
+
+}  // namespace marisa
+
+#endif  // MARISA_KEY_H_

data/ext/marisa/lib/marisa/keyset.cc

@@ -0,0 +1,181 @@
+#include <new>
+
+#include "marisa/keyset.h"
+
+namespace marisa {
+
+Keyset::Keyset()
+    : base_blocks_(), base_blocks_size_(0), base_blocks_capacity_(0),
+      extra_blocks_(), extra_blocks_size_(0), extra_blocks_capacity_(0),
+      key_blocks_(), key_blocks_size_(0), key_blocks_capacity_(0),
+      ptr_(NULL), avail_(0), size_(0), total_length_(0) {}
+
+void Keyset::push_back(const Key &key) {
+  MARISA_DEBUG_IF(size_ == MARISA_SIZE_MAX, MARISA_SIZE_ERROR);
+
+  char * const key_ptr = reserve(key.length());
+  for (std::size_t i = 0; i < key.length(); ++i) {
+    key_ptr[i] = key[i];
+  }
+
+  Key &new_key = key_blocks_[size_ / KEY_BLOCK_SIZE][size_ % KEY_BLOCK_SIZE];
+  new_key.set_str(key_ptr, key.length());
+  new_key.set_id(key.id());
+  ++size_;
+  total_length_ += new_key.length();
+}
+
+void Keyset::push_back(const Key &key, char end_marker) {
+  MARISA_DEBUG_IF(size_ == MARISA_SIZE_MAX, MARISA_SIZE_ERROR);
+
+  if ((size_ / KEY_BLOCK_SIZE) == key_blocks_size_) {
+    append_key_block();
+  }
+
+  char * const key_ptr = reserve(key.length() + 1);
+  for (std::size_t i = 0; i < key.length(); ++i) {
+    key_ptr[i] = key[i];
+  }
+  key_ptr[key.length()] = end_marker;
+
+  Key &new_key = key_blocks_[size_ / KEY_BLOCK_SIZE][size_ % KEY_BLOCK_SIZE];
+  new_key.set_str(key_ptr, key.length());
+  new_key.set_id(key.id());
+  ++size_;
+  total_length_ += new_key.length();
+}
+
+void Keyset::push_back(const char *str) {
+  MARISA_DEBUG_IF(size_ == MARISA_SIZE_MAX, MARISA_SIZE_ERROR);
+  MARISA_THROW_IF(str == NULL, MARISA_NULL_ERROR);
+
+  std::size_t length = 0;
+  while (str[length] != '\0') {
+    ++length;
+  }
+  push_back(str, length);
+}
+
+void Keyset::push_back(const char *ptr, std::size_t length, float weight) {
+  MARISA_DEBUG_IF(size_ == MARISA_SIZE_MAX, MARISA_SIZE_ERROR);
+  MARISA_THROW_IF((ptr == NULL) && (length != 0), MARISA_NULL_ERROR);
+  MARISA_THROW_IF(length > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+
+  char * const key_ptr = reserve(length);
+  for (std::size_t i = 0; i < length; ++i) {
+    key_ptr[i] = ptr[i];
+  }
+
+  Key &key = key_blocks_[size_ / KEY_BLOCK_SIZE][size_ % KEY_BLOCK_SIZE];
+  key.set_str(key_ptr, length);
+  key.set_weight(weight);
+  ++size_;
+  total_length_ += length;
+}
+
+void Keyset::reset() {
+  base_blocks_size_ = 0;
+  extra_blocks_size_ = 0;
+  ptr_ = NULL;
+  avail_ = 0;
+  size_ = 0;
+  total_length_ = 0;
+}
+
+void Keyset::clear() {
+  Keyset().swap(*this);
+}
+
+void Keyset::swap(Keyset &rhs) {
+  base_blocks_.swap(rhs.base_blocks_);
+  marisa::swap(base_blocks_size_, rhs.base_blocks_size_);
+  marisa::swap(base_blocks_capacity_, rhs.base_blocks_capacity_);
+  extra_blocks_.swap(rhs.extra_blocks_);
+  marisa::swap(extra_blocks_size_, rhs.extra_blocks_size_);
+  marisa::swap(extra_blocks_capacity_, rhs.extra_blocks_capacity_);
+  key_blocks_.swap(rhs.key_blocks_);
+  marisa::swap(key_blocks_size_, rhs.key_blocks_size_);
+  marisa::swap(key_blocks_capacity_, rhs.key_blocks_capacity_);
+  marisa::swap(ptr_, rhs.ptr_);
+  marisa::swap(avail_, rhs.avail_);
+  marisa::swap(size_, rhs.size_);
+  marisa::swap(total_length_, rhs.total_length_);
+}
+
+char *Keyset::reserve(std::size_t size) {
+  if ((size_ / KEY_BLOCK_SIZE) == key_blocks_size_) {
+    append_key_block();
+  }
+
+  if (size > EXTRA_BLOCK_SIZE) {
+    append_extra_block(size);
+    return extra_blocks_[extra_blocks_size_ - 1].get();
+  } else {
+    if (size > avail_) {
+      append_base_block();
+    }
+    ptr_ += size;
+    avail_ -= size;
+    return ptr_ - size;
+  }
+}
+
+void Keyset::append_base_block() {
+  if (base_blocks_size_ == base_blocks_capacity_) {
+    const std::size_t new_capacity =
+        (base_blocks_size_ != 0) ? (base_blocks_size_ * 2) : 1;
+    scoped_array<scoped_array<char> > new_blocks(
+        new (std::nothrow) scoped_array<char>[new_capacity]);
+    MARISA_THROW_IF(new_blocks.get() == NULL, MARISA_MEMORY_ERROR);
+    for (std::size_t i = 0; i < base_blocks_size_; ++i) {
+      base_blocks_[i].swap(new_blocks[i]);
+    }
+    base_blocks_.swap(new_blocks);
+    base_blocks_capacity_ = new_capacity;
+  }
+  if (base_blocks_[base_blocks_size_].get() == NULL) {
+    scoped_array<char> new_block(new (std::nothrow) char[BASE_BLOCK_SIZE]);
+    MARISA_THROW_IF(new_block.get() == NULL, MARISA_MEMORY_ERROR);
+    base_blocks_[base_blocks_size_].swap(new_block);
+  }
+  ptr_ = base_blocks_[base_blocks_size_++].get();
+  avail_ = BASE_BLOCK_SIZE;
+}
+
+void Keyset::append_extra_block(std::size_t size) {
+  if (extra_blocks_size_ == extra_blocks_capacity_) {
+    const std::size_t new_capacity =
+        (extra_blocks_size_ != 0) ? (extra_blocks_size_ * 2) : 1;
+    scoped_array<scoped_array<char> > new_blocks(
+        new (std::nothrow) scoped_array<char>[new_capacity]);
+    MARISA_THROW_IF(new_blocks.get() == NULL, MARISA_MEMORY_ERROR);
+    for (std::size_t i = 0; i < extra_blocks_size_; ++i) {
+      extra_blocks_[i].swap(new_blocks[i]);
+    }
+    extra_blocks_.swap(new_blocks);
+    extra_blocks_capacity_ = new_capacity;
+  }
+  scoped_array<char> new_block(new (std::nothrow) char[size]);
+  MARISA_THROW_IF(new_block.get() == NULL, MARISA_MEMORY_ERROR);
+  extra_blocks_[extra_blocks_size_++].swap(new_block);
+}
+
+void Keyset::append_key_block() {
+  if (key_blocks_size_ == key_blocks_capacity_) {
+    const std::size_t new_capacity =
+        (key_blocks_size_ != 0) ? (key_blocks_size_ * 2) : 1;
+    scoped_array<scoped_array<Key> > new_blocks(
+        new (std::nothrow) scoped_array<Key>[new_capacity]);
+    MARISA_THROW_IF(new_blocks.get() == NULL, MARISA_MEMORY_ERROR);
+    for (std::size_t i = 0; i < key_blocks_size_; ++i) {
+      key_blocks_[i].swap(new_blocks[i]);
+    }
+    key_blocks_.swap(new_blocks);
+    key_blocks_capacity_ = new_capacity;
+  }
+  scoped_array<Key> new_block(new (std::nothrow) Key[KEY_BLOCK_SIZE]);
+  MARISA_THROW_IF(new_block.get() == NULL, MARISA_MEMORY_ERROR);
+  key_blocks_[key_blocks_size_++].swap(new_block);
+}
+
+}  // namespace marisa