melisa 0.1.0

data/ext/marisa/lib/marisa/grimoire/io/reader.h

@@ -0,0 +1,66 @@
+#ifndef MARISA_GRIMOIRE_IO_READER_H_
+#define MARISA_GRIMOIRE_IO_READER_H_
+
+#include <cstdio>
+#include <iostream>
+
+#include "marisa/base.h"
+
+namespace marisa {
+namespace grimoire {
+namespace io {
+
+class Reader {
+ public:
+  Reader();
+  ~Reader();
+
+  void open(const char *filename);
+  void open(std::FILE *file);
+  void open(int fd);
+  void open(std::istream &stream);
+
+  template <typename T>
+  void read(T *obj) {
+    MARISA_THROW_IF(obj == NULL, MARISA_NULL_ERROR);
+    read_data(obj, sizeof(T));
+  }
+
+  template <typename T>
+  void read(T *objs, std::size_t num_objs) {
+    MARISA_THROW_IF((objs == NULL) && (num_objs != 0), MARISA_NULL_ERROR);
+    MARISA_THROW_IF(num_objs > (MARISA_SIZE_MAX / sizeof(T)),
+        MARISA_SIZE_ERROR);
+    read_data(objs, sizeof(T) * num_objs);
+  }
+
+  void seek(std::size_t size);
+
+  bool is_open() const;
+
+  void clear();
+  void swap(Reader &rhs);
+
+ private:
+  std::FILE *file_;
+  int fd_;
+  std::istream *stream_;
+  bool needs_fclose_;
+
+  void open_(const char *filename);
+  void open_(std::FILE *file);
+  void open_(int fd);
+  void open_(std::istream &stream);
+
+  void read_data(void *buf, std::size_t size);
+
+  // Disallows copy and assignment.
+  Reader(const Reader &);
+  Reader &operator=(const Reader &);
+};
+
+}  // namespace io
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_IO_READER_H_

data/ext/marisa/lib/marisa/grimoire/io/writer.cc

@@ -0,0 +1,148 @@
+#include <stdio.h>
+
+#ifdef _WIN32
+ #include <io.h>
+#else  // _WIN32
+ #include <unistd.h>
+#endif  // _WIN32
+
+#include <limits>
+
+#include "marisa/grimoire/io/writer.h"
+
+namespace marisa {
+namespace grimoire {
+namespace io {
+
+Writer::Writer()
+    : file_(NULL), fd_(-1), stream_(NULL), needs_fclose_(false) {}
+
+Writer::~Writer() {
+  if (needs_fclose_) {
+    ::fclose(file_);
+  }
+}
+
+void Writer::open(const char *filename) {
+  MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR);
+
+  Writer temp;
+  temp.open_(filename);
+  swap(temp);
+}
+
+void Writer::open(std::FILE *file) {
+  MARISA_THROW_IF(file == NULL, MARISA_NULL_ERROR);
+
+  Writer temp;
+  temp.open_(file);
+  swap(temp);
+}
+
+void Writer::open(int fd) {
+  MARISA_THROW_IF(fd == -1, MARISA_CODE_ERROR);
+
+  Writer temp;
+  temp.open_(fd);
+  swap(temp);
+}
+
+void Writer::open(std::ostream &stream) {
+  Writer temp;
+  temp.open_(stream);
+  swap(temp);
+}
+
+void Writer::clear() {
+  Writer().swap(*this);
+}
+
+void Writer::swap(Writer &rhs) {
+  marisa::swap(file_, rhs.file_);
+  marisa::swap(fd_, rhs.fd_);
+  marisa::swap(stream_, rhs.stream_);
+  marisa::swap(needs_fclose_, rhs.needs_fclose_);
+}
+
+void Writer::seek(std::size_t size) {
+  MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR);
+  if (size == 0) {
+    return;
+  } else if (size <= 16) {
+    const char buf[16] = {};
+    write_data(buf, size);
+  } else {
+    const char buf[1024] = {};
+    do {
+      const std::size_t count = (size < sizeof(buf)) ? size : sizeof(buf);
+      write_data(buf, count);
+      size -= count;
+    } while (size != 0);
+  }
+}
+
+bool Writer::is_open() const {
+  return (file_ != NULL) || (fd_ != -1) || (stream_ != NULL);
+}
+
+void Writer::open_(const char *filename) {
+  std::FILE *file = NULL;
+#ifdef _MSC_VER
+  MARISA_THROW_IF(::fopen_s(&file, filename, "wb") != 0, MARISA_IO_ERROR);
+#else  // _MSC_VER
+  file = ::fopen(filename, "wb");
+  MARISA_THROW_IF(file == NULL, MARISA_IO_ERROR);
+#endif  // _MSC_VER
+  file_ = file;
+  needs_fclose_ = true;
+}
+
+void Writer::open_(std::FILE *file) {
+  file_ = file;
+}
+
+void Writer::open_(int fd) {
+  fd_ = fd;
+}
+
+void Writer::open_(std::ostream &stream) {
+  stream_ = &stream;
+}
+
+void Writer::write_data(const void *data, std::size_t size) {
+  MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR);
+  if (size == 0) {
+    return;
+  } else if (fd_ != -1) {
+    while (size != 0) {
+#ifdef _WIN32
+      static const std::size_t CHUNK_SIZE =
+          std::numeric_limits<int>::max();
+      const unsigned int count = (size < CHUNK_SIZE) ? size : CHUNK_SIZE;
+      const int size_written = ::_write(fd_, data, count);
+#else  // _WIN32
+      static const std::size_t CHUNK_SIZE =
+          std::numeric_limits< ::ssize_t>::max();
+      const ::size_t count = (size < CHUNK_SIZE) ? size : CHUNK_SIZE;
+      const ::ssize_t size_written = ::write(fd_, data, count);
+#endif  // _WIN32
+      MARISA_THROW_IF(size_written <= 0, MARISA_IO_ERROR);
+      data = static_cast<const char *>(data) + size_written;
+      size -= size_written;
+    }
+  } else if (file_ != NULL) {
+    MARISA_THROW_IF(::fwrite(data, 1, size, file_) != size, MARISA_IO_ERROR);
+    MARISA_THROW_IF(::fflush(file_) != 0, MARISA_IO_ERROR);
+  } else if (stream_ != NULL) {
+    try {
+      MARISA_THROW_IF(!stream_->write(static_cast<const char *>(data), size),
+          MARISA_IO_ERROR);
+    } catch (const std::ios_base::failure &) {
+      MARISA_THROW(MARISA_IO_ERROR, "std::ios_base::failure");
+    }
+  }
+}
+
+}  // namespace io
+}  // namespace grimoire
+}  // namespace marisa

data/ext/marisa/lib/marisa/grimoire/io/writer.h

@@ -0,0 +1,65 @@
+#ifndef MARISA_GRIMOIRE_IO_WRITER_H_
+#define MARISA_GRIMOIRE_IO_WRITER_H_
+
+#include <cstdio>
+#include <iostream>
+
+#include "marisa/base.h"
+
+namespace marisa {
+namespace grimoire {
+namespace io {
+
+class Writer {
+ public:
+  Writer();
+  ~Writer();
+
+  void open(const char *filename);
+  void open(std::FILE *file);
+  void open(int fd);
+  void open(std::ostream &stream);
+
+  template <typename T>
+  void write(const T &obj) {
+    write_data(&obj, sizeof(T));
+  }
+
+  template <typename T>
+  void write(const T *objs, std::size_t num_objs) {
+    MARISA_THROW_IF((objs == NULL) && (num_objs != 0), MARISA_NULL_ERROR);
+    MARISA_THROW_IF(num_objs > (MARISA_SIZE_MAX / sizeof(T)),
+                    MARISA_SIZE_ERROR);
+    write_data(objs, sizeof(T) * num_objs);
+  }
+
+  void seek(std::size_t size);
+
+  bool is_open() const;
+
+  void clear();
+  void swap(Writer &rhs);
+
+ private:
+  std::FILE *file_;
+  int fd_;
+  std::ostream *stream_;
+  bool needs_fclose_;
+
+  void open_(const char *filename);
+  void open_(std::FILE *file);
+  void open_(int fd);
+  void open_(std::ostream &stream);
+
+  void write_data(const void *data, std::size_t size);
+
+  // Disallows copy and assignment.
+  Writer(const Writer &);
+  Writer &operator=(const Writer &);
+};
+
+}  // namespace io
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_IO_WRITER_H_

data/ext/marisa/lib/marisa/grimoire/trie.h

@@ -0,0 +1,16 @@
+#ifndef MARISA_GRIMOIRE_TRIE_H_
+#define MARISA_GRIMOIRE_TRIE_H_
+
+#include "marisa/grimoire/trie/state.h"
+#include "marisa/grimoire/trie/louds-trie.h"
+
+namespace marisa {
+namespace grimoire {
+
+using trie::State;
+using trie::LoudsTrie;
+
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_TRIE_H_

data/ext/marisa/lib/marisa/grimoire/trie/cache.h

@@ -0,0 +1,81 @@
+#ifndef MARISA_GRIMOIRE_TRIE_CACHE_H_
+#define MARISA_GRIMOIRE_TRIE_CACHE_H_
+
+#include <cfloat>
+
+#include "marisa/base.h"
+
+namespace marisa {
+namespace grimoire {
+namespace trie {
+
+class Cache {
+ public:
+  Cache() : parent_(0), child_(0), union_() {
+    union_.weight = FLT_MIN;
+  }
+  Cache(const Cache &cache)
+      : parent_(cache.parent_), child_(cache.child_), union_(cache.union_) {}
+
+  Cache &operator=(const Cache &cache) {
+    parent_ = cache.parent_;
+    child_ = cache.child_;
+    union_ = cache.union_;
+    return *this;
+  }
+
+  void set_parent(std::size_t parent) {
+    MARISA_DEBUG_IF(parent > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    parent_ = (UInt32)parent;
+  }
+  void set_child(std::size_t child) {
+    MARISA_DEBUG_IF(child > MARISA_UINT32_MAX, MARISA_SIZE_ERROR);
+    child_ = (UInt32)child;
+  }
+  void set_base(UInt8 base) {
+    union_.link = (union_.link & ~0xFFU) | base;
+  }
+  void set_extra(std::size_t extra) {
+    MARISA_DEBUG_IF(extra > (MARISA_UINT32_MAX >> 8), MARISA_SIZE_ERROR);
+    union_.link = (UInt32)((union_.link & 0xFFU) | (extra << 8));
+  }
+  void set_weight(float weight) {
+    union_.weight = weight;
+  }
+
+  std::size_t parent() const {
+    return parent_;
+  }
+  std::size_t child() const {
+    return child_;
+  }
+  UInt8 base() const {
+    return (UInt8)(union_.link & 0xFFU);
+  }
+  std::size_t extra() const {
+    return union_.link >> 8;
+  }
+  char label() const {
+    return (char)base();
+  }
+  std::size_t link() const {
+    return union_.link;
+  }
+  float weight() const {
+    return union_.weight;
+  }
+
+ private:
+  UInt32 parent_;
+  UInt32 child_;
+  union Union {
+    UInt32 link;
+    float weight;
+  } union_;
+};
+
+}  // namespace trie
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_TRIE_CACHE_H_

data/ext/marisa/lib/marisa/grimoire/trie/config.h

@@ -0,0 +1,155 @@
+#ifndef MARISA_GRIMOIRE_TRIE_CONFIG_H_
+#define MARISA_GRIMOIRE_TRIE_CONFIG_H_
+
+#include "marisa/base.h"
+
+namespace marisa {
+namespace grimoire {
+namespace trie {
+
+class Config {
+ public:
+  Config()
+      : num_tries_(MARISA_DEFAULT_NUM_TRIES),
+        cache_level_(MARISA_DEFAULT_CACHE),
+        tail_mode_(MARISA_DEFAULT_TAIL),
+        node_order_(MARISA_DEFAULT_ORDER) {}
+
+  void parse(int config_flags) {
+    Config temp;
+    temp.parse_(config_flags);
+    swap(temp);
+  }
+
+  int flags() const {
+    return (int)num_tries_ | tail_mode_ | node_order_;
+  }
+
+  std::size_t num_tries() const {
+    return num_tries_;
+  }
+  CacheLevel cache_level() const {
+    return cache_level_;
+  }
+  TailMode tail_mode() const {
+    return tail_mode_;
+  }
+  NodeOrder node_order() const {
+    return node_order_;
+  }
+
+  void clear() {
+    Config().swap(*this);
+  }
+  void swap(Config &rhs) {
+    marisa::swap(num_tries_, rhs.num_tries_);
+    marisa::swap(cache_level_, rhs.cache_level_);
+    marisa::swap(tail_mode_, rhs.tail_mode_);
+    marisa::swap(node_order_, rhs.node_order_);
+  }
+
+ private:
+  std::size_t num_tries_;
+  CacheLevel cache_level_;
+  TailMode tail_mode_;
+  NodeOrder node_order_;
+
+  void parse_(int config_flags) {
+    MARISA_THROW_IF((config_flags & ~MARISA_CONFIG_MASK) != 0,
+        MARISA_CODE_ERROR);
+
+    parse_num_tries(config_flags);
+    parse_cache_level(config_flags);
+    parse_tail_mode(config_flags);
+    parse_node_order(config_flags);
+  }
+
+  void parse_num_tries(int config_flags) {
+    const int num_tries = config_flags & MARISA_NUM_TRIES_MASK;
+    if (num_tries != 0) {
+      num_tries_ = num_tries;
+    }
+  }
+
+  void parse_cache_level(int config_flags) {
+    switch (config_flags & MARISA_CACHE_LEVEL_MASK) {
+      case 0: {
+        cache_level_ = MARISA_DEFAULT_CACHE;
+        break;
+      }
+      case MARISA_HUGE_CACHE: {
+        cache_level_ = MARISA_HUGE_CACHE;
+        break;
+      }
+      case MARISA_LARGE_CACHE: {
+        cache_level_ = MARISA_LARGE_CACHE;
+        break;
+      }
+      case MARISA_NORMAL_CACHE: {
+        cache_level_ = MARISA_NORMAL_CACHE;
+        break;
+      }
+      case MARISA_SMALL_CACHE: {
+        cache_level_ = MARISA_SMALL_CACHE;
+        break;
+      }
+      case MARISA_TINY_CACHE: {
+        cache_level_ = MARISA_TINY_CACHE;
+        break;
+      }
+      default: {
+        MARISA_THROW(MARISA_CODE_ERROR, "undefined cache level");
+      }
+    }
+  }
+
+  void parse_tail_mode(int config_flags) {
+    switch (config_flags & MARISA_TAIL_MODE_MASK) {
+      case 0: {
+        tail_mode_ = MARISA_DEFAULT_TAIL;
+        break;
+      }
+      case MARISA_TEXT_TAIL: {
+        tail_mode_ = MARISA_TEXT_TAIL;
+        break;
+      }
+      case MARISA_BINARY_TAIL: {
+        tail_mode_ = MARISA_BINARY_TAIL;
+        break;
+      }
+      default: {
+        MARISA_THROW(MARISA_CODE_ERROR, "undefined tail mode");
+      }
+    }
+  }
+
+  void parse_node_order(int config_flags) {
+    switch (config_flags & MARISA_NODE_ORDER_MASK) {
+      case 0: {
+        node_order_ = MARISA_DEFAULT_ORDER;
+        break;
+      }
+      case MARISA_LABEL_ORDER: {
+        node_order_ = MARISA_LABEL_ORDER;
+        break;
+      }
+      case MARISA_WEIGHT_ORDER: {
+        node_order_ = MARISA_WEIGHT_ORDER;
+        break;
+      }
+      default: {
+        MARISA_THROW(MARISA_CODE_ERROR, "undefined node order");
+      }
+    }
+  }
+
+  // Disallows copy and assignment.
+  Config(const Config &);
+  Config &operator=(const Config &);
+};
+
+}  // namespace trie
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_TRIE_CONFIG_H_