melisa 0.1.0

data/ext/marisa/lib/marisa/grimoire/intrin.h

@@ -0,0 +1,115 @@
+#ifndef MARISA_GRIMOIRE_INTRIN_H_
+#define MARISA_GRIMOIRE_INTRIN_H_
+
+#include "marisa/base.h"
+
+#if defined(__x86_64__) || defined(_M_X64)
+ #define MARISA_X64
+#elif defined(__i386__) || defined(_M_IX86)
+ #define MARISA_X86
+#else  // defined(__i386__) || defined(_M_IX86)
+ #ifdef MARISA_USE_POPCNT
+  #undef MARISA_USE_POPCNT
+ #endif  // MARISA_USE_POPCNT
+ #ifdef MARISA_USE_SSE4A
+  #undef MARISA_USE_SSE4A
+ #endif  // MARISA_USE_SSE4A
+ #ifdef MARISA_USE_SSE4
+  #undef MARISA_USE_SSE4
+ #endif  // MARISA_USE_SSE4
+ #ifdef MARISA_USE_SSE4_2
+  #undef MARISA_USE_SSE4_2
+ #endif  // MARISA_USE_SSE4_2
+ #ifdef MARISA_USE_SSE4_1
+  #undef MARISA_USE_SSE4_1
+ #endif  // MARISA_USE_SSE4_1
+ #ifdef MARISA_USE_SSSE3
+  #undef MARISA_USE_SSSE3
+ #endif  // MARISA_USE_SSSE3
+ #ifdef MARISA_USE_SSE3
+  #undef MARISA_USE_SSE3
+ #endif  // MARISA_USE_SSE3
+ #ifdef MARISA_USE_SSE2
+  #undef MARISA_USE_SSE2
+ #endif  // MARISA_USE_SSE2
+#endif  // defined(__i386__) || defined(_M_IX86)
+
+#ifdef MARISA_USE_POPCNT
+ #ifndef MARISA_USE_SSE3
+  #define MARISA_USE_SSE3
+ #endif  // MARISA_USE_SSE3
+ #ifdef _MSC_VER
+  #include <intrin.h>
+ #else  // _MSC_VER
+  #include <popcntintrin.h>
+ #endif  // _MSC_VER
+#endif  // MARISA_USE_POPCNT
+
+#ifdef MARISA_USE_SSE4A
+ #ifndef MARISA_USE_SSE3
+  #define MARISA_USE_SSE3
+ #endif  // MARISA_USE_SSE3
+ #ifndef MARISA_USE_POPCNT
+  #define MARISA_USE_POPCNT
+ #endif  // MARISA_USE_POPCNT
+#endif  // MARISA_USE_SSE4A
+
+#ifdef MARISA_USE_SSE4
+ #ifndef MARISA_USE_SSE4_2
+  #define MARISA_USE_SSE4_2
+ #endif  // MARISA_USE_SSE4_2
+#endif  // MARISA_USE_SSE4
+
+#ifdef MARISA_USE_SSE4_2
+ #ifndef MARISA_USE_SSE4_1
+  #define MARISA_USE_SSE4_1
+ #endif  // MARISA_USE_SSE4_1
+ #ifndef MARISA_USE_POPCNT
+  #define MARISA_USE_POPCNT
+ #endif  // MARISA_USE_POPCNT
+#endif  // MARISA_USE_SSE4_2
+
+#ifdef MARISA_USE_SSE4_1
+ #ifndef MARISA_USE_SSSE3
+  #define MARISA_USE_SSSE3
+ #endif  // MARISA_USE_SSSE3
+#endif  // MARISA_USE_SSE4_1
+
+#ifdef MARISA_USE_SSSE3
+ #ifndef MARISA_USE_SSE3
+  #define MARISA_USE_SSE3
+ #endif  // MARISA_USE_SSE3
+ #ifdef MARISA_X64
+  #define MARISA_X64_SSSE3
+ #else  // MARISA_X64
+  #define MARISA_X86_SSSE3
+ #endif  // MAIRSA_X64
+ #include <tmmintrin.h>
+#endif  // MARISA_USE_SSSE3
+
+#ifdef MARISA_USE_SSE3
+ #ifndef MARISA_USE_SSE2
+  #define MARISA_USE_SSE2
+ #endif  // MARISA_USE_SSE2
+#endif  // MARISA_USE_SSE3
+
+#ifdef MARISA_USE_SSE2
+ #ifdef MARISA_X64
+  #define MARISA_X64_SSE2
+ #else  // MARISA_X64
+  #define MARISA_X86_SSE2
+ #endif  // MAIRSA_X64
+ #include <emmintrin.h>
+#endif  // MARISA_USE_SSE2
+
+#ifdef _MSC_VER
+ #if MARISA_WORD_SIZE == 64
+  #include <intrin.h>
+  #pragma intrinsic(_BitScanForward64)
+ #else  // MARISA_WORD_SIZE == 64
+  #include <intrin.h>
+  #pragma intrinsic(_BitScanForward)
+ #endif  // MARISA_WORD_SIZE == 64
+#endif  // _MSC_VER
+
+#endif  // MARISA_GRIMOIRE_INTRIN_H_

data/ext/marisa/lib/marisa/grimoire/io.h

@@ -0,0 +1,18 @@
+#ifndef MARISA_GRIMOIRE_IO_H_
+#define MARISA_GRIMOIRE_IO_H_
+
+#include "marisa/grimoire/io/mapper.h"
+#include "marisa/grimoire/io/reader.h"
+#include "marisa/grimoire/io/writer.h"
+
+namespace marisa {
+namespace grimoire {
+
+using io::Mapper;
+using io::Reader;
+using io::Writer;
+
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_IO_H_

data/ext/marisa/lib/marisa/grimoire/io/mapper.cc

@@ -0,0 +1,163 @@
+#if (defined _WIN32) || (defined _WIN64)
+ #include <sys/types.h>
+ #include <sys/stat.h>
+ #include <windows.h>
+#else  // (defined _WIN32) || (defined _WIN64)
+ #include <sys/mman.h>
+ #include <sys/stat.h>
+ #include <sys/types.h>
+ #include <fcntl.h>
+ #include <unistd.h>
+#endif  // (defined _WIN32) || (defined _WIN64)
+
+#include "marisa/grimoire/io/mapper.h"
+
+namespace marisa {
+namespace grimoire {
+namespace io {
+
+#if (defined _WIN32) || (defined _WIN64)
+Mapper::Mapper()
+    : ptr_(NULL), origin_(NULL), avail_(0), size_(0),
+      file_(NULL), map_(NULL) {}
+#else  // (defined _WIN32) || (defined _WIN64)
+Mapper::Mapper()
+    : ptr_(NULL), origin_(MAP_FAILED), avail_(0), size_(0), fd_(-1) {}
+#endif  // (defined _WIN32) || (defined _WIN64)
+
+#if (defined _WIN32) || (defined _WIN64)
+Mapper::~Mapper() {
+  if (origin_ != NULL) {
+    ::UnmapViewOfFile(origin_);
+  }
+
+  if (map_ != NULL) {
+    ::CloseHandle(map_);
+  }
+
+  if (file_ != NULL) {
+    ::CloseHandle(file_);
+  }
+}
+#else  // (defined _WIN32) || (defined _WIN64)
+Mapper::~Mapper() {
+  if (origin_ != MAP_FAILED) {
+    ::munmap(origin_, size_);
+  }
+
+  if (fd_ != -1) {
+    ::close(fd_);
+  }
+}
+#endif  // (defined _WIN32) || (defined _WIN64)
+
+void Mapper::open(const char *filename) {
+  MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR);
+
+  Mapper temp;
+  temp.open_(filename);
+  swap(temp);
+}
+
+void Mapper::open(const void *ptr, std::size_t size) {
+  MARISA_THROW_IF((ptr == NULL) && (size != 0), MARISA_NULL_ERROR);
+
+  Mapper temp;
+  temp.open_(ptr, size);
+  swap(temp);
+}
+
+void Mapper::seek(std::size_t size) {
+  MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR);
+  MARISA_THROW_IF(size > avail_, MARISA_IO_ERROR);
+
+  map_data(size);
+}
+
+bool Mapper::is_open() const {
+  return ptr_ != NULL;
+}
+
+void Mapper::clear() {
+  Mapper().swap(*this);
+}
+
+void Mapper::swap(Mapper &rhs) {
+  marisa::swap(ptr_, rhs.ptr_);
+  marisa::swap(avail_, rhs.avail_);
+  marisa::swap(origin_, rhs.origin_);
+  marisa::swap(size_, rhs.size_);
+#if (defined _WIN32) || (defined _WIN64)
+  marisa::swap(file_, rhs.file_);
+  marisa::swap(map_, rhs.map_);
+#else  // (defined _WIN32) || (defined _WIN64)
+  marisa::swap(fd_, rhs.fd_);
+#endif  // (defined _WIN32) || (defined _WIN64)
+}
+
+const void *Mapper::map_data(std::size_t size) {
+  MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR);
+  MARISA_THROW_IF(size > avail_, MARISA_IO_ERROR);
+
+  const char * const data = static_cast<const char *>(ptr_);
+  ptr_ = data + size;
+  avail_ -= size;
+  return data;
+}
+
+#if (defined _WIN32) || (defined _WIN64)
+ #ifdef __MSVCRT_VERSION__
+  #if __MSVCRT_VERSION__ >= 0x0601
+   #define MARISA_HAS_STAT64
+  #endif  // __MSVCRT_VERSION__ >= 0x0601
+ #endif  // __MSVCRT_VERSION__
+void Mapper::open_(const char *filename) {
+ #ifdef MARISA_HAS_STAT64
+  struct __stat64 st;
+  MARISA_THROW_IF(::_stat64(filename, &st) != 0, MARISA_IO_ERROR);
+ #else  // MARISA_HAS_STAT64
+  struct _stat st;
+  MARISA_THROW_IF(::_stat(filename, &st) != 0, MARISA_IO_ERROR);
+ #endif  // MARISA_HAS_STAT64
+  MARISA_THROW_IF((UInt64)st.st_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR);
+  size_ = (std::size_t)st.st_size;
+
+  file_ = ::CreateFileA(filename, GENERIC_READ, FILE_SHARE_READ,
+      NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
+  MARISA_THROW_IF(file_ == INVALID_HANDLE_VALUE, MARISA_IO_ERROR);
+
+  map_ = ::CreateFileMapping(file_, NULL, PAGE_READONLY, 0, 0, NULL);
+  MARISA_THROW_IF(map_ == NULL, MARISA_IO_ERROR);
+
+  origin_ = ::MapViewOfFile(map_, FILE_MAP_READ, 0, 0, 0);
+  MARISA_THROW_IF(origin_ == NULL, MARISA_IO_ERROR);
+
+  ptr_ = static_cast<const char *>(origin_);
+  avail_ = size_;
+}
+#else  // (defined _WIN32) || (defined _WIN64)
+void Mapper::open_(const char *filename) {
+  struct stat st;
+  MARISA_THROW_IF(::stat(filename, &st) != 0, MARISA_IO_ERROR);
+  MARISA_THROW_IF((UInt64)st.st_size > MARISA_SIZE_MAX, MARISA_SIZE_ERROR);
+  size_ = (std::size_t)st.st_size;
+
+  fd_ = ::open(filename, O_RDONLY);
+  MARISA_THROW_IF(fd_ == -1, MARISA_IO_ERROR);
+
+  origin_ = ::mmap(NULL, size_, PROT_READ, MAP_SHARED, fd_, 0);
+  MARISA_THROW_IF(origin_ == MAP_FAILED, MARISA_IO_ERROR);
+
+  ptr_ = static_cast<const char *>(origin_);
+  avail_ = size_;
+}
+#endif  // (defined _WIN32) || (defined _WIN64)
+
+void Mapper::open_(const void *ptr, std::size_t size) {
+  ptr_ = ptr;
+  avail_ = size;
+}
+
+}  // namespace io
+}  // namespace grimoire
+}  // namespace marisa

data/ext/marisa/lib/marisa/grimoire/io/mapper.h

@@ -0,0 +1,67 @@
+#ifndef MARISA_GRIMOIRE_IO_MAPPER_H_
+#define MARISA_GRIMOIRE_IO_MAPPER_H_
+
+#include <cstdio>
+
+#include "marisa/base.h"
+
+namespace marisa {
+namespace grimoire {
+namespace io {
+
+class Mapper {
+ public:
+  Mapper();
+  ~Mapper();
+
+  void open(const char *filename);
+  void open(const void *ptr, std::size_t size);
+
+  template <typename T>
+  void map(T *obj) {
+    MARISA_THROW_IF(obj == NULL, MARISA_NULL_ERROR);
+    *obj = *static_cast<const T *>(map_data(sizeof(T)));
+  }
+
+  template <typename T>
+  void map(const T **objs, std::size_t num_objs) {
+    MARISA_THROW_IF((objs == NULL) && (num_objs != 0), MARISA_NULL_ERROR);
+    MARISA_THROW_IF(num_objs > (MARISA_SIZE_MAX / sizeof(T)),
+        MARISA_SIZE_ERROR);
+    *objs = static_cast<const T *>(map_data(sizeof(T) * num_objs));
+  }
+
+  void seek(std::size_t size);
+
+  bool is_open() const;
+
+  void clear();
+  void swap(Mapper &rhs);
+
+ private:
+  const void *ptr_;
+  void *origin_;
+  std::size_t avail_;
+  std::size_t size_;
+#if (defined _WIN32) || (defined _WIN64)
+  void *file_;
+  void *map_;
+#else  // (defined _WIN32) || (defined _WIN64)
+  int fd_;
+#endif  // (defined _WIN32) || (defined _WIN64)
+
+  void open_(const char *filename);
+  void open_(const void *ptr, std::size_t size);
+
+  const void *map_data(std::size_t size);
+
+  // Disallows copy and assignment.
+  Mapper(const Mapper &);
+  Mapper &operator=(const Mapper &);
+};
+
+}  // namespace io
+}  // namespace grimoire
+}  // namespace marisa
+
+#endif  // MARISA_GRIMOIRE_IO_MAPPER_H_

data/ext/marisa/lib/marisa/grimoire/io/reader.cc

@@ -0,0 +1,147 @@
+#include <stdio.h>
+
+#ifdef _WIN32
+ #include <io.h>
+#else  // _WIN32
+ #include <unistd.h>
+#endif  // _WIN32
+
+#include <limits>
+
+#include "marisa/grimoire/io/reader.h"
+
+namespace marisa {
+namespace grimoire {
+namespace io {
+
+Reader::Reader()
+    : file_(NULL), fd_(-1), stream_(NULL), needs_fclose_(false) {}
+
+Reader::~Reader() {
+  if (needs_fclose_) {
+    ::fclose(file_);
+  }
+}
+
+void Reader::open(const char *filename) {
+  MARISA_THROW_IF(filename == NULL, MARISA_NULL_ERROR);
+
+  Reader temp;
+  temp.open_(filename);
+  swap(temp);
+}
+
+void Reader::open(std::FILE *file) {
+  MARISA_THROW_IF(file == NULL, MARISA_NULL_ERROR);
+
+  Reader temp;
+  temp.open_(file);
+  swap(temp);
+}
+
+void Reader::open(int fd) {
+  MARISA_THROW_IF(fd == -1, MARISA_CODE_ERROR);
+
+  Reader temp;
+  temp.open_(fd);
+  swap(temp);
+}
+
+void Reader::open(std::istream &stream) {
+  Reader temp;
+  temp.open_(stream);
+  swap(temp);
+}
+
+void Reader::clear() {
+  Reader().swap(*this);
+}
+
+void Reader::swap(Reader &rhs) {
+  marisa::swap(file_, rhs.file_);
+  marisa::swap(fd_, rhs.fd_);
+  marisa::swap(stream_, rhs.stream_);
+  marisa::swap(needs_fclose_, rhs.needs_fclose_);
+}
+
+void Reader::seek(std::size_t size) {
+  MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR);
+  if (size == 0) {
+    return;
+  } else if (size <= 16) {
+    char buf[16];
+    read_data(buf, size);
+  } else {
+    char buf[1024];
+    while (size != 0) {
+      const std::size_t count = (size < sizeof(buf)) ? size : sizeof(buf);
+      read_data(buf, count);
+      size -= count;
+    }
+  }
+}
+
+bool Reader::is_open() const {
+  return (file_ != NULL) || (fd_ != -1) || (stream_ != NULL);
+}
+
+void Reader::open_(const char *filename) {
+  std::FILE *file = NULL;
+#ifdef _MSC_VER
+  MARISA_THROW_IF(::fopen_s(&file, filename, "rb") != 0, MARISA_IO_ERROR);
+#else  // _MSC_VER
+  file = ::fopen(filename, "rb");
+  MARISA_THROW_IF(file == NULL, MARISA_IO_ERROR);
+#endif  // _MSC_VER
+  file_ = file;
+  needs_fclose_ = true;
+}
+
+void Reader::open_(std::FILE *file) {
+  file_ = file;
+}
+
+void Reader::open_(int fd) {
+  fd_ = fd;
+}
+
+void Reader::open_(std::istream &stream) {
+  stream_ = &stream;
+}
+
+void Reader::read_data(void *buf, std::size_t size) {
+  MARISA_THROW_IF(!is_open(), MARISA_STATE_ERROR);
+  if (size == 0) {
+    return;
+  } else if (fd_ != -1) {
+    while (size != 0) {
+#ifdef _WIN32
+      static const std::size_t CHUNK_SIZE =
+          std::numeric_limits<int>::max();
+      const unsigned int count = (size < CHUNK_SIZE) ? size : CHUNK_SIZE;
+      const int size_read = ::_read(fd_, buf, count);
+#else  // _WIN32
+      static const std::size_t CHUNK_SIZE =
+          std::numeric_limits< ::ssize_t>::max();
+      const ::size_t count = (size < CHUNK_SIZE) ? size : CHUNK_SIZE;
+      const ::ssize_t size_read = ::read(fd_, buf, count);
+#endif  // _WIN32
+      MARISA_THROW_IF(size_read <= 0, MARISA_IO_ERROR);
+      buf = static_cast<char *>(buf) + size_read;
+      size -= size_read;
+    }
+  } else if (file_ != NULL) {
+    MARISA_THROW_IF(::fread(buf, 1, size, file_) != size, MARISA_IO_ERROR);
+  } else if (stream_ != NULL) {
+    try {
+      MARISA_THROW_IF(!stream_->read(static_cast<char *>(buf), size),
+          MARISA_IO_ERROR);
+    } catch (const std::ios_base::failure &) {
+      MARISA_THROW(MARISA_IO_ERROR, "std::ios_base::failure");
+    }
+  }
+}
+
+}  // namespace io
+}  // namespace grimoire
+}  // namespace marisa