cppjieba_rb 0.2.1

data/ext/cppjieba/include/cppjieba/Unicode.hpp

@@ -0,0 +1,215 @@
+#ifndef CPPJIEBA_UNICODE_H
+#define CPPJIEBA_UNICODE_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string>
+#include <vector>
+#include <ostream>
+#include "limonp/LocalVector.hpp"
+
+namespace cppjieba {
+
+using std::string;
+using std::vector;
+
+typedef uint32_t Rune;
+
+struct Word {
+  string word;
+  uint32_t offset;
+  Word(const string& w, uint32_t o)
+   : word(w), offset(o) {
+  }
+}; // struct Word
+
+inline std::ostream& operator << (std::ostream& os, const Word& w) {
+  return os << "{\"word\": \"" << w.word << "\", \"offset\": " << w.offset << "}";
+}
+
+struct RuneStr {
+  Rune rune;
+  uint32_t offset;
+  uint32_t len;
+  RuneStr(): rune(0), offset(0), len(0) {
+  }
+  RuneStr(Rune r, uint32_t o, uint32_t l)
+    : rune(r), offset(o), len(l) {
+  }
+}; // struct RuneStr
+
+inline std::ostream& operator << (std::ostream& os, const RuneStr& r) {
+  return os << "{\"rune\": \"" << r.rune << "\", \"offset\": " << r.offset << ", \"len\": " << r.len << "}";
+}
+
+typedef limonp::LocalVector<Rune> Unicode;
+typedef limonp::LocalVector<struct RuneStr> RuneStrArray;
+
+// [left, right]
+struct WordRange {
+  RuneStrArray::const_iterator left;
+  RuneStrArray::const_iterator right;
+  WordRange(RuneStrArray::const_iterator l, RuneStrArray::const_iterator r)
+   : left(l), right(r) {
+  }
+  size_t Length() const {
+    return right - left + 1;
+  }
+  bool IsAllAscii() const {
+    for (RuneStrArray::const_iterator iter = left; iter <= right; ++iter) {
+      if (iter->rune >= 0x80) {
+        return false;
+      }
+    }
+    return true;
+  }
+}; // struct WordRange
+
+struct RuneStrLite {
+  uint32_t rune;
+  uint32_t len;
+  RuneStrLite(): rune(0), len(0) {
+  }
+  RuneStrLite(uint32_t r, uint32_t l): rune(r), len(l) {
+  }
+}; // struct RuneStrLite
+
+inline RuneStrLite DecodeRuneInString(const char* str, size_t len) {
+  RuneStrLite rp(0, 0);
+  if (str == NULL || len == 0) {
+    return rp;
+  }
+  if (!(str[0] & 0x80)) { // 0xxxxxxx
+    // 7bit, total 7bit
+    rp.rune = (uint8_t)(str[0]) & 0x7f;
+    rp.len = 1;
+  } else if ((uint8_t)str[0] <= 0xdf &&  1 < len) { 
+    // 110xxxxxx
+    // 5bit, total 5bit
+    rp.rune = (uint8_t)(str[0]) & 0x1f;
+
+    // 6bit, total 11bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[1]) & 0x3f;
+    rp.len = 2;
+  } else if((uint8_t)str[0] <= 0xef && 2 < len) { // 1110xxxxxx
+    // 4bit, total 4bit
+    rp.rune = (uint8_t)(str[0]) & 0x0f;
+
+    // 6bit, total 10bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[1]) & 0x3f;
+
+    // 6bit, total 16bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[2]) & 0x3f;
+
+    rp.len = 3;
+  } else if((uint8_t)str[0] <= 0xf7 && 3 < len) { // 11110xxxx
+    // 3bit, total 3bit
+    rp.rune = (uint8_t)(str[0]) & 0x07;
+
+    // 6bit, total 9bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[1]) & 0x3f;
+
+    // 6bit, total 15bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[2]) & 0x3f;
+
+    // 6bit, total 21bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[3]) & 0x3f;
+
+    rp.len = 4;
+  } else {
+    rp.rune = 0;
+    rp.len = 0;
+  }
+  return rp;
+}
+
+inline bool DecodeRunesInString(const char* s, size_t len, RuneStrArray& runes) {
+  runes.clear();
+  runes.reserve(len / 2);
+  for (size_t i = 0; i < len;) {
+    RuneStrLite rp = DecodeRuneInString(s + i, len - i);
+    if (rp.len == 0) {
+      runes.clear();
+      return false;
+    }
+    RuneStr x(rp.rune, i, rp.len);
+    runes.push_back(x);
+    i += rp.len;
+  }
+  return true;
+}
+
+inline bool DecodeRunesInString(const string& s, RuneStrArray& runes) {
+  return DecodeRunesInString(s.c_str(), s.size(), runes);
+}
+
+inline bool DecodeRunesInString(const char* s, size_t len, Unicode& unicode) {
+  unicode.clear();
+  RuneStrArray runes;
+  if (!DecodeRunesInString(s, len, runes)) {
+    return false;
+  }
+  unicode.reserve(runes.size());
+  for (size_t i = 0; i < runes.size(); i++) {
+    unicode.push_back(runes[i].rune);
+  }
+  return true;
+}
+
+inline bool IsSingleWord(const string& str) {
+  RuneStrLite rp = DecodeRuneInString(str.c_str(), str.size());
+  return rp.len == str.size();
+}
+
+inline bool DecodeRunesInString(const string& s, Unicode& unicode) {
+  return DecodeRunesInString(s.c_str(), s.size(), unicode);
+}
+
+inline Unicode DecodeRunesInString(const string& s) {
+  Unicode result;
+  DecodeRunesInString(s, result);
+  return result;
+}
+
+
+// [left, right]
+inline Word GetWordFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {
+  assert(right->offset >= left->offset);
+  uint32_t len = right->offset - left->offset + right->len;
+  return Word(s.substr(left->offset, len), left->offset);
+}
+
+inline string GetStringFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {
+  assert(right->offset >= left->offset);
+  uint32_t len = right->offset - left->offset + right->len;
+  return s.substr(left->offset, len);
+}
+
+inline void GetWordsFromWordRanges(const string& s, const vector<WordRange>& wrs, vector<Word>& words) {
+  for (size_t i = 0; i < wrs.size(); i++) {
+    words.push_back(GetWordFromRunes(s, wrs[i].left, wrs[i].right));
+  }
+}
+
+inline vector<Word> GetWordsFromWordRanges(const string& s, const vector<WordRange>& wrs) {
+  vector<Word> result;
+  GetWordsFromWordRanges(s, wrs, result);
+  return result;
+}
+
+inline void GetStringsFromWords(const vector<Word>& words, vector<string>& strs) {
+  strs.resize(words.size());
+  for (size_t i = 0; i < words.size(); ++i) {
+    strs[i] = words[i].word;
+  }
+}
+
+} // namespace cppjieba
+
+#endif // CPPJIEBA_UNICODE_H

data/ext/cppjieba/test/CMakeLists.txt

@@ -0,0 +1,5 @@
+SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR})
+
+ADD_EXECUTABLE(demo demo.cpp)
+ADD_EXECUTABLE(load_test load_test.cpp)
+ADD_SUBDIRECTORY(unittest)

data/ext/cppjieba/test/demo.cpp

@@ -0,0 +1,80 @@
+#include "cppjieba/Jieba.hpp"
+
+using namespace std;
+
+const char* const DICT_PATH = "../dict/jieba.dict.utf8";
+const char* const HMM_PATH = "../dict/hmm_model.utf8";
+const char* const USER_DICT_PATH = "../dict/user.dict.utf8";
+const char* const IDF_PATH = "../dict/idf.utf8";
+const char* const STOP_WORD_PATH = "../dict/stop_words.utf8";
+
+int main(int argc, char** argv) {
+  cppjieba::Jieba jieba(DICT_PATH,
+        HMM_PATH,
+        USER_DICT_PATH,
+        IDF_PATH,
+        STOP_WORD_PATH);
+  vector<string> words;
+  vector<cppjieba::Word> jiebawords;
+  string s;
+  string result;
+
+  s = "他来到了网易杭研大厦";
+  cout << s << endl;
+  cout << "[demo] Cut With HMM" << endl;
+  jieba.Cut(s, words, true);
+  cout << limonp::Join(words.begin(), words.end(), "/") << endl;
+
+  cout << "[demo] Cut Without HMM " << endl;
+  jieba.Cut(s, words, false);
+  cout << limonp::Join(words.begin(), words.end(), "/") << endl;
+
+  s = "我来到北京清华大学";
+  cout << s << endl;
+  cout << "[demo] CutAll" << endl;
+  jieba.CutAll(s, words);
+  cout << limonp::Join(words.begin(), words.end(), "/") << endl;
+
+  s = "小明硕士毕业于中国科学院计算所，后在日本京都大学深造";
+  cout << s << endl;
+  cout << "[demo] CutForSearch" << endl;
+  jieba.CutForSearch(s, words);
+  cout << limonp::Join(words.begin(), words.end(), "/") << endl;
+
+  cout << "[demo] Insert User Word" << endl;
+  jieba.Cut("男默女泪", words);
+  cout << limonp::Join(words.begin(), words.end(), "/") << endl;
+  jieba.InsertUserWord("男默女泪");
+  jieba.Cut("男默女泪", words);
+  cout << limonp::Join(words.begin(), words.end(), "/") << endl;
+
+  cout << "[demo] CutForSearch Word With Offset" << endl;
+  jieba.CutForSearch(s, jiebawords, true);
+  cout << jiebawords << endl;
+
+  cout << "[demo] Lookup Tag for Single Token" << endl;
+  const int DemoTokenMaxLen = 32;
+  char DemoTokens[][DemoTokenMaxLen] = {"拖拉机", "CEO", "123", "。"};
+  vector<pair<string, string> > LookupTagres(sizeof(DemoTokens) / DemoTokenMaxLen);
+  vector<pair<string, string> >::iterator it;
+  for (it = LookupTagres.begin(); it != LookupTagres.end(); it++) {
+	it->first = DemoTokens[it - LookupTagres.begin()];
+	it->second = jieba.LookupTag(it->first);
+  }
+  cout << LookupTagres << endl;
+
+  cout << "[demo] Tagging" << endl;
+  vector<pair<string, string> > tagres;
+  s = "我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。";
+  jieba.Tag(s, tagres);
+  cout << s << endl;
+  cout << tagres << endl;;
+
+  cout << "[demo] Keyword Extraction" << endl;
+  const size_t topk = 5;
+  vector<cppjieba::KeywordExtractor::Word> keywordres;
+  jieba.extractor.Extract(s, keywordres, topk);
+  cout << s << endl;
+  cout << keywordres << endl;
+  return EXIT_SUCCESS;
+}

data/ext/cppjieba/test/load_test.cpp

@@ -0,0 +1,54 @@
+#include <iostream>
+#include <ctime>
+#include <fstream>
+#include "cppjieba/MPSegment.hpp"
+#include "cppjieba/HMMSegment.hpp"
+#include "cppjieba/MixSegment.hpp"
+#include "cppjieba/KeywordExtractor.hpp"
+#include "limonp/Colors.hpp"
+
+using namespace cppjieba;
+
+void Cut(size_t times = 50) {
+  MixSegment seg("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8");
+  vector<string> res;
+  string doc;
+  ifstream ifs("../test/testdata/weicheng.utf8");
+  assert(ifs);
+  doc << ifs;
+  long beginTime = clock();
+  for (size_t i = 0; i < times; i ++) {
+    printf("process [%3.0lf %%]\r", 100.0*(i+1)/times);
+    fflush(stdout);
+    res.clear();
+    seg.Cut(doc, res);
+  }
+  printf("\n");
+  long endTime = clock();
+  ColorPrintln(GREEN, "Cut: [%.3lf seconds]time consumed.", double(endTime - beginTime)/CLOCKS_PER_SEC);
+}
+
+void Extract(size_t times = 400) {
+  KeywordExtractor Extractor("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
+  vector<string> words;
+  string doc;
+  ifstream ifs("../test/testdata/review.100");
+  assert(ifs);
+  doc << ifs;
+  long beginTime = clock();
+  for (size_t i = 0; i < times; i ++) {
+    printf("process [%3.0lf %%]\r", 100.0*(i+1)/times);
+    fflush(stdout);
+    words.clear();
+    Extractor.Extract(doc, words, 5);
+  }
+  printf("\n");
+  long endTime = clock();
+  ColorPrintln(GREEN, "Extract: [%.3lf seconds]time consumed.", double(endTime - beginTime)/CLOCKS_PER_SEC);
+}
+
+int main(int argc, char ** argv) {
+  Cut();
+  Extract();
+  return EXIT_SUCCESS;
+}

data/ext/cppjieba/test/testdata/curl.res

@@ -0,0 +1 @@
+["南京市", "长江大桥"]