jieba-rb 5.0.0

data/ext/cppjieba/include/cppjieba/MPSegment.hpp

@@ -0,0 +1,137 @@
+#ifndef CPPJIEBA_MPSEGMENT_H
+#define CPPJIEBA_MPSEGMENT_H
+
+#include <algorithm>
+#include <set>
+#include <cassert>
+#include "limonp/Logging.hpp"
+#include "DictTrie.hpp"
+#include "SegmentTagged.hpp"
+#include "PosTagger.hpp"
+
+namespace cppjieba {
+
+class MPSegment: public SegmentTagged {
+ public:
+  MPSegment(const string& dictPath, const string& userDictPath = "")
+    : dictTrie_(new DictTrie(dictPath, userDictPath)), isNeedDestroy_(true) {
+  }
+  MPSegment(const DictTrie* dictTrie)
+    : dictTrie_(dictTrie), isNeedDestroy_(false) {
+    assert(dictTrie_);
+  }
+  ~MPSegment() {
+    if (isNeedDestroy_) {
+      delete dictTrie_;
+    }
+  }
+
+  void Cut(const string& sentence, vector<string>& words) const {
+    Cut(sentence, words, MAX_WORD_LENGTH);
+  }
+
+  void Cut(const string& sentence,
+        vector<string>& words,
+        size_t max_word_len) const {
+    vector<Word> tmp;
+    Cut(sentence, tmp, max_word_len);
+    GetStringsFromWords(tmp, words);
+  }
+  void Cut(const string& sentence, 
+        vector<Word>& words, 
+        size_t max_word_len = MAX_WORD_LENGTH) const {
+    PreFilter pre_filter(symbols_, sentence);
+    PreFilter::Range range;
+    vector<WordRange> wrs;
+    wrs.reserve(sentence.size()/2);
+    while (pre_filter.HasNext()) {
+      range = pre_filter.Next();
+      Cut(range.begin, range.end, wrs, max_word_len);
+    }
+    words.clear();
+    words.reserve(wrs.size());
+    GetWordsFromWordRanges(sentence, wrs, words);
+  }
+  void Cut(RuneStrArray::const_iterator begin,
+           RuneStrArray::const_iterator end,
+           vector<WordRange>& words,
+           size_t max_word_len = MAX_WORD_LENGTH) const {
+    vector<Dag> dags;
+    dictTrie_->Find(begin, 
+          end, 
+          dags,
+          max_word_len);
+    CalcDP(dags);
+    CutByDag(begin, end, dags, words);
+  }
+
+  const DictTrie* GetDictTrie() const {
+    return dictTrie_;
+  }
+
+  bool Tag(const string& src, vector<pair<string, string> >& res) const {
+    return tagger_.Tag(src, res, *this);
+  }
+
+  bool IsUserDictSingleChineseWord(const Rune& value) const {
+    return dictTrie_->IsUserDictSingleChineseWord(value);
+  }
+ private:
+  void CalcDP(vector<Dag>& dags) const {
+    size_t nextPos;
+    const DictUnit* p;
+    double val;
+
+    for (vector<Dag>::reverse_iterator rit = dags.rbegin(); rit != dags.rend(); rit++) {
+      rit->pInfo = NULL;
+      rit->weight = MIN_DOUBLE;
+      assert(!rit->nexts.empty());
+      for (LocalVector<pair<size_t, const DictUnit*> >::const_iterator it = rit->nexts.begin(); it != rit->nexts.end(); it++) {
+        nextPos = it->first;
+        p = it->second;
+        val = 0.0;
+        if (nextPos + 1 < dags.size()) {
+          val += dags[nextPos + 1].weight;
+        }
+
+        if (p) {
+          val += p->weight;
+        } else {
+          val += dictTrie_->GetMinWeight();
+        }
+        if (val > rit->weight) {
+          rit->pInfo = p;
+          rit->weight = val;
+        }
+      }
+    }
+  }
+  void CutByDag(RuneStrArray::const_iterator begin, 
+        RuneStrArray::const_iterator end, 
+        const vector<Dag>& dags, 
+        vector<WordRange>& words) const {
+    size_t i = 0;
+    while (i < dags.size()) {
+      const DictUnit* p = dags[i].pInfo;
+      if (p) {
+        assert(p->word.size() >= 1);
+        WordRange wr(begin + i, begin + i + p->word.size() - 1);
+        words.push_back(wr);
+        i += p->word.size();
+      } else { //single chinese word
+        WordRange wr(begin + i, begin + i);
+        words.push_back(wr);
+        i++;
+      }
+    }
+  }
+
+  const DictTrie* dictTrie_;
+  bool isNeedDestroy_;
+  PosTagger tagger_;
+
+}; // class MPSegment
+
+} // namespace cppjieba
+
+#endif

data/ext/cppjieba/include/cppjieba/MixSegment.hpp

@@ -0,0 +1,109 @@
+#ifndef CPPJIEBA_MIXSEGMENT_H
+#define CPPJIEBA_MIXSEGMENT_H
+
+#include <cassert>
+#include "MPSegment.hpp"
+#include "HMMSegment.hpp"
+#include "limonp/StringUtil.hpp"
+#include "PosTagger.hpp"
+
+namespace cppjieba {
+class MixSegment: public SegmentTagged {
+ public:
+  MixSegment(const string& mpSegDict, const string& hmmSegDict, 
+        const string& userDict = "") 
+    : mpSeg_(mpSegDict, userDict), 
+      hmmSeg_(hmmSegDict) {
+  }
+  MixSegment(const DictTrie* dictTrie, const HMMModel* model) 
+    : mpSeg_(dictTrie), hmmSeg_(model) {
+  }
+  ~MixSegment() {
+  }
+
+  void Cut(const string& sentence, vector<string>& words) const {
+    Cut(sentence, words, true);
+  }
+  void Cut(const string& sentence, vector<string>& words, bool hmm) const {
+    vector<Word> tmp;
+    Cut(sentence, tmp, hmm);
+    GetStringsFromWords(tmp, words);
+  }
+  void Cut(const string& sentence, vector<Word>& words, bool hmm = true) const {
+    PreFilter pre_filter(symbols_, sentence);
+    PreFilter::Range range;
+    vector<WordRange> wrs;
+    wrs.reserve(sentence.size() / 2);
+    while (pre_filter.HasNext()) {
+      range = pre_filter.Next();
+      Cut(range.begin, range.end, wrs, hmm);
+    }
+    words.clear();
+    words.reserve(wrs.size());
+    GetWordsFromWordRanges(sentence, wrs, words);
+  }
+
+  void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<WordRange>& res, bool hmm) const {
+    if (!hmm) {
+      mpSeg_.Cut(begin, end, res);
+      return;
+    }
+    vector<WordRange> words;
+    assert(end >= begin);
+    words.reserve(end - begin);
+    mpSeg_.Cut(begin, end, words);
+
+    vector<WordRange> hmmRes;
+    hmmRes.reserve(end - begin);
+    for (size_t i = 0; i < words.size(); i++) {
+      //if mp Get a word, it's ok, put it into result
+      if (words[i].left != words[i].right || (words[i].left == words[i].right && mpSeg_.IsUserDictSingleChineseWord(words[i].left->rune))) {
+        res.push_back(words[i]);
+        continue;
+      }
+
+      // if mp Get a single one and it is not in userdict, collect it in sequence
+      size_t j = i;
+      while (j < words.size() && words[j].left == words[j].right && !mpSeg_.IsUserDictSingleChineseWord(words[j].left->rune)) {
+        j++;
+      }
+
+      // Cut the sequence with hmm
+      assert(j - 1 >= i);
+      // TODO
+      hmmSeg_.Cut(words[i].left, words[j - 1].left + 1, hmmRes);
+      //put hmm result to result
+      for (size_t k = 0; k < hmmRes.size(); k++) {
+        res.push_back(hmmRes[k]);
+      }
+
+      //clear tmp vars
+      hmmRes.clear();
+
+      //let i jump over this piece
+      i = j - 1;
+    }
+  }
+
+  const DictTrie* GetDictTrie() const {
+    return mpSeg_.GetDictTrie();
+  }
+
+  bool Tag(const string& src, vector<pair<string, string> >& res) const {
+    return tagger_.Tag(src, res, *this);
+  }
+
+  string LookupTag(const string &str) const {
+    return tagger_.LookupTag(str, *this);
+  }
+
+ private:
+  MPSegment mpSeg_;
+  HMMSegment hmmSeg_;
+  PosTagger tagger_;
+
+}; // class MixSegment
+
+} // namespace cppjieba
+
+#endif

data/ext/cppjieba/include/cppjieba/PosTagger.hpp

@@ -0,0 +1,77 @@
+#ifndef CPPJIEBA_POS_TAGGING_H
+#define CPPJIEBA_POS_TAGGING_H
+
+#include "limonp/StringUtil.hpp"
+#include "SegmentTagged.hpp"
+#include "DictTrie.hpp"
+
+namespace cppjieba {
+using namespace limonp;
+
+static const char* const POS_M = "m";
+static const char* const POS_ENG = "eng";
+static const char* const POS_X = "x";
+
+class PosTagger {
+ public:
+  PosTagger() {
+  }
+  ~PosTagger() {
+  }
+
+  bool Tag(const string& src, vector<pair<string, string> >& res, const SegmentTagged& segment) const {
+    vector<string> CutRes;
+    segment.Cut(src, CutRes);
+
+    for (vector<string>::iterator itr = CutRes.begin(); itr != CutRes.end(); ++itr) {
+      res.push_back(make_pair(*itr, LookupTag(*itr, segment)));
+    }
+    return !res.empty();
+  }
+
+  string LookupTag(const string &str, const SegmentTagged& segment) const {
+    const DictUnit *tmp = NULL;
+    RuneStrArray runes;
+    const DictTrie * dict = segment.GetDictTrie();
+    assert(dict != NULL);
+      if (!DecodeRunesInString(str, runes)) {
+        XLOG(ERROR) << "Decode failed.";
+        return POS_X;
+      }
+      tmp = dict->Find(runes.begin(), runes.end());
+      if (tmp == NULL || tmp->tag.empty()) {
+        return SpecialRule(runes);
+      } else {
+        return tmp->tag;
+      }
+  }
+
+ private:
+  const char* SpecialRule(const RuneStrArray& unicode) const {
+    size_t m = 0;
+    size_t eng = 0;
+    for (size_t i = 0; i < unicode.size() && eng < unicode.size() / 2; i++) {
+      if (unicode[i].rune < 0x80) {
+        eng ++;
+        if ('0' <= unicode[i].rune && unicode[i].rune <= '9') {
+          m++;
+        }
+      }
+    }
+    // ascii char is not found
+    if (eng == 0) {
+      return POS_X;
+    }
+    // all the ascii is number char
+    if (m == eng) {
+      return POS_M;
+    }
+    // the ascii chars contain english letter
+    return POS_ENG;
+  }
+
+}; // class PosTagger
+
+} // namespace cppjieba
+
+#endif

data/ext/cppjieba/include/cppjieba/PreFilter.hpp

@@ -0,0 +1,54 @@
+#ifndef CPPJIEBA_PRE_FILTER_H
+#define CPPJIEBA_PRE_FILTER_H
+
+#include "Trie.hpp"
+#include "limonp/Logging.hpp"
+
+namespace cppjieba {
+
+class PreFilter {
+ public:
+  //TODO use WordRange instead of Range
+  struct Range {
+    RuneStrArray::const_iterator begin;
+    RuneStrArray::const_iterator end;
+  }; // struct Range
+
+  PreFilter(const unordered_set<Rune>& symbols, 
+        const string& sentence)
+    : symbols_(symbols) {
+    if (!DecodeRunesInString(sentence, sentence_)) {
+      XLOG(ERROR) << "decode failed. "; 
+    }
+    cursor_ = sentence_.begin();
+  }
+  ~PreFilter() {
+  }
+  bool HasNext() const {
+    return cursor_ != sentence_.end();
+  }
+  Range Next() {
+    Range range;
+    range.begin = cursor_;
+    while (cursor_ != sentence_.end()) {
+      if (IsIn(symbols_, cursor_->rune)) {
+        if (range.begin == cursor_) {
+          cursor_ ++;
+        }
+        range.end = cursor_;
+        return range;
+      }
+      cursor_ ++;
+    }
+    range.end = sentence_.end();
+    return range;
+  }
+ private:
+  RuneStrArray::const_iterator cursor_;
+  RuneStrArray sentence_;
+  const unordered_set<Rune>& symbols_;
+}; // class PreFilter
+
+} // namespace cppjieba
+
+#endif // CPPJIEBA_PRE_FILTER_H

data/ext/cppjieba/include/cppjieba/QuerySegment.hpp

@@ -0,0 +1,90 @@
+#ifndef CPPJIEBA_QUERYSEGMENT_H
+#define CPPJIEBA_QUERYSEGMENT_H
+
+#include <algorithm>
+#include <set>
+#include <cassert>
+#include "limonp/Logging.hpp"
+#include "DictTrie.hpp"
+#include "SegmentBase.hpp"
+#include "FullSegment.hpp"
+#include "MixSegment.hpp"
+#include "Unicode.hpp"
+#include "DictTrie.hpp"
+
+namespace cppjieba {
+class QuerySegment: public SegmentBase {
+ public:
+  QuerySegment(const string& dict, const string& model, const string& userDict = "")
+    : mixSeg_(dict, model, userDict),
+      trie_(mixSeg_.GetDictTrie()) {
+  }
+  QuerySegment(const DictTrie* dictTrie, const HMMModel* model)
+    : mixSeg_(dictTrie, model), trie_(dictTrie) {
+  }
+  ~QuerySegment() {
+  }
+
+  void Cut(const string& sentence, vector<string>& words) const {
+    Cut(sentence, words, true);
+  }
+  void Cut(const string& sentence, vector<string>& words, bool hmm) const {
+    vector<Word> tmp;
+    Cut(sentence, tmp, hmm);
+    GetStringsFromWords(tmp, words);
+  }
+  void Cut(const string& sentence, vector<Word>& words, bool hmm = true) const {
+    PreFilter pre_filter(symbols_, sentence);
+    PreFilter::Range range;
+    vector<WordRange> wrs;
+    wrs.reserve(sentence.size()/2);
+    while (pre_filter.HasNext()) {
+      range = pre_filter.Next();
+      Cut(range.begin, range.end, wrs, hmm);
+    }
+    words.clear();
+    words.reserve(wrs.size());
+    GetWordsFromWordRanges(sentence, wrs, words);
+  }
+  void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<WordRange>& res, bool hmm) const {
+    //use mix Cut first
+    vector<WordRange> mixRes;
+    mixSeg_.Cut(begin, end, mixRes, hmm);
+
+    vector<WordRange> fullRes;
+    for (vector<WordRange>::const_iterator mixResItr = mixRes.begin(); mixResItr != mixRes.end(); mixResItr++) {
+      if (mixResItr->Length() > 2) {
+        for (size_t i = 0; i + 1 < mixResItr->Length(); i++) {
+          WordRange wr(mixResItr->left + i, mixResItr->left + i + 1);
+          if (trie_->Find(wr.left, wr.right + 1) != NULL) {
+            res.push_back(wr);
+          }
+        }
+      }
+      if (mixResItr->Length() > 3) {
+        for (size_t i = 0; i + 2 < mixResItr->Length(); i++) {
+          WordRange wr(mixResItr->left + i, mixResItr->left + i + 2);
+          if (trie_->Find(wr.left, wr.right + 1) != NULL) {
+            res.push_back(wr);
+          }
+        }
+      }
+      res.push_back(*mixResItr);
+    }
+  }
+ private:
+  bool IsAllAscii(const Unicode& s) const {
+   for(size_t i = 0; i < s.size(); i++) {
+     if (s[i] >= 0x80) {
+       return false;
+     }
+   }
+   return true;
+  }
+  MixSegment mixSeg_;
+  const DictTrie* trie_;
+}; // QuerySegment
+
+} // namespace cppjieba
+
+#endif