cppjieba_rb 0.3.3 → 0.4.0

data/ext/cppjieba/include/cppjieba/HMMSegment.hpp

@@ -1,190 +0,0 @@
-#ifndef CPPJIBEA_HMMSEGMENT_H
-#define CPPJIBEA_HMMSEGMENT_H
-
-#include <iostream>
-#include <fstream>
-#include <memory.h>
-#include <cassert>
-#include "HMMModel.hpp"
-#include "SegmentBase.hpp"
-
-namespace cppjieba {
-class HMMSegment: public SegmentBase {
- public:
-  HMMSegment(const string& filePath)
-  : model_(new HMMModel(filePath)), isNeedDestroy_(true) {
-  }
-  HMMSegment(const HMMModel* model) 
-  : model_(model), isNeedDestroy_(false) {
-  }
-  ~HMMSegment() {
-    if (isNeedDestroy_) {
-      delete model_;
-    }
-  }
-
-  void Cut(const string& sentence, 
-        vector<string>& words) const {
-    vector<Word> tmp;
-    Cut(sentence, tmp);
-    GetStringsFromWords(tmp, words);
-  }
-  void Cut(const string& sentence, 
-        vector<Word>& words) const {
-    PreFilter pre_filter(symbols_, sentence);
-    PreFilter::Range range;
-    vector<WordRange> wrs;
-    wrs.reserve(sentence.size()/2);
-    while (pre_filter.HasNext()) {
-      range = pre_filter.Next();
-      Cut(range.begin, range.end, wrs);
-    }
-    words.clear();
-    words.reserve(wrs.size());
-    GetWordsFromWordRanges(sentence, wrs, words);
-  }
-  void Cut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<WordRange>& res) const {
-    RuneStrArray::const_iterator left = begin;
-    RuneStrArray::const_iterator right = begin;
-    while (right != end) {
-      if (right->rune < 0x80) {
-        if (left != right) {
-          InternalCut(left, right, res);
-        }
-        left = right;
-        do {
-          right = SequentialLetterRule(left, end);
-          if (right != left) {
-            break;
-          }
-          right = NumbersRule(left, end);
-          if (right != left) {
-            break;
-          }
-          right ++;
-        } while (false);
-        WordRange wr(left, right - 1);
-        res.push_back(wr);
-        left = right;
-      } else {
-        right++;
-      }
-    }
-    if (left != right) {
-      InternalCut(left, right, res);
-    }
-  }
- private:
-  // sequential letters rule
-  RuneStrArray::const_iterator SequentialLetterRule(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const {
-    Rune x = begin->rune;
-    if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z')) {
-      begin ++;
-    } else {
-      return begin;
-    }
-    while (begin != end) {
-      x = begin->rune;
-      if (('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z') || ('0' <= x && x <= '9')) {
-        begin ++;
-      } else {
-        break;
-      }
-    }
-    return begin;
-  }
-  //
-  RuneStrArray::const_iterator NumbersRule(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end) const {
-    Rune x = begin->rune;
-    if ('0' <= x && x <= '9') {
-      begin ++;
-    } else {
-      return begin;
-    }
-    while (begin != end) {
-      x = begin->rune;
-      if ( ('0' <= x && x <= '9') || x == '.') {
-        begin++;
-      } else {
-        break;
-      }
-    }
-    return begin;
-  }
-  void InternalCut(RuneStrArray::const_iterator begin, RuneStrArray::const_iterator end, vector<WordRange>& res) const {
-    vector<size_t> status;
-    Viterbi(begin, end, status);
-
-    RuneStrArray::const_iterator left = begin;
-    RuneStrArray::const_iterator right;
-    for (size_t i = 0; i < status.size(); i++) {
-      if (status[i] % 2) { //if (HMMModel::E == status[i] || HMMModel::S == status[i])
-        right = begin + i + 1;
-        WordRange wr(left, right - 1);
-        res.push_back(wr);
-        left = right;
-      }
-    }
-  }
-
-  void Viterbi(RuneStrArray::const_iterator begin, 
-        RuneStrArray::const_iterator end, 
-        vector<size_t>& status) const {
-    size_t Y = HMMModel::STATUS_SUM;
-    size_t X = end - begin;
-
-    size_t XYSize = X * Y;
-    size_t now, old, stat;
-    double tmp, endE, endS;
-
-    vector<int> path(XYSize);
-    vector<double> weight(XYSize);
-
-    //start
-    for (size_t y = 0; y < Y; y++) {
-      weight[0 + y * X] = model_->startProb[y] + model_->GetEmitProb(model_->emitProbVec[y], begin->rune, MIN_DOUBLE);
-      path[0 + y * X] = -1;
-    }
-
-    double emitProb;
-
-    for (size_t x = 1; x < X; x++) {
-      for (size_t y = 0; y < Y; y++) {
-        now = x + y*X;
-        weight[now] = MIN_DOUBLE;
-        path[now] = HMMModel::E; // warning
-        emitProb = model_->GetEmitProb(model_->emitProbVec[y], (begin+x)->rune, MIN_DOUBLE);
-        for (size_t preY = 0; preY < Y; preY++) {
-          old = x - 1 + preY * X;
-          tmp = weight[old] + model_->transProb[preY][y] + emitProb;
-          if (tmp > weight[now]) {
-            weight[now] = tmp;
-            path[now] = preY;
-          }
-        }
-      }
-    }
-
-    endE = weight[X-1+HMMModel::E*X];
-    endS = weight[X-1+HMMModel::S*X];
-    stat = 0;
-    if (endE >= endS) {
-      stat = HMMModel::E;
-    } else {
-      stat = HMMModel::S;
-    }
-
-    status.resize(X);
-    for (int x = X -1 ; x >= 0; x--) {
-      status[x] = stat;
-      stat = path[x + stat*X];
-    }
-  }
-
-  const HMMModel* model_;
-  bool isNeedDestroy_;
-}; // class HMMSegment
-
-} // namespace cppjieba
-
-#endif

data/ext/cppjieba/include/cppjieba/Jieba.hpp

@@ -1,130 +0,0 @@
-#ifndef CPPJIEAB_JIEBA_H
-#define CPPJIEAB_JIEBA_H
-
-#include "QuerySegment.hpp"
-#include "KeywordExtractor.hpp"
-
-namespace cppjieba {
-
-class Jieba {
- public:
-  Jieba(const string& dict_path, 
-        const string& model_path,
-        const string& user_dict_path, 
-        const string& idfPath, 
-        const string& stopWordPath) 
-    : dict_trie_(dict_path, user_dict_path),
-      model_(model_path),
-      mp_seg_(&dict_trie_),
-      hmm_seg_(&model_),
-      mix_seg_(&dict_trie_, &model_),
-      full_seg_(&dict_trie_),
-      query_seg_(&dict_trie_, &model_),
-      extractor(&dict_trie_, &model_, idfPath, stopWordPath) {
-  }
-  ~Jieba() {
-  }
-
-  struct LocWord {
-    string word;
-    size_t begin;
-    size_t end;
-  }; // struct LocWord
-
-  void Cut(const string& sentence, vector<string>& words, bool hmm = true) const {
-    mix_seg_.Cut(sentence, words, hmm);
-  }
-  void Cut(const string& sentence, vector<Word>& words, bool hmm = true) const {
-    mix_seg_.Cut(sentence, words, hmm);
-  }
-  void CutAll(const string& sentence, vector<string>& words) const {
-    full_seg_.Cut(sentence, words);
-  }
-  void CutAll(const string& sentence, vector<Word>& words) const {
-    full_seg_.Cut(sentence, words);
-  }
-  void CutForSearch(const string& sentence, vector<string>& words, bool hmm = true) const {
-    query_seg_.Cut(sentence, words, hmm);
-  }
-  void CutForSearch(const string& sentence, vector<Word>& words, bool hmm = true) const {
-    query_seg_.Cut(sentence, words, hmm);
-  }
-  void CutHMM(const string& sentence, vector<string>& words) const {
-    hmm_seg_.Cut(sentence, words);
-  }
-  void CutHMM(const string& sentence, vector<Word>& words) const {
-    hmm_seg_.Cut(sentence, words);
-  }
-  void CutSmall(const string& sentence, vector<string>& words, size_t max_word_len) const {
-    mp_seg_.Cut(sentence, words, max_word_len);
-  }
-  void CutSmall(const string& sentence, vector<Word>& words, size_t max_word_len) const {
-    mp_seg_.Cut(sentence, words, max_word_len);
-  }
-  
-  void Tag(const string& sentence, vector<pair<string, string> >& words) const {
-    mix_seg_.Tag(sentence, words);
-  }
-  string LookupTag(const string &str) const {
-    return mix_seg_.LookupTag(str);
-  }
-  bool InsertUserWord(const string& word, const string& tag = UNKNOWN_TAG) {
-    return dict_trie_.InsertUserWord(word, tag);
-  }
-
-  bool InsertUserWord(const string& word,int freq, const string& tag = UNKNOWN_TAG) {
-    return dict_trie_.InsertUserWord(word,freq, tag);
-  }
-
-  bool Find(const string& word)
-  {
-    return dict_trie_.Find(word);
-  }
-
-  void ResetSeparators(const string& s) {
-    //TODO
-    mp_seg_.ResetSeparators(s);
-    hmm_seg_.ResetSeparators(s);
-    mix_seg_.ResetSeparators(s);
-    full_seg_.ResetSeparators(s);
-    query_seg_.ResetSeparators(s);
-  }
-
-  const DictTrie* GetDictTrie() const {
-    return &dict_trie_;
-  } 
-  
-  const HMMModel* GetHMMModel() const {
-    return &model_;
-  }
-
-  void LoadUserDict(const vector<string>& buf)  {
-    dict_trie_.LoadUserDict(buf);
-  }
-
-  void LoadUserDict(const set<string>& buf)  {
-    dict_trie_.LoadUserDict(buf);
-  }
-
-  void LoadUserDict(const string& path)  {
-    dict_trie_.LoadUserDict(path);
-  }
-
- private:
-  DictTrie dict_trie_;
-  HMMModel model_;
-  
-  // They share the same dict trie and model
-  MPSegment mp_seg_;
-  HMMSegment hmm_seg_;
-  MixSegment mix_seg_;
-  FullSegment full_seg_;
-  QuerySegment query_seg_;
-
- public:
-  KeywordExtractor extractor;
-}; // class Jieba
-
-} // namespace cppjieba
-
-#endif // CPPJIEAB_JIEBA_H

data/ext/cppjieba/include/cppjieba/KeywordExtractor.hpp

@@ -1,153 +0,0 @@
-#ifndef CPPJIEBA_KEYWORD_EXTRACTOR_H
-#define CPPJIEBA_KEYWORD_EXTRACTOR_H
-
-#include <cmath>
-#include <set>
-#include "MixSegment.hpp"
-
-namespace cppjieba {
-
-using namespace limonp;
-using namespace std;
-
-/*utf8*/
-class KeywordExtractor {
- public:
-  struct Word {
-    string word;
-    vector<size_t> offsets;
-    double weight;
-  }; // struct Word
-
-  KeywordExtractor(const string& dictPath, 
-        const string& hmmFilePath, 
-        const string& idfPath, 
-        const string& stopWordPath, 
-        const string& userDict = "") 
-    : segment_(dictPath, hmmFilePath, userDict) {
-    LoadIdfDict(idfPath);
-    LoadStopWordDict(stopWordPath);
-  }
-  KeywordExtractor(const DictTrie* dictTrie, 
-        const HMMModel* model,
-        const string& idfPath, 
-        const string& stopWordPath) 
-    : segment_(dictTrie, model) {
-    LoadIdfDict(idfPath);
-    LoadStopWordDict(stopWordPath);
-  }
-  ~KeywordExtractor() {
-  }
-
-  void Extract(const string& sentence, vector<string>& keywords, size_t topN) const {
-    vector<Word> topWords;
-    Extract(sentence, topWords, topN);
-    for (size_t i = 0; i < topWords.size(); i++) {
-      keywords.push_back(topWords[i].word);
-    }
-  }
-
-  void Extract(const string& sentence, vector<pair<string, double> >& keywords, size_t topN) const {
-    vector<Word> topWords;
-    Extract(sentence, topWords, topN);
-    for (size_t i = 0; i < topWords.size(); i++) {
-      keywords.push_back(pair<string, double>(topWords[i].word, topWords[i].weight));
-    }
-  }
-
-  void Extract(const string& sentence, vector<Word>& keywords, size_t topN) const {
-    vector<string> words;
-    segment_.Cut(sentence, words);
-
-    map<string, Word> wordmap;
-    size_t offset = 0;
-    for (size_t i = 0; i < words.size(); ++i) {
-      size_t t = offset;
-      offset += words[i].size();
-      if (IsSingleWord(words[i]) || stopWords_.find(words[i]) != stopWords_.end()) {
-        continue;
-      }
-      wordmap[words[i]].offsets.push_back(t);
-      wordmap[words[i]].weight += 1.0;
-    }
-    if (offset != sentence.size()) {
-      XLOG(ERROR) << "words illegal";
-      return;
-    }
-
-    keywords.clear();
-    keywords.reserve(wordmap.size());
-    for (map<string, Word>::iterator itr = wordmap.begin(); itr != wordmap.end(); ++itr) {
-      unordered_map<string, double>::const_iterator cit = idfMap_.find(itr->first);
-      if (cit != idfMap_.end()) {
-        itr->second.weight *= cit->second;
-      } else {
-        itr->second.weight *= idfAverage_;
-      }
-      itr->second.word = itr->first;
-      keywords.push_back(itr->second);
-    }
-    topN = min(topN, keywords.size());
-    partial_sort(keywords.begin(), keywords.begin() + topN, keywords.end(), Compare);
-    keywords.resize(topN);
-  }
- private:
-  void LoadIdfDict(const string& idfPath) {
-    ifstream ifs(idfPath.c_str());
-    XCHECK(ifs.is_open()) << "open " << idfPath << " failed";
-    string line ;
-    vector<string> buf;
-    double idf = 0.0;
-    double idfSum = 0.0;
-    size_t lineno = 0;
-    for (; getline(ifs, line); lineno++) {
-      buf.clear();
-      if (line.empty()) {
-        XLOG(ERROR) << "lineno: " << lineno << " empty. skipped.";
-        continue;
-      }
-      Split(line, buf, " ");
-      if (buf.size() != 2) {
-        XLOG(ERROR) << "line: " << line << ", lineno: " << lineno << " empty. skipped.";
-        continue;
-      }
-      idf = atof(buf[1].c_str());
-      idfMap_[buf[0]] = idf;
-      idfSum += idf;
-
-    }
-
-    assert(lineno);
-    idfAverage_ = idfSum / lineno;
-    assert(idfAverage_ > 0.0);
-  }
-  void LoadStopWordDict(const string& filePath) {
-    ifstream ifs(filePath.c_str());
-    XCHECK(ifs.is_open()) << "open " << filePath << " failed";
-    string line ;
-    while (getline(ifs, line)) {
-      stopWords_.insert(line);
-    }
-    assert(stopWords_.size());
-  }
-
-  static bool Compare(const Word& lhs, const Word& rhs) {
-    return lhs.weight > rhs.weight;
-  }
-
-  MixSegment segment_;
-  unordered_map<string, double> idfMap_;
-  double idfAverage_;
-
-  unordered_set<string> stopWords_;
-}; // class KeywordExtractor
-
-inline ostream& operator << (ostream& os, const KeywordExtractor::Word& word) {
-  return os << "{\"word\": \"" << word.word << "\", \"offset\": " << word.offsets << ", \"weight\": " << word.weight << "}"; 
-}
-
-} // namespace cppjieba
-
-#endif
-
-