jieba_rb 0.0.1

data/ext/cppjieba/src/HMMSegment.hpp

@@ -0,0 +1,394 @@
+#ifndef CPPJIBEA_HMMSEGMENT_H
+#define CPPJIBEA_HMMSEGMENT_H
+
+#include <iostream>
+#include <fstream>
+#include <memory.h>
+#include <cassert>
+#include "Limonp/StringUtil.hpp"
+#include "Limonp/Logger.hpp"
+#include "TransCode.hpp"
+#include "ISegment.hpp"
+#include "SegmentBase.hpp"
+#include "DictTrie.hpp"
+
+namespace CppJieba
+{
+    using namespace Limonp;
+    typedef unordered_map<uint16_t, double> EmitProbMap;
+    class HMMSegment: public SegmentBase
+    {
+        public:
+            /*
+             * STATUS:
+             * 0:B, 1:E, 2:M, 3:S
+             * */
+            enum {B = 0, E = 1, M = 2, S = 3, STATUS_SUM = 4};
+        private:
+            char _statMap[STATUS_SUM];
+            double _startProb[STATUS_SUM];
+            double _transProb[STATUS_SUM][STATUS_SUM];
+            EmitProbMap _emitProbB;
+            EmitProbMap _emitProbE;
+            EmitProbMap _emitProbM;
+            EmitProbMap _emitProbS;
+            vector<EmitProbMap* > _emitProbVec;
+
+        public:
+            HMMSegment(){}
+            explicit HMMSegment(const string& filePath)
+            {
+                LIMONP_CHECK(init(filePath));
+            }
+            virtual ~HMMSegment(){}
+        public:
+            bool init(const string& filePath)
+            {
+                memset(_startProb, 0, sizeof(_startProb));
+                memset(_transProb, 0, sizeof(_transProb));
+                _statMap[0] = 'B';
+                _statMap[1] = 'E';
+                _statMap[2] = 'M';
+                _statMap[3] = 'S';
+                _emitProbVec.push_back(&_emitProbB);
+                _emitProbVec.push_back(&_emitProbE);
+                _emitProbVec.push_back(&_emitProbM);
+                _emitProbVec.push_back(&_emitProbS);
+                LIMONP_CHECK(_loadModel(filePath.c_str()));
+                LogInfo("HMMSegment init(%s) ok.", filePath.c_str());
+                return true;
+            }
+        public:
+            using SegmentBase::cut;
+        public:
+            bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res)const 
+            {
+                Unicode::const_iterator left = begin;
+                Unicode::const_iterator right = begin;
+                while(right != end)
+                {
+                    if(*right < 0x80) 
+                    {
+                        if(left != right && !_cut(left, right, res))
+                        {
+                            return false;
+                        }
+                        left = right;
+                        do {
+                            right = _sequentialLetterRule(left, end);
+                            if(right != left)
+                            {
+                                break;
+                            }
+                            right = _numbersRule(left, end);
+                            if(right != left)
+                            {
+                                break;
+                            }
+                            right ++;
+                        } while(false);
+                        res.push_back(Unicode(left, right));
+                        left = right;
+                    }
+                    else
+                    {
+                        right++;
+                    }
+                }
+                if(left != right && !_cut(left, right, res))
+                {
+                    return false;
+                }
+                return true;
+            }
+        private:
+            // sequential letters rule
+            Unicode::const_iterator _sequentialLetterRule(Unicode::const_iterator begin, Unicode::const_iterator end) const
+            {
+                Unicode::value_type x;
+                while(begin != end)
+                {
+                    x = *begin;
+                    if(('a' <= x && x <= 'z') || ('A' <= x && x <= 'Z'))
+                    {
+                        begin ++;
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+                return begin;
+            }
+            // 
+            Unicode::const_iterator _numbersRule(Unicode::const_iterator begin, Unicode::const_iterator end) const
+            {
+                Unicode::value_type x = *begin;
+                if('0' <= x && x <= '9')
+                {
+                    begin ++;
+                }
+                else
+                {
+                    return begin;
+                }
+                while(begin != end)
+                {
+                    x = *begin;
+                    if( ('0' <= x && x <= '9') || x == '.')
+                    {
+                        begin++;
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
+                return begin;
+            }
+            bool _cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const 
+            {
+                vector<size_t> status; 
+                if(!_viterbi(begin, end, status))
+                {
+                    LogError("_viterbi failed.");
+                    return false;
+                }
+
+                Unicode::const_iterator left = begin;
+                Unicode::const_iterator right;
+                for(size_t i = 0; i < status.size(); i++)
+                {
+                    if(status[i] % 2) //if(E == status[i] || S == status[i])
+                    {
+                        right = begin + i + 1;
+                        res.push_back(Unicode(left, right));
+                        left = right;
+                    }
+                }
+                return true;
+            }
+        public:
+            virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res)const
+            {
+                if(begin == end)
+                {
+                    return false;
+                }
+                vector<Unicode> words;
+                words.reserve(end - begin);
+                if(!cut(begin, end, words))
+                {
+                    return false;
+                }
+                size_t offset = res.size();
+                res.resize(res.size() + words.size());
+                for(size_t i = 0; i < words.size(); i++)
+                {
+                    if(!TransCode::encode(words[i], res[offset + i]))
+                    {
+                        LogError("encode failed.");
+                    }
+                }
+                return true;
+            }
+
+        private:
+            bool _viterbi(Unicode::const_iterator begin, Unicode::const_iterator end, vector<size_t>& status)const
+            {
+                if(begin == end)
+                {
+                    return false;
+                }
+
+                size_t Y = STATUS_SUM;
+                size_t X = end - begin;
+
+                size_t XYSize = X * Y;
+                size_t now, old, stat;
+                double tmp, endE, endS;
+
+                vector<int> path(XYSize);
+                vector<double> weight(XYSize);
+
+                //start
+                for(size_t y = 0; y < Y; y++)
+                {
+                    weight[0 + y * X] = _startProb[y] + _getEmitProb(_emitProbVec[y], *begin, MIN_DOUBLE);
+                    path[0 + y * X] = -1;
+                }
+
+
+                double emitProb;
+
+                for(size_t x = 1; x < X; x++)
+                {
+                    for(size_t y = 0; y < Y; y++)
+                    {
+                        now = x + y*X;
+                        weight[now] = MIN_DOUBLE;
+                        path[now] = E; // warning
+                        emitProb = _getEmitProb(_emitProbVec[y], *(begin+x), MIN_DOUBLE);
+                        for(size_t preY = 0; preY < Y; preY++)
+                        {
+                            old = x - 1 + preY * X;
+                            tmp = weight[old] + _transProb[preY][y] + emitProb;
+                            if(tmp > weight[now])
+                            {
+                                weight[now] = tmp;
+                                path[now] = preY;
+                            }
+                        }
+                    }
+                }
+
+                endE = weight[X-1+E*X];
+                endS = weight[X-1+S*X];
+                stat = 0;
+                if(endE >= endS)
+                {
+                    stat = E;
+                }
+                else
+                {
+                    stat = S;
+                }
+
+                status.resize(X);
+                for(int x = X -1 ; x >= 0; x--)
+                {
+                    status[x] = stat;
+                    stat = path[x + stat*X];
+                }
+
+                return true;
+            }
+            bool _loadModel(const char* const filePath)
+            {
+                LogDebug("loadModel [%s] start ...", filePath);
+                ifstream ifile(filePath);
+                string line;
+                vector<string> tmp;
+                vector<string> tmp2;
+                //load _startProb
+                if(!_getLine(ifile, line))
+                {
+                    return false;
+                }
+                split(line, tmp, " ");
+                if(tmp.size() != STATUS_SUM)
+                {
+                    LogError("start_p illegal");
+                    return false;
+                }
+                for(size_t j = 0; j< tmp.size(); j++)
+                {
+                    _startProb[j] = atof(tmp[j].c_str());
+                }
+
+                //load _transProb
+                for(size_t i = 0; i < STATUS_SUM; i++)
+                {
+                    if(!_getLine(ifile, line))
+                    {
+                        return false;
+                    }
+                    split(line, tmp, " ");
+                    if(tmp.size() != STATUS_SUM)
+                    {
+                        LogError("trans_p illegal");
+                        return false;
+                    }
+                    for(size_t j =0; j < STATUS_SUM; j++)
+                    {
+                        _transProb[i][j] = atof(tmp[j].c_str());
+                    }
+                }
+
+                //load _emitProbB
+                if(!_getLine(ifile, line) || !_loadEmitProb(line, _emitProbB))
+                {
+                    return false;
+                }
+
+                //load _emitProbE
+                if(!_getLine(ifile, line) || !_loadEmitProb(line, _emitProbE))
+                {
+                    return false;
+                }
+
+                //load _emitProbM
+                if(!_getLine(ifile, line) || !_loadEmitProb(line, _emitProbM))
+                {
+                    return false;
+                }
+
+                //load _emitProbS
+                if(!_getLine(ifile, line) || !_loadEmitProb(line, _emitProbS))
+                {
+                    return false;
+                }
+
+                LogDebug("loadModel [%s] end.", filePath);
+
+                return true;
+            }
+            bool _getLine(ifstream& ifile, string& line)
+            {
+                while(getline(ifile, line))
+                {
+                    trim(line);
+                    if(line.empty())
+                    {
+                        continue;
+                    }
+                    if(startsWith(line, "#"))
+                    {
+                        continue;
+                    }
+                    return true;
+                }
+                return false;
+            }
+            bool _loadEmitProb(const string& line, EmitProbMap& mp)
+            {
+                if(line.empty())
+                {
+                    return false;
+                }
+                vector<string> tmp, tmp2;
+                Unicode unicode;
+                split(line, tmp, ",");
+                for(size_t i = 0; i < tmp.size(); i++)
+                {
+                    split(tmp[i], tmp2, ":");
+                    if(2 != tmp2.size())
+                    {
+                        LogError("_emitProb illegal.");
+                        return false;
+                    }
+                    if(!TransCode::decode(tmp2[0], unicode) || unicode.size() != 1)
+                    {
+                        LogError("TransCode failed.");
+                        return false;
+                    }
+                    mp[unicode[0]] = atof(tmp2[1].c_str());
+                }
+                return true;
+            }
+            double _getEmitProb(const EmitProbMap* ptMp, uint16_t key, double defVal)const 
+            {
+                EmitProbMap::const_iterator cit = ptMp->find(key);
+                if(cit == ptMp->end())
+                {
+                    return defVal;
+                }
+                return cit->second;
+
+            }
+
+
+    };
+}
+
+#endif

data/ext/cppjieba/src/ISegment.hpp

@@ -0,0 +1,17 @@
+#ifndef CPPJIEBA_SEGMENTINTERFACE_H
+#define CPPJIEBA_SEGMENTINTERFACE_H
+
+
+namespace CppJieba
+{
+    class ISegment
+    {
+        public:
+            virtual ~ISegment(){};
+        public:
+            virtual bool cut(Unicode::const_iterator begin , Unicode::const_iterator end, vector<string>& res) const = 0;
+            virtual bool cut(const string& str, vector<string>& res) const = 0;
+    };
+}
+
+#endif

data/ext/cppjieba/src/KeywordExtractor.hpp

@@ -0,0 +1,173 @@
+#ifndef CPPJIEBA_KEYWORD_EXTRACTOR_H
+#define CPPJIEBA_KEYWORD_EXTRACTOR_H
+
+#include "MixSegment.hpp"
+#include <cmath>
+#include <set>
+
+namespace CppJieba
+{
+    using namespace Limonp;
+
+    /*utf8*/
+    class KeywordExtractor
+    {
+        private:
+            MixSegment _segment;
+        private:
+            unordered_map<string, double> _idfMap;
+            double _idfAverage;
+
+            unordered_set<string> _stopWords;
+        public:
+            KeywordExtractor(){};
+            KeywordExtractor(const string& dictPath, const string& hmmFilePath, const string& idfPath, const string& stopWordPath)
+            {
+                LIMONP_CHECK(init(dictPath, hmmFilePath, idfPath, stopWordPath));
+            };
+            ~KeywordExtractor(){};
+
+        public:
+            bool init(const string& dictPath, const string& hmmFilePath, const string& idfPath, const string& stopWordPath)
+            {
+                _loadIdfDict(idfPath);
+                _loadStopWordDict(stopWordPath);
+                LIMONP_CHECK(_segment.init(dictPath, hmmFilePath));
+                return true;
+            };
+        public:
+
+            bool extract(const string& str, vector<string>& keywords, size_t topN) const
+            {
+                vector<pair<string, double> > topWords;
+                if(!extract(str, topWords, topN))
+                {
+                    return false;
+                }
+                for(size_t i = 0; i < topWords.size(); i++)
+                {
+                    keywords.push_back(topWords[i].first);
+                }
+                return true;
+            }
+
+            bool extract(const string& str, vector<pair<string, double> >& keywords, size_t topN) const
+            {
+                vector<string> words;
+                if(!_segment.cut(str, words))
+                {
+                    LogError("segment cut(%s) failed.", str.c_str());
+                    return false;
+                }
+
+                map<string, double> wordmap;
+                for(vector<string>::iterator iter = words.begin(); iter != words.end(); iter++)
+                {
+                    if(_isSingleWord(*iter))
+                    {
+                        continue;
+                    }
+                    wordmap[*iter] += 1.0;
+                }
+
+                for(map<string, double>::iterator itr = wordmap.begin(); itr != wordmap.end(); )
+                {
+                    if(_stopWords.end() != _stopWords.find(itr->first))
+                    {
+                        wordmap.erase(itr++);
+                        continue;
+                    }
+
+                    unordered_map<string, double>::const_iterator cit = _idfMap.find(itr->first);
+                    if(cit != _idfMap.end())
+                    {
+                        itr->second *= cit->second;
+                    }
+                    else
+                    {
+                        itr->second *= _idfAverage;
+                    }
+                    itr ++;
+                }
+
+                keywords.clear();
+                std::copy(wordmap.begin(), wordmap.end(), std::inserter(keywords, keywords.begin()));
+                topN = min(topN, keywords.size());
+                partial_sort(keywords.begin(), keywords.begin() + topN, keywords.end(), _cmp);
+                keywords.resize(topN);
+                return true;
+            }
+        private:
+            void _loadIdfDict(const string& idfPath)
+            {
+                ifstream ifs(idfPath.c_str());
+                if(!ifs)
+                {
+                    LogError("open %s failed.", idfPath.c_str());
+                    assert(false);
+                }
+                string line ;
+                vector<string> buf;
+                double idf = 0.0;
+                double idfSum = 0.0;
+                size_t lineno = 0;
+                for(;getline(ifs, line); lineno++)
+                {
+                    buf.clear();
+                    if(line.empty())
+                    {
+                        LogError("line[%d] empty. skipped.", lineno);
+                        continue;
+                    }
+                    if(!split(line, buf, " ") || buf.size() != 2)
+                    {
+                        LogError("line %d [%s] illegal. skipped.", lineno, line.c_str());
+                        continue;
+                    }
+                    idf = atof(buf[1].c_str());
+                    _idfMap[buf[0]] = idf;
+                    idfSum += idf;
+
+                } 
+
+                assert(lineno);
+                _idfAverage = idfSum / lineno;
+                assert(_idfAverage > 0.0);
+            }
+            void _loadStopWordDict(const string& filePath)
+            {
+                ifstream ifs(filePath.c_str());
+                if(!ifs)
+                {
+                    LogError("open %s failed.", filePath.c_str());
+                    assert(false);
+                }
+                string line ;
+                while(getline(ifs, line))
+                {
+                    _stopWords.insert(line);
+                }
+                assert(_stopWords.size());
+            }
+        private:
+            bool _isSingleWord(const string& str) const
+            {
+                Unicode unicode;
+                TransCode::decode(str, unicode);
+                if(unicode.size() == 1)
+                  return true;
+                return false;
+            }
+
+        private:
+            static bool _cmp(const pair<string, double>& lhs, const pair<string, double>& rhs)
+            {
+                return lhs.second > rhs.second;
+            }
+            
+    };
+}
+
+#endif
+
+

data/ext/cppjieba/src/Limonp/ArgvContext.hpp

@@ -0,0 +1,84 @@
+/************************************
+ * file enc : ascii
+ * author   : wuyanyi09@gmail.com
+ ************************************/
+
+#ifndef LIMONP_ARGV_FUNCTS_H
+#define LIMONP_ARGV_FUNCTS_H
+
+#include <set>
+#include <sstream>
+#include "StringUtil.hpp"
+
+namespace Limonp
+{
+    using namespace std;
+    class ArgvContext
+    {
+        public :
+            ArgvContext(int argc, const char* const * argv)
+            {
+
+                for(int i = 0; i < argc; i++)
+                {
+                    if(startsWith(argv[i], "-"))
+                    {
+                        if(i + 1 < argc && !startsWith(argv[i + 1], "-"))
+                        {
+                            mpss_[argv[i]] = argv[i+1];
+                            i++;
+                        }
+                        else
+                        {
+                            sset_.insert(argv[i]);
+                        }
+                    }
+                    else
+                    {
+                        args_.push_back(argv[i]);
+                    }
+                }
+            }
+            ~ArgvContext(){};
+        public:
+            friend ostream& operator << (ostream& os, const ArgvContext& args); 
+            string operator [](size_t i) const
+            {
+                if(i < args_.size())
+                {
+                    return args_[i];
+                }
+                return "";
+            }
+            string operator [](const string& key) const
+            {
+                map<string, string>::const_iterator it = mpss_.find(key);
+                if(it != mpss_.end())
+                {
+                    return it->second;
+                }
+                return "";
+            }
+        public:
+            bool hasKey(const string& key) const
+            {
+                if(mpss_.find(key) != mpss_.end() || sset_.find(key) != sset_.end())
+                {
+                    return true;
+                }
+                return false;
+            }
+        private:
+            vector<string> args_;
+            map<string, string> mpss_;
+            set<string> sset_; 
+
+    };
+
+    inline ostream& operator << (ostream& os, const ArgvContext& args)
+    {
+        return os<<args.args_<<args.mpss_<<args.sset_;
+    }
+}
+
+#endif