jieba_rb 0.0.1

data/ext/cppjieba/src/MPSegment.hpp

@@ -0,0 +1,148 @@
+#ifndef CPPJIEBA_MPSEGMENT_H
+#define CPPJIEBA_MPSEGMENT_H
+
+#include <algorithm>
+#include <set>
+#include <cassert>
+#include "Limonp/Logger.hpp"
+#include "DictTrie.hpp"
+#include "ISegment.hpp"
+#include "SegmentBase.hpp"
+
+namespace CppJieba
+{
+
+    class MPSegment: public SegmentBase
+    {
+        private:
+            DictTrie _dictTrie;
+
+        public:
+            MPSegment(){};
+            MPSegment(const string& dictPath, const string& userDictPath = "")
+            {
+                LIMONP_CHECK(init(dictPath, userDictPath));
+            };
+            virtual ~MPSegment(){};
+        public:
+            bool init(const string& dictPath, const string& userDictPath = "")
+            {
+                LIMONP_CHECK(_dictTrie.init(dictPath, userDictPath));
+                LogInfo("MPSegment init(%s) ok", dictPath.c_str());
+                return true;
+            }
+            bool isUserDictSingleChineseWord(const Unicode::value_type & value) const
+            {
+                return _dictTrie.isUserDictSingleChineseWord(value);
+            }
+        public:
+            using SegmentBase::cut;
+            virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res)const
+            {
+                if(begin == end)
+                {
+                    return false;
+                }
+
+                vector<Unicode> words;
+                words.reserve(end - begin);
+                if(!cut(begin, end, words))
+                {
+                    return false;
+                }
+                size_t offset = res.size();
+                res.resize(res.size() + words.size());
+                for(size_t i = 0; i < words.size(); i++)
+                {
+                    if(!TransCode::encode(words[i], res[i + offset]))
+                    {
+                        LogError("encode failed.");
+                        res[i + offset].clear();
+                    }
+                }
+                return true;
+            }
+
+            bool cut(Unicode::const_iterator begin , Unicode::const_iterator end, vector<Unicode>& res) const
+            {
+                if(end == begin)
+                {
+                    return false;
+                }
+                vector<SegmentChar> segmentChars;
+
+                _dictTrie.find(begin, end, segmentChars);
+
+                _calcDP(segmentChars);
+
+                _cut(segmentChars, res);
+
+                return true;
+            }
+            const DictTrie* getDictTrie() const 
+            {
+                return &_dictTrie;
+            }
+
+        private:
+            void _calcDP(vector<SegmentChar>& segmentChars) const
+            {
+                size_t nextPos;
+                const DictUnit* p;
+                double val;
+
+                for(ssize_t i = segmentChars.size() - 1; i >= 0; i--)
+                {
+                    segmentChars[i].pInfo = NULL;
+                    segmentChars[i].weight = MIN_DOUBLE;
+                    assert(!segmentChars[i].dag.empty());
+                    for(DagType::const_iterator it = segmentChars[i].dag.begin(); it != segmentChars[i].dag.end(); it++)
+                    {
+                        nextPos = it->first;
+                        p = it->second;
+                        val = 0.0;
+                        if(nextPos + 1 < segmentChars.size())
+                        {
+                            val += segmentChars[nextPos + 1].weight;
+                        }
+
+                        if(p)
+                        {
+                            val += p->weight; 
+                        }
+                        else
+                        {
+                            val += _dictTrie.getMinWeight();
+                        }
+                        if(val > segmentChars[i].weight)
+                        {
+                            segmentChars[i].pInfo = p;
+                            segmentChars[i].weight = val;
+                        }
+                    }
+                }
+            }
+            void _cut(const vector<SegmentChar>& segmentChars, vector<Unicode>& res) const
+            {
+                size_t i = 0;
+                while(i < segmentChars.size())
+                {
+                    const DictUnit* p = segmentChars[i].pInfo;
+                    if(p)
+                    {
+                        res.push_back(p->word);
+                        i += p->word.size();
+                    }
+                    else//single chinese word
+                    {
+                        res.push_back(Unicode(1, segmentChars[i].uniCh));
+                        i++;
+                    }
+                }
+            }
+
+
+    };
+}
+
+#endif

data/ext/cppjieba/src/MixSegment.hpp

@@ -0,0 +1,121 @@
+#ifndef CPPJIEBA_MIXSEGMENT_H
+#define CPPJIEBA_MIXSEGMENT_H
+
+#include <cassert>
+#include "MPSegment.hpp"
+#include "HMMSegment.hpp"
+#include "Limonp/StringUtil.hpp"
+
+namespace CppJieba
+{
+    class MixSegment: public SegmentBase
+    {
+        private:
+            MPSegment _mpSeg;
+            HMMSegment _hmmSeg;
+        public:
+            MixSegment(){};
+            MixSegment(const string& mpSegDict, const string& hmmSegDict, const string& userDict = "")
+            {
+                LIMONP_CHECK(init(mpSegDict, hmmSegDict, userDict));
+            }
+            virtual ~MixSegment(){}
+        public:
+            bool init(const string& mpSegDict, const string& hmmSegDict, const string& userDict = "")
+            {
+                LIMONP_CHECK(_mpSeg.init(mpSegDict, userDict));
+                LIMONP_CHECK(_hmmSeg.init(hmmSegDict));
+                LogInfo("MixSegment init(%s, %s)", mpSegDict.c_str(), hmmSegDict.c_str());
+                return true;
+            }
+        public:
+            using SegmentBase::cut;
+        public:
+            virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const
+            {
+                vector<Unicode> words;
+                words.reserve(end - begin);
+                if(!_mpSeg.cut(begin, end, words))
+                {
+                    LogError("mpSeg cutDAG failed.");
+                    return false;
+                }
+
+                vector<Unicode> hmmRes;
+                hmmRes.reserve(end - begin);
+                Unicode piece;
+                piece.reserve(end - begin);
+                for (size_t i = 0, j = 0; i < words.size(); i++)
+                {
+                    //if mp get a word, it's ok, put it into result
+                    if (1 != words[i].size() || (words[i].size() == 1 && _mpSeg.isUserDictSingleChineseWord(words[i][0])))
+                    {
+                        res.push_back(words[i]);
+                        continue;
+                    }
+
+                    // if mp get a single one and it is not in userdict, collect it in sequence
+                    j = i;
+                    while (j < words.size() && 1 == words[j].size() && !_mpSeg.isUserDictSingleChineseWord(words[j][0]))
+                    {
+                        piece.push_back(words[j][0]);
+                        j++;
+                    }
+
+                    // cut the sequence with hmm
+                    if (!_hmmSeg.cut(piece.begin(), piece.end(), hmmRes))
+                    {
+                        LogError("_hmmSeg cut failed.");
+                        return false;
+                    }
+
+                    //put hmm result to result
+                    for (size_t k = 0; k < hmmRes.size(); k++)
+                    {
+                        res.push_back(hmmRes[k]);
+                    }
+
+                    //clear tmp vars
+                    piece.clear();
+                    hmmRes.clear();
+
+                    //let i jump over this piece
+                    i = j - 1;
+                }
+                return true;
+            }
+
+            virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res)const
+            {
+                if(begin == end)
+                {
+                    return false;
+                }
+
+                vector<Unicode> uRes;
+                uRes.reserve(end - begin);
+                if (!cut(begin, end, uRes))
+                {
+                    return false;
+                }
+
+                size_t offset = res.size();
+                res.resize(res.size() + uRes.size());
+                for(size_t i = 0; i < uRes.size(); i ++, offset++)
+                {
+                    if(!TransCode::encode(uRes[i], res[offset]))
+                    {
+                        LogError("encode failed.");
+                    }
+                }
+                return true;
+            }
+
+            const DictTrie* getDictTrie() const 
+            {
+                return _mpSeg.getDictTrie();
+            }
+    };
+}
+
+#endif

data/ext/cppjieba/src/PosTagger.hpp

@@ -0,0 +1,109 @@
+#ifndef CPPJIEBA_POS_TAGGING_H
+#define CPPJIEBA_POS_TAGGING_H
+
+#include "MixSegment.hpp"
+#include "Limonp/StringUtil.hpp"
+#include "DictTrie.hpp"
+
+namespace CppJieba
+{
+    using namespace Limonp;
+
+    static const char* const POS_M = "m";
+    static const char* const POS_ENG = "eng";
+    static const char* const POS_X = "x";
+
+    class PosTagger
+    {
+        private:
+            MixSegment _segment;
+            const DictTrie * _dictTrie;
+
+        public:
+            PosTagger()
+            {}
+            PosTagger(
+                const string& dictPath, 
+                const string& hmmFilePath,
+                const string& userDictPath = ""
+            )
+            {
+                init(dictPath, hmmFilePath, userDictPath);
+            };
+            ~PosTagger(){};
+        public:
+            void init(
+                const string& dictPath, 
+                const string& hmmFilePath,
+                const string& userDictPath = ""
+            )
+            {
+                LIMONP_CHECK(_segment.init(dictPath, hmmFilePath, userDictPath));
+                _dictTrie = _segment.getDictTrie();
+                LIMONP_CHECK(_dictTrie);
+            };
+            
+
+            bool tag(const string& src, vector<pair<string, string> >& res) const
+            {
+                vector<string> cutRes;
+                if (!_segment.cut(src, cutRes))
+                {
+                    LogError("_mixSegment cut failed");
+                    return false;
+                }
+
+                const DictUnit *tmp = NULL;
+                Unicode unico;
+                for (vector<string>::iterator itr = cutRes.begin(); itr != cutRes.end(); ++itr)
+                {
+                    if (!TransCode::decode(*itr, unico))
+                    {
+                        LogError("decode failed.");
+                        return false;
+                    }
+                    tmp = _dictTrie->find(unico.begin(), unico.end());
+                    if(tmp == NULL || tmp->tag.empty())
+                    {
+                        res.push_back(make_pair(*itr, _specialRule(unico)));
+                    }
+                    else
+                    {
+                        res.push_back(make_pair(*itr, tmp->tag));
+                    }
+                }
+                return !res.empty();
+            }
+        private:
+            const char* _specialRule(const Unicode& unicode) const
+            {
+                size_t m = 0;
+                size_t eng = 0;
+                for(size_t i = 0; i < unicode.size() && eng < unicode.size() / 2; i++) 
+                {
+                    if(unicode[i] < 0x80)
+                    {
+                        eng ++;
+                        if('0' <= unicode[i] && unicode[i] <= '9')
+                        {
+                            m++;
+                        }
+                    }
+                }
+                // ascii char is not found
+                if(eng == 0)
+                {
+                    return POS_X;
+                }
+                // all the ascii is number char
+                if(m == eng)
+                {
+                    return POS_M;
+                }
+                // the ascii chars contain english letter
+                return POS_ENG;
+            }
+    };
+}
+
+#endif

data/ext/cppjieba/src/QuerySegment.hpp

@@ -0,0 +1,123 @@
+#ifndef CPPJIEBA_QUERYSEGMENT_H
+#define CPPJIEBA_QUERYSEGMENT_H
+
+#include <algorithm>
+#include <set>
+#include <cassert>
+#include "Limonp/Logger.hpp"
+#include "DictTrie.hpp"
+#include "ISegment.hpp"
+#include "SegmentBase.hpp"
+#include "FullSegment.hpp"
+#include "MixSegment.hpp"
+#include "TransCode.hpp"
+#include "DictTrie.hpp"
+
+namespace CppJieba
+{
+    class QuerySegment: public SegmentBase
+    {
+    private:
+        MixSegment _mixSeg;
+        FullSegment _fullSeg;
+        size_t _maxWordLen;
+
+    public:
+        QuerySegment(){};
+        QuerySegment(const string& dict, const string& model, size_t maxWordLen)
+        {
+            init(dict, model, maxWordLen);
+        };
+        virtual ~QuerySegment(){};
+    public:
+        bool init(const string& dict, const string& model, size_t maxWordLen)
+        {
+            LIMONP_CHECK(_mixSeg.init(dict, model));
+            LIMONP_CHECK(_fullSeg.init(_mixSeg.getDictTrie()));
+            assert(maxWordLen);
+            _maxWordLen = maxWordLen;
+            return true;
+        }
+
+    public:
+        using SegmentBase::cut;
+
+    public:
+        bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<Unicode>& res) const
+        {
+            if (begin >= end)
+            {
+                LogError("begin >= end");
+                return false;
+            }
+
+            //use mix cut first
+            vector<Unicode> mixRes;
+            if (!_mixSeg.cut(begin, end, mixRes))
+            {
+                LogError("_mixSeg cut failed.");
+                return false;
+            }
+
+            vector<Unicode> fullRes;
+            for (vector<Unicode>::const_iterator mixResItr = mixRes.begin(); mixResItr != mixRes.end(); mixResItr++)
+            {
+                
+                // if it's too long, cut with _fullSeg, put fullRes in res
+                if (mixResItr->size() > _maxWordLen)
+                {
+                    if (_fullSeg.cut(mixResItr->begin(), mixResItr->end(), fullRes))
+                    {
+                       for (vector<Unicode>::const_iterator fullResItr = fullRes.begin(); fullResItr != fullRes.end(); fullResItr++)
+                       {
+                           res.push_back(*fullResItr);
+                       }
+
+                       //clear tmp res
+                       fullRes.clear();
+                    }
+                }
+                else // just use the mix result
+                {
+                    res.push_back(*mixResItr);
+                }
+            }
+
+            return true;
+        }
+
+
+        bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res) const
+        {
+            if (begin >= end)
+            {
+                LogError("begin >= end");
+                return false;
+            }
+
+            vector<Unicode> uRes;
+            if (!cut(begin, end, uRes))
+            {
+                LogError("get unicode cut result error.");
+                return false;
+            }
+
+            string tmp;
+            for (vector<Unicode>::const_iterator uItr = uRes.begin(); uItr != uRes.end(); uItr++)
+            {
+                if (TransCode::encode(*uItr, tmp))
+                {
+                    res.push_back(tmp);
+                }
+                else
+                {
+                    LogError("encode failed.");
+                }
+            }
+
+            return true;
+        }
+    };
+}
+
+#endif

data/ext/cppjieba/src/SegmentBase.hpp

@@ -0,0 +1,78 @@
+#ifndef CPPJIEBA_SEGMENTBASE_H
+#define CPPJIEBA_SEGMENTBASE_H
+
+#include "TransCode.hpp"
+#include "Limonp/Logger.hpp"
+#include "Limonp/NonCopyable.hpp"
+#include "Limonp/HandyMacro.hpp"
+#include "ISegment.hpp"
+#include <cassert>
+
+
+namespace CppJieba
+{
+    using namespace Limonp;
+
+    //const char* const SPECIAL_CHARS = " \t\n";
+#ifndef CPPJIEBA_GBK
+    const UnicodeValueType SPECIAL_SYMBOL[] = {32u, 9u, 10u, 12290u, 65292u};  
+#else
+    const UnicodeValueType SPECIAL_SYMBOL[] = {32u, 9u, 10u};  
+#endif
+
+    class SegmentBase: public ISegment, public NonCopyable
+    {
+        public:
+            SegmentBase(){_loadSpecialSymbols();};
+            virtual ~SegmentBase(){};
+        private:
+            unordered_set<UnicodeValueType> _specialSymbols;
+        private:
+            void _loadSpecialSymbols()
+            {
+                size_t size = sizeof(SPECIAL_SYMBOL)/sizeof(*SPECIAL_SYMBOL);
+                for(size_t i = 0; i < size; i ++)
+                {
+                    _specialSymbols.insert(SPECIAL_SYMBOL[i]);
+                }
+                assert(_specialSymbols.size());
+            }
+
+        public:
+            virtual bool cut(Unicode::const_iterator begin, Unicode::const_iterator end, vector<string>& res) const = 0;
+            virtual bool cut(const string& str, vector<string>& res) const
+            {
+                res.clear();
+
+                Unicode unicode;
+                unicode.reserve(str.size());
+
+                TransCode::decode(str, unicode);
+                
+                Unicode::const_iterator left = unicode.begin();
+                Unicode::const_iterator right;
+                
+                for(right = unicode.begin(); right != unicode.end(); right++)
+                {
+                    if(isIn(_specialSymbols, *right))
+                    {
+                        if(left != right)
+                        {
+                            cut(left, right, res);
+                        }
+                        res.resize(res.size() + 1);
+                        TransCode::encode(right, right + 1, res.back());
+                        left = right + 1;
+                    }
+                }
+                if(left != right)
+                {
+                    cut(left, right, res);
+                }
+                
+                return true;
+            }
+    };
+}
+
+#endif

data/ext/cppjieba/src/TransCode.hpp

@@ -0,0 +1,63 @@
+/************************************
+ * file enc : utf-8
+ * author   : wuyanyi09@gmail.com
+ ************************************/
+#ifndef CPPJIEBA_TRANSCODE_H
+#define CPPJIEBA_TRANSCODE_H
+
+
+#include "Limonp/StringUtil.hpp"
+#include "Limonp/LocalVector.hpp"
+
+namespace CppJieba
+{
+
+    using namespace Limonp;
+    typedef uint16_t UnicodeValueType;
+    typedef Limonp::LocalVector<UnicodeValueType> Unicode;
+    namespace TransCode
+    {
+        inline bool decode(const string& str, Unicode& res)
+        {
+#ifdef CPPJIEBA_GBK
+            return gbkTrans(str, res);
+#else
+            return utf8ToUnicode(str, res);
+#endif
+        }
+
+        inline bool encode(Unicode::const_iterator begin, Unicode::const_iterator end, string& res)
+        {
+#ifdef CPPJIEBA_GBK
+            return gbkTrans(begin, end, res);
+#else
+            return unicodeToUtf8(begin, end, res);
+#endif
+        }
+        
+        inline bool encode(const Unicode& uni, string& res)
+        {
+            return encode(uni.begin(), uni.end(), res);
+        }
+
+        // compiler is expected to optimized this function to avoid return value copy
+        inline string encode(Unicode::const_iterator begin, Unicode::const_iterator end) 
+        {
+            string res;
+            res.reserve(end - begin);
+            encode(begin, end, res);
+            return res;
+        }
+
+        // compiler is expected to optimized this function to avoid return value copy
+        inline Unicode decode(const string& str)
+        {
+            Unicode unicode;
+            unicode.reserve(str.size());
+            decode(str, unicode);
+            return unicode;
+        }
+    }
+}
+
+#endif