jieba_rb 0.0.1

data/ext/cppjieba/src/Trie.hpp

@@ -0,0 +1,298 @@
+#ifndef CPPJIEBA_TRIE_HPP
+#define CPPJIEBA_TRIE_HPP
+
+#include "Limonp/StdExtension.hpp"
+#include <vector>
+#include <queue>
+
+namespace CppJieba
+{
+    using namespace std;
+
+    struct DictUnit
+    {
+        Unicode word;
+        double weight; 
+        string tag;
+    };
+
+    // for debugging
+    inline ostream & operator << (ostream& os, const DictUnit& unit)
+    {
+        string s;
+        s << unit.word;
+        return os << string_format("%s %s %.3lf", s.c_str(), unit.tag.c_str(), unit.weight);
+    }
+
+    typedef LocalVector<std::pair<size_t, const DictUnit*> > DagType;
+
+    struct SegmentChar 
+    {
+        uint16_t uniCh;
+        DagType dag;
+        const DictUnit * pInfo;
+        double weight;
+        size_t nextPos;
+        SegmentChar():uniCh(0), pInfo(NULL), weight(0.0), nextPos(0)
+        {}
+        ~SegmentChar() 
+        {}
+    };
+
+    typedef Unicode::value_type TrieKey;
+
+    class TrieNode
+    {
+        public:
+            typedef unordered_map<TrieKey,  TrieNode*> NextMap;
+        public:
+            TrieNode * fail;
+            NextMap * next;
+            const DictUnit * ptValue;
+        public:
+            TrieNode(): fail(NULL), next(NULL), ptValue(NULL) 
+            {}
+            const TrieNode * findNext(TrieKey key) const
+            {
+                if(next == NULL)
+                {
+                    return NULL;
+                }
+                typename NextMap::const_iterator iter = next->find(key);
+                if(iter == next->end()) 
+                {
+                    return NULL;
+                }
+                return iter->second;
+            }
+    };
+
+    class Trie
+    {
+        private:
+            TrieNode* _root;
+        public:
+            Trie(const vector<Unicode>& keys, const vector<const DictUnit*> & valuePointers)
+            {
+                _root = new TrieNode;
+                _createTrie(keys, valuePointers);
+                _build();// build automation
+            }
+            ~Trie()
+            {
+                if(_root)
+                {
+                    _deleteNode(_root);
+                }
+            }
+        public:
+            const DictUnit* find(typename Unicode::const_iterator begin, typename Unicode::const_iterator end) const
+            {
+                typename TrieNode::NextMap::const_iterator citer;
+                const TrieNode* ptNode = _root;
+                for(typename Unicode::const_iterator it = begin; it != end; it++)
+                {// build automation
+                    assert(ptNode);
+                    if(NULL == ptNode->next || ptNode->next->end() == (citer = ptNode->next->find(*it)))
+                    {
+                        return NULL;
+                    }
+                    ptNode = citer->second;
+                }
+                return ptNode->ptValue;
+            }
+            // aho-corasick-automation 
+            void find(
+                        typename Unicode::const_iterator begin, 
+                        typename Unicode::const_iterator end, 
+                        vector<struct SegmentChar>& res
+                        ) const
+            {
+                res.resize(end - begin);
+                const TrieNode * now = _root;
+                //typename TrieNode::NextMap::const_iterator iter;
+                const TrieNode* node;
+                // compiler will complain warnings if only "i < end - begin" .
+                for (size_t i = 0; i < size_t(end - begin); i++) 
+                {
+                    Unicode::value_type ch = *(begin + i);
+                    res[i].uniCh = ch;
+                    assert(res[i].dag.empty());
+                    res[i].dag.push_back(pair<typename vector<Unicode >::size_type, const DictUnit* >(i, NULL));
+                    bool flag = false;
+
+                    // rollback
+                    while( now != _root )
+                    {
+                        node = now->findNext(ch);
+                        if (node != NULL) 
+                        {
+                            flag = true;
+                            break;
+                        }
+                        else 
+                        {
+                            now = now->fail;
+                        }
+                    }
+
+                    if(!flag)
+                    {
+                        node = now->findNext(ch);
+                    }
+                    if(node == NULL) 
+                    {
+                        now = _root;
+                    } 
+                    else 
+                    {
+                        now = node;
+                        const TrieNode * temp = now;
+                        while(temp != _root) 
+                        {
+                            if (temp->ptValue) 
+                            {
+                                size_t pos = i - temp->ptValue->word.size() + 1;
+                                res[pos].dag.push_back(pair<typename vector<Unicode >::size_type, const DictUnit* >(i, temp->ptValue));
+                                if(pos == i) 
+                                {
+                                    res[pos].dag[0].second = temp->ptValue;
+                                }
+                            }
+                            temp = temp->fail;
+                            assert(temp);
+                        }
+                    }
+                }
+            }
+            bool find(
+                        typename Unicode::const_iterator begin, 
+                        typename Unicode::const_iterator end, 
+                        DagType & res,
+                        size_t offset = 0) const
+            {
+                const TrieNode * ptNode = _root;
+                typename TrieNode::NextMap::const_iterator citer;
+                for(typename Unicode::const_iterator itr = begin; itr != end ; itr++)
+                {
+                    assert(ptNode);
+                    if(NULL == ptNode->next || ptNode->next->end() == (citer = ptNode->next->find(*itr)))
+                    {
+                        break;
+                    }
+                    ptNode = citer->second;
+                    if(ptNode->ptValue)
+                    {
+                        if(itr == begin && res.size() == 1) // first singleword
+                        {
+                            res[0].second = ptNode->ptValue;
+                        }
+                        else
+                        {
+                            res.push_back(pair<typename vector<Unicode >::size_type, const DictUnit* >(itr - begin + offset, ptNode->ptValue));
+                        }
+                    }
+                }
+                return !res.empty();
+            }
+        private:
+            void _build()
+            {
+                queue<TrieNode*> que;
+                assert(_root->ptValue == NULL);
+                assert(_root->next);
+                _root->fail = NULL;
+                for(typename TrieNode::NextMap::iterator iter = _root->next->begin(); iter != _root->next->end(); iter++) {
+                    iter->second->fail = _root;
+                    que.push(iter->second);
+                }
+                TrieNode* back = NULL;
+                typename TrieNode::NextMap::iterator backiter;
+                while(!que.empty()) {
+                    TrieNode * now = que.front();
+                    que.pop();
+                    if(now->next == NULL) {
+                        continue;
+                    }
+                    for(typename TrieNode::NextMap::iterator iter = now->next->begin(); iter != now->next->end(); iter++) {
+                        back = now->fail;
+                        while(back != NULL) {
+                            if(back->next && (backiter = back->next->find(iter->first)) != back->next->end()) 
+                            {
+                                iter->second->fail = backiter->second;
+                                break;
+                            }
+                            back = back->fail;
+                        }
+                        if(back == NULL) {
+                            iter->second->fail = _root;
+                        }
+                        que.push(iter->second);
+                    }
+                }
+            }
+        private:
+            void _createTrie(const vector<Unicode>& keys, const vector<const DictUnit*> & valuePointers)
+            {
+                if(valuePointers.empty() || keys.empty())
+                {
+                    return;
+                }
+                assert(keys.size() == valuePointers.size());
+
+                for(size_t i = 0; i < keys.size(); i++)
+                {
+                    _insertNode(keys[i], valuePointers[i]);
+                }
+            }
+        private:
+            void _insertNode(const Unicode& key, const DictUnit* ptValue)
+            {
+                TrieNode* ptNode  = _root;
+
+                typename TrieNode::NextMap::const_iterator kmIter;
+
+                for(typename Unicode::const_iterator citer = key.begin(); citer != key.end(); citer++)
+                {
+                    if(NULL == ptNode->next)
+                    {
+                        ptNode->next = new typename TrieNode::NextMap;
+                    }
+                    kmIter = ptNode->next->find(*citer);
+                    if(ptNode->next->end() == kmIter)
+                    {
+                        TrieNode * nextNode = new TrieNode;
+                        nextNode->next = NULL;
+                        nextNode->ptValue = NULL;
+
+                        (*ptNode->next)[*citer] = nextNode;
+                        ptNode = nextNode;
+                    }
+                    else
+                    {
+                        ptNode = kmIter->second;
+                    }
+                }
+                ptNode->ptValue = ptValue;
+            }
+            void _deleteNode(TrieNode* node)
+            {
+                if(!node)
+                {
+                    return;
+                }
+                if(node->next)
+                {
+                    typename TrieNode::NextMap::iterator it;
+                    for(it = node->next->begin(); it != node->next->end(); it++)
+                    {
+                        _deleteNode(it->second);
+                    }
+                    delete node->next;
+                }
+                delete node;
+            }
+    };
+}
+
+#endif

data/ext/cppjieba/test/CMakeLists.txt

@@ -0,0 +1,7 @@
+SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR})
+
+ADD_EXECUTABLE(segment.demo segment_demo.cpp)
+ADD_EXECUTABLE(keyword.demo keyword_demo.cpp)
+ADD_EXECUTABLE(tagging.demo tagging_demo.cpp)
+ADD_EXECUTABLE(load_test load_test.cpp)
+ADD_SUBDIRECTORY(unittest)

data/ext/cppjieba/test/keyword_demo.cpp

@@ -0,0 +1,16 @@
+#include "../src/KeywordExtractor.hpp"
+using namespace CppJieba;
+
+int main(int argc, char ** argv)
+{
+    KeywordExtractor extractor("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
+    string s("我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。");
+    vector<pair<string, double> > wordweights;
+    vector<string> words;
+    size_t topN = 5;
+    extractor.extract(s, wordweights, topN);
+    cout<< s << '\n' << wordweights << endl;
+    extractor.extract(s, words, topN);
+    cout<< s << '\n' << words << endl;
+    return EXIT_SUCCESS;
+}

data/ext/cppjieba/test/load_test.cpp

@@ -0,0 +1,56 @@
+#include <iostream>
+#include <ctime>
+#include <fstream>
+#include "../src/MPSegment.hpp"
+#include "../src/HMMSegment.hpp"
+#include "../src/MixSegment.hpp"
+#include "../src/KeywordExtractor.hpp"
+
+using namespace CppJieba;
+
+void cut(size_t times = 50)
+{
+    MixSegment seg("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8");
+    vector<string> res;
+    string doc;
+    ifstream ifs("../test/testdata/weicheng.utf8");
+    assert(ifs);
+    doc << ifs;
+    long beginTime = clock();
+    for(size_t i = 0; i < times; i ++)
+    {
+        printf("process [%3.0lf %%]\r", 100.0*(i+1)/times);
+        fflush(stdout);
+        res.clear();
+        seg.cut(doc, res);
+    }
+    long endTime = clock();
+    printf("\ncut: [%.3lf seconds]time consumed.\n", double(endTime - beginTime)/CLOCKS_PER_SEC);
+}
+
+void extract(size_t times = 400)
+{
+    KeywordExtractor extractor("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
+    vector<string> words;
+    string doc;
+    ifstream ifs("../test/testdata/review.100");
+    assert(ifs);
+    doc << ifs;
+    long beginTime = clock();
+    for(size_t i = 0; i < times; i ++)
+    {
+        printf("process [%3.0lf %%]\r", 100.0*(i+1)/times);
+        fflush(stdout);
+        words.clear();
+        extractor.extract(doc, words, 5);
+    }
+    long endTime = clock();
+    printf("\nextract: [%.3lf seconds]time consumed.\n", double(endTime - beginTime)/CLOCKS_PER_SEC);
+}
+
+int main(int argc, char ** argv)
+{
+    cut();
+    extract();
+    return EXIT_SUCCESS;
+}

data/ext/cppjieba/test/segment_demo.cpp

@@ -0,0 +1,59 @@
+#include <iostream>
+#include <fstream>
+
+#define LOGGER_LEVEL LL_WARN
+
+#include "../src/MPSegment.hpp"
+#include "../src/HMMSegment.hpp"
+#include "../src/MixSegment.hpp"
+
+using namespace CppJieba;
+
+const char * const TEST_FILE = "../test/testdata/testlines.utf8";
+const char * const JIEBA_DICT_FILE = "../dict/jieba.dict.utf8";
+const char * const HMM_DICT_FILE = "../dict/hmm_model.utf8";
+const char * const USER_DICT_FILE = "../dict/user.dict.utf8";
+
+void cut(const ISegment& seg, const char * const filePath)
+{
+    ifstream ifile(filePath);
+    vector<string> words;
+    string line;
+    string res;
+    while(getline(ifile, line))
+    {
+        if(!line.empty())
+        {
+            words.clear();
+            seg.cut(line, words);
+            join(words.begin(), words.end(), res, "/");
+            cout<< res <<endl;
+        }
+    }
+}
+
+
+int main(int argc, char ** argv)
+{
+    {
+        printf("\e[32m%s\e[0m\n", "[demo] MPSegment"); // colorful
+        MPSegment seg(JIEBA_DICT_FILE);
+        cut(seg, TEST_FILE);
+    }
+    {
+        printf("\e[32m%s\e[0m\n", "[demo] HMMSegment"); // colorful
+        HMMSegment seg(HMM_DICT_FILE);
+        cut(seg, TEST_FILE);
+    }
+    {
+        printf("\e[32m%s\e[0m\n", "[demo] MixSegment"); // colorful
+        MixSegment seg(JIEBA_DICT_FILE, HMM_DICT_FILE);
+        cut(seg, TEST_FILE);
+    }
+    {
+        printf("\e[32m%s\e[0m\n", "[demo] MixSegment with UserDict"); // colorful
+        MixSegment seg(JIEBA_DICT_FILE, HMM_DICT_FILE, USER_DICT_FILE);
+        cut(seg, TEST_FILE);
+    }
+    return EXIT_SUCCESS;
+}

data/ext/cppjieba/test/servertest/go_load_test.sh

@@ -0,0 +1,2 @@
+# go get github.com/aszxqw/go_http_load
+go_http_load -method=GET -get_urls="../test/testdata/load_test.urls" -loop_count=500 -goroutines=2

data/ext/cppjieba/test/servertest/load_test.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python
+# coding:utf-8
+import time
+import urllib2
+import threading
+from Queue import Queue
+from time import sleep
+import sys
+
+# 性能测试页面
+#PERF_TEST_URL = "http://10.2.66.38/?yyid=-1&suv=1309231700203264&callback=xxxxx"
+URLS = [line for line in open("../test/testdata/load_test.urls", "r")]
+
+# 配置:压力测试
+THREAD_NUM = 10            # 并发线程总数
+ONE_WORKER_NUM = 500       # 每个线程的循环次数
+LOOP_SLEEP = 0.01      # 每次请求时间间隔(秒)
+
+# 配置:模拟运行状态
+#THREAD_NUM = 10        # 并发线程总数
+#ONE_WORKER_NUM = 10      # 每个线程的循环次数
+#LOOP_SLEEP = 0        # 每次请求时间间隔(秒)
+
+
+# 出错数
+ERROR_NUM = 0
+
+
+#具体的处理函数，负责处理单个任务
+def doWork(index, url):
+    t = threading.currentThread()
+    #print "["+t.name+" "+str(index)+"] "+PERF_TEST_URL
+
+    try:
+        html = urllib2.urlopen(url).read()
+    except urllib2.URLError, e:
+        print "["+t.name+" "+str(index)+"] "
+        print e
+        global ERROR_NUM
+        ERROR_NUM += 1
+
+
+#这个是工作进程，负责不断从队列取数据并处理
+def working():
+    t = threading.currentThread()
+    print "["+t.name+"] Sub Thread Begin"
+
+    i = 0
+    while i < ONE_WORKER_NUM:
+        i += 1
+        doWork(i, URLS[i % len(URLS)])
+        sleep(LOOP_SLEEP)
+
+    print "["+t.name+"] Sub Thread End"
+
+
+def main():
+    #doWork(0)
+    #return
+    
+    t1 = time.time()
+
+    Threads = []
+
+    # 创建线程
+    for i in range(THREAD_NUM):
+        t = threading.Thread(target=working, name="T"+str(i))
+        t.setDaemon(True)
+        Threads.append(t)
+
+    for t in Threads:
+        t.start()
+
+    for t in Threads:
+        t.join()
+
+    print "main thread end"
+
+    t2 = time.time()
+    print "========================================"
+    #print "URL:", PERF_TEST_URL
+    print "任务数量:", THREAD_NUM, "*", ONE_WORKER_NUM, "=", THREAD_NUM*ONE_WORKER_NUM
+    print "总耗时(秒):", t2-t1
+    print "每次请求耗时(秒):", (t2-t1) / (THREAD_NUM*ONE_WORKER_NUM)
+    print "每秒承载请求数:", 1 / ((t2-t1) / (THREAD_NUM*ONE_WORKER_NUM))
+    print "错误数量:", ERROR_NUM
+
+
+if __name__ == "__main__": 
+    main()
+

data/ext/cppjieba/test/servertest/run_curl.sh

@@ -0,0 +1,11 @@
+CURL_RES=../test/testdata/curl.res
+TMP=curl.res.tmp
+curl -s "http://127.0.0.1:11200/?key=南京市长江大桥" >> $TMP
+if diff $TMP $CURL_RES >> /dev/null
+then
+    echo "ok";
+else
+    echo "failed."
+fi
+
+rm $TMP

data/ext/cppjieba/test/tagging_demo.cpp

@@ -0,0 +1,12 @@
+#include "../src/PosTagger.hpp"
+using namespace CppJieba;
+
+int main(int argc, char ** argv)
+{
+    PosTagger tagger("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../dict/user.dict.utf8");
+    string s("我是蓝翔技工拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上总经理，出任CEO，迎娶白富美，走上人生巅峰。");
+    vector<pair<string, string> > res;
+    tagger.tag(s, res);
+    cout << res << endl;
+    return EXIT_SUCCESS;
+}

data/ext/cppjieba/test/testdata/curl.res

@@ -0,0 +1 @@
+["南京市", "长江大桥"]

data/ext/cppjieba/test/testdata/jieba.dict.0.1.utf8

@@ -0,0 +1,93 @@
+龙鸣狮吼 3 nr
+龙齐诺 2 nr
+龙齿 3 n
+龚 176 nr
+龚世萍 2 nr
+龚书铎 2 nr
+龚二人 2 nr
+龚云甫 3 nr
+龚伟强 5 nr
+龚先生 4 nr
+龚光杰 44 nr
+龚古尔 24 nr
+龚子敬 2 nr
+龚孝升 12 nr
+龚学平 2 nr
+龚完敬 5 nr
+龚定庵 3 nr
+龚定敬 2 nr
+龚宝铨 5 nr
+龚家村 3 nr
+龚建国 29 nr
+龚德俊 6 nr
+龚心瀚 3 nr
+龚志国 2 nr
+龚意田 3 nr
+龚慈恩 3 nr
+龚施茜 3 nr
+龚晓犁 2 nr
+龚普洛 3 nr
+龚智超 7 nr
+龚松林 10 nr
+龚永明 3 nr
+龚永泉 5 nr
+龚泽艺 256 nr
+龚睿 8 nrfg
+龚祖同 2 nr
+龚秋婷 3 nr
+龚老爷 2 nr
+龚育之 19 nr
+龚自珍 28 nr
+龚蓓苾 3 nr
+龚虹嘉 3 nr
+龚诗嘉 3 nr
+龛 223 ng
+龜 2 zg
+龟 903 ns
+龟儿子 123 n
+龟兆 3 nz
+龟兹 215 ns
+龟兹王 3 nrt
+龟冷搘床 3 v
+龟冷支床 3 n
+龟卜 3 n
+龟厌不告 3 l
+龟壳 33 n
+龟壳花 3 n
+龟头 34 n
+龟头炎 3 n
+龟山 23 ns
+龟山乡 3 ns
+龟山岛 3 ns
+龟年鹤寿 3 ns
+龟年鹤算 3 l
+龟文 3 nz
+龟文写迹 3 n
+龟文鸟迹 3 n
+龟板 10 n
+龟毛免角 3 n
+龟毛兔角 3 n
+龟溪 3 ns
+龟玉 3 nz
+龟王 3 nz
+龟甲 92 ns
+龟甲胶 3 nz
+龟筮 3 n
+龟纹 3 n
+龟缩 29 v
+龟肉 3 n
+龟背 21 n
+龟背竹 3 n
+龟苓膏 3 n
+龟苗 3 n
+龟裂 34 v
+龟足 5 v
+龟鉴 2 n
+龟镜 3 nz
+龟鳖 3 n
+龟鹤遐寿 3 l
+龟龄鹤算 3 n
+龟龙片甲 3 nz
+龟龙麟凤 3 ns
+龠 5 g
+龢 732 zg