cppjieba_rb 0.2.1

data/ext/cppjieba/test/unittest/CMakeLists.txt

@@ -0,0 +1,24 @@
+SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/test)
+SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)
+
+INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/deps/gtest/include)
+
+ADD_DEFINITIONS(-DLOGGING_LEVEL=LL_WARNING)
+
+ADD_EXECUTABLE(test.run 
+    gtest_main.cpp 
+    keyword_extractor_test.cpp 
+    trie_test.cpp 
+    segments_test.cpp 
+    pos_tagger_test.cpp
+    jieba_test.cpp
+    pre_filter_test.cpp
+    unicode_test.cpp
+    textrank_test.cpp
+)
+
+if(MSVC)
+	TARGET_LINK_LIBRARIES(test.run gtest)
+else()
+	TARGET_LINK_LIBRARIES(test.run gtest pthread)
+endif()

data/ext/cppjieba/test/unittest/gtest_main.cpp

@@ -0,0 +1,39 @@
+// Copyright 2006, Google Inc.
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions are
+// met:
+//
+//     * Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+//     * Redistributions in binary form must reproduce the above
+// copyright notice, this list of conditions and the following disclaimer
+// in the documentation and/or other materials provided with the
+// distribution.
+//     * Neither the name of Google Inc. nor the names of its
+// contributors may be used to endorse or promote products derived from
+// this software without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+#include <iostream>
+
+#include "gtest/gtest.h"
+
+GTEST_API_ int main(int argc, char **argv) {
+  std::cout << "Running main() from gtest_main.cc\n";
+
+  testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}

data/ext/cppjieba/test/unittest/jieba_test.cpp

@@ -0,0 +1,133 @@
+#include "cppjieba/Jieba.hpp"
+#include "gtest/gtest.h"
+
+using namespace cppjieba;
+
+TEST(JiebaTest, Test1) {
+  cppjieba::Jieba jieba("../dict/jieba.dict.utf8",
+                        "../dict/hmm_model.utf8",
+                        "../dict/user.dict.utf8",
+                        "../dict/idf.utf8",
+                        "../dict/stop_words.utf8");
+  vector<string> words;
+  string result;
+
+  jieba.Cut("他来到了网易杭研大厦", words);
+  result << words;
+  ASSERT_EQ("[\"他\", \"来到\", \"了\", \"网易\", \"杭研\", \"大厦\"]", result);
+
+  jieba.Cut("我来自北京邮电大学。", words, false);
+  result << words;
+  ASSERT_EQ("[\"我\", \"来自\", \"北京邮电大学\", \"。\"]", result);
+
+  jieba.CutSmall("南京市长江大桥", words, 3);
+  ASSERT_EQ("[\"南京市\", \"长江\", \"大桥\"]", result << words);
+
+  jieba.CutHMM("我来自北京邮电大学。。。学号123456", words);
+  result << words;
+  ASSERT_EQ("[\"我来\", \"自北京\", \"邮电大学\", \"。\", \"。\", \"。\", \"学号\", \"123456\"]", result);
+
+  jieba.Cut("我来自北京邮电大学。。。学号123456，用AK47", words);
+  result << words;
+  ASSERT_EQ("[\"我\", \"来自\", \"北京邮电大学\", \"。\", \"。\", \"。\", \"学号\", \"123456\", \"，\", \"用\", \"AK47\"]", result);
+
+  jieba.CutAll("我来自北京邮电大学", words);
+  result << words;
+  ASSERT_EQ(result, "[\"我\", \"来自\", \"北京\", \"北京邮电\", \"北京邮电大学\", \"邮电\", \"邮电大学\", \"电大\", \"大学\"]");
+
+  jieba.CutForSearch("他来到了网易杭研大厦", words);
+  result << words;
+  ASSERT_EQ("[\"他\", \"来到\", \"了\", \"网易\", \"杭研\", \"大厦\"]", result);
+
+}
+TEST(JiebaTest, WordTest) {
+  cppjieba::Jieba jieba("../dict/jieba.dict.utf8",
+                        "../dict/hmm_model.utf8",
+                        "../dict/user.dict.utf8",
+                        "../dict/idf.utf8",
+                        "../dict/stop_words.utf8");
+  vector<Word> words;
+  string result;
+
+  jieba.Cut("他来到了网易杭研大厦", words);
+  result << words;
+  ASSERT_EQ("[{\"word\": \"\xE4\xBB\x96\", \"offset\": 0}, {\"word\": \"\xE6\x9D\xA5\xE5\x88\xB0\", \"offset\": 3}, {\"word\": \"\xE4\xBA\x86\", \"offset\": 9}, {\"word\": \"\xE7\xBD\x91\xE6\x98\x93\", \"offset\": 12}, {\"word\": \"\xE6\x9D\xAD\xE7\xA0\x94\", \"offset\": 18}, {\"word\": \"\xE5\xA4\xA7\xE5\x8E\xA6\", \"offset\": 24}]", result);
+
+  jieba.Cut("我来自北京邮电大学。", words, false);
+  result << words;
+  //ASSERT_EQ("[\"我\", \"来自\", \"北京邮电大学\", \"。\"]", result);
+  ASSERT_EQ("[{\"word\": \"\xE6\x88\x91\", \"offset\": 0}, {\"word\": \"\xE6\x9D\xA5\xE8\x87\xAA\", \"offset\": 3}, {\"word\": \"\xE5\x8C\x97\xE4\xBA\xAC\xE9\x82\xAE\xE7\x94\xB5\xE5\xA4\xA7\xE5\xAD\xA6\", \"offset\": 9}, {\"word\": \"\xE3\x80\x82\", \"offset\": 27}]", result);
+
+  jieba.CutSmall("南京市长江大桥", words, 3);
+  //ASSERT_EQ("[\"南京市\", \"长江\", \"大桥\"]", result << words);
+  ASSERT_EQ("[{\"word\": \"\xE5\x8D\x97\xE4\xBA\xAC\xE5\xB8\x82\", \"offset\": 0}, {\"word\": \"\xE9\x95\xBF\xE6\xB1\x9F\", \"offset\": 9}, {\"word\": \"\xE5\xA4\xA7\xE6\xA1\xA5\", \"offset\": 15}]", result << words);
+
+  jieba.CutHMM("我来自北京邮电大学。。。学号123456", words);
+  result << words;
+  ASSERT_EQ("[{\"word\": \"\xE6\x88\x91\xE6\x9D\xA5\", \"offset\": 0}, {\"word\": \"\xE8\x87\xAA\xE5\x8C\x97\xE4\xBA\xAC\", \"offset\": 6}, {\"word\": \"\xE9\x82\xAE\xE7\x94\xB5\xE5\xA4\xA7\xE5\xAD\xA6\", \"offset\": 15}, {\"word\": \"\xE3\x80\x82\", \"offset\": 27}, {\"word\": \"\xE3\x80\x82\", \"offset\": 30}, {\"word\": \"\xE3\x80\x82\", \"offset\": 33}, {\"word\": \"\xE5\xAD\xA6\xE5\x8F\xB7\", \"offset\": 36}, {\"word\": \"123456\", \"offset\": 42}]", result);
+
+  jieba.Cut("我来自北京邮电大学。。。学号123456，用AK47", words);
+  result << words;
+  //ASSERT_EQ("[\"我\", \"来自\", \"北京邮电大学\", \"。\", \"。\", \"。\", \"学号\", \"123456\", \"，\", \"用\", \"AK47\"]", result);
+  ASSERT_EQ("[{\"word\": \"\xE6\x88\x91\", \"offset\": 0}, {\"word\": \"\xE6\x9D\xA5\xE8\x87\xAA\", \"offset\": 3}, {\"word\": \"\xE5\x8C\x97\xE4\xBA\xAC\xE9\x82\xAE\xE7\x94\xB5\xE5\xA4\xA7\xE5\xAD\xA6\", \"offset\": 9}, {\"word\": \"\xE3\x80\x82\", \"offset\": 27}, {\"word\": \"\xE3\x80\x82\", \"offset\": 30}, {\"word\": \"\xE3\x80\x82\", \"offset\": 33}, {\"word\": \"\xE5\xAD\xA6\xE5\x8F\xB7\", \"offset\": 36}, {\"word\": \"123456\", \"offset\": 42}, {\"word\": \"\xEF\xBC\x8C\", \"offset\": 48}, {\"word\": \"\xE7\x94\xA8\", \"offset\": 51}, {\"word\": \"AK47\", \"offset\": 54}]", result);
+
+  jieba.CutAll("我来自北京邮电大学", words);
+  result << words;
+  //ASSERT_EQ(result, "[\"我\", \"来自\", \"北京\", \"北京邮电\", \"北京邮电大学\", \"邮电\", \"邮电大学\", \"电大\", \"大学\"]");
+  ASSERT_EQ("[{\"word\": \"\xE6\x88\x91\", \"offset\": 0}, {\"word\": \"\xE6\x9D\xA5\xE8\x87\xAA\", \"offset\": 3}, {\"word\": \"\xE5\x8C\x97\xE4\xBA\xAC\", \"offset\": 9}, {\"word\": \"\xE5\x8C\x97\xE4\xBA\xAC\xE9\x82\xAE\xE7\x94\xB5\", \"offset\": 9}, {\"word\": \"\xE5\x8C\x97\xE4\xBA\xAC\xE9\x82\xAE\xE7\x94\xB5\xE5\xA4\xA7\xE5\xAD\xA6\", \"offset\": 9}, {\"word\": \"\xE9\x82\xAE\xE7\x94\xB5\", \"offset\": 15}, {\"word\": \"\xE9\x82\xAE\xE7\x94\xB5\xE5\xA4\xA7\xE5\xAD\xA6\", \"offset\": 15}, {\"word\": \"\xE7\x94\xB5\xE5\xA4\xA7\", \"offset\": 18}, {\"word\": \"\xE5\xA4\xA7\xE5\xAD\xA6\", \"offset\": 21}]", result);
+
+  jieba.CutForSearch("他来到了网易杭研大厦", words);
+  result << words;
+  //ASSERT_EQ("[\"他\", \"来到\", \"了\", \"网易\", \"杭研\", \"大厦\"]", result);
+  ASSERT_EQ("[{\"word\": \"\xE4\xBB\x96\", \"offset\": 0}, {\"word\": \"\xE6\x9D\xA5\xE5\x88\xB0\", \"offset\": 3}, {\"word\": \"\xE4\xBA\x86\", \"offset\": 9}, {\"word\": \"\xE7\xBD\x91\xE6\x98\x93\", \"offset\": 12}, {\"word\": \"\xE6\x9D\xAD\xE7\xA0\x94\", \"offset\": 18}, {\"word\": \"\xE5\xA4\xA7\xE5\x8E\xA6\", \"offset\": 24}]", result);
+}
+
+TEST(JiebaTest, InsertUserWord) {
+  cppjieba::Jieba jieba("../dict/jieba.dict.utf8",
+                        "../dict/hmm_model.utf8",
+                        "../dict/user.dict.utf8",
+                        "../dict/idf.utf8",
+                        "../dict/stop_words.utf8");
+  vector<string> words;
+  string result;
+
+  jieba.Cut("男默女泪", words);
+  result << words;
+  ASSERT_EQ("[\"男默\", \"女泪\"]", result);
+
+  ASSERT_TRUE(jieba.InsertUserWord("男默女泪"));
+
+  jieba.Cut("男默女泪", words);
+  result << words;
+  ASSERT_EQ("[\"男默女泪\"]", result);
+
+  for (size_t i = 0; i < 100; i++) {
+    string newWord;
+    newWord << rand();
+    ASSERT_TRUE(jieba.InsertUserWord(newWord));
+    jieba.Cut(newWord, words);
+    result << words;
+    ASSERT_EQ(result, StringFormat("[\"%s\"]", newWord.c_str()));
+  }
+
+  ASSERT_TRUE(jieba.InsertUserWord("同一个世界，同一个梦想"));
+  jieba.Cut("同一个世界，同一个梦想", words);
+  result = Join(words.begin(), words.end(), "/");
+  ASSERT_EQ(result, "同一个/世界/，/同一个/梦想");
+
+  jieba.ResetSeparators("");
+
+  jieba.Cut("同一个世界，同一个梦想", words);
+  result = Join(words.begin(), words.end(), "/");
+  ASSERT_EQ(result, "同一个世界，同一个梦想");
+
+  {
+    string s("一部iPhone6");
+    string res;
+    vector<KeywordExtractor::Word> wordweights;
+    size_t topN = 5;
+    jieba.extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"iPhone6\", \"offset\": [6], \"weight\": 11.7392}, {\"word\": \"\xE4\xB8\x80\xE9\x83\xA8\", \"offset\": [0], \"weight\": 6.47592}]");
+  }
+}

data/ext/cppjieba/test/unittest/keyword_extractor_test.cpp

@@ -0,0 +1,79 @@
+#include "cppjieba/KeywordExtractor.hpp"
+#include "gtest/gtest.h"
+
+using namespace cppjieba;
+
+TEST(KeywordExtractorTest, Test1) {
+  KeywordExtractor Extractor("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
+
+  {
+    string s("你好世界世界而且而且");
+    string res;
+    size_t topN = 5;
+
+    {
+      vector<string> words;
+      Extractor.Extract(s, words, topN);
+      res << words;
+      ASSERT_EQ(res, "[\"世界\", \"你好\"]");
+    }
+
+    {
+      vector<pair<string, double> > words;
+      Extractor.Extract(s, words, topN);
+      res << words;
+      ASSERT_EQ(res, "[世界:8.73506, 你好:7.95788]");
+    }
+
+    {
+      vector<KeywordExtractor::Word> words;
+      Extractor.Extract(s, words, topN);
+      res << words;
+      ASSERT_EQ(res, "[{\"word\": \"\xE4\xB8\x96\xE7\x95\x8C\", \"offset\": [6, 12], \"weight\": 8.73506}, {\"word\": \"\xE4\xBD\xA0\xE5\xA5\xBD\", \"offset\": [0], \"weight\": 7.95788}]");
+    }
+  }
+
+  {
+    string s("我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。");
+    string res;
+    vector<KeywordExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"CEO\", \"offset\": [93], \"weight\": 11.7392}, {\"word\": \"\xE5\x8D\x87\xE8\x81\x8C\", \"offset\": [72], \"weight\": 10.8562}, {\"word\": \"\xE5\x8A\xA0\xE8\x96\xAA\", \"offset\": [78], \"weight\": 10.6426}, {\"word\": \"\xE6\x89\x8B\xE6\x89\xB6\xE6\x8B\x96\xE6\x8B\x89\xE6\x9C\xBA\", \"offset\": [21], \"weight\": 10.0089}, {\"word\": \"\xE5\xB7\x85\xE5\xB3\xB0\", \"offset\": [111], \"weight\": 9.49396}]");
+  }
+
+  {
+    string s("一部iPhone6");
+    string res;
+    vector<KeywordExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"iPhone6\", \"offset\": [6], \"weight\": 11.7392}, {\"word\": \"\xE4\xB8\x80\xE9\x83\xA8\", \"offset\": [0], \"weight\": 6.47592}]");
+  }
+}
+
+TEST(KeywordExtractorTest, Test2) {
+  KeywordExtractor Extractor("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8", "../test/testdata/userdict.utf8");
+
+  {
+    string s("蓝翔优秀毕业生");
+    string res;
+    vector<KeywordExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"\xE8\x93\x9D\xE7\xBF\x94\", \"offset\": [0], \"weight\": 11.7392}, {\"word\": \"\xE6\xAF\x95\xE4\xB8\x9A\xE7\x94\x9F\", \"offset\": [12], \"weight\": 8.13549}, {\"word\": \"\xE4\xBC\x98\xE7\xA7\x80\", \"offset\": [6], \"weight\": 6.78347}]");
+  }
+
+  {
+    string s("一部iPhone6");
+    string res;
+    vector<KeywordExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"iPhone6\", \"offset\": [6], \"weight\": 11.7392}, {\"word\": \"\xE4\xB8\x80\xE9\x83\xA8\", \"offset\": [0], \"weight\": 6.47592}]");
+  }
+}

data/ext/cppjieba/test/unittest/pos_tagger_test.cpp

@@ -0,0 +1,41 @@
+#include "cppjieba/MixSegment.hpp"
+#include "gtest/gtest.h"
+
+using namespace cppjieba;
+
+static const char * const QUERY_TEST1 = "我是蓝翔技工拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上总经理，出任CEO，迎娶白富美，走上人生巅峰。";
+static const char * const ANS_TEST1 = "[我:r, 是:v, 蓝翔:x, 技工:n, 拖拉机:n, 学院:n, 手扶拖拉机:n, 专业:n, 的:uj, 。:x, 不用:v, 多久:m, ，:x, 我:r, 就:d, 会:v, 升职:v, 加薪:nr, ，:x, 当上:t, 总经理:n, ，:x, 出任:v, CEO:eng, ，:x, 迎娶:v, 白富:x, 美:ns, ，:x, 走上:v, 人生:n, 巅峰:n, 。:x]";
+static const char * const QUERY_TEST2 = "我是蓝翔技工拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上总经理，出任CEO，迎娶白富美，走上人生巅峰。";
+static const char * const ANS_TEST2 = "[我:r, 是:v, 蓝翔:nz, 技工:n, 拖拉机:n, 学院:n, 手扶拖拉机:n, 专业:n, 的:uj, 。:x, 不用:v, 多久:m, ，:x, 我:r, 就:d, 会:v, 升职:v, 加薪:nr, ，:x, 当上:t, 总经理:n, ，:x, 出任:v, CEO:eng, ，:x, 迎娶:v, 白富:x, 美:ns, ，:x, 走上:v, 人生:n, 巅峰:n, 。:x]";
+
+static const char * const QUERY_TEST3 = "iPhone6手机的最大特点是很容易弯曲。";
+static const char * const ANS_TEST3 = "[iPhone6:eng, 手机:n, 的:uj, 最大:a, 特点:n, 是:v, 很:zg, 容易:a, 弯曲:v, 。:x]";
+//static const char * const ANS_TEST3 = "";
+
+TEST(PosTaggerTest, Test) {
+  MixSegment tagger("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8");
+  {
+    vector<pair<string, string> > res;
+    tagger.Tag(QUERY_TEST1, res);
+    string s;
+    s << res;
+    ASSERT_TRUE(s == ANS_TEST1);
+  }
+}
+TEST(PosTagger, TestUserDict) {
+  MixSegment tagger("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../test/testdata/userdict.utf8");
+  {
+    vector<pair<string, string> > res;
+    tagger.Tag(QUERY_TEST2, res);
+    string s;
+    s << res;
+    ASSERT_EQ(s, ANS_TEST2);
+  }
+  {
+    vector<pair<string, string> > res;
+    tagger.Tag(QUERY_TEST3, res);
+    string s;
+    s << res;
+    ASSERT_EQ(s, ANS_TEST3);
+  }
+}

data/ext/cppjieba/test/unittest/pre_filter_test.cpp

@@ -0,0 +1,43 @@
+#include "gtest/gtest.h"
+#include "cppjieba/PreFilter.hpp"
+#include "limonp/StringUtil.hpp"
+
+using namespace cppjieba;
+
+TEST(PreFilterTest, Test1) {
+  unordered_set<Rune> symbol;
+  symbol.insert(65292u); // "，"
+  symbol.insert(12290u); // "。"
+  string expected;
+  string res;
+
+  {
+    string s = "你好，美丽的，世界";
+    PreFilter filter(symbol, s);
+    expected = "你好/，/美丽的/，/世界";
+    ASSERT_TRUE(filter.HasNext());
+    vector<string> words;
+    while (filter.HasNext()) {
+      PreFilter::Range range;
+      range = filter.Next();
+      words.push_back(GetStringFromRunes(s, range.begin, range.end - 1));
+    }
+    res = limonp::Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(res, expected);
+  }
+
+  {
+    string s = "我来自北京邮电大学。。。学号123456，用AK47";
+    PreFilter filter(symbol, s);
+    expected = "我来自北京邮电大学/。/。/。/学号123456/，/用AK47";
+    ASSERT_TRUE(filter.HasNext());
+    vector<string> words;
+    while (filter.HasNext()) {
+      PreFilter::Range range;
+      range = filter.Next();
+      words.push_back(GetStringFromRunes(s, range.begin, range.end - 1));
+    }
+    res = limonp::Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(res, expected);
+  }
+}

data/ext/cppjieba/test/unittest/segments_test.cpp

@@ -0,0 +1,256 @@
+#include "cppjieba/SegmentBase.hpp"
+#include "cppjieba/MixSegment.hpp"
+#include "cppjieba/MPSegment.hpp"
+#include "cppjieba/HMMSegment.hpp"
+#include "cppjieba/FullSegment.hpp"
+#include "cppjieba/QuerySegment.hpp"
+#include "gtest/gtest.h"
+
+using namespace cppjieba;
+
+TEST(MixSegmentTest, Test1) {
+  MixSegment segment("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8");;
+  string sentence;
+  vector<string> words;
+  string actual;
+  string expected;
+
+  {
+    sentence = "我来自北京邮电大学。。。学号123456，用AK47";
+    expected = "我/来自/北京邮电大学/。/。/。/学号/123456/，/用/AK47";
+    segment.Cut(sentence, words);
+    actual = Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(actual, expected);
+  }
+
+  {
+    sentence = "B超 T恤";
+    expected = "B超/ /T恤";
+    segment.Cut(sentence, words);
+    actual = Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(actual, expected);
+  }
+
+  {
+    sentence = "他来到了网易杭研大厦";
+    expected = "他/来到/了/网易/杭/研/大厦";
+    segment.Cut(sentence, words, false);
+    actual = Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(actual, expected);
+  }
+
+  {
+    sentence = "他来到了网易杭研大厦";
+    expected = "他/来到/了/网易/杭研/大厦";
+    segment.Cut(sentence, words);
+    actual = Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(actual, expected);
+  }
+}
+
+TEST(MixSegmentTest, NoUserDict) {
+  MixSegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8");
+  const char* str = "令狐冲是云计算方面的专家";
+  vector<string> words;
+  segment.Cut(str, words);
+  string res;
+  ASSERT_EQ("[\"令狐冲\", \"是\", \"云\", \"计算\", \"方面\", \"的\", \"专家\"]", res << words);
+
+}
+TEST(MixSegmentTest, UserDict) {
+  MixSegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", "../dict/user.dict.utf8");
+  {
+    const char* str = "令狐冲是云计算方面的专家";
+    vector<string> words;
+    segment.Cut(str, words);
+    string res;
+    ASSERT_EQ("[\"令狐冲\", \"是\", \"云计算\", \"方面\", \"的\", \"专家\"]", res << words);
+  }
+  {
+    const char* str = "小明先就职于IBM,后在日本京都大学深造";
+    vector<string> words;
+    segment.Cut(str, words);
+    string res;
+    res << words;
+    ASSERT_EQ("[\"小明\", \"先\", \"就职\", \"于\", \"IBM\", \",\", \"后\", \"在\", \"日本\", \"京都大学\", \"深造\"]", res);
+  }
+  {
+    const char* str = "IBM,3.14";
+    vector<string> words;
+    segment.Cut(str, words);
+    string res;
+    res << words;
+    ASSERT_EQ("[\"IBM\", \",\", \"3.14\"]", res);
+  }
+}
+TEST(MixSegmentTest, TestUserDict) {
+  MixSegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", 
+        "../test/testdata/userdict.utf8");
+  vector<string> words;
+  string res;
+
+  segment.Cut("令狐冲是云计算方面的专家", words);
+  ASSERT_EQ("[\"令狐冲\", \"是\", \"云计算\", \"方面\", \"的\", \"专家\"]", res << words);
+
+  segment.Cut("小明先就职于IBM,后在日本京都大学深造", words);
+  res << words;
+  ASSERT_EQ("[\"小明\", \"先\", \"就职\", \"于\", \"I\", \"B\", \"M\", \",\", \"后\", \"在\", \"日本\", \"京都大学\", \"深造\"]", res);
+
+  segment.Cut("IBM,3.14", words);
+  res << words;
+  ASSERT_EQ("[\"I\", \"B\", \"M\", \",\", \"3.14\"]", res);
+
+  segment.Cut("忽如一夜春风来，千树万树梨花开", words);
+  res = limonp::Join(words.begin(), words.end(), "/");
+  ASSERT_EQ("忽如一夜春风来/，/千树/万树/梨花/开", res);
+
+  // rand input
+  {
+    const size_t ITERATION = 16;
+    const size_t MAX_LEN = 256;
+    string s;
+    srand(time(NULL));
+
+    for (size_t i = 0; i < ITERATION; i++) {
+      size_t len = rand() % MAX_LEN;
+      s.resize(len);
+      for (size_t j = 0; j < len; j++) {
+        s[rand() % len] = rand();
+      }
+      segment.Cut(s, words);
+    }
+  }
+}
+
+TEST(MixSegmentTest, TestMultiUserDict) {
+  MixSegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", 
+        "../test/testdata/userdict.utf8;../test/testdata/userdict.2.utf8");
+  vector<string> words;
+  string res;
+
+  segment.Cut("忽如一夜春风来，千树万树梨花开", words);
+  res = limonp::Join(words.begin(), words.end(), "/");
+  ASSERT_EQ("忽如一夜春风来/，/千树万树梨花开", res);
+}
+
+TEST(MPSegmentTest, Test1) {
+  MPSegment segment("../dict/jieba.dict.utf8");;
+  string s;
+  vector<string> words;
+  segment.Cut("我来自北京邮电大学。", words);
+  ASSERT_EQ("[\"我\", \"来自\", \"北京邮电大学\", \"。\"]", s << words);
+
+  segment.Cut("B超 T恤", words);
+  ASSERT_EQ(s << words, "[\"B超\", \" \", \"T恤\"]");
+
+  segment.Cut("南京市长江大桥", words);
+  ASSERT_EQ("[\"南京市\", \"长江大桥\"]", s << words);
+
+  // MaxWordLen
+  segment.Cut("南京市长江大桥", words, 3);
+  ASSERT_EQ("[\"南京市\", \"长江\", \"大桥\"]", s << words);
+
+  segment.Cut("南京市长江大桥", words, 0);
+  ASSERT_EQ("[\"南\", \"京\", \"市\", \"长\", \"江\", \"大\", \"桥\"]", s << words);
+
+  segment.Cut("湖南长沙市天心区", words);
+  s = Join(words.begin(), words.end(), "/");
+  ASSERT_EQ("湖南长沙市/天心区", s);
+
+  segment.Cut("湖南长沙市天心区", words, 3);
+  s = Join(words.begin(), words.end(), "/");
+  ASSERT_EQ("湖南/长沙市/天心区", s);
+}
+
+TEST(HMMSegmentTest, Test1) {
+  HMMSegment segment("../dict/hmm_model.utf8");;
+  {
+    const char* str = "我来自北京邮电大学。。。学号123456";
+    const char* res[] = {"我来", "自北京", "邮电大学", "。", "。", "。", "学号", "123456"};
+    vector<string> words;
+    segment.Cut(str, words);
+    ASSERT_EQ(words, vector<string>(res, res + sizeof(res)/sizeof(res[0])));
+  }
+
+  {
+    const char* str = "IBM,1.2,123";
+    const char* res[] = {"IBM", ",", "1.2", ",", "123"};
+    vector<string> words;
+    segment.Cut(str, words);
+    ASSERT_EQ(words, vector<string>(res, res + sizeof(res)/sizeof(res[0])));
+  }
+}
+
+TEST(FullSegment, Test1) {
+  FullSegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8");
+  vector<string> words;
+  string s;
+
+  segment.Cut("我来自北京邮电大学", words);
+  s << words;
+  ASSERT_EQ(s, "[\"我\", \"来自\", \"北京\", \"北京邮电大学\", \"邮电\", \"电大\", \"大学\"]");
+
+  
+  segment.Cut("上市公司CEO", words);
+  s << words;
+  ASSERT_EQ(s, "[\"上市\", \"公司\", \"C\", \"E\", \"O\"]");
+}
+
+TEST(QuerySegment, Test1) {
+  QuerySegment segment("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "");
+  vector<string> words;
+  string s1, s2;
+
+  segment.Cut("小明硕士毕业于中国科学院计算所，后在日本京都大学深造", words);
+  s1 = Join(words.begin(), words.end(), "/");
+  s2 = "小明/硕士/毕业/于/中国/科学/学院/科学院/中国科学院/计算/计算所/，/后/在/日本/京都/大学/日本京都大学/深造";
+  ASSERT_EQ(s1, s2);
+
+  segment.Cut("亲口交代", words);
+  s1 = Join(words.begin(), words.end(), "/");
+  s2 = "亲口/交代";
+  ASSERT_EQ(s1, s2);
+
+  segment.Cut("他心理健康", words);
+  s1 = Join(words.begin(), words.end(), "/");
+  s2 = "他/心理/健康/心理健康";
+  ASSERT_EQ(s1, s2);
+}
+
+TEST(QuerySegment, Test2) {
+  QuerySegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", "../test/testdata/userdict.utf8|../test/testdata/userdict.english");
+  vector<string> words;
+  string s1, s2;
+
+  {
+    segment.Cut("小明硕士毕业于中国科学院计算所，后在日本京都大学深造", words);
+    s1 = Join(words.begin(), words.end(), "/");
+    s2 = "小明/硕士/毕业/于/中国/科学/学院/科学院/中国科学院/计算/计算所/，/后/在/日本/京都/大学/京都大学/深造";
+    ASSERT_EQ(s1, s2);
+  }
+
+  {
+    segment.Cut("小明硕士毕业于中国科学院计算所iPhone6", words);
+    s1 = Join(words.begin(), words.end(), "/");
+    s2 = "小明/硕士/毕业/于/中国/科学/学院/科学院/中国科学院/计算/计算所/iPhone6";
+    ASSERT_EQ(s1, s2);
+  }
+
+  {
+    segment.Cut("中国科学院", words);
+    s1 = Join(words.begin(), words.end(), "/");
+    s2 = "中国/科学/学院/科学院/中国科学院";
+    ASSERT_EQ(s1, s2);
+  }
+
+}
+
+TEST(MPSegmentTest, Unicode32) {
+  string s("天气很好，🙋 我们去郊游。");
+  vector<string> words;
+
+  MPSegment segment("../dict/jieba.dict.utf8");;
+  segment.Cut(s, words);
+
+  ASSERT_EQ(Join(words.begin(), words.end(), "/"), "天气/很/好/，/🙋/ /我们/去/郊游/。");
+}