cppjieba_rb 0.4.0 → 0.4.1

data/ext/cppjieba/test/unittest/keyword_extractor_test.cpp

@@ -0,0 +1,79 @@
+#include "cppjieba/KeywordExtractor.hpp"
+#include "gtest/gtest.h"
+
+using namespace cppjieba;
+
+TEST(KeywordExtractorTest, Test1) {
+  KeywordExtractor Extractor("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8");
+
+  {
+    string s("你好世界世界而且而且");
+    string res;
+    size_t topN = 5;
+
+    {
+      vector<string> words;
+      Extractor.Extract(s, words, topN);
+      res << words;
+      ASSERT_EQ(res, "[\"世界\", \"你好\"]");
+    }
+
+    {
+      vector<pair<string, double> > words;
+      Extractor.Extract(s, words, topN);
+      res << words;
+      ASSERT_EQ(res, "[世界:8.73506, 你好:7.95788]");
+    }
+
+    {
+      vector<KeywordExtractor::Word> words;
+      Extractor.Extract(s, words, topN);
+      res << words;
+      ASSERT_EQ(res, "[{\"word\": \"\xE4\xB8\x96\xE7\x95\x8C\", \"offset\": [6, 12], \"weight\": 8.73506}, {\"word\": \"\xE4\xBD\xA0\xE5\xA5\xBD\", \"offset\": [0], \"weight\": 7.95788}]");
+    }
+  }
+
+  {
+    string s("我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。");
+    string res;
+    vector<KeywordExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"CEO\", \"offset\": [93], \"weight\": 11.7392}, {\"word\": \"\xE5\x8D\x87\xE8\x81\x8C\", \"offset\": [72], \"weight\": 10.8562}, {\"word\": \"\xE5\x8A\xA0\xE8\x96\xAA\", \"offset\": [78], \"weight\": 10.6426}, {\"word\": \"\xE6\x89\x8B\xE6\x89\xB6\xE6\x8B\x96\xE6\x8B\x89\xE6\x9C\xBA\", \"offset\": [21], \"weight\": 10.0089}, {\"word\": \"\xE5\xB7\x85\xE5\xB3\xB0\", \"offset\": [111], \"weight\": 9.49396}]");
+  }
+
+  {
+    string s("一部iPhone6");
+    string res;
+    vector<KeywordExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"iPhone6\", \"offset\": [6], \"weight\": 11.7392}, {\"word\": \"\xE4\xB8\x80\xE9\x83\xA8\", \"offset\": [0], \"weight\": 6.47592}]");
+  }
+}
+
+TEST(KeywordExtractorTest, Test2) {
+  KeywordExtractor Extractor("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", "../dict/idf.utf8", "../dict/stop_words.utf8", "../test/testdata/userdict.utf8");
+
+  {
+    string s("蓝翔优秀毕业生");
+    string res;
+    vector<KeywordExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"\xE8\x93\x9D\xE7\xBF\x94\", \"offset\": [0], \"weight\": 11.7392}, {\"word\": \"\xE6\xAF\x95\xE4\xB8\x9A\xE7\x94\x9F\", \"offset\": [12], \"weight\": 8.13549}, {\"word\": \"\xE4\xBC\x98\xE7\xA7\x80\", \"offset\": [6], \"weight\": 6.78347}]");
+  }
+
+  {
+    string s("一部iPhone6");
+    string res;
+    vector<KeywordExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"iPhone6\", \"offset\": [6], \"weight\": 11.7392}, {\"word\": \"\xE4\xB8\x80\xE9\x83\xA8\", \"offset\": [0], \"weight\": 6.47592}]");
+  }
+}

data/ext/cppjieba/test/unittest/pos_tagger_test.cpp

@@ -0,0 +1,41 @@
+#include "cppjieba/MixSegment.hpp"
+#include "gtest/gtest.h"
+
+using namespace cppjieba;
+
+static const char * const QUERY_TEST1 = "我是蓝翔技工拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上总经理，出任CEO，迎娶白富美，走上人生巅峰。";
+static const char * const ANS_TEST1 = "[我:r, 是:v, 蓝翔:x, 技工:n, 拖拉机:n, 学院:n, 手扶拖拉机:n, 专业:n, 的:uj, 。:x, 不用:v, 多久:m, ，:x, 我:r, 就:d, 会:v, 升职:v, 加薪:nr, ，:x, 当上:t, 总经理:n, ，:x, 出任:v, CEO:eng, ，:x, 迎娶:v, 白富:x, 美:ns, ，:x, 走上:v, 人生:n, 巅峰:n, 。:x]";
+static const char * const QUERY_TEST2 = "我是蓝翔技工拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上总经理，出任CEO，迎娶白富美，走上人生巅峰。";
+static const char * const ANS_TEST2 = "[我:r, 是:v, 蓝翔:nz, 技工:n, 拖拉机:n, 学院:n, 手扶拖拉机:n, 专业:n, 的:uj, 。:x, 不用:v, 多久:m, ，:x, 我:r, 就:d, 会:v, 升职:v, 加薪:nr, ，:x, 当上:t, 总经理:n, ，:x, 出任:v, CEO:eng, ，:x, 迎娶:v, 白富:x, 美:ns, ，:x, 走上:v, 人生:n, 巅峰:n, 。:x]";
+
+static const char * const QUERY_TEST3 = "iPhone6手机的最大特点是很容易弯曲。";
+static const char * const ANS_TEST3 = "[iPhone6:eng, 手机:n, 的:uj, 最大:a, 特点:n, 是:v, 很:zg, 容易:a, 弯曲:v, 。:x]";
+//static const char * const ANS_TEST3 = "";
+
+TEST(PosTaggerTest, Test) {
+  MixSegment tagger("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8");
+  {
+    vector<pair<string, string> > res;
+    tagger.Tag(QUERY_TEST1, res);
+    string s;
+    s << res;
+    ASSERT_TRUE(s == ANS_TEST1);
+  }
+}
+TEST(PosTagger, TestUserDict) {
+  MixSegment tagger("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "../test/testdata/userdict.utf8");
+  {
+    vector<pair<string, string> > res;
+    tagger.Tag(QUERY_TEST2, res);
+    string s;
+    s << res;
+    ASSERT_EQ(s, ANS_TEST2);
+  }
+  {
+    vector<pair<string, string> > res;
+    tagger.Tag(QUERY_TEST3, res);
+    string s;
+    s << res;
+    ASSERT_EQ(s, ANS_TEST3);
+  }
+}

data/ext/cppjieba/test/unittest/pre_filter_test.cpp

@@ -0,0 +1,43 @@
+#include "gtest/gtest.h"
+#include "cppjieba/PreFilter.hpp"
+#include "limonp/StringUtil.hpp"
+
+using namespace cppjieba;
+
+TEST(PreFilterTest, Test1) {
+  unordered_set<Rune> symbol;
+  symbol.insert(65292u); // "，"
+  symbol.insert(12290u); // "。"
+  string expected;
+  string res;
+
+  {
+    string s = "你好，美丽的，世界";
+    PreFilter filter(symbol, s);
+    expected = "你好/，/美丽的/，/世界";
+    ASSERT_TRUE(filter.HasNext());
+    vector<string> words;
+    while (filter.HasNext()) {
+      PreFilter::Range range;
+      range = filter.Next();
+      words.push_back(GetStringFromRunes(s, range.begin, range.end - 1));
+    }
+    res = limonp::Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(res, expected);
+  }
+
+  {
+    string s = "我来自北京邮电大学。。。学号123456，用AK47";
+    PreFilter filter(symbol, s);
+    expected = "我来自北京邮电大学/。/。/。/学号123456/，/用AK47";
+    ASSERT_TRUE(filter.HasNext());
+    vector<string> words;
+    while (filter.HasNext()) {
+      PreFilter::Range range;
+      range = filter.Next();
+      words.push_back(GetStringFromRunes(s, range.begin, range.end - 1));
+    }
+    res = limonp::Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(res, expected);
+  }
+}

data/ext/cppjieba/test/unittest/segments_test.cpp

@@ -0,0 +1,256 @@
+#include "cppjieba/SegmentBase.hpp"
+#include "cppjieba/MixSegment.hpp"
+#include "cppjieba/MPSegment.hpp"
+#include "cppjieba/HMMSegment.hpp"
+#include "cppjieba/FullSegment.hpp"
+#include "cppjieba/QuerySegment.hpp"
+#include "gtest/gtest.h"
+
+using namespace cppjieba;
+
+TEST(MixSegmentTest, Test1) {
+  MixSegment segment("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8");;
+  string sentence;
+  vector<string> words;
+  string actual;
+  string expected;
+
+  {
+    sentence = "我来自北京邮电大学。。。学号123456，用AK47";
+    expected = "我/来自/北京邮电大学/。/。/。/学号/123456/，/用/AK47";
+    segment.Cut(sentence, words);
+    actual = Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(actual, expected);
+  }
+
+  {
+    sentence = "B超 T恤";
+    expected = "B超/ /T恤";
+    segment.Cut(sentence, words);
+    actual = Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(actual, expected);
+  }
+
+  {
+    sentence = "他来到了网易杭研大厦";
+    expected = "他/来到/了/网易/杭/研/大厦";
+    segment.Cut(sentence, words, false);
+    actual = Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(actual, expected);
+  }
+
+  {
+    sentence = "他来到了网易杭研大厦";
+    expected = "他/来到/了/网易/杭研/大厦";
+    segment.Cut(sentence, words);
+    actual = Join(words.begin(), words.end(), "/");
+    ASSERT_EQ(actual, expected);
+  }
+}
+
+TEST(MixSegmentTest, NoUserDict) {
+  MixSegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8");
+  const char* str = "令狐冲是云计算方面的专家";
+  vector<string> words;
+  segment.Cut(str, words);
+  string res;
+  ASSERT_EQ("[\"令狐冲\", \"是\", \"云\", \"计算\", \"方面\", \"的\", \"专家\"]", res << words);
+
+}
+TEST(MixSegmentTest, UserDict) {
+  MixSegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", "../dict/user.dict.utf8");
+  {
+    const char* str = "令狐冲是云计算方面的专家";
+    vector<string> words;
+    segment.Cut(str, words);
+    string res;
+    ASSERT_EQ("[\"令狐冲\", \"是\", \"云计算\", \"方面\", \"的\", \"专家\"]", res << words);
+  }
+  {
+    const char* str = "小明先就职于IBM,后在日本京都大学深造";
+    vector<string> words;
+    segment.Cut(str, words);
+    string res;
+    res << words;
+    ASSERT_EQ("[\"小明\", \"先\", \"就职\", \"于\", \"IBM\", \",\", \"后\", \"在\", \"日本\", \"京都大学\", \"深造\"]", res);
+  }
+  {
+    const char* str = "IBM,3.14";
+    vector<string> words;
+    segment.Cut(str, words);
+    string res;
+    res << words;
+    ASSERT_EQ("[\"IBM\", \",\", \"3.14\"]", res);
+  }
+}
+TEST(MixSegmentTest, TestUserDict) {
+  MixSegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", 
+        "../test/testdata/userdict.utf8");
+  vector<string> words;
+  string res;
+
+  segment.Cut("令狐冲是云计算方面的专家", words);
+  ASSERT_EQ("[\"令狐冲\", \"是\", \"云计算\", \"方面\", \"的\", \"专家\"]", res << words);
+
+  segment.Cut("小明先就职于IBM,后在日本京都大学深造", words);
+  res << words;
+  ASSERT_EQ("[\"小明\", \"先\", \"就职\", \"于\", \"I\", \"B\", \"M\", \",\", \"后\", \"在\", \"日本\", \"京都大学\", \"深造\"]", res);
+
+  segment.Cut("IBM,3.14", words);
+  res << words;
+  ASSERT_EQ("[\"I\", \"B\", \"M\", \",\", \"3.14\"]", res);
+
+  segment.Cut("忽如一夜春风来，千树万树梨花开", words);
+  res = limonp::Join(words.begin(), words.end(), "/");
+  ASSERT_EQ("忽如一夜春风来/，/千树/万树/梨花/开", res);
+
+  // rand input
+  {
+    const size_t ITERATION = 16;
+    const size_t MAX_LEN = 256;
+    string s;
+    srand(time(NULL));
+
+    for (size_t i = 0; i < ITERATION; i++) {
+      size_t len = rand() % MAX_LEN;
+      s.resize(len);
+      for (size_t j = 0; j < len; j++) {
+        s[rand() % len] = rand();
+      }
+      segment.Cut(s, words);
+    }
+  }
+}
+
+TEST(MixSegmentTest, TestMultiUserDict) {
+  MixSegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", 
+        "../test/testdata/userdict.utf8;../test/testdata/userdict.2.utf8");
+  vector<string> words;
+  string res;
+
+  segment.Cut("忽如一夜春风来，千树万树梨花开", words);
+  res = limonp::Join(words.begin(), words.end(), "/");
+  ASSERT_EQ("忽如一夜春风来/，/千树万树梨花开", res);
+}
+
+TEST(MPSegmentTest, Test1) {
+  MPSegment segment("../dict/jieba.dict.utf8");;
+  string s;
+  vector<string> words;
+  segment.Cut("我来自北京邮电大学。", words);
+  ASSERT_EQ("[\"我\", \"来自\", \"北京邮电大学\", \"。\"]", s << words);
+
+  segment.Cut("B超 T恤", words);
+  ASSERT_EQ(s << words, "[\"B超\", \" \", \"T恤\"]");
+
+  segment.Cut("南京市长江大桥", words);
+  ASSERT_EQ("[\"南京市\", \"长江大桥\"]", s << words);
+
+  // MaxWordLen
+  segment.Cut("南京市长江大桥", words, 3);
+  ASSERT_EQ("[\"南京市\", \"长江\", \"大桥\"]", s << words);
+
+  segment.Cut("南京市长江大桥", words, 0);
+  ASSERT_EQ("[\"南\", \"京\", \"市\", \"长\", \"江\", \"大\", \"桥\"]", s << words);
+
+  segment.Cut("湖南长沙市天心区", words);
+  s = Join(words.begin(), words.end(), "/");
+  ASSERT_EQ("湖南长沙市/天心区", s);
+
+  segment.Cut("湖南长沙市天心区", words, 3);
+  s = Join(words.begin(), words.end(), "/");
+  ASSERT_EQ("湖南/长沙市/天心区", s);
+}
+
+TEST(HMMSegmentTest, Test1) {
+  HMMSegment segment("../dict/hmm_model.utf8");;
+  {
+    const char* str = "我来自北京邮电大学。。。学号123456";
+    const char* res[] = {"我来", "自北京", "邮电大学", "。", "。", "。", "学号", "123456"};
+    vector<string> words;
+    segment.Cut(str, words);
+    ASSERT_EQ(words, vector<string>(res, res + sizeof(res)/sizeof(res[0])));
+  }
+
+  {
+    const char* str = "IBM,1.2,123";
+    const char* res[] = {"IBM", ",", "1.2", ",", "123"};
+    vector<string> words;
+    segment.Cut(str, words);
+    ASSERT_EQ(words, vector<string>(res, res + sizeof(res)/sizeof(res[0])));
+  }
+}
+
+TEST(FullSegment, Test1) {
+  FullSegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8");
+  vector<string> words;
+  string s;
+
+  segment.Cut("我来自北京邮电大学", words);
+  s << words;
+  ASSERT_EQ(s, "[\"我\", \"来自\", \"北京\", \"北京邮电大学\", \"邮电\", \"电大\", \"大学\"]");
+
+  
+  segment.Cut("上市公司CEO", words);
+  s << words;
+  ASSERT_EQ(s, "[\"上市\", \"公司\", \"C\", \"E\", \"O\"]");
+}
+
+TEST(QuerySegment, Test1) {
+  QuerySegment segment("../dict/jieba.dict.utf8", "../dict/hmm_model.utf8", "");
+  vector<string> words;
+  string s1, s2;
+
+  segment.Cut("小明硕士毕业于中国科学院计算所，后在日本京都大学深造", words);
+  s1 = Join(words.begin(), words.end(), "/");
+  s2 = "小明/硕士/毕业/于/中国/科学/学院/科学院/中国科学院/计算/计算所/，/后/在/日本/京都/大学/日本京都大学/深造";
+  ASSERT_EQ(s1, s2);
+
+  segment.Cut("亲口交代", words);
+  s1 = Join(words.begin(), words.end(), "/");
+  s2 = "亲口/交代";
+  ASSERT_EQ(s1, s2);
+
+  segment.Cut("他心理健康", words);
+  s1 = Join(words.begin(), words.end(), "/");
+  s2 = "他/心理/健康/心理健康";
+  ASSERT_EQ(s1, s2);
+}
+
+TEST(QuerySegment, Test2) {
+  QuerySegment segment("../test/testdata/extra_dict/jieba.dict.small.utf8", "../dict/hmm_model.utf8", "../test/testdata/userdict.utf8|../test/testdata/userdict.english");
+  vector<string> words;
+  string s1, s2;
+
+  {
+    segment.Cut("小明硕士毕业于中国科学院计算所，后在日本京都大学深造", words);
+    s1 = Join(words.begin(), words.end(), "/");
+    s2 = "小明/硕士/毕业/于/中国/科学/学院/科学院/中国科学院/计算/计算所/，/后/在/日本/京都/大学/京都大学/深造";
+    ASSERT_EQ(s1, s2);
+  }
+
+  {
+    segment.Cut("小明硕士毕业于中国科学院计算所iPhone6", words);
+    s1 = Join(words.begin(), words.end(), "/");
+    s2 = "小明/硕士/毕业/于/中国/科学/学院/科学院/中国科学院/计算/计算所/iPhone6";
+    ASSERT_EQ(s1, s2);
+  }
+
+  {
+    segment.Cut("中国科学院", words);
+    s1 = Join(words.begin(), words.end(), "/");
+    s2 = "中国/科学/学院/科学院/中国科学院";
+    ASSERT_EQ(s1, s2);
+  }
+
+}
+
+TEST(MPSegmentTest, Unicode32) {
+  string s("天气很好，🙋 我们去郊游。");
+  vector<string> words;
+
+  MPSegment segment("../dict/jieba.dict.utf8");;
+  segment.Cut(s, words);
+
+  ASSERT_EQ(Join(words.begin(), words.end(), "/"), "天气/很/好/，/🙋/ /我们/去/郊游/。");
+}

data/ext/cppjieba/test/unittest/textrank_test.cpp

@@ -0,0 +1,86 @@
+#include "cppjieba/TextRankExtractor.hpp"
+#include "gtest/gtest.h"
+
+using namespace cppjieba;
+
+TEST(TextRankExtractorTest, Test1) {
+  TextRankExtractor Extractor(
+    "../test/testdata/extra_dict/jieba.dict.small.utf8",
+    "../dict/hmm_model.utf8", 
+    "../dict/stop_words.utf8");
+  {
+    string s("你好世界世界而且而且");
+    string res;
+    size_t topN = 5;
+
+    {
+      vector<string> words;
+      Extractor.Extract(s, words, topN);
+      res << words;
+      ASSERT_EQ(res, "[\"世界\", \"你好\"]");
+    }
+
+    {
+      vector<pair<string, double> > words;
+      Extractor.Extract(s, words, topN);
+      res << words;
+      ASSERT_EQ(res, "[世界:1, 你好:0.519787]");
+    }
+
+    {
+      vector<TextRankExtractor::Word> words;
+      Extractor.Extract(s, words, topN);
+      res << words;
+      ASSERT_EQ(res, "[{\"word\": \"世界\", \"offset\": [6, 12], \"weight\": 1}, {\"word\": \"你好\", \"offset\": [0], \"weight\": 0.519787}]");
+    }
+  }
+
+  { 
+    string s("\xe6\x88\x91\xe6\x98\xaf\xe6\x8b\x96\xe6\x8b\x89\xe6\x9c\xba\xe5\xad\xa6\xe9\x99\xa2\xe6\x89\x8b\xe6\x89\xb6\xe6\x8b\x96\xe6\x8b\x89\xe6\x9c\xba\xe4\xb8\x93\xe4\xb8\x9a\xe7\x9a\x84\xe3\x80\x82\xe4\xb8\x8d\xe7\x94\xa8\xe5\xa4\x9a\xe4\xb9\x85\xef\xbc\x8c\xe6\x88\x91\xe5\xb0\xb1\xe4\xbc\x9a\xe5\x8d\x87\xe8\x81\x8c\xe5\x8a\xa0\xe8\x96\xaa\xef\xbc\x8c\xe5\xbd\x93\xe4\xb8\x8a CEO\xef\xbc\x8c\xe8\xb5\xb0\xe4\xb8\x8a\xe4\xba\xba\xe7\x94\x9f\xe5\xb7\x85\xe5\xb3\xb0");
+    string res;
+    vector<TextRankExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"当上\", \"offset\": [87], \"weight\": 1}, {\"word\": \"不用\", \"offset\": [48], \"weight\": 0.989848}, {\"word\": \"多久\", \"offset\": [54], \"weight\": 0.985126}, {\"word\": \"加薪\", \"offset\": [78], \"weight\": 0.983046}, {\"word\": \"升职\", \"offset\": [72], \"weight\": 0.980278}]");
+    //ASSERT_EQ(res, "[{\"word\": \"专业\", \"offset\": [36], \"weight\": 1}, {\"word\": \"CEO\", \"offset\": [94], \"weight\": 0.95375}, {\"word\": \"手扶拖拉机\", \"offset\": [21], \"weight\": 0.801701}, {\"word\": \"当上\", \"offset\": [87], \"weight\": 0.798968}, {\"word\": \"走上\", \"offset\": [100], \"weight\": 0.775505}]");
+  }
+
+  {
+    string s("一部iPhone6");
+    string res;
+    vector<TextRankExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"一部\", \"offset\": [0], \"weight\": 1}, {\"word\": \"iPhone6\", \"offset\": [6], \"weight\": 0.996126}]");
+  }
+}
+
+TEST(TextRankExtractorTest, Test2) {
+  TextRankExtractor Extractor(
+    "../test/testdata/extra_dict/jieba.dict.small.utf8",
+    "../dict/hmm_model.utf8",
+    "../dict/stop_words.utf8",
+    "../test/testdata/userdict.utf8");
+
+  {
+    string s("\xe8\x93\x9d\xe7\xbf\x94\xe4\xbc\x98\xe7\xa7\x80\xe6\xaf\x95\xe4\xb8\x9a\xe7\x94\x9f");
+    string res;
+    vector<TextRankExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"蓝翔\", \"offset\": [0], \"weight\": 1}, {\"word\": \"毕业生\", \"offset\": [12], \"weight\": 0.996685}, {\"word\": \"优秀\", \"offset\": [6], \"weight\": 0.992994}]");
+  }
+
+  {
+    string s("一部iPhone6");
+    string res;
+    vector<TextRankExtractor::Word> wordweights;
+    size_t topN = 5;
+    Extractor.Extract(s, wordweights, topN);
+    res << wordweights;
+    ASSERT_EQ(res, "[{\"word\": \"一部\", \"offset\": [0], \"weight\": 1}, {\"word\": \"iPhone6\", \"offset\": [6], \"weight\": 0.996126}]");
+  }
+}