nodejieba-plus 3.5.8

package/submodules/cppjieba/deps/limonp/include/limonp/StringUtil.hpp

@@ -0,0 +1,386 @@
+/************************************
+ * file enc : ascii
+ * author   : wuyanyi09@gmail.com
+ ************************************/
+#ifndef LIMONP_STR_FUNCTS_H
+#define LIMONP_STR_FUNCTS_H
+#include <fstream>
+#include <iostream>
+#include <string>
+#include <vector>
+#include <algorithm>
+#include <cctype>
+#include <map>
+#include <cassert>
+#include <ctime>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <memory.h>
+#include <functional>
+#include <locale>
+#include <sstream>
+#include <sys/types.h>
+#include <iterator>
+#include <algorithm>
+#include "StdExtension.hpp"
+
+namespace limonp {
+using namespace std;
+inline string StringFormat(const char* fmt, ...) {
+  int size = 256;
+  std::string str;
+  va_list ap;
+  while (1) {
+    str.resize(size);
+    va_start(ap, fmt);
+    int n = vsnprintf((char *)str.c_str(), size, fmt, ap);
+    va_end(ap);
+    if (n > -1 && n < size) {
+      str.resize(n);
+      return str;
+    }
+    if (n > -1)
+      size = n + 1;
+    else
+      size *= 2;
+  }
+  return str;
+}
+
+template<class T>
+void Join(T begin, T end, string& res, const string& connector) {
+  if(begin == end) {
+    return;
+  }
+  stringstream ss;
+  ss<<*begin;
+  begin++;
+  while(begin != end) {
+    ss << connector << *begin;
+    begin ++;
+  }
+  res = ss.str();
+}
+
+template<class T>
+string Join(T begin, T end, const string& connector) {
+  string res;
+  Join(begin ,end, res, connector);
+  return res;
+}
+
+inline string& Upper(string& str) {
+  transform(str.begin(), str.end(), str.begin(), (int (*)(int))toupper);
+  return str;
+}
+
+inline string& Lower(string& str) {
+  transform(str.begin(), str.end(), str.begin(), (int (*)(int))tolower);
+  return str;
+}
+
+inline bool IsSpace(unsigned c) {
+  // when passing large int as the argument of isspace, it core dump, so here need a type cast.
+  return c > 0xff ? false : std::isspace(c & 0xff) != 0;
+}
+
+inline std::string& LTrim(std::string &s) {
+    s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) {
+        return !std::isspace(ch);
+    }));
+    return s;
+}
+
+inline std::string& RTrim(std::string &s) {
+  s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) {
+      return !std::isspace(ch);
+  }).base(), s.end());
+  return s;
+}
+
+inline std::string& Trim(std::string &s) {
+  return LTrim(RTrim(s));
+}
+
+inline std::string& LTrim(std::string& s, char x) {
+    s.erase(s.begin(), std::find_if(s.begin(), s.end(),
+      [x](unsigned char c) { return !std::isspace(c) && c != x; }));
+    return s;
+}
+
+inline std::string& RTrim(std::string& s, char x) {
+    s.erase(std::find_if(s.rbegin(), s.rend(),
+        [x](unsigned char c) { return !std::isspace(c) && c != x; }).base(), s.end());
+    return s;
+}
+
+inline std::string& Trim(std::string &s, char x) {
+  return LTrim(RTrim(s, x), x);
+}
+
+inline void Split(const string& src, vector<string>& res, const string& pattern, size_t maxsplit = string::npos) {
+  res.clear();
+  size_t Start = 0;
+  size_t end = 0;
+  string sub;
+  while(Start < src.size()) {
+    end = src.find_first_of(pattern, Start);
+    if(string::npos == end || res.size() >= maxsplit) {
+      sub = src.substr(Start);
+      res.push_back(sub);
+      return;
+    }
+    sub = src.substr(Start, end - Start);
+    res.push_back(sub);
+    Start = end + 1;
+  }
+  return;
+}
+
+inline vector<string> Split(const string& src, const string& pattern, size_t maxsplit = string::npos) {
+  vector<string> res;
+  Split(src, res, pattern, maxsplit);
+  return res;
+}
+
+inline bool StartsWith(const string& str, const string& prefix) {
+  if(prefix.length() > str.length()) {
+    return false;
+  }
+  return 0 == str.compare(0, prefix.length(), prefix);
+}
+
+inline bool EndsWith(const string& str, const string& suffix) {
+  if(suffix.length() > str.length()) {
+    return false;
+  }
+  return 0 == str.compare(str.length() -  suffix.length(), suffix.length(), suffix);
+}
+
+inline bool IsInStr(const string& str, char ch) {
+  return str.find(ch) != string::npos;
+}
+
+inline uint16_t TwocharToUint16(char high, char low) {
+  return (((uint16_t(high) & 0x00ff ) << 8) | (uint16_t(low) & 0x00ff));
+}
+
+template <class Uint16Container>
+bool Utf8ToUnicode(const char * const str, size_t len, Uint16Container& vec) {
+  if(!str) {
+    return false;
+  }
+  char ch1, ch2;
+  uint16_t tmp;
+  vec.clear();
+  for(size_t i = 0; i < len;) {
+    if(!(str[i] & 0x80)) { // 0xxxxxxx
+      vec.push_back(str[i]);
+      i++;
+    } else if ((uint8_t)str[i] <= 0xdf && i + 1 < len) { // 110xxxxxx
+      ch1 = (str[i] >> 2) & 0x07;
+      ch2 = (str[i+1] & 0x3f) | ((str[i] & 0x03) << 6 );
+      tmp = (((uint16_t(ch1) & 0x00ff ) << 8) | (uint16_t(ch2) & 0x00ff));
+      vec.push_back(tmp);
+      i += 2;
+    } else if((uint8_t)str[i] <= 0xef && i + 2 < len) {
+      ch1 = ((uint8_t)str[i] << 4) | ((str[i+1] >> 2) & 0x0f );
+      ch2 = (((uint8_t)str[i+1]<<6) & 0xc0) | (str[i+2] & 0x3f);
+      tmp = (((uint16_t(ch1) & 0x00ff ) << 8) | (uint16_t(ch2) & 0x00ff));
+      vec.push_back(tmp);
+      i += 3;
+    } else {
+      return false;
+    }
+  }
+  return true;
+}
+
+template <class Uint16Container>
+bool Utf8ToUnicode(const string& str, Uint16Container& vec) {
+  return Utf8ToUnicode(str.c_str(), str.size(), vec);
+}
+
+template <class Uint32Container>
+bool Utf8ToUnicode32(const string& str, Uint32Container& vec) {
+  uint32_t tmp;
+  vec.clear();
+  for(size_t i = 0; i < str.size();) {
+    if(!(str[i] & 0x80)) { // 0xxxxxxx
+      // 7bit, total 7bit
+      tmp = (uint8_t)(str[i]) & 0x7f;
+      i++;
+    } else if ((uint8_t)str[i] <= 0xdf && i + 1 < str.size()) { // 110xxxxxx
+      // 5bit, total 5bit
+      tmp = (uint8_t)(str[i]) & 0x1f;
+
+      // 6bit, total 11bit
+      tmp <<= 6;
+      tmp |= (uint8_t)(str[i+1]) & 0x3f;
+      i += 2;
+    } else if((uint8_t)str[i] <= 0xef && i + 2 < str.size()) { // 1110xxxxxx
+      // 4bit, total 4bit
+      tmp = (uint8_t)(str[i]) & 0x0f;
+
+      // 6bit, total 10bit
+      tmp <<= 6;
+      tmp |= (uint8_t)(str[i+1]) & 0x3f;
+
+      // 6bit, total 16bit
+      tmp <<= 6;
+      tmp |= (uint8_t)(str[i+2]) & 0x3f;
+
+      i += 3;
+    } else if((uint8_t)str[i] <= 0xf7 && i + 3 < str.size()) { // 11110xxxx
+      // 3bit, total 3bit
+      tmp = (uint8_t)(str[i]) & 0x07;
+
+      // 6bit, total 9bit
+      tmp <<= 6;
+      tmp |= (uint8_t)(str[i+1]) & 0x3f;
+
+      // 6bit, total 15bit
+      tmp <<= 6;
+      tmp |= (uint8_t)(str[i+2]) & 0x3f;
+
+      // 6bit, total 21bit
+      tmp <<= 6;
+      tmp |= (uint8_t)(str[i+3]) & 0x3f;
+
+      i += 4;
+    } else {
+      return false;
+    }
+    vec.push_back(tmp);
+  }
+  return true;
+}
+
+template <class Uint32ContainerConIter>
+void Unicode32ToUtf8(Uint32ContainerConIter begin, Uint32ContainerConIter end, string& res) {
+  res.clear();
+  uint32_t ui;
+  while(begin != end) {
+    ui = *begin;
+    if(ui <= 0x7f) {
+      res += char(ui);
+    } else if(ui <= 0x7ff) {
+      res += char(((ui >> 6) & 0x1f) | 0xc0);
+      res += char((ui & 0x3f) | 0x80);
+    } else if(ui <= 0xffff) {
+      res += char(((ui >> 12) & 0x0f) | 0xe0);
+      res += char(((ui >> 6) & 0x3f) | 0x80);
+      res += char((ui & 0x3f) | 0x80);
+    } else {
+      res += char(((ui >> 18) & 0x03) | 0xf0);
+      res += char(((ui >> 12) & 0x3f) | 0x80);
+      res += char(((ui >> 6) & 0x3f) | 0x80);
+      res += char((ui & 0x3f) | 0x80);
+    }
+    begin ++;
+  }
+}
+
+template <class Uint16ContainerConIter>
+void UnicodeToUtf8(Uint16ContainerConIter begin, Uint16ContainerConIter end, string& res) {
+  res.clear();
+  uint16_t ui;
+  while(begin != end) {
+    ui = *begin;
+    if(ui <= 0x7f) {
+      res += char(ui);
+    } else if(ui <= 0x7ff) {
+      res += char(((ui>>6) & 0x1f) | 0xc0);
+      res += char((ui & 0x3f) | 0x80);
+    } else {
+      res += char(((ui >> 12) & 0x0f )| 0xe0);
+      res += char(((ui>>6) & 0x3f )| 0x80 );
+      res += char((ui & 0x3f) | 0x80);
+    }
+    begin ++;
+  }
+}
+
+
+template <class Uint16Container>
+bool GBKTrans(const char* const str, size_t len, Uint16Container& vec) {
+  vec.clear();
+  if(!str) {
+    return true;
+  }
+  size_t i = 0;
+  while(i < len) {
+    if(0 == (str[i] & 0x80)) {
+      vec.push_back(uint16_t(str[i]));
+      i++;
+    } else {
+      if(i + 1 < len) { //&& (str[i+1] & 0x80))
+        uint16_t tmp = (((uint16_t(str[i]) & 0x00ff ) << 8) | (uint16_t(str[i+1]) & 0x00ff));
+        vec.push_back(tmp);
+        i += 2;
+      } else {
+        return false;
+      }
+    }
+  }
+  return true;
+}
+
+template <class Uint16Container>
+bool GBKTrans(const string& str, Uint16Container& vec) {
+  return GBKTrans(str.c_str(), str.size(), vec);
+}
+
+template <class Uint16ContainerConIter>
+void GBKTrans(Uint16ContainerConIter begin, Uint16ContainerConIter end, string& res) {
+  res.clear();
+  //pair<char, char> pa;
+  char first, second;
+  while(begin != end) {
+    //pa = uint16ToChar2(*begin);
+    first = ((*begin)>>8) & 0x00ff;
+    second = (*begin) & 0x00ff;
+    if(first & 0x80) {
+      res += first;
+      res += second;
+    } else {
+      res += second;
+    }
+    begin++;
+  }
+}
+
+/*
+ * format example: "%Y-%m-%d %H:%M:%S"
+ */
+inline void GetTime(const string& format, string&  timeStr) {
+  time_t timeNow;
+  time(&timeNow);
+
+  struct tm tmNow;
+
+  #if defined(_WIN32) || defined(_WIN64)
+  errno_t e = localtime_s(&tmNow, &timeNow);
+  assert(e == 0);
+  #else
+  struct tm * tm_tmp = localtime_r(&timeNow, &tmNow);
+  assert(tm_tmp != nullptr);
+  #endif
+
+  timeStr.resize(64);
+  
+  size_t len = strftime((char*)timeStr.c_str(), timeStr.size(), format.c_str(), &tmNow);
+  
+  timeStr.resize(len);
+}
+
+inline string PathJoin(const string& path1, const string& path2) {
+  if(EndsWith(path1, "/")) {
+    return path1 + path2;
+  }
+  return path1 + "/" + path2;
+}
+
+}
+#endif

package/submodules/cppjieba/deps/limonp/test/CMakeLists.txt

@@ -0,0 +1,8 @@
+SET(EXECUTABLE_OUTPUT_PATH ${PROJECT_BINARY_DIR}/test)
+SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/test/lib)
+
+INCLUDE_DIRECTORIES(${PROJECT_SOURCE_DIR}/include)
+
+ADD_EXECUTABLE(demo demo.cpp)
+
+ADD_SUBDIRECTORY(unittest)

package/submodules/cppjieba/deps/limonp/test/demo.cpp

@@ -0,0 +1,40 @@
+#include "limonp/StringUtil.hpp"
+#include "limonp/Logging.hpp"
+
+using namespace std;
+
+#define print(x) std::cout << x << std::endl
+
+int main(int argc, char** argv) {
+  string strname = "hello, world";
+  print(strname); //hello, world
+  map<string, int> mp;
+  mp["hello"] = 1;
+  mp["world"] = 2;
+  print(mp); // {"hello":1, "world":2}
+
+  string res;
+  res << mp;
+  print(res); // {"hello":1, "world":2}
+
+  string str;
+  str = limonp::StringFormat("%s, %s", "hello", "world");
+  print(str); //hello, world
+
+  const char * a[] = {"hello", "world"};
+  str = limonp::Join(a, a + sizeof(a)/sizeof(*a), ",");
+  print(str); //hello,world
+
+  str = "hello, world";
+  vector<string> buf;
+  limonp::Split(str, buf, ",");
+  print(buf); //["hello", " world"]
+
+  int arr[] = {1,10,100,1000};
+  vector<int> vec_i(arr, arr + sizeof(arr)/sizeof(arr[0]));
+  print(vec_i); //[1, 10, 100, 1000]
+
+  XLOG(INFO) << "hello, world"; //2014-04-05 20:52:37 demo.cpp:35 INFO hello, world
+  //In the same way, `LOG(DEBUG),LOG(WARNING),LOG(ERROR),LOG(FATAL)`.
+  return EXIT_SUCCESS;
+}

package/submodules/cppjieba/deps/limonp/test/testdata/1.conf

@@ -0,0 +1,5 @@
+
+key1 = val1
+##this is comment
+
+key2=val2

package/submodules/cppjieba/deps/limonp/test/testdata/StdExtension.data

@@ -0,0 +1,3 @@
+key1 = val1
+##this is comment
+key2=val2

package/submodules/cppjieba/deps/limonp/test/testdata/dict.gbk

@@ -0,0 +1,50 @@
+AT&T 3 nz
+B�� 3 n
+c# 3 nz
+C# 3 nz
+c++ 3 nz
+C++ 3 nz
+T�� 4 n
+һ 217830 m
+һһ 1670 m
+һһ�� 11 m
+һһ�� 3 m
+һһ�� 8 m
+һһ�о� 34 i
+һһ�� 9 m
+һһ��Ӧ 43 l
+һһ�� 2 m
+һһ���� 4 l
+һ�� 18 d
+һ����ʶ 3 i
+һ���� 3 m
+һ����� 24 m
+һ�� 22 m
+һ�߰˲� 3 l
+һ�� 442 m
+һ��һǧ 4 m
+һ��һǧ��ٶ�ʮ�� 2 m
+һ��һǧ�˰ٰ�ʮ�� 2 m
+һ��һǧ��� 2 m
+һ��һǧ���ʮ��� 4 m
+һ����ǧ 5 m
+һ����ǧ�� 2 m
+һ����ǧ�� 2 m
+һ����ǧ�໧ 2 m
+һ���� 4 m
+һ������ 4 m
+һ����ǧ 8 m
+һ����ǧ���һʮ�� 2 m
+һ����ǧ��ٽ� 4 m
+һ����ǧ���� 2 m
+һ����ǧ�� 2 m
+һ���� 124 m
+һ������ 4 m
+һ����ǧ 3 m
+һ��� 62 m
+һ���ǧ 2 m
+һ���ǧ�� 2 m
+һ��� 10 m
+һ���ǧ 7 m
+һ���ǧ�� 2 m
+һ���ǧ��� 4 m

package/submodules/cppjieba/deps/limonp/test/testdata/dict.utf8

@@ -0,0 +1,50 @@
+AT&T 3 nz
+B超 3 n
+c# 3 nz
+C# 3 nz
+c++ 3 nz
+C++ 3 nz
+T恤 4 n
+一 217830 m
+一一 1670 m
+一一二 11 m
+一一例 3 m
+一一分 8 m
+一一列举 34 i
+一一对 9 m
+一一对应 43 l
+一一记 2 m
+一一道来 4 l
+一丁 18 d
+一丁不识 3 i
+一丁点 3 m
+一丁点儿 24 m
+一七 22 m
+一七八不 3 l
+一万 442 m
+一万一千 4 m
+一万一千五百二十颗 2 m
+一万一千八百八十斤 2 m
+一万一千多间 2 m
+一万一千零九十五册 4 m
+一万七千 5 m
+一万七千余 2 m
+一万七千多 2 m
+一万七千多户 2 m
+一万万 4 m
+一万万两 4 m
+一万三千 8 m
+一万三千五百一十七 2 m
+一万三千五百斤 4 m
+一万三千余种 2 m
+一万三千块 2 m
+一万两 124 m
+一万两万 4 m
+一万两千 3 m
+一万个 62 m
+一万九千 2 m
+一万九千余 2 m
+一万二 10 m
+一万二千 7 m
+一万二千两 2 m
+一万二千五百 4 m

package/submodules/cppjieba/deps/limonp/test/testdata/io_testfile

@@ -0,0 +1,2 @@
+line1
+line2

package/submodules/cppjieba/deps/limonp/test/testdata/jieba.dict.0.1.utf8

@@ -0,0 +1,93 @@
+龙鸣狮吼 3 nr
+龙齐诺 2 nr
+龙齿 3 n
+龚 176 nr
+龚世萍 2 nr
+龚书铎 2 nr
+龚二人 2 nr
+龚云甫 3 nr
+龚伟强 5 nr
+龚先生 4 nr
+龚光杰 44 nr
+龚古尔 24 nr
+龚子敬 2 nr
+龚孝升 12 nr
+龚学平 2 nr
+龚完敬 5 nr
+龚定庵 3 nr
+龚定敬 2 nr
+龚宝铨 5 nr
+龚家村 3 nr
+龚建国 29 nr
+龚德俊 6 nr
+龚心瀚 3 nr
+龚志国 2 nr
+龚意田 3 nr
+龚慈恩 3 nr
+龚施茜 3 nr
+龚晓犁 2 nr
+龚普洛 3 nr
+龚智超 7 nr
+龚松林 10 nr
+龚永明 3 nr
+龚永泉 5 nr
+龚泽艺 256 nr
+龚睿 8 nrfg
+龚祖同 2 nr
+龚秋婷 3 nr
+龚老爷 2 nr
+龚育之 19 nr
+龚自珍 28 nr
+龚蓓苾 3 nr
+龚虹嘉 3 nr
+龚诗嘉 3 nr
+龛 223 ng
+龜 2 zg
+龟 903 ns
+龟儿子 123 n
+龟兆 3 nz
+龟兹 215 ns
+龟兹王 3 nrt
+龟冷搘床 3 v
+龟冷支床 3 n
+龟卜 3 n
+龟厌不告 3 l
+龟壳 33 n
+龟壳花 3 n
+龟头 34 n
+龟头炎 3 n
+龟山 23 ns
+龟山乡 3 ns
+龟山岛 3 ns
+龟年鹤寿 3 ns
+龟年鹤算 3 l
+龟文 3 nz
+龟文写迹 3 n
+龟文鸟迹 3 n
+龟板 10 n
+龟毛免角 3 n
+龟毛兔角 3 n
+龟溪 3 ns
+龟玉 3 nz
+龟王 3 nz
+龟甲 92 ns
+龟甲胶 3 nz
+龟筮 3 n
+龟纹 3 n
+龟缩 29 v
+龟肉 3 n
+龟背 21 n
+龟背竹 3 n
+龟苓膏 3 n
+龟苗 3 n
+龟裂 34 v
+龟足 5 v
+龟鉴 2 n
+龟镜 3 nz
+龟鳖 3 n
+龟鹤遐寿 3 l
+龟龄鹤算 3 n
+龟龙片甲 3 nz
+龟龙麟凤 3 ns
+龠 5 g
+龢 732 zg