@ohos-ports/nodejieba 0.0.0-stage → 3.5.8-beta.0

package/submodules/cppjieba/include/cppjieba/Unicode.hpp

@@ -0,0 +1,227 @@
+#ifndef CPPJIEBA_UNICODE_H
+#define CPPJIEBA_UNICODE_H
+
+#include <stdint.h>
+#include <stdlib.h>
+#include <string>
+#include <vector>
+#include <ostream>
+#include "limonp/LocalVector.hpp"
+
+namespace cppjieba {
+
+using std::string;
+using std::vector;
+
+typedef uint32_t Rune;
+
+struct Word {
+  string word;
+  uint32_t offset;
+  uint32_t unicode_offset;
+  uint32_t unicode_length;
+  Word(const string& w, uint32_t o)
+   : word(w), offset(o) {
+  }
+  Word(const string& w, uint32_t o, uint32_t unicode_offset, uint32_t unicode_length)
+          : word(w), offset(o), unicode_offset(unicode_offset), unicode_length(unicode_length) {
+  }
+}; // struct Word
+
+inline std::ostream& operator << (std::ostream& os, const Word& w) {
+  return os << "{\"word\": \"" << w.word << "\", \"offset\": " << w.offset << "}";
+}
+
+struct RuneStr {
+  Rune rune;
+  uint32_t offset;
+  uint32_t len;
+  uint32_t unicode_offset;
+  uint32_t unicode_length;
+  RuneStr(): rune(0), offset(0), len(0), unicode_offset(0), unicode_length(0) {
+  }
+  RuneStr(Rune r, uint32_t o, uint32_t l)
+    : rune(r), offset(o), len(l), unicode_offset(0), unicode_length(0) {
+  }
+  RuneStr(Rune r, uint32_t o, uint32_t l, uint32_t unicode_offset, uint32_t unicode_length)
+          : rune(r), offset(o), len(l), unicode_offset(unicode_offset), unicode_length(unicode_length) {
+  }
+}; // struct RuneStr
+
+inline std::ostream& operator << (std::ostream& os, const RuneStr& r) {
+  return os << "{\"rune\": \"" << r.rune << "\", \"offset\": " << r.offset << ", \"len\": " << r.len << "}";
+}
+
+typedef limonp::LocalVector<Rune> Unicode;
+typedef limonp::LocalVector<struct RuneStr> RuneStrArray;
+
+// [left, right]
+struct WordRange {
+  RuneStrArray::const_iterator left;
+  RuneStrArray::const_iterator right;
+  WordRange(RuneStrArray::const_iterator l, RuneStrArray::const_iterator r)
+   : left(l), right(r) {
+  }
+  size_t Length() const {
+    return right - left + 1;
+  }
+  bool IsAllAscii() const {
+    for (RuneStrArray::const_iterator iter = left; iter <= right; ++iter) {
+      if (iter->rune >= 0x80) {
+        return false;
+      }
+    }
+    return true;
+  }
+}; // struct WordRange
+
+struct RuneStrLite {
+  uint32_t rune;
+  uint32_t len;
+  RuneStrLite(): rune(0), len(0) {
+  }
+  RuneStrLite(uint32_t r, uint32_t l): rune(r), len(l) {
+  }
+}; // struct RuneStrLite
+
+inline RuneStrLite DecodeUTF8ToRune(const char* str, size_t len) {
+  RuneStrLite rp(0, 0);
+  if (str == NULL || len == 0) {
+    return rp;
+  }
+  if (!(str[0] & 0x80)) { // 0xxxxxxx
+    // 7bit, total 7bit
+    rp.rune = (uint8_t)(str[0]) & 0x7f;
+    rp.len = 1;
+  } else if ((uint8_t)str[0] <= 0xdf &&  1 < len) { 
+    // 110xxxxxx
+    // 5bit, total 5bit
+    rp.rune = (uint8_t)(str[0]) & 0x1f;
+
+    // 6bit, total 11bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[1]) & 0x3f;
+    rp.len = 2;
+  } else if((uint8_t)str[0] <= 0xef && 2 < len) { // 1110xxxxxx
+    // 4bit, total 4bit
+    rp.rune = (uint8_t)(str[0]) & 0x0f;
+
+    // 6bit, total 10bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[1]) & 0x3f;
+
+    // 6bit, total 16bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[2]) & 0x3f;
+
+    rp.len = 3;
+  } else if((uint8_t)str[0] <= 0xf7 && 3 < len) { // 11110xxxx
+    // 3bit, total 3bit
+    rp.rune = (uint8_t)(str[0]) & 0x07;
+
+    // 6bit, total 9bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[1]) & 0x3f;
+
+    // 6bit, total 15bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[2]) & 0x3f;
+
+    // 6bit, total 21bit
+    rp.rune <<= 6;
+    rp.rune |= (uint8_t)(str[3]) & 0x3f;
+
+    rp.len = 4;
+  } else {
+    rp.rune = 0;
+    rp.len = 0;
+  }
+  return rp;
+}
+
+inline bool DecodeUTF8RunesInString(const char* s, size_t len, RuneStrArray& runes) {
+  runes.clear();
+  runes.reserve(len / 2);
+  for (uint32_t i = 0, j = 0; i < len;) {
+    RuneStrLite rp = DecodeUTF8ToRune(s + i, len - i);
+    if (rp.len == 0) {
+      runes.clear();
+      return false;
+    }
+    RuneStr x(rp.rune, i, rp.len, j, 1);
+    runes.push_back(x);
+    i += rp.len;
+    ++j;
+  }
+  return true;
+}
+
+inline bool DecodeUTF8RunesInString(const string& s, RuneStrArray& runes) {
+  return DecodeUTF8RunesInString(s.c_str(), s.size(), runes);
+}
+
+inline bool DecodeUTF8RunesInString(const char* s, size_t len, Unicode& unicode) {
+  unicode.clear();
+  RuneStrArray runes;
+  if (!DecodeUTF8RunesInString(s, len, runes)) {
+    return false;
+  }
+  unicode.reserve(runes.size());
+  for (size_t i = 0; i < runes.size(); i++) {
+    unicode.push_back(runes[i].rune);
+  }
+  return true;
+}
+
+inline bool IsSingleWord(const string& str) {
+  RuneStrLite rp = DecodeUTF8ToRune(str.c_str(), str.size());
+  return rp.len == str.size();
+}
+
+inline bool DecodeUTF8RunesInString(const string& s, Unicode& unicode) {
+  return DecodeUTF8RunesInString(s.c_str(), s.size(), unicode);
+}
+
+inline Unicode DecodeUTF8RunesInString(const string& s) {
+  Unicode result;
+  DecodeUTF8RunesInString(s, result);
+  return result;
+}
+
+
+// [left, right]
+inline Word GetWordFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {
+  assert(right->offset >= left->offset);
+  uint32_t len = right->offset - left->offset + right->len;
+  uint32_t unicode_length = right->unicode_offset - left->unicode_offset + right->unicode_length;
+  return Word(s.substr(left->offset, len), left->offset, left->unicode_offset, unicode_length);
+}
+
+inline string GetStringFromRunes(const string& s, RuneStrArray::const_iterator left, RuneStrArray::const_iterator right) {
+  assert(right->offset >= left->offset);
+  uint32_t len = right->offset - left->offset + right->len;
+  return s.substr(left->offset, len);
+}
+
+inline void GetWordsFromWordRanges(const string& s, const vector<WordRange>& wrs, vector<Word>& words) {
+  for (size_t i = 0; i < wrs.size(); i++) {
+    words.push_back(GetWordFromRunes(s, wrs[i].left, wrs[i].right));
+  }
+}
+
+inline vector<Word> GetWordsFromWordRanges(const string& s, const vector<WordRange>& wrs) {
+  vector<Word> result;
+  GetWordsFromWordRanges(s, wrs, result);
+  return result;
+}
+
+inline void GetStringsFromWords(const vector<Word>& words, vector<string>& strs) {
+  strs.resize(words.size());
+  for (size_t i = 0; i < words.size(); ++i) {
+    strs[i] = words[i].word;
+  }
+}
+
+} // namespace cppjieba
+
+#endif // CPPJIEBA_UNICODE_H

package/test/load_dict_test.js

@@ -0,0 +1,14 @@
+var should = require("should");
+var nodejieba = require("../index.js");
+
+describe("nodejieba", function() {
+  var userDict = __dirname + '/testdata/userdict.utf8';
+  nodejieba.load({
+    userDict: userDict,
+  });
+  it('nodejieba.cut("红掌拨清波")', function() {
+    nodejieba.cut("红掌拨清波").should.eql([
+      '红掌拨清波',
+    ]);
+  });
+});

package/test/missing_binding_test.js

@@ -0,0 +1,42 @@
+var fs = require("fs");
+var os = require("os");
+var path = require("path");
+var should = require("should");
+
+describe("nodejieba missing binding", function() {
+  it("loads without the native binding until first use", function() {
+    var fixtureDir = fs.mkdtempSync(path.join(os.tmpdir(), "nodejieba-missing-binding-"));
+    try {
+      var fixtureIndex = path.join(fixtureDir, "index.js");
+
+      fs.copyFileSync(path.join(__dirname, "..", "index.js"), fixtureIndex);
+
+      var nodejieba = require(fixtureIndex);
+
+      should(nodejieba.DEFAULT_DICT).type("string");
+      should(nodejieba.cut).type("function");
+
+      var firstError;
+      var secondError;
+
+      try {
+        nodejieba.cut("南京市长江大桥");
+      } catch (err) {
+        firstError = err;
+      }
+
+      try {
+        nodejieba.cut("南京市长江大桥");
+      } catch (err) {
+        secondError = err;
+      }
+
+      should(firstError).be.an.Error();
+      firstError.message.should.match(/nodejieba native binding was not found/);
+      firstError.code.should.equal("BINDING_NOT_FOUND");
+      should(secondError).equal(firstError);
+    } finally {
+      fs.rmSync(fixtureDir, { recursive: true, force: true });
+    }
+  });
+});

package/test/test.js

@@ -0,0 +1,366 @@
+var should = require("should");
+var nodejieba = require("../index.js");
+
+describe("nodejieba", function() {
+
+  var sentence = "我是拖拉机学院手扶拖拉机专业的。不用多久，我就会升职加薪，当上CEO，走上人生巅峰。";
+
+  it("nodejieba.cut(sentence, true)", function() {
+    nodejieba.cut(sentence, true).should.eql([ '我',
+      '是',
+      '拖拉机',
+      '学院',
+      '手扶拖拉机',
+      '专业',
+      '的',
+      '。',
+      '不用',
+      '多久',
+      '，',
+      '我',
+      '就',
+      '会',
+      '升职',
+      '加薪',
+      '，',
+      '当上',
+      'CEO',
+      '，',
+      '走上',
+      '人生',
+      '巅峰',
+      '。' ]);
+  });
+
+  it("nodejieba.cut('南京市长江大桥')", function() {
+    nodejieba.cut('南京市长江大桥').should.eql([ '南京市', '长江大桥' ]);
+  });
+
+  it("nodejieba.cut('南京市长江大桥')", function() {
+    nodejieba.cut('南京市长江大桥').should.eql([ '南京市', '长江大桥' ]);
+  });
+
+  it("nodejieba.cutHMM('南京市长江大桥')", function() {
+    nodejieba.cutHMM('南京市长江大桥').should.eql([ '南京市', '长江大桥' ]);
+  });
+
+  it("nodejieba.cut('南京市长江大桥', true)", function() {
+    nodejieba.cut('南京市长江大桥', true).should.eql([ '南京市', '长江大桥' ]);
+  });
+
+  it("nodejieba.cut('南京长江大桥')", function() {
+    nodejieba.cut('南京长江大桥').should.eql([ '南京长江大桥' ]);
+  });
+
+  it("nodejieba.cut('南京长江大桥')", function() {
+    nodejieba.cut('南京长江大桥').should.eql([ '南京长江大桥' ]);
+  });
+
+  it("nodejieba.cutHMM('南京长江大桥')", function() {
+    nodejieba.cutHMM('南京长江大桥').should.eql([ '南京长', '江大桥' ]);
+  });
+
+  it("nodejieba.cut('南京长江大桥', true)", function() {
+    nodejieba.cut('南京长江大桥', true).should.eql([ '南京长江大桥' ]);
+  });
+
+  it('nodejieba.cut(sentence)', function() {
+    nodejieba.cut(sentence).should.eql([ '我',
+      '是',
+      '拖拉机',
+      '学院',
+      '手扶拖拉机',
+      '专业',
+      '的',
+      '。',
+      '不用',
+      '多久',
+      '，',
+      '我',
+      '就',
+      '会',
+      '升职',
+      '加薪',
+      '，',
+      '当',
+      '上',
+      'C',
+      'E',
+      'O',
+      '，',
+      '走上',
+      '人生',
+      '巅峰',
+      '。' ]);
+  })
+
+  it('nodejieba.cutHMM(sentence)', function() {
+    nodejieba.cutHMM(sentence).should.eql([ '我',
+      '是',
+      '拖拉机',
+      '学院',
+      '手',
+      '扶',
+      '拖拉机',
+      '专业',
+      '的',
+      '。',
+      '不用',
+      '多久',
+      '，',
+      '我',
+      '就',
+      '会升',
+      '职加薪',
+      '，',
+      '当上',
+      'CEO',
+      '，',
+      '走上',
+      '人生',
+      '巅峰',
+      '。' ]);
+  });
+
+  it('nodejieba.cut(sentence, true)', function() {
+    nodejieba.cut(sentence, true).should.eql([ '我',
+      '是',
+      '拖拉机',
+      '学院',
+      '手扶拖拉机',
+      '专业',
+      '的',
+      '。',
+      '不用',
+      '多久',
+      '，',
+      '我',
+      '就',
+      '会',
+      '升职',
+      '加薪',
+      '，',
+      '当上',
+      'CEO',
+      '，',
+      '走上',
+      '人生',
+      '巅峰',
+      '。' ]);
+  });
+
+  it('nodejieba.cutAll(sentence)', function() {
+    nodejieba.cutAll(sentence).should.eql([ '我',
+      '是',
+      '拖拉',
+      '拖拉机',
+      '学院',
+      '手扶',
+      '手扶拖拉机',
+      '拖拉',
+      '拖拉机',
+      '专业',
+      '的',
+      '。',
+      '不用',
+      '多久',
+      '，',
+      '我',
+      '就',
+      '会升',
+      '升职',
+      '加薪',
+      '，',
+      '当上',
+      'C',
+      'E',
+      'O',
+      '，',
+      '走上',
+      '人生',
+      '巅峰',
+      '。' ]);
+  });
+
+  it('nodejieba.cutForSearch(sentence)', function() {
+    nodejieba.cutForSearch(sentence).should.eql([ '我',
+      '是',
+      '拖拉',
+      '拖拉机',
+      '学院',
+      '手扶',
+      '拖拉',
+      '拖拉机',
+      '手扶拖拉机',
+      '专业',
+      '的',
+      '。',
+      '不用',
+      '多久',
+      '，',
+      '我',
+      '就',
+      '会',
+      '升职',
+      '加薪',
+      '，',
+      '当',
+      '上',
+      'C',
+      'E',
+      'O',
+      '，',
+      '走上',
+      '人生',
+      '巅峰',
+      '。' ]);
+  });
+
+  it('nodejieba.cutForSearch(sentence, true)', function() {
+    nodejieba.cutForSearch(sentence, true).should.eql([ '我',
+      '是',
+      '拖拉',
+      '拖拉机',
+      '学院',
+      '手扶',
+      '拖拉',
+      '拖拉机',
+      '手扶拖拉机',
+      '专业',
+      '的',
+      '。',
+      '不用',
+      '多久',
+      '，',
+      '我',
+      '就',
+      '会',
+      '升职',
+      '加薪',
+      '，',
+      '当上',
+      'CEO',
+      '，',
+      '走上',
+      '人生',
+      '巅峰',
+      '。' ]);
+  });
+
+  it('nodejieba.tag(sentence)', function() {
+    nodejieba.tag(sentence).should.eql([ { word: '我', tag: 'r' },
+      { word: '是', tag: 'v' },
+      { word: '拖拉机', tag: 'n' },
+      { word: '学院', tag: 'n' },
+      { word: '手扶拖拉机', tag: 'n' },
+      { word: '专业', tag: 'n' },
+      { word: '的', tag: 'uj' },
+      { word: '。', tag: 'x' },
+      { word: '不用', tag: 'v' },
+      { word: '多久', tag: 'm' },
+      { word: '，', tag: 'x' },
+      { word: '我', tag: 'r' },
+      { word: '就', tag: 'd' },
+      { word: '会', tag: 'v' },
+      { word: '升职', tag: 'v' },
+      { word: '加薪', tag: 'nr' },
+      { word: '，', tag: 'x' },
+      { word: '当上', tag: 't' },
+      { word: 'CEO', tag: 'eng' },
+      { word: '，', tag: 'x' },
+      { word: '走上', tag: 'v' },
+      { word: '人生', tag: 'n' },
+      { word: '巅峰', tag: 'n' },
+      { word: '。', tag: 'x' } ]);
+  });
+
+  // float compare error in multi platform (macos, windows)
+  //it('nodejieba.extract(sentence, 5)', function() {
+  //  nodejieba.extract(sentence, 5).should.eql([
+  //        {
+  //            "weight": 11.739204307083542,
+  //            "word": "CEO"
+  //        },
+  //        {
+  //            "weight": 10.8561552143,
+  //            "word": "升职"
+  //        },
+  //        {
+  //            "weight": 10.642581114,
+  //            "word": "加薪"
+  //        },
+  //        {
+  //            "weight": 10.0088573539,
+  //            "word": "手扶拖拉机"
+  //        },
+  //        {
+  //            "weight": 9.49395840471,
+  //            "word": "巅峰"
+  //        }]);
+  //});
+
+  //it('nodejieba.textRankExtract(sentence, 5)', function() {
+  //  nodejieba.textRankExtract(sentence, 5).should.eql([
+  //      	  {
+  //            "weight": 1,
+  //      	      "word": "当上"
+  //      	  },
+  //      	  {
+  //            "weight": 0.9898479330698993,
+  //      	      "word": "不用"
+  //      	  },
+  //      	  {
+  //      	      "weight": 0.9851260595435759,
+  //      	      "word": "多久"
+  //      	  },
+  //      	  {
+  //      	      "weight": 0.9830464899847804,
+  //      	      "word": "加薪"
+  //      	  },
+  //      	  {
+  //      	      "weight": 0.9802777682279076,
+  //      	      "word": "升职"
+  //      	  }]);
+  //});
+
+  it('nodejieba.cut("红掌拨清波")', function() {
+    nodejieba.cut("红掌拨清波").should.eql([
+      '红掌',
+      '拨',
+      '清波',
+    ]);
+  });
+
+  it('nodejieba.cut("男默女泪")', function() {
+    nodejieba.cut("男默女泪").should.eql([ '男', '默',
+      '女', '泪' ]);
+  });
+  it('nodejieba.insertWord("男默女泪")', function() {
+    nodejieba.insertWord("男默女泪").should.eql(true);
+  });
+  it('nodejieba.cut("男默女泪")', function() {
+    nodejieba.cut("男默女泪").should.eql([ '男默女泪' ]);
+  });
+
+  it('nodejieba.cut("今天天气很好，🙋 我们去郊游。")', function() {
+    nodejieba.cut("今天天气很好，🙋 我们去郊游。").should.eql([ '今天天气', '很', '好', '，', '🙋', ' ', '我们', '去', '郊游', '。' ]);
+  });
+
+  it('nodejieba.cutSmall("南京市长江大桥", 3)', function() {
+    nodejieba.cutSmall("南京市长江大桥", 3).should.eql([ '南京市', 
+    '长江',
+    '大桥']);
+  });
+
+  it('nodejieba.cut("区块链")', function() {
+    nodejieba.cut("区块链").should.eql(['区块链']);
+  });
+  it('nodejieba.tag("区块链")', function() {
+    nodejieba.tag("区块链").should.eql([ { word: '区块链', tag: 'nz' }]);
+  });
+  it('nodejieba.cut("访问www.baidu.com进行搜索")',function(){
+    nodejieba.cut("访问www.baidu.com进行搜索", true).should.eql([
+        '访问', 'www', '.', 'baidu', '.', 'com', '进行', '搜索',
+    ]);
+  });
+});

package/test/testdata/userdict.utf8

@@ -0,0 +1 @@
+红掌拨清波

package/types/index.d.ts

@@ -0,0 +1,30 @@
+declare module "nodejieba" {
+  export interface LoadOptions {
+    dict?: string;
+    hmmDict?: string;
+    userDict?: string;
+    idfDict?: string;
+    stopWordDict?: string;
+  }
+  
+  export interface TagResult {
+    word: string;
+    tag: string;
+  }
+  
+  export interface ExtractResult {
+    word: string;
+    weight: number;
+  }
+
+  export function load(dict?: LoadOptions): void;
+  export function cut(sentence: string, strict?: boolean): string[];
+  export function cutHMM(sentence: string): string[];
+  export function cutAll(sentence: string): string[];
+  export function cutForSearch(sentence: string, strict?: boolean): string[];
+  export function tag(sentence: string): TagResult[];
+  export function extract(sentence: string, threshold: number): ExtractResult[];
+  export function textRankExtract(sentence: string, threshold: number): ExtractResult[];
+  export function insertWord(word: string, tag?: string): boolean;
+  export function cutSmall(sentence: string, small: number): string[];
+}