nodejieba-plus 3.5.8

package/lib/nodejieba.cpp

@@ -0,0 +1,218 @@
+#include "nodejieba.h"
+
+#include "utils.h"
+
+#include "cppjieba/Jieba.hpp"
+#include "cppjieba/KeywordExtractor.hpp"
+#include "cppjieba/TextRankExtractor.hpp"
+
+NodeJieba::NodeJieba(Napi::Env env, Napi::Object exports) {
+  DefineAddon(exports, {
+    InstanceMethod("load", &NodeJieba::load),
+    InstanceMethod("cut", &NodeJieba::cut),
+    InstanceMethod("cutAll", &NodeJieba::cutAll),
+    InstanceMethod("cutHMM", &NodeJieba::cutHMM),
+    InstanceMethod("cutForSearch", &NodeJieba::cutForSearch),
+    InstanceMethod("cutSmall", &NodeJieba::cutSmall),
+    InstanceMethod("tag", &NodeJieba::tag),
+    InstanceMethod("extract", &NodeJieba::extract),
+    InstanceMethod("textRankExtract", &NodeJieba::textRankExtract),
+    InstanceMethod("insertWord", &NodeJieba::insertWord)
+  });
+}
+
+Napi::Value NodeJieba::load(const Napi::CallbackInfo& info) {
+  if (info.Length() != 5) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+
+  std::string dictPath = info[0].As<Napi::String>();
+  std::string modelPath = info[1].As<Napi::String>();
+  std::string userDictPath = info[2].As<Napi::String>();
+  std::string idfPath = info[3].As<Napi::String>();
+  std::string stopWordsPath = info[4].As<Napi::String>();
+
+  delete _jieba_handle;
+  _jieba_handle = new cppjieba::Jieba(dictPath,
+                                  modelPath,
+                                  userDictPath,
+                                  idfPath,
+                                  stopWordsPath);
+
+  delete _text_rank_extractor_handle;
+  _text_rank_extractor_handle = new cppjieba
+		      ::TextRankExtractor(dictPath,
+                                  modelPath,
+                                  stopWordsPath,
+                                  userDictPath);
+
+  return Napi::Boolean::New(info.Env(), true);
+}
+
+Napi::Value NodeJieba::insertWord(const Napi::CallbackInfo& info) {
+  if(info.Length() < 1) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+  std::string word = info[0].As<Napi::String>();
+  std::string tag = "x";
+  if(info.Length() > 1) {
+      tag = info[1].As<Napi::String>();
+  }
+
+  if( !_jieba_handle ){
+    Napi::Error::New(info.Env(), "Before calling any other function you have to call load() first").ThrowAsJavaScriptException();
+  }
+
+  if(!_jieba_handle->InsertUserWord(word, tag)) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+
+  return Napi::Boolean::New(info.Env(), true);
+}
+
+Napi::Value NodeJieba::cut(const Napi::CallbackInfo& info) {
+  if (info.Length() == 0) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+  std::string sentence = info[0].As<Napi::String>();
+  bool useHMM = false;
+  if (info.Length() > 1) {
+    useHMM = info[1].As<Napi::Boolean>();
+  }
+
+  if( !_jieba_handle ){
+    Napi::Error::New(info.Env(), "Before calling any other function you have to call load() first").ThrowAsJavaScriptException();
+  }
+
+  std::vector<std::string> words;
+  _jieba_handle->Cut(sentence, words, useHMM);
+  Napi::Array outArray;
+  WrapVector(info.Env(), words, outArray);
+  return outArray;
+}
+
+Napi::Value NodeJieba::cutHMM(const Napi::CallbackInfo& info) {
+  if (info.Length() == 0) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+  std::string sentence = info[0].As<Napi::String>();
+
+  if( !_jieba_handle ){
+    Napi::Error::New(info.Env(), "Before calling any other function you have to call load() first").ThrowAsJavaScriptException();
+  }
+
+  std::vector<std::string> words;
+  _jieba_handle->CutHMM(sentence, words);
+  Napi::Array outArray;
+  WrapVector(info.Env(), words, outArray);
+  return outArray;
+}
+
+Napi::Value NodeJieba::cutAll(const Napi::CallbackInfo& info) {
+  if (info.Length() == 0) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+  std::string sentence = info[0].As<Napi::String>();
+
+  if( !_jieba_handle ){
+    Napi::Error::New(info.Env(), "Before calling any other function you have to call load() first").ThrowAsJavaScriptException();
+  }
+
+  std::vector<std::string> words;
+  _jieba_handle->CutAll(sentence, words);
+  Napi::Array outArray;
+  WrapVector(info.Env(), words, outArray);
+  return outArray;
+}
+
+Napi::Value NodeJieba::cutForSearch(const Napi::CallbackInfo& info) {
+  if (info.Length() == 0) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+  std::string sentence = info[0].As<Napi::String>();
+  bool useHMM = false;
+  if (info.Length() > 1) {
+    useHMM = info[1].As<Napi::Boolean>();
+  }
+
+  if( !_jieba_handle ){
+    Napi::Error::New(info.Env(), "Before calling any other function you have to call load() first").ThrowAsJavaScriptException();
+  }
+
+  std::vector<std::string> words;
+  _jieba_handle->CutForSearch(sentence, words, useHMM);
+  Napi::Array outArray;
+  WrapVector(info.Env(), words, outArray);
+  return outArray;
+}
+
+Napi::Value NodeJieba::cutSmall(const Napi::CallbackInfo& info) {
+  if (info.Length() < 2) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+  std::string sentence = info[0].As<Napi::String>();
+  size_t word_len_limit = info[1].As<Napi::Number>().Int32Value();
+
+  if( !_jieba_handle ){
+    Napi::Error::New(info.Env(), "Before calling any other function you have to call load() first").ThrowAsJavaScriptException();
+  }
+
+  std::vector<std::string> words;
+  _jieba_handle->CutSmall(sentence, words, word_len_limit);
+  Napi::Array outArray;
+  WrapVector(info.Env(), words, outArray);
+  return outArray;
+}
+
+Napi::Value NodeJieba::tag(const Napi::CallbackInfo& info) {
+  if (info.Length() == 0) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+  std::string sentence = info[0].As<Napi::String>();
+
+  if( !_jieba_handle ){
+    Napi::Error::New(info.Env(), "Before calling any other function you have to call load() first").ThrowAsJavaScriptException();
+  }
+
+  std::vector<std::pair<std::string, std::string> > words;
+  _jieba_handle->Tag(sentence, words);
+  Napi::Array outArray;
+  WrapPairVector(info.Env(), words, outArray);
+  return outArray;
+}
+
+Napi::Value NodeJieba::extract(const Napi::CallbackInfo& info) {
+  if (info.Length() != 2) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+  std::string sentence = info[0].As<Napi::String>();
+  size_t topN = info[1].As<Napi::Number>().Int32Value();
+
+  if( !_jieba_handle ){
+    Napi::Error::New(info.Env(), "Before calling any other function you have to call load() first").ThrowAsJavaScriptException();
+  }
+
+  std::vector<std::pair<std::string, double> > words;
+  _jieba_handle->extractor.Extract(sentence, words, topN);
+  Napi::Array outArray;
+  WrapPairVector(info.Env(), words, outArray);
+  return outArray;
+}
+
+Napi::Value NodeJieba::textRankExtract(const Napi::CallbackInfo& info) {
+  if (info.Length() != 2) {
+    return Napi::Boolean::New(info.Env(), false);
+  }
+  std::string sentence = info[0].As<Napi::String>();
+  size_t topN = info[1].As<Napi::Number>().Int32Value();
+
+  if( !_text_rank_extractor_handle ){
+    Napi::Error::New(info.Env(), "Before calling any other function you have to call load() first").ThrowAsJavaScriptException();
+  }
+
+  std::vector<std::pair<std::string, double> > words;
+  _text_rank_extractor_handle->Extract(sentence, words, topN);
+  Napi::Array outArray;
+  WrapPairVector(info.Env(), words, outArray);
+  return outArray;
+}

package/lib/nodejieba.h

@@ -0,0 +1,28 @@
+#ifndef NODEJIBEA_SRC_NODEJIEBA_H
+#define NODEJIBEA_SRC_NODEJIEBA_H
+
+#include <napi.h>
+
+namespace cppjieba{ class Jieba; class TextRankExtractor; }
+
+class NodeJieba : public Napi::Addon<NodeJieba> {
+public:
+  NodeJieba(Napi::Env env, Napi::Object exports);
+
+private:
+  Napi::Value load(const Napi::CallbackInfo& info);
+  Napi::Value cut(const Napi::CallbackInfo& info);
+  Napi::Value cutAll(const Napi::CallbackInfo& info);
+  Napi::Value cutHMM(const Napi::CallbackInfo& info);
+  Napi::Value cutForSearch(const Napi::CallbackInfo& info);
+  Napi::Value cutSmall(const Napi::CallbackInfo& info);
+  Napi::Value tag(const Napi::CallbackInfo& info);
+  Napi::Value extract(const Napi::CallbackInfo& info);
+  Napi::Value textRankExtract(const Napi::CallbackInfo& info);
+  Napi::Value insertWord(const Napi::CallbackInfo& info);
+
+  cppjieba::Jieba* _jieba_handle{nullptr};
+  cppjieba::TextRankExtractor* _text_rank_extractor_handle{nullptr};
+};
+
+#endif // NODEJIBEA_SRC_NODEJIEBA_H

package/lib/utils.h

@@ -0,0 +1,47 @@
+#ifndef NODEJIEBA_SRC_UTLS_H
+#define NODEJIEBA_SRC_UTLS_H
+
+#include <string>
+#include <utility>
+#include <vector>
+
+inline void WrapVector(Napi::Env env, std::vector<std::string> &ov, Napi::Array &array) {
+  array = Napi::Array::New(env, ov.size());
+  for(size_t i = 0; i < ov.size(); i++) {
+      array.Set(i, Napi::String::New(env, ov[i]));
+  }
+}
+
+inline void WrapPairVector(Napi::Env env, std::vector<std::pair<std::string,double> > &ov, Napi::Array &array) {
+  array = Napi::Array::New(env, ov.size());
+  for(size_t i = 0; i < ov.size(); i++) {
+    Napi::Object obj = Napi::Object::New(env);
+    Napi::Value k;
+    Napi::Value v;
+    k = Napi::String::New(env, "word");
+    v = Napi::String::New(env, ov[i].first);
+    obj.Set(k, v);
+    k = Napi::String::New(env, "weight");
+    v = Napi::Number::New(env, ov[i].second);
+    obj.Set(k, v);
+    array.Set(i, obj);
+  }
+}
+
+inline void WrapPairVector(Napi::Env env, std::vector<std::pair<std::string,std::string> > &ov, Napi::Array &array) {
+  array = Napi::Array::New(env, ov.size());
+  for(size_t i = 0; i < ov.size(); i++) {
+    Napi::Object obj = Napi::Object::New(env);
+    Napi::Value k;
+    Napi::Value v;
+    k = Napi::String::New(env, "word");
+    v = Napi::String::New(env, ov[i].first);
+    obj.Set(k, v);
+    k = Napi::String::New(env, "tag");
+    v = Napi::String::New(env, ov[i].second);
+    obj.Set(k, v);
+    array.Set(i, obj);
+  }
+}
+
+#endif

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "nodejieba-plus",
+  "description": "chinese word segmentation for node",
+  "version": "3.5.8",
+  "author": "Yanyi Wu <wuyanyi09@foxmail.com>",
+  "maintainers": [
+    "Yanyi Wu <wuyanyi09@foxmail.com>"
+  ],
+  "main": "./index.js",
+  "typings": "types/index.d.ts",
+  "engines": {
+    "node": ">= 14.0.0"
+  },
+  "repository": {
+    "type": "git",
+    "url": "http://github.com/yanyiwu/nodejieba.git"
+  },
+  "keywords": [
+    "chinese",
+    "segment",
+    "cppjieba",
+    "jieba",
+    "中文分词",
+    "结巴分词"
+  ],
+  "dependencies": {
+    "@mapbox/node-pre-gyp": "^1.0.10",
+    "node-addon-api": "^5.1.0"
+  },
+  "devDependencies": {
+    "mocha": "~10.7.3",
+    "should": "~13.2.3",
+    "typescript": "^5.0.4"
+  },
+  "scripts": {
+    "test": "mocha --timeout 10s -R spec test/test.js && mocha --timeout 10s -R spec test/load_dict_test.js && mocha --timeout 10s -R spec test/missing_binding_test.js",
+    "install": "npx @mapbox/node-pre-gyp install --fallback-to-build",
+    "rebuild": "npx @mapbox/node-pre-gyp rebuild"
+  },
+  "binary": {
+    "module_name": "nodejieba",
+    "module_path": "./build/Release/",
+    "host": "https://github.com/yanyiwu/nodejieba/releases/download/",
+    "remote_path": "v{version}",
+    "package_name": "{module_name}-v{version}-{node_abi}-{platform}-{arch}-{libc}.tar.gz"
+  },
+  "license": "MIT"
+}

package/submodules/cppjieba/.github/workflows/cmake.yml

@@ -0,0 +1,51 @@
+name: CMake
+
+on:
+  push:
+  pull_request:
+
+env:
+  # Customize the CMake build type here (Release, Debug, RelWithDebInfo, etc.)
+  BUILD_TYPE: Release
+
+jobs:
+  build:
+    # The CMake configure and build commands are platform agnostic and should work equally well on Windows or Mac.
+    # You can convert this to a matrix build if you need cross-platform coverage.
+    # See: https://docs.github.com/en/free-pro-team@latest/actions/learn-github-actions/managing-complex-workflows#using-a-build-matrix
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [
+          ubuntu-20.04, 
+          ubuntu-22.04, 
+          macos-13,
+          macos-14,
+          windows-2019,
+          windows-2022,
+        ]  
+        cpp_version: [11, 14, 17, 20]
+
+    steps:
+      - name: Check out repository code
+        uses: actions/checkout@v2
+        with:
+            submodules: recursive
+
+      - name: Configure CMake
+        # Configure CMake in a 'build' subdirectory. `CMAKE_BUILD_TYPE` is only required if you are using a single-configuration generator such as make.
+        # See https://cmake.org/cmake/help/latest/variable/CMAKE_BUILD_TYPE.html?highlight=cmake_build_type
+        # run: cmake -B ${{github.workspace}}/build -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+        run: cmake -B ${{github.workspace}}/build -DBUILD_TESTING=ON -DCMAKE_CXX_STANDARD=${{matrix.cpp_version}} -DCMAKE_BUILD_TYPE=${{env.BUILD_TYPE}}
+
+      - name: Build
+        # Build your program with the given configuration
+        # run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
+        run: cmake --build ${{github.workspace}}/build --config ${{env.BUILD_TYPE}}
+
+      - name: Test
+        working-directory: ${{github.workspace}}/build
+        # Execute tests defined by the CMake configuration.  
+        # See https://cmake.org/cmake/help/latest/manual/ctest.1.html for more detail
+        run: ctest -C ${{env.BUILD_TYPE}} --verbose
+      

package/submodules/cppjieba/.github/workflows/stale-issues.yml

@@ -0,0 +1,24 @@
+name: Close Stale Issues
+
+on:
+  schedule:
+    - cron: '0 0 3 * *'  # Monthly on the 3rd day of the month at midnight
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+
+    steps:
+    - uses: actions/stale@v5
+      with:
+        repo-token: ${{ secrets.GITHUB_TOKEN }}
+        stale-issue-message: 'This issue has not been updated for over 1 year and will be marked as stale. If the issue still exists, please comment or update the issue, otherwise it will be closed after 7 days.'
+        close-issue-message: 'This issue has been automatically closed due to inactivity. If the issue still exists, please reopen it.'
+        days-before-issue-stale: 365
+        days-before-issue-close: 7
+        stale-issue-label: 'Stale'
+        exempt-issue-labels: 'pinned,security'
+        operations-per-run: 100

package/submodules/cppjieba/.gitmodules

@@ -0,0 +1,3 @@
+[submodule "deps/limonp"]
+	path = deps/limonp
+	url = https://github.com/yanyiwu/limonp.git