gatsby-source-notion-churnotion 1.0.98 → 1.1.0

package/dist/onPostBootstrap.js

@@ -8,16 +8,9 @@ const crypto_1 = __importDefault(require("crypto"));
 const natural_1 = require("natural");
 const constants_1 = require("./constants");
 const compute_cosine_similarity_1 = __importDefault(require("compute-cosine-similarity"));
-const node_nlp_1 = require("node-nlp");
-const manager = new node_nlp_1.NlpManager({ languages: ["ko"] });
-const getTokens = async (doc) => {
-    const result = await manager.process("ko", doc);
-    if (!result.entities) {
-        return [];
-    }
-    return result.entities
-        .map((entity) => entity.utteranceText)
-        .filter((text) => text.length > 1);
+const tokenizer = new natural_1.WordTokenizer();
+const getTokens = (doc) => {
+    return tokenizer.tokenize(doc);
 };
 const vector_similarity_memo = new Map();
 const md5 = (str) => {
@@ -68,11 +61,8 @@ const getTextFromRawText = async (doc) => {
         .replace(/http[^ ]+/g, "")
         .replace(/[\#\!\(\)\*\_\[\]\|\=\>\+\`\:\-]/g, "");
 };
-const getSpaceSeparatedDoc = async (doc) => {
-    if (!doc.trim()) {
-        return "";
-    }
-    const tokens = await getTokens(doc);
+const getSpaceSeparatedDoc = (doc) => {
+    const tokens = getTokens(doc);
     return tokens.join(" ");
 };
 const onPostBootstrap = async ({ actions, getNode, getNodesByType, createNodeId, reporter, cache }, options) => {

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gatsby-source-notion-churnotion",
   "description": "Gatsby plugin that can connect with One Notion Database RECURSIVELY using official API",
-  "version": "1.0.98",
+  "version": "1.1.0",
   "skipLibCheck": true,
   "license": "0BSD",
   "main": "./dist/gatsby-node.js",
@@ -48,7 +48,6 @@
     "metascraper-url": "^5.45.25",
     "natural": "^8.0.1",
     "node-fetch": "^3.3.2",
-    "node-nlp": "^5.0.0-alpha.4",
     "notion-to-md": "^3.1.1",
     "notion-types": "^7.1.5",
     "typescript": "^5.7.2"