gatsby-source-notion-churnotion 1.0.99 → 1.1.1

package/dist/onPostBootstrap.js

@@ -8,17 +8,9 @@ const crypto_1 = __importDefault(require("crypto"));
 const natural_1 = require("natural");
 const constants_1 = require("./constants");
 const compute_cosine_similarity_1 = __importDefault(require("compute-cosine-similarity"));
-const node_nlp_1 = require("node-nlp");
-const manager = new node_nlp_1.NlpManager({ languages: ["ko"] });
-const getTokens = async (doc) => {
-    const result = await manager.process("ko", doc);
-    console.dir(result);
-    if (!result.entities) {
-        return [];
-    }
-    return result.entities
-        .map((entity) => entity.utteranceText)
-        .filter((text) => text.length > 1);
+const tokenizer = new natural_1.WordTokenizer();
+const getTokens = (doc) => {
+    return tokenizer.tokenize(doc);
 };
 const vector_similarity_memo = new Map();
 const md5 = (str) => {
@@ -69,11 +61,8 @@ const getTextFromRawText = async (doc) => {
         .replace(/http[^ ]+/g, "")
         .replace(/[\#\!\(\)\*\_\[\]\|\=\>\+\`\:\-]/g, "");
 };
-const getSpaceSeparatedDoc = async (doc) => {
-    if (!doc.trim()) {
-        return "";
-    }
-    const tokens = await getTokens(doc);
+const getSpaceSeparatedDoc = (doc) => {
+    const tokens = getTokens(doc);
     return tokens.join(" ");
 };
 const onPostBootstrap = async ({ actions, getNode, getNodesByType, createNodeId, reporter, cache }, options) => {
@@ -86,8 +75,6 @@ const onPostBootstrap = async ({ actions, getNode, getNodesByType, createNodeId,
     docs.map(async (doc) => {
         if (doc.text) {
             const key = `${md5(doc.text)}-doc`;
-            reporter.info(`[TEST1] ${doc.text}`);
-            reporter.info(`[TEST2] ${await getTextFromRawText(doc.text)}`);
             const cached_ssd = await cache.get(key);
             if (cached_ssd !== undefined) {
                 tfidf.addDocument(cached_ssd);
@@ -114,6 +101,7 @@ const onPostBootstrap = async ({ actions, getNode, getNodesByType, createNodeId,
         });
     });
     const bow_vectors = new Map();
+    reporter.info(`[DEBUG] BOW Vectors: ${JSON.stringify([...bow_vectors.entries()])}`);
     docs.forEach((x, i) => {
         if (bow_vectors === null)
             return;
@@ -126,12 +114,13 @@ const onPostBootstrap = async ({ actions, getNode, getNodesByType, createNodeId,
         const related_nodes = getRelatedPosts(node.id, bow_vectors)
             .slice(1)
             .map((id) => getNode(id));
+        reporter.info(`[DEBUG] Related posts for node ${node.id}: ${JSON.stringify(related_nodes)}`);
         const digest = `${node.id} - ${constants_1.NODE_TYPE.RelatedPost}`;
         actions.createNode({
             id: createNodeId(digest),
             parent: node.id,
             internal: {
-                type: `related${constants_1.NODE_TYPE.RelatedPost}s`,
+                type: constants_1.NODE_TYPE.RelatedPost,
                 contentDigest: digest,
             },
             posts: related_nodes,

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gatsby-source-notion-churnotion",
   "description": "Gatsby plugin that can connect with One Notion Database RECURSIVELY using official API",
-  "version": "1.0.99",
+  "version": "1.1.1",
   "skipLibCheck": true,
   "license": "0BSD",
   "main": "./dist/gatsby-node.js",
@@ -48,7 +48,6 @@
     "metascraper-url": "^5.45.25",
     "natural": "^8.0.1",
     "node-fetch": "^3.3.2",
-    "node-nlp": "^5.0.0-alpha.4",
     "notion-to-md": "^3.1.1",
     "notion-types": "^7.1.5",
     "typescript": "^5.7.2"