gatsby-source-notion-churnotion 1.1.35 → 1.1.36

package/dist/onPostBootstrap.js

@@ -65,9 +65,10 @@ const getSpaceSeparatedDoc = (doc) => {
     const tokens = getTokens(doc);
     return tokens.join(" ");
 };
-const onPostBootstrap = async ({ getNodesByType, actions, reporter, }) => {
-    const { createNodeField } = actions;
+const onPostBootstrap = async ({ getNodesByType, actions, reporter, createNodeId, cache, }) => {
+    const { createNodeField, createNode } = actions;
     reporter.info(`Creating explicit relationships between nodes...`);
+    // 1. Book과 Post 간의 관계 설정
     // Get all Book and Post nodes
     const books = getNodesByType(constants_1.NODE_TYPE.Book);
     const posts = getNodesByType(constants_1.NODE_TYPE.Post);
@@ -93,6 +94,112 @@ const onPostBootstrap = async ({ getNodesByType, actions, reporter, }) => {
         });
         reporter.info(`Added ${relatedPostIds.length} posts to book: ${book.book_name}`);
     });
-    reporter.info(`Relationship creation completed`);
+    reporter.info(`Book-Post relationship creation completed`);
+    // 2. 관련 포스트 기능 구현
+    reporter.info(`Creating related posts...`);
+    // 유효한 텍스트가 있는 포스트 필터링
+    const docsWithText = posts
+        .filter((post) => post.rawText && post.rawText.trim() !== "")
+        .map((post) => ({
+        id: post.id,
+        text: post.rawText || "",
+    }));
+    if (docsWithText.length === 0) {
+        reporter.warn(`No posts with valid text content found for related posts calculation`);
+        // 빈 관련 포스트 노드라도 생성
+        posts.forEach((post) => {
+            const digest = `${post.id} - ${constants_1.NODE_TYPE.RelatedPost}`;
+            createNode({
+                id: createNodeId(digest),
+                parent: post.id,
+                internal: {
+                    type: constants_1.NODE_TYPE.RelatedPost,
+                    contentDigest: digest,
+                },
+                posts: [],
+            });
+        });
+        return;
+    }
+    reporter.info(`Processing ${docsWithText.length} posts for related content`);
+    // TF-IDF 계산 준비
+    const tfidf = new natural_1.TfIdf();
+    // 텍스트 전처리 및 TF-IDF 문서 추가
+    for (const doc of docsWithText) {
+        if (!doc.text)
+            continue;
+        const cacheKey = `${md5(doc.text)}-doc`;
+        let processedText;
+        try {
+            // 캐시에서 전처리된 문서 가져오기 시도
+            const cachedText = await cache.get(cacheKey);
+            if (cachedText) {
+                processedText = cachedText;
+            }
+            else {
+                // 텍스트 전처리
+                const cleanedText = await getTextFromRawText(doc.text);
+                processedText = getSpaceSeparatedDoc(cleanedText);
+                await cache.set(cacheKey, processedText);
+            }
+            // TF-IDF에 문서 추가
+            tfidf.addDocument(processedText);
+        }
+        catch (err) {
+            reporter.warn(`Error processing text for post ${doc.id}: ${err}`);
+            tfidf.addDocument(""); // 오류 방지를 위해 빈 문서 추가
+        }
+    }
+    const docTerms = Array.from({ length: docsWithText.length }, (_, i) => {
+        try {
+            return tfidf.listTerms(i)
+                .map((x) => ({ ...x, tfidf: x.tf * x.idf }))
+                .sort((x, y) => y.tfidf - x.tfidf);
+        }
+        catch (err) {
+            reporter.warn(`Error listing terms for document index ${i}: ${err}`);
+            return [];
+        }
+    });
+    // 모든 키워드 수집 및 TF-IDF 맵 생성
+    const allKeywords = new Set();
+    const tfidfMapForEachDoc = [];
+    docTerms.forEach((terms, i) => {
+        tfidfMapForEachDoc[i] = new Map();
+        terms.slice(0, 30).forEach((term) => {
+            allKeywords.add(term.term);
+            tfidfMapForEachDoc[i].set(term.term, term.tfidf);
+        });
+    });
+    // 각 문서의 BOW 벡터 생성
+    const bowVectors = new Map();
+    docsWithText.forEach((doc, i) => {
+        const vector = Array.from(allKeywords).map((keyword) => tfidfMapForEachDoc[i].get(keyword) || 0);
+        bowVectors.set(doc.id, vector);
+    });
+    // 각 포스트에 대해 관련 포스트 노드 생성
+    posts.forEach((post) => {
+        let relatedNodeIds = [];
+        if (bowVectors.has(post.id)) {
+            try {
+                relatedNodeIds = getRelatedPosts(post.id, bowVectors);
+            }
+            catch (err) {
+                reporter.warn(`Error getting related posts for ${post.id}: ${err}`);
+            }
+        }
+        const digest = `${post.id} - ${constants_1.NODE_TYPE.RelatedPost}`;
+        createNode({
+            id: createNodeId(digest),
+            parent: post.id,
+            internal: {
+                type: constants_1.NODE_TYPE.RelatedPost,
+                contentDigest: digest,
+            },
+            posts: relatedNodeIds,
+        });
+        reporter.info(`Created related posts node for: ${post.title} with ${relatedNodeIds.length} related posts`);
+    });
+    reporter.info(`Related posts creation completed`);
 };
 exports.onPostBootstrap = onPostBootstrap;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gatsby-source-notion-churnotion",
   "description": "Gatsby plugin that can connect with One Notion Database RECURSIVELY using official API",
-  "version": "1.1.35",
+  "version": "1.1.36",
   "skipLibCheck": true,
   "license": "0BSD",
   "main": "./dist/gatsby-node.js",