gatsby-source-notion-churnotion 1.1.32 → 1.1.36

package/README.md

@@ -6,6 +6,33 @@ This plugin recursively collects categories from a single Notion database, which
 
 If you're considering Notion as your CMS for Gatsby, this plugin could be a great choice as it supports recursive category collection.
 
+## What's New in v1.1.35
+
+- **Major Overhaul of Node Relationship Handling**:
+  - Completely rewrote the Book-Post relationship mechanism
+  - Added custom GraphQL resolver for childrenChurnotion field
+  - Improved recursive database traversal logic
+  - Optimized batch processing with smaller batches (20 pages at a time)
+  - Added detailed logging for better debuggability
+  - Simplified code structure with direct database page processing
+
+## What's New in v1.1.34
+
+- **Fixed GraphQL Query Error**:
+  - Fixed "Cannot return null for non-nullable field Churnotion.rawText" error
+  - Changed rawText field to be nullable in schema definition
+  - Added fallback empty string for rawText in Post node creation
+  - Improved error handling for description field
+
+## What's New in v1.1.33
+
+- **Fixed childrenChurnotion Field Issues**:
+  - Completely redesigned how Book-Post relationships are handled
+  - Added explicit node fields for childrenChurnotion
+  - Implemented robust node relationship creation in onPostBootstrap
+  - Fixed schema definition by using Fields type
+  - Resolved persistent "Field childrenChurnotion is not defined" error
+
 ## What's New in v1.1.32
 
 - **Fixed Gatsby Schema Relationship Bug**:
@@ -110,9 +137,4 @@ query MyQuery {
     }
   }
 }
-```
-
-This will return results in MDX format, as shown below:
-
-
-![alt text](readme3.png)
+```

package/dist/api/getPages.js

@@ -37,12 +37,12 @@ const getPages = async ({ databaseId, reporter, getCache, actions, createNode, c
                 const result = await notionService.queryDatabase(databaseId);
                 hasMore = false; // Notion API가 페이지네이션을 완전히 지원하지 않으므로 일단 한 번만 처리
                 if (result?.results?.length) {
-                    reporter.info(`[SUCCESS] total pages > ${result.results.length}`);
+                    reporter.info(`[SUCCESS] Database ${databaseId} has ${result.results.length} pages`);
                     // 페이지 ID 목록 수집
                     const pageIds = result.results.map((page) => page.id);
-                    // 페이지 블록들을 병렬로 가져오기 - 최대 50개씩 배치 처리
-                    for (let i = 0; i < pageIds.length; i += 50) {
-                        const batch = pageIds.slice(i, i + 50);
+                    // 페이지 블록들을 병렬로 가져오기 - 최대 20개씩 배치 처리
+                    for (let i = 0; i < pageIds.length; i += 20) {
+                        const batch = pageIds.slice(i, i + 20);
                         reporter.info(`[BATCH] Processing pages ${i + 1} to ${i + batch.length} of ${pageIds.length}`);
                         const batchBlocks = await notionService.getMultiplePagesBlocks(batch);
                         // 페이지 데이터와 블록 결합
@@ -59,106 +59,100 @@ const getPages = async ({ databaseId, reporter, getCache, actions, createNode, c
                     }
                 }
             }
-            // 모든 페이지 병렬 처리
-            await Promise.all(pagesToProcess.map(async ({ page, blocks }) => {
+            reporter.info(`[PROCESS] Processing ${pagesToProcess.length} pages from database ${databaseId}`);
+            // 모든 페이지 처리
+            for (const { page, blocks } of pagesToProcess) {
                 try {
-                    // Timeout 설정으로 너무 오래 걸리는 페이지는 건너뛰기
-                    await (0, timeLimit_1.timeLimit)(processPageData(page, blocks, parentCategoryId, categoryPath, tagMap, parentCategoryUrl), 30000, // 30초 제한
-                    `Processing page ${page.id} timed out after 30 seconds`);
+                    // 첫 번째 블록이 child_database인지 확인
+                    if (blocks?.[0]?.type === `child_database`) {
+                        // 카테고리 처리
+                        const categoryJsonData = blocks[0];
+                        const title = categoryJsonData.child_database?.title || `Unnamed Category`;
+                        const slug = (0, slugify_1.slugify)(title) || `no-title-${categoryJsonData.id}`;
+                        if (!title) {
+                            reporter.warn(`[WARNING] Category without a title detected: ${categoryJsonData.id}`);
+                        }
+                        const nodeId = createNodeId(`${categoryJsonData.id}-category`);
+                        const categoryUrl = `${parentCategoryUrl}/${slug}`;
+                        const categoryNode = {
+                            id: nodeId,
+                            category_name: title,
+                            parent: parentCategoryId,
+                            slug: slug,
+                            children: [],
+                            internal: {
+                                type: constants_1.NODE_TYPE.Category,
+                                contentDigest: crypto_1.default
+                                    .createHash(`md5`)
+                                    .update(JSON.stringify(categoryJsonData))
+                                    .digest(`hex`),
+                            },
+                            url: `${constants_1.COMMON_URI}/${constants_1.CATEGORY_URI}${categoryUrl}`,
+                            books: [],
+                        };
+                        await createNode(categoryNode);
+                        // Book 관계 처리
+                        const bookRelations = page.properties?.book?.relation || null;
+                        if (bookRelations) {
+                            bookRelations.forEach((relation) => {
+                                const bookId = relation.id;
+                                const bookNodeId = createNodeId(`${bookId}-book`);
+                                const bookNode = getNode(bookNodeId);
+                                if (bookNode) {
+                                    createParentChildLink({
+                                        parent: categoryNode,
+                                        child: bookNode,
+                                    });
+                                    const updatedBookNode = {
+                                        ...bookNode,
+                                        book_category: categoryNode.id,
+                                        internal: {
+                                            type: bookNode.internal.type,
+                                            contentDigest: crypto_1.default
+                                                .createHash(`md5`)
+                                                .update(JSON.stringify(bookNode))
+                                                .digest(`hex`),
+                                        },
+                                    };
+                                    createNode(updatedBookNode);
+                                    reporter.info(`[SUCCESS] Linked Category-Book: ${categoryNode.category_name} -> child: ${bookNode.book_name}`);
+                                }
+                            });
+                        }
+                        // 부모-자식 카테고리 관계 설정
+                        if (parentCategoryId && categoryNode) {
+                            const parentNode = getNode(parentCategoryId);
+                            if (parentNode) {
+                                createParentChildLink({
+                                    parent: parentNode,
+                                    child: categoryNode,
+                                });
+                                reporter.info(`[SUCCESS] Linked parent: ${parentNode.category_name} -> child: ${categoryNode.category_name}`);
+                            }
+                            else {
+                                reporter.warn(`[WARNING] Parent node not found for ID: ${parentCategoryId}`);
+                            }
+                        }
+                        const newCategoryPath = [...categoryPath, categoryNode];
+                        // 해당 데이터베이스의 하위 페이지들을 처리
+                        // 여기서 재귀적으로 자식 데이터베이스 처리
+                        await processDatabase(categoryJsonData.id, nodeId, newCategoryPath, categoryUrl);
+                    }
+                    else {
+                        // 일반 포스트 처리
+                        await (0, timeLimit_1.timeLimit)(processPost(page, blocks, parentCategoryId, categoryPath, tagMap, parentCategoryUrl), 30000, // 30초 제한
+                        `Processing post ${page.id} timed out after 30 seconds`);
+                    }
                 }
                 catch (error) {
                     reporter.warn(`[WARNING] Error processing page ${page.id}: ${error}`);
                 }
-            }));
+            }
         }
         catch (error) {
             reporter.error(`[ERROR] Processing database ${databaseId} failed: ${error}`);
         }
     };
-    /**
-     * 페이지 데이터 처리 메서드
-     */
-    const processPageData = async (page, pageBlocks, parentCategoryId, categoryPath, tagMap, parentCategoryUrl) => {
-        // 첫 번째 블록이 child_database인지 확인
-        if (pageBlocks?.[0]?.type === `child_database`) {
-            await processCategory(page, pageBlocks, parentCategoryId, categoryPath, tagMap, parentCategoryUrl);
-        }
-        else {
-            await processPost(page, pageBlocks, parentCategoryId, categoryPath, tagMap, parentCategoryUrl);
-        }
-    };
-    /**
-     * 카테고리 처리 메서드
-     */
-    const processCategory = async (page, pageBlocks, parentCategoryId, categoryPath, tagMap, parentCategoryUrl) => {
-        const categoryJsonData = pageBlocks[0];
-        const title = categoryJsonData.child_database?.title || `Unnamed Category`;
-        const slug = (0, slugify_1.slugify)(title) || `no-title-${categoryJsonData.id}`;
-        if (!title) {
-            reporter.warn(`[WARNING] Category without a title detected: ${categoryJsonData.id}`);
-        }
-        const nodeId = createNodeId(`${categoryJsonData.id}-category`);
-        const categoryUrl = `${parentCategoryUrl}/${slug}`;
-        const categoryNode = {
-            id: nodeId,
-            category_name: title,
-            parent: parentCategoryId,
-            slug: slug,
-            children: [],
-            internal: {
-                type: constants_1.NODE_TYPE.Category,
-                contentDigest: crypto_1.default
-                    .createHash(`md5`)
-                    .update(JSON.stringify(categoryJsonData))
-                    .digest(`hex`),
-            },
-            url: `${constants_1.COMMON_URI}/${constants_1.CATEGORY_URI}${categoryUrl}`,
-            books: [],
-        };
-        await createNode(categoryNode);
-        const bookRelations = page.properties?.book?.relation || null;
-        if (bookRelations) {
-            bookRelations.forEach((relation) => {
-                const bookId = relation.id;
-                const bookNodeId = createNodeId(`${bookId}-book`);
-                const bookNode = getNode(bookNodeId);
-                if (bookNode) {
-                    createParentChildLink({
-                        parent: categoryNode,
-                        child: bookNode,
-                    });
-                    const updatedBookNode = {
-                        ...bookNode,
-                        book_category: categoryNode.id,
-                        internal: {
-                            type: bookNode.internal.type,
-                            contentDigest: crypto_1.default
-                                .createHash(`md5`)
-                                .update(JSON.stringify(bookNode))
-                                .digest(`hex`),
-                        },
-                    };
-                    createNode(updatedBookNode);
-                    reporter.info(`[SUCCESS] Linked Category-Book: ${categoryNode.category_name} -> child: ${bookNode.book_name}`);
-                }
-            });
-        }
-        if (parentCategoryId && categoryNode) {
-            const parentNode = getNode(parentCategoryId);
-            if (parentNode) {
-                createParentChildLink({
-                    parent: parentNode,
-                    child: categoryNode,
-                });
-                reporter.info(`[SUCCESS] Linked parent: ${parentNode.category_name} -> child: ${categoryNode.category_name}`);
-            }
-            else {
-                reporter.warn(`[WARNING] Parent node not found for ID: ${parentCategoryId}`);
-            }
-        }
-        const newCategoryPath = [...categoryPath, categoryNode];
-        await processDatabase(categoryJsonData.id, nodeId, newCategoryPath, categoryUrl);
-    };
     /**
      * 포스트 처리 메서드
      */
@@ -224,7 +218,7 @@ const getPages = async ({ databaseId, reporter, getCache, actions, createNode, c
             update_date: (0, formatDate_1.useFormatDate)(page.last_edited_time),
             version: page.properties?.version?.number || null,
             description: page.properties?.description?.rich_text?.[0]?.plain_text ||
-                rawText.substring(0, 400),
+                (rawText ? rawText.substring(0, 400) : ""),
             slug: slug,
             category_list: categoryPath,
             children: [],
@@ -240,8 +234,10 @@ const getPages = async ({ databaseId, reporter, getCache, actions, createNode, c
             url: `${constants_1.COMMON_URI}/${constants_1.POST_URI}/${slug}`,
             thumbnail: imageNode,
             parent: parentCategoryId,
+            rawText: rawText || "",
         };
         await createNode(postNode);
+        reporter.info(`[SUCCESS] Created post node: ${title} (${nodeId})`);
         if (parentCategoryId) {
             const parentNode = getNode(parentCategoryId);
             if (parentNode) {

package/dist/createSchemaCustomization.js

@@ -2,9 +2,64 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createSchemaCustomization = void 0;
 const constants_1 = require("./constants");
-const createSchemaCustomization = ({ actions }) => {
+const createSchemaCustomization = ({ actions, schema }) => {
     const { createTypes } = actions;
-    createTypes(`
+    createTypes([
+        schema.buildObjectType({
+            name: constants_1.NODE_TYPE.Book,
+            interfaces: ["Node"],
+            fields: {
+                id: "ID!",
+                book_name: "String!",
+                create_date: {
+                    type: "Date!",
+                    extensions: {
+                        dateformat: {},
+                    },
+                },
+                update_date: {
+                    type: "Date!",
+                    extensions: {
+                        dateformat: {},
+                    },
+                },
+                children: {
+                    type: `[${constants_1.NODE_TYPE.Post}]`,
+                    extensions: {
+                        link: { by: "book", from: "id" },
+                    },
+                },
+                childrenChurnotion: {
+                    type: `[${constants_1.NODE_TYPE.Post}]`,
+                    resolve: (source, args, context) => {
+                        return context.nodeModel.runQuery({
+                            query: {
+                                filter: {
+                                    book: { eq: source.id },
+                                },
+                            },
+                            type: constants_1.NODE_TYPE.Post,
+                            firstOnly: false,
+                        });
+                    },
+                },
+                url: "String!",
+                book_category: {
+                    type: constants_1.NODE_TYPE.Category,
+                    extensions: {
+                        link: { by: "id", from: "book_category" },
+                    },
+                },
+                book_image: {
+                    type: "File",
+                    extensions: {
+                        link: { by: "id", from: "book_image" },
+                    },
+                },
+                description: "String!",
+            },
+        }),
+        `
         type ${constants_1.NODE_TYPE.Post} implements Node {
             id: ID!
             category: ${constants_1.NODE_TYPE.Category}! @link(by: "id", from: "category")
@@ -22,7 +77,7 @@ const createSchemaCustomization = ({ actions }) => {
             category_list: [${constants_1.NODE_TYPE.Category}]
             url: String!
             thumbnail: File @link(by: "id", from: "thumbnail")
-            rawText: String!
+            rawText: String
         }
         
         type ${constants_1.NODE_TYPE.Tag} implements Node {
@@ -46,17 +101,8 @@ const createSchemaCustomization = ({ actions }) => {
             childrenNBook: [${constants_1.NODE_TYPE.Book}] @link(by: "book_category", from: "id")
         }
 
-        type ${constants_1.NODE_TYPE.Book} implements Node {
-            id: ID!
-            book_name: String!
-            create_date: Date! @dateformat
-            update_date: Date! @dateformat
-            children: [${constants_1.NODE_TYPE.Post}] @link(by: "book", from: "id")
-            childrenChurnotion: [${constants_1.NODE_TYPE.Post}] @link(by: "book", from: "id")
-            url: String!
-            book_category: ${constants_1.NODE_TYPE.Category} @link(by: "id", from: "book_category")
-            book_image: File @link(by: "id", from: "book_image")
-            description: String!
+        type Fields {
+            childrenChurnotion: [${constants_1.NODE_TYPE.Post}] @link(by: "id")
         }
 
         type ${constants_1.NODE_TYPE.Metadata} implements Node {
@@ -70,6 +116,7 @@ const createSchemaCustomization = ({ actions }) => {
         type ${constants_1.NODE_TYPE.RelatedPost} implements Node {
             posts: [${constants_1.NODE_TYPE.Post}] @link(by: "id")
         }
-    `);
+    `,
+    ]);
 };
 exports.createSchemaCustomization = createSchemaCustomization;

package/dist/onPostBootstrap.d.ts

@@ -1,2 +1,2 @@
 import { GatsbyNode } from "gatsby";
-export declare const onPostBootstrap: GatsbyNode[`onPostBootstrap`];
+export declare const onPostBootstrap: GatsbyNode["onPostBootstrap"];

package/dist/onPostBootstrap.js

@@ -65,59 +65,141 @@ const getSpaceSeparatedDoc = (doc) => {
     const tokens = getTokens(doc);
     return tokens.join(" ");
 };
-const onPostBootstrap = async ({ actions, getNode, getNodesByType, createNodeId, reporter, cache }, options) => {
-    const nodes = getNodesByType(constants_1.NODE_TYPE.Post);
-    const docs = nodes
-        .map((node) => ({ id: node.id, text: node.rawText }))
-        .filter((doc) => doc.text?.trim() !== "");
+const onPostBootstrap = async ({ getNodesByType, actions, reporter, createNodeId, cache, }) => {
+    const { createNodeField, createNode } = actions;
+    reporter.info(`Creating explicit relationships between nodes...`);
+    // 1. Book과 Post 간의 관계 설정
+    // Get all Book and Post nodes
+    const books = getNodesByType(constants_1.NODE_TYPE.Book);
+    const posts = getNodesByType(constants_1.NODE_TYPE.Post);
+    // Create a map of book ID to related posts
+    const bookPostMap = new Map();
+    // Populate the map
+    posts.forEach((post) => {
+        if (post.book) {
+            if (!bookPostMap.has(post.book)) {
+                bookPostMap.set(post.book, []);
+            }
+            bookPostMap.get(post.book).push(post.id);
+        }
+    });
+    // Create explicit fields for each book
+    books.forEach((book) => {
+        const relatedPostIds = bookPostMap.get(book.id) || [];
+        // Add childrenChurnotion field explicitly
+        createNodeField({
+            node: book,
+            name: "childrenChurnotion",
+            value: relatedPostIds,
+        });
+        reporter.info(`Added ${relatedPostIds.length} posts to book: ${book.book_name}`);
+    });
+    reporter.info(`Book-Post relationship creation completed`);
+    // 2. 관련 포스트 기능 구현
+    reporter.info(`Creating related posts...`);
+    // 유효한 텍스트가 있는 포스트 필터링
+    const docsWithText = posts
+        .filter((post) => post.rawText && post.rawText.trim() !== "")
+        .map((post) => ({
+        id: post.id,
+        text: post.rawText || "",
+    }));
+    if (docsWithText.length === 0) {
+        reporter.warn(`No posts with valid text content found for related posts calculation`);
+        // 빈 관련 포스트 노드라도 생성
+        posts.forEach((post) => {
+            const digest = `${post.id} - ${constants_1.NODE_TYPE.RelatedPost}`;
+            createNode({
+                id: createNodeId(digest),
+                parent: post.id,
+                internal: {
+                    type: constants_1.NODE_TYPE.RelatedPost,
+                    contentDigest: digest,
+                },
+                posts: [],
+            });
+        });
+        return;
+    }
+    reporter.info(`Processing ${docsWithText.length} posts for related content`);
+    // TF-IDF 계산 준비
     const tfidf = new natural_1.TfIdf();
-    // tfidf
-    docs.map(async (doc) => {
-        if (doc.text) {
-            const key = `${md5(doc.text)}-doc`;
-            const cached_ssd = await cache.get(key);
-            if (cached_ssd !== undefined) {
-                tfidf.addDocument(cached_ssd);
+    // 텍스트 전처리 및 TF-IDF 문서 추가
+    for (const doc of docsWithText) {
+        if (!doc.text)
+            continue;
+        const cacheKey = `${md5(doc.text)}-doc`;
+        let processedText;
+        try {
+            // 캐시에서 전처리된 문서 가져오기 시도
+            const cachedText = await cache.get(cacheKey);
+            if (cachedText) {
+                processedText = cachedText;
             }
             else {
-                const ssd = await getSpaceSeparatedDoc(await getTextFromRawText(doc.text));
-                tfidf.addDocument(ssd);
-                await cache.set(key, ssd);
+                // 텍스트 전처리
+                const cleanedText = await getTextFromRawText(doc.text);
+                processedText = getSpaceSeparatedDoc(cleanedText);
+                await cache.set(cacheKey, processedText);
             }
+            // TF-IDF에 문서 추가
+            tfidf.addDocument(processedText);
+        }
+        catch (err) {
+            reporter.warn(`Error processing text for post ${doc.id}: ${err}`);
+            tfidf.addDocument(""); // 오류 방지를 위해 빈 문서 추가
+        }
+    }
+    const docTerms = Array.from({ length: docsWithText.length }, (_, i) => {
+        try {
+            return tfidf.listTerms(i)
+                .map((x) => ({ ...x, tfidf: x.tf * x.idf }))
+                .sort((x, y) => y.tfidf - x.tfidf);
+        }
+        catch (err) {
+            reporter.warn(`Error listing terms for document index ${i}: ${err}`);
+            return [];
         }
     });
-    const doc_terms = docs.map((_, i) => tfidf.listTerms(i)
-        .map((x) => ({ ...x, tfidf: x.tf * x.idf }))
-        .sort((x, y) => y.tfidf - x.tfidf));
-    const all_keywords = new Set();
-    const tfidf_map_for_each_doc = [];
-    doc_terms.forEach((x, i) => {
-        tfidf_map_for_each_doc[i] = new Map();
-        x.slice(0, 30).forEach((x) => {
-            all_keywords.add(x.term);
-            tfidf_map_for_each_doc[i].set(x.term, x.tfidf);
+    // 모든 키워드 수집 및 TF-IDF 맵 생성
+    const allKeywords = new Set();
+    const tfidfMapForEachDoc = [];
+    docTerms.forEach((terms, i) => {
+        tfidfMapForEachDoc[i] = new Map();
+        terms.slice(0, 30).forEach((term) => {
+            allKeywords.add(term.term);
+            tfidfMapForEachDoc[i].set(term.term, term.tfidf);
         });
     });
-    const bow_vectors = new Map();
-    docs.forEach((x, i) => {
-        if (bow_vectors === null)
-            return;
-        bow_vectors.set(x.id, Array.from(all_keywords)
-            .map((x) => tfidf_map_for_each_doc[i].get(x))
-            .map((x) => (x === undefined ? 0 : x)));
+    // 각 문서의 BOW 벡터 생성
+    const bowVectors = new Map();
+    docsWithText.forEach((doc, i) => {
+        const vector = Array.from(allKeywords).map((keyword) => tfidfMapForEachDoc[i].get(keyword) || 0);
+        bowVectors.set(doc.id, vector);
     });
-    nodes.forEach((node) => {
-        const relatedNodeIds = getRelatedPosts(node.id, bow_vectors);
-        const digest = `${node.id} - ${constants_1.NODE_TYPE.RelatedPost}`;
-        actions.createNode({
+    // 각 포스트에 대해 관련 포스트 노드 생성
+    posts.forEach((post) => {
+        let relatedNodeIds = [];
+        if (bowVectors.has(post.id)) {
+            try {
+                relatedNodeIds = getRelatedPosts(post.id, bowVectors);
+            }
+            catch (err) {
+                reporter.warn(`Error getting related posts for ${post.id}: ${err}`);
+            }
+        }
+        const digest = `${post.id} - ${constants_1.NODE_TYPE.RelatedPost}`;
+        createNode({
             id: createNodeId(digest),
-            parent: node.id,
+            parent: post.id,
             internal: {
                 type: constants_1.NODE_TYPE.RelatedPost,
                 contentDigest: digest,
             },
             posts: relatedNodeIds,
         });
+        reporter.info(`Created related posts node for: ${post.title} with ${relatedNodeIds.length} related posts`);
     });
+    reporter.info(`Related posts creation completed`);
 };
 exports.onPostBootstrap = onPostBootstrap;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gatsby-source-notion-churnotion",
   "description": "Gatsby plugin that can connect with One Notion Database RECURSIVELY using official API",
-  "version": "1.1.32",
+  "version": "1.1.36",
   "skipLibCheck": true,
   "license": "0BSD",
   "main": "./dist/gatsby-node.js",