gatsby-source-notion-churnotion 1.0.0

package/README.md

@@ -0,0 +1,3 @@
+# plugin
+
+The main focus of the overall tutorial. This contains the whole source plugin.

package/dist/api/connector.d.ts

@@ -0,0 +1,9 @@
+import { NotionToMarkdown } from "notion-to-md";
+/**
+ * Page to Markdown Instance
+ */
+export declare const n2m: NotionToMarkdown;
+/**
+ * Notion API Instance
+ */
+export declare const instance: import("axios").AxiosInstance;

package/dist/api/connector.js

@@ -0,0 +1,28 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.instance = exports.n2m = void 0;
+const client_1 = require("@notionhq/client");
+const axios_1 = __importDefault(require("axios"));
+const notion_to_md_1 = require("notion-to-md");
+const constants_1 = require("../constants");
+const notion = new client_1.Client({
+    auth: process.env.GATSBY_INTEGRATION_TOKEN,
+});
+/**
+ * Page to Markdown Instance
+ */
+exports.n2m = new notion_to_md_1.NotionToMarkdown({ notionClient: notion });
+/**
+ * Notion API Instance
+ */
+exports.instance = axios_1.default.create({
+    baseURL: `https://api.notion.com/v1/`,
+    headers: {
+        "Content-Type": `application/json`,
+        "Notion-Version": constants_1.NOTION_API_VERSION,
+        Authorization: `Bearer ${process.env.GATSBY_INTEGRATION_TOKEN}`,
+    },
+});

package/dist/api/getPage.d.ts

@@ -0,0 +1,2 @@
+import { IGetPagesParams } from "../types";
+export declare const getPages: ({ databaseId, reporter, getCache, actions, createNode, createNodeId, createParentChildLink, getNode, }: IGetPagesParams) => Promise<void>;

package/dist/api/getPage.js

@@ -0,0 +1,143 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getPages = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+const constants_1 = require("../constants");
+const fetchData_1 = require("../util/fetchData");
+const imageProcessor_1 = require("../util/imageProcessor");
+const slugify_1 = require("../util/slugify");
+const connector_1 = require("./connector");
+const getPages = async ({ databaseId, reporter, getCache, actions, createNode, createNodeId, createParentChildLink, getNode, }) => {
+    let hasMore = true;
+    /**
+     * 데이터베이스 내에 페이지들을 읽어서 재귀적으로 추가하는 서브 메서드드
+     * @param databaseId 데이터베이스 아이디
+     * @param parentCategoryId 부모 데이터베이스 아이디
+     */
+    const processDatabase = async (databaseId, parentCategoryId = null) => {
+        try {
+            while (hasMore) {
+                const databaseUrl = `databases/${databaseId}/query`;
+                const body = {};
+                const result = await (0, fetchData_1.fetchPostWithRetry)(databaseUrl, body);
+                if (result?.results?.length) {
+                    reporter.info(`[SUCCESS] total pages > ${result.results.length}`);
+                }
+                for (const page of result.results) {
+                    reporter.info(`[CHECK!!!] page: ${page.id}`);
+                    const pageUrl = `blocks/${page.id}/children?page_size=100`;
+                    // 페이지 데이터
+                    const pageData = await (0, fetchData_1.fetchGetWithRetry)(pageUrl);
+                    if (pageData.results[0].type === `child_database`) {
+                        const categoryJsonData = pageData.results[0];
+                        const title = categoryJsonData.child_database?.title || `Unnamed Category`;
+                        const slug = (0, slugify_1.slugify)(title);
+                        if (!title) {
+                            reporter.warn(`[WARNING] Category without a title detected: ${categoryJsonData.id}`);
+                        }
+                        const nodeId = createNodeId(`${categoryJsonData.id}-category`);
+                        const categoryNode = {
+                            id: nodeId,
+                            category_name: title,
+                            parent_id: parentCategoryId,
+                            slug: slug || `no-title-${categoryJsonData.id}`,
+                            children: [],
+                            internal: {
+                                type: constants_1.NODE_TYPE.Category,
+                                contentDigest: crypto_1.default
+                                    .createHash(`md5`)
+                                    .update(JSON.stringify(categoryJsonData))
+                                    .digest(`hex`),
+                            },
+                        };
+                        createNode(categoryNode);
+                        if (parentCategoryId && categoryNode) {
+                            const parentNode = getNode(parentCategoryId); // Gatsby에서 노드를 검색
+                            if (parentNode) {
+                                createParentChildLink({
+                                    parent: parentNode,
+                                    child: categoryNode,
+                                });
+                                reporter.info(`[SUCCESS] Linked parent: ${parentNode.category_name} -> child: ${categoryNode.category_name}`);
+                            }
+                            else {
+                                reporter.warn(`[WARNING] Parent node not found for ID: ${parentCategoryId}`);
+                            }
+                        }
+                        await processDatabase(categoryJsonData.id, nodeId);
+                    }
+                    else {
+                        // 페이지인 경우
+                        const title = page.properties?.[`이름`]?.title?.[0]?.plain_text || `Unnamed`;
+                        const slug = (0, slugify_1.slugify)(title);
+                        if (!title) {
+                            reporter.warn(`[WARNING] Category without a title detected: ${page.id}`);
+                        }
+                        const nodeId = createNodeId(`${page.id}-page`);
+                        // Tag 노드 만들기
+                        if (page.properties.tags && page.properties.tags.multi_select) {
+                            page.properties.tags.multi_select.map((tagData) => createNode({
+                                id: createNodeId(`${tagData.id}-tag`),
+                                tag_name: tagData.name,
+                                color: tagData.color,
+                                internal: {
+                                    type: constants_1.NODE_TYPE.Tag,
+                                    contentDigest: crypto_1.default
+                                        .createHash(`md5`)
+                                        .update(JSON.stringify(tagData.id))
+                                        .digest(`hex`),
+                                },
+                            }));
+                        }
+                        const bookId = page.properties?.book?.relation?.[0]?.id || null;
+                        const markdownContent = await connector_1.n2m.pageToMarkdown(page.id);
+                        await (0, imageProcessor_1.processBlocks)(markdownContent, actions, getCache, createNodeId, reporter);
+                        const postNode = {
+                            id: nodeId,
+                            category_id: parentCategoryId,
+                            book_id: bookId,
+                            title: title,
+                            content: markdownContent,
+                            create_date: page.created_time,
+                            update_date: page.last_edited_time,
+                            version: page.properties?.version?.rich_text?.[0]?.plain_text || null,
+                            description: null,
+                            slug: slug || `no-title-${nodeId}`,
+                            children: [],
+                            internal: {
+                                type: constants_1.NODE_TYPE.Post,
+                                contentDigest: crypto_1.default
+                                    .createHash(`md5`)
+                                    .update(JSON.stringify(nodeId))
+                                    .digest(`hex`),
+                            },
+                        };
+                        await createNode(postNode);
+                        if (parentCategoryId && postNode) {
+                            const parentNode = getNode(parentCategoryId); // Gatsby에서 노드를 검색
+                            if (parentNode) {
+                                createParentChildLink({
+                                    parent: parentNode,
+                                    child: postNode,
+                                });
+                                reporter.info(`[SUCCESS] Linked parent: ${parentNode.category_name} -> child: ${postNode.title}`);
+                            }
+                            else {
+                                reporter.warn(`[WARNING] Parent node not found for ID: ${parentCategoryId}`);
+                            }
+                        }
+                    }
+                }
+                hasMore = result.has_more;
+            }
+        }
+        catch (error) {
+            reporter.error(`[ERROR] fetching page`);
+        }
+    };
+    await processDatabase(databaseId);
+};
+exports.getPages = getPages;

package/dist/constants.d.ts

@@ -0,0 +1,9 @@
+export declare const NOTION_API_VERSION = "2022-06-28";
+export declare const NOTION_LIMIT_ERROR = 429;
+export declare const PLUGIN_NAME = "gatsby-source-notion-churnotion";
+export declare const NODE_TYPE: {
+    Post: string;
+    Category: string;
+    Tag: string;
+    Book: string;
+};

package/dist/constants.js

@@ -0,0 +1,13 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.NODE_TYPE = exports.PLUGIN_NAME = exports.NOTION_LIMIT_ERROR = exports.NOTION_API_VERSION = void 0;
+exports.NOTION_API_VERSION = `2022-06-28`;
+exports.NOTION_LIMIT_ERROR = 429;
+exports.PLUGIN_NAME = `gatsby-source-notion-churnotion`;
+//////////////////////////////////////////////////
+exports.NODE_TYPE = {
+    Post: `Churnotion`,
+    Category: `NCategory`,
+    Tag: `NTag`,
+    Book: `NBook`,
+};

package/dist/createSchemaCustomization.d.ts

@@ -0,0 +1,2 @@
+import { GatsbyNode } from "gatsby";
+export declare const createSchemaCustomization: GatsbyNode[`createSchemaCustomization`];

package/dist/createSchemaCustomization.js

@@ -0,0 +1,43 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createSchemaCustomization = void 0;
+const constants_1 = require("./constants");
+const createSchemaCustomization = ({ actions }) => {
+    const { createTypes } = actions;
+    createTypes(`
+        type ${constants_1.NODE_TYPE.Post} implements Node {
+            id: ID!
+            category_id: ${constants_1.NODE_TYPE.Category} @link(by: "id")
+            tags: ${constants_1.NODE_TYPE.Tag} @link(by: "id")
+            book_id: ${constants_1.NODE_TYPE.Book} @link(by: "id")
+            title: String
+            content: [JSON]
+            create_date: Date! @dateformat
+            update_date: Date! @dateformat
+            version: Int
+            description: String
+            slug: String
+        }
+        
+        type ${constants_1.NODE_TYPE.Tag} implements Node {
+            id: ID!
+            tag_name: String!
+            color: String!
+        }
+
+        type ${constants_1.NODE_TYPE.Category} implements Node {
+            id: ID!
+            parent_id: ${constants_1.NODE_TYPE.Category} @link(by: "id")
+            category_name: String!
+            slug: String!
+        }
+
+        type ${constants_1.NODE_TYPE.Book} implements Node {
+            id: ID!
+            book_name: String!
+            create_date: Date! @dateformat
+            update_date: Date! @dateformat
+        }
+    `);
+};
+exports.createSchemaCustomization = createSchemaCustomization;

package/dist/gatsby-node.d.ts

@@ -0,0 +1,4 @@
+export type { IPluginOptions } from "./types";
+export { onPluginInit } from "./on-plugin-init";
+export { sourceNodes } from "./source-nodes";
+export { createSchemaCustomization } from "./createSchemaCustomization";

package/dist/gatsby-node.js

@@ -0,0 +1,9 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createSchemaCustomization = exports.sourceNodes = exports.onPluginInit = void 0;
+var on_plugin_init_1 = require("./on-plugin-init");
+Object.defineProperty(exports, "onPluginInit", { enumerable: true, get: function () { return on_plugin_init_1.onPluginInit; } });
+var source_nodes_1 = require("./source-nodes");
+Object.defineProperty(exports, "sourceNodes", { enumerable: true, get: function () { return source_nodes_1.sourceNodes; } });
+var createSchemaCustomization_1 = require("./createSchemaCustomization");
+Object.defineProperty(exports, "createSchemaCustomization", { enumerable: true, get: function () { return createSchemaCustomization_1.createSchemaCustomization; } });

package/dist/on-plugin-init.d.ts

@@ -0,0 +1,2 @@
+import type { GatsbyNode } from "gatsby";
+export declare const onPluginInit: GatsbyNode[`onPluginInit`];

package/dist/on-plugin-init.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.onPluginInit = void 0;
+const onPluginInit = ({ reporter }) => {
+    reporter.info(`Example plugin loaded...`);
+};
+exports.onPluginInit = onPluginInit;

package/dist/source-nodes.d.ts

@@ -0,0 +1,2 @@
+import type { GatsbyNode } from "gatsby";
+export declare const sourceNodes: GatsbyNode["sourceNodes"];

package/dist/source-nodes.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sourceNodes = void 0;
+const getPage_1 = require("./api/getPage");
+const sourceNodes = async (gatsbyApi, options) => {
+    const { actions, reporter, createNodeId, getNode, getCache } = gatsbyApi;
+    const { createNode, createParentChildLink } = actions;
+    const { token, databaseId } = options;
+    if (!token || !databaseId) {
+        reporter.error(`[ERROR] Missing Notion API token or database ID.`);
+        return;
+    }
+    reporter.info(`[INFO] Fetching pages from Notion database...`);
+    try {
+        await (0, getPage_1.getPages)({
+            token,
+            databaseId,
+            reporter,
+            getCache,
+            actions,
+            createNode,
+            createNodeId,
+            createParentChildLink,
+            getNode,
+        });
+    }
+    catch (e) {
+        reporter.error(`[ERROR] ${e}`);
+    }
+};
+exports.sourceNodes = sourceNodes;

package/dist/util/fetchData.d.ts

@@ -0,0 +1,18 @@
+/**
+ * POST request with retry logic using axios instance
+ * @param {string} url - API endpoint URL
+ * @param {any} body - Request body
+ * @param {number} tryCount - Current retry count
+ * @param {number} maxRetries - Maximum retry attempts
+ * @returns {Promise<any>}
+ */
+export declare const fetchPostWithRetry: (url: string, body: any, tryCount?: number, maxRetries?: number) => Promise<any>;
+/**
+ * General GET/PUT/DELETE request with retry logic using axios instance
+ * @param {string} url - API endpoint URL
+ * @param {any} options - Axios request options
+ * @param {number} tryCount - Current retry count
+ * @param {number} maxRetries - Maximum retry attempts
+ * @returns {Promise<any>}
+ */
+export declare const fetchGetWithRetry: (url: string, options?: any, tryCount?: number, maxRetries?: number) => Promise<any>;

package/dist/util/fetchData.js

@@ -0,0 +1,80 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.fetchGetWithRetry = exports.fetchPostWithRetry = void 0;
+const connector_1 = require("../api/connector");
+const constants_1 = require("../constants");
+const timeLimit_1 = require("./timeLimit");
+/**
+ * POST request with retry logic using axios instance
+ * @param {string} url - API endpoint URL
+ * @param {any} body - Request body
+ * @param {number} tryCount - Current retry count
+ * @param {number} maxRetries - Maximum retry attempts
+ * @returns {Promise<any>}
+ */
+const fetchPostWithRetry = async (url, body, tryCount = 0, maxRetries = 5) => {
+    try {
+        const response = await connector_1.instance.post(url, body);
+        // Return data if the request was successful
+        return response.data;
+    }
+    catch (error) {
+        const status = error.response?.status;
+        const message = error.response?.data?.message || `Unknown error`;
+        // Handle rate limit errors with exponential backoff
+        if (status === constants_1.NOTION_LIMIT_ERROR && tryCount < maxRetries) {
+            const delay = Math.pow(2, tryCount) * 1000; // Exponential backoff: 1s, 2s, 4s...
+            console.log(`[${constants_1.PLUGIN_NAME}] Rate limit hit. Retrying in ${delay / 1000}s...`);
+            await (0, timeLimit_1.sleep)(delay);
+            return (0, exports.fetchPostWithRetry)(url, body, tryCount + 1, maxRetries);
+        }
+        // Handle unexpected errors with retry
+        if (tryCount < maxRetries) {
+            const delay = Math.pow(2, tryCount) * 1000; // Exponential backoff
+            console.log(`[${constants_1.PLUGIN_NAME}] Unexpected error. Retrying in ${delay / 1000}s...`);
+            await (0, timeLimit_1.sleep)(delay);
+            return (0, exports.fetchPostWithRetry)(url, body, tryCount + 1, maxRetries);
+        }
+        // Log and throw the error if all retries fail
+        console.error(`[${constants_1.PLUGIN_NAME}] Failed after ${tryCount} retries:`, message);
+        throw error;
+    }
+};
+exports.fetchPostWithRetry = fetchPostWithRetry;
+/**
+ * General GET/PUT/DELETE request with retry logic using axios instance
+ * @param {string} url - API endpoint URL
+ * @param {any} options - Axios request options
+ * @param {number} tryCount - Current retry count
+ * @param {number} maxRetries - Maximum retry attempts
+ * @returns {Promise<any>}
+ */
+const fetchGetWithRetry = async (url, options = {}, tryCount = 0, maxRetries = 5) => {
+    try {
+        const response = await connector_1.instance.get(url);
+        // Return data if the request was successful
+        return response.data;
+    }
+    catch (error) {
+        const status = error.response?.status;
+        const message = error.response?.data?.message || `Unknown error`;
+        // Handle rate limit errors with exponential backoff
+        if (status === constants_1.NOTION_LIMIT_ERROR && tryCount < maxRetries) {
+            const delay = Math.pow(2, tryCount) * 1000; // Exponential backoff: 1s, 2s, 4s...
+            console.log(`[${constants_1.PLUGIN_NAME}] Rate limit hit. Retrying in ${delay / 1000}s...`);
+            await (0, timeLimit_1.sleep)(delay);
+            return (0, exports.fetchGetWithRetry)(url, options, tryCount + 1, maxRetries);
+        }
+        // Handle unexpected errors with retry
+        if (tryCount < maxRetries) {
+            const delay = Math.pow(2, tryCount) * 1000; // Exponential backoff
+            console.log(`[${constants_1.PLUGIN_NAME}] Unexpected error. Retrying in ${delay / 1000}s...`);
+            await (0, timeLimit_1.sleep)(delay);
+            return (0, exports.fetchGetWithRetry)(url, options, tryCount + 1, maxRetries);
+        }
+        // Log and throw the error if all retries fail
+        console.error(`[${constants_1.PLUGIN_NAME}] Failed after ${tryCount} retries:`, message);
+        throw error;
+    }
+};
+exports.fetchGetWithRetry = fetchGetWithRetry;

package/dist/util/imageProcessor.d.ts

@@ -0,0 +1,3 @@
+import { Actions, GatsbyCache, Reporter } from "gatsby";
+import { MdBlock } from "notion-to-md/build/types";
+export declare const processBlocks: (blocks: MdBlock[], actions: Actions, getCache: (this: void, id: string) => GatsbyCache, createNodeId: (this: void, input: string) => string, reporter: Reporter) => Promise<void>;

package/dist/util/imageProcessor.js

@@ -0,0 +1,47 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.processBlocks = void 0;
+const gatsby_source_filesystem_1 = require("gatsby-source-filesystem");
+const path_1 = __importDefault(require("path"));
+const crypto_1 = __importDefault(require("crypto"));
+const processBlocks = async (blocks, actions, getCache, createNodeId, reporter) => {
+    const { createNode } = actions;
+    for (const block of blocks) {
+        // 이미지 블록 처리
+        if (block.type === `image` &&
+            typeof block.parent === `string` &&
+            block.parent.includes(`http`)) {
+            const match = block.parent.match(/\((https?:\/\/.*\.(?:png|jpg|jpeg|gif|webp))\)/);
+            if (match) {
+                const imageUrl = match[1]; // 이미지 URL 추출
+                try {
+                    // 해싱된 파일명 생성
+                    const fileExtension = path_1.default.extname(imageUrl); // 확장자 추출
+                    const hashedFileName = crypto_1.default.createHash(`md5`).update(imageUrl).digest(`hex`) +
+                        fileExtension;
+                    // 파일 생성
+                    const fileNode = await (0, gatsby_source_filesystem_1.createRemoteFileNode)({
+                        url: imageUrl,
+                        parentNodeId: block.blockId, // 블록 ID를 부모로 설정
+                        getCache,
+                        createNode,
+                        createNodeId,
+                        name: hashedFileName,
+                    });
+                    if (fileNode) {
+                        const relativePath = `/static/images/${hashedFileName}`; // 해싱된 파일명을 사용
+                        block.parent = `![${hashedFileName}](${relativePath})`; // Markdown 형식으로 업데이트
+                        reporter.info(`[SUCCESS] Updated block with new hashed image path: ${block.parent}`);
+                    }
+                }
+                catch (error) {
+                    reporter.warn(`[WARNING] Failed to download image: ${imageUrl}`);
+                }
+            }
+        }
+    }
+};
+exports.processBlocks = processBlocks;

package/dist/util/slugify.d.ts

@@ -0,0 +1 @@
+export declare const slugify: (text: string) => string;

package/dist/util/slugify.js

@@ -0,0 +1,7 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.slugify = void 0;
+const slugify = (text) => {
+    return text.toLowerCase().replace(/\s+/g, `-`);
+};
+exports.slugify = slugify;

package/dist/util/timeLimit.d.ts

@@ -0,0 +1,5 @@
+/**
+ * Sleep for a given number of milliseconds
+ * @param {number} ms - Milliseconds to sleep
+ */
+export declare const sleep: (ms: number) => Promise<unknown>;

package/dist/util/timeLimit.js

@@ -0,0 +1,11 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.sleep = void 0;
+/**
+ * Sleep for a given number of milliseconds
+ * @param {number} ms - Milliseconds to sleep
+ */
+const sleep = (ms) => {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+};
+exports.sleep = sleep;

package/gatsby-node.js

@@ -0,0 +1 @@
+module.exports = require(`./dist/gatsby-node`);

package/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "gatsby-source-notion-churnotion",
+  "description": "Gatsby plugin that can connect with One Notion Database RECURSIVELY using official API",
+  "version": "1.0.0",
+  "skipLibCheck": true,
+  "license": "0BSD",
+  "main": "./dist/gatsby-node.js",
+  "files": [
+    "./dist/*",
+    "gatsby-node.js"
+  ],
+  "scripts": {
+    "clean": "del-cli dist",
+    "build": "tsc",
+    "develop": "tsc --watch",
+    "test": "jest",
+    "prepare": "npm run clean && npm run build"
+  },
+  "keywords": [
+    "gatsby",
+    "gatsby-plugin",
+    "notion",
+    "gatsby-notion",
+    "recursive"
+  ],
+  "author": "Churnobyl <tjdcjfals3@naver.com> (https://github.com/Churnobyl)",
+  "homepage": "https://github.com/Churnobyl/gatsby-source-notion-churnotion",
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/Churnobyl/gatsby-source-notion-churnotion"
+  },
+  "engines": {
+    "node": ">=18.0.0"
+  },
+  "dependencies": {
+    "@notionhq/client": "^2.2.15",
+    "@types/node": "^22.10.2",
+    "axios": "^1.7.9",
+    "gatsby-source-filesystem": "^5.14.0",
+    "gatsby-transformer-json": "^5.14.0",
+    "node-fetch": "^3.3.2",
+    "notion-to-md": "^3.1.1",
+    "typescript": "^5.7.2"
+  },
+  "devDependencies": {
+    "gatsby": "^5.14.0"
+  },
+  "peerDependencies": {
+    "gatsby": "^5.14.0",
+    "react": "^18.0.0",
+    "react-dom": "^18.0.0"
+  }
+}