gatsby-source-notion-churnotion 1.0.74 → 1.0.76

package/dist/api/getBooks.d.ts

@@ -1,2 +1,2 @@
 import { IGetBooksParams } from "../types";
-export declare const getBooks: ({ bookDatabaseId, reporter, createNode, createNodeId, getNode, }: IGetBooksParams) => Promise<void>;
+export declare const getBooks: ({ bookDatabaseId, reporter, createNode, createNodeId, getNode, cache, }: IGetBooksParams) => Promise<void>;

package/dist/api/getBooks.js

@@ -7,16 +7,26 @@ exports.getBooks = void 0;
 const crypto_1 = __importDefault(require("crypto"));
 const constants_1 = require("../constants");
 const fetchData_1 = require("../util/fetchData");
-const getBooks = async ({ bookDatabaseId, reporter, createNode, createNodeId, getNode, }) => {
+const getBooks = async ({ bookDatabaseId, reporter, createNode, createNodeId, getNode, cache, }) => {
     const databaseUrl = `databases/${bookDatabaseId}/query`;
-    const body = {};
-    const result = await (0, fetchData_1.fetchPostWithRetry)(databaseUrl, body);
+    const cacheKey = `booksDatabase-${bookDatabaseId}`;
+    let result = await cache.get(cacheKey);
+    if (!result) {
+        const body = {};
+        result = await (0, fetchData_1.fetchPostWithRetry)(databaseUrl, body);
+        await cache.set(cacheKey, result);
+    }
     if (result?.results?.length) {
         reporter.info(`[SUCCESS] total BOOK pages > ${result.results.length}`);
     }
     for (const page of result.results) {
         reporter.info(`[CHECK] BOOK page: ${page.id}`);
         const nodeId = createNodeId(`${page.id}-book`);
+        const cachedNode = await cache.get(nodeId);
+        if (cachedNode) {
+            reporter.info(`[CACHE HIT] Skipping already created node: ${nodeId}`);
+            continue;
+        }
         const slug = page.properties?.slug?.rich_text?.[0]?.plain_text || `unnamed-slug`;
         const categoryId = page.properties?.category?.relation?.[0]?.id || null;
         let book_category = null;
@@ -43,6 +53,7 @@ const getBooks = async ({ bookDatabaseId, reporter, createNode, createNodeId, ge
         };
         reporter.info(`[DEBUG] Book ${bookNode.book_name} has book_category: ${bookNode.book_category}`);
         createNode(bookNode);
+        await cache.set(nodeId, bookNode);
     }
 };
 exports.getBooks = getBooks;

package/dist/api/getPages.d.ts

@@ -1,2 +1,2 @@
 import { IGetPagesParams } from "../types";
-export declare const getPages: ({ databaseId, reporter, getCache, actions, createNode, createNodeId, createParentChildLink, getNode, }: IGetPagesParams) => Promise<void>;
+export declare const getPages: ({ databaseId, reporter, getCache, actions, createNode, createNodeId, createParentChildLink, getNode, cache, }: IGetPagesParams) => Promise<void>;

package/dist/api/getPages.js

@@ -7,9 +7,9 @@ exports.getPages = void 0;
 const crypto_1 = __importDefault(require("crypto"));
 const constants_1 = require("../constants");
 const fetchData_1 = require("../util/fetchData");
-const slugify_1 = require("../util/slugify");
 const processor_1 = require("../util/processor");
-const getPages = async ({ databaseId, reporter, getCache, actions, createNode, createNodeId, createParentChildLink, getNode, }) => {
+const slugify_1 = require("../util/slugify");
+const getPages = async ({ databaseId, reporter, getCache, actions, createNode, createNodeId, createParentChildLink, getNode, cache, }) => {
     /**
      * 데이터베이스 내에 페이지들을 읽어서 재귀적으로 추가하는 서브 메서드드
      * @param databaseId 데이터베이스 아이디
@@ -139,7 +139,7 @@ const getPages = async ({ databaseId, reporter, getCache, actions, createNode, c
                             });
                         }
                         const bookId = page.properties?.book?.relation?.[0]?.id || null;
-                        const [imageNode, tableOfContents, updatedBlocks, rawText] = await (0, processor_1.processor)(pageData.results, actions, getCache, createNodeId, reporter);
+                        const [imageNode, tableOfContents, updatedBlocks, rawText] = await (0, processor_1.processor)(pageData.results, actions, getCache, createNodeId, reporter, cache);
                         const postNode = {
                             id: nodeId,
                             category: parentCategoryId,

package/dist/source-nodes.js

@@ -4,7 +4,7 @@ exports.sourceNodes = void 0;
 const getPages_1 = require("./api/getPages");
 const getBooks_1 = require("./api/getBooks");
 const sourceNodes = async (gatsbyApi, options) => {
-    const { actions, reporter, createNodeId, getNode, getCache } = gatsbyApi;
+    const { actions, reporter, createNodeId, getNode, getCache, cache } = gatsbyApi;
     const { createNode, createParentChildLink } = actions;
     const { token, databaseId, bookDatabaseId } = options;
     if (!token || !databaseId) {
@@ -19,6 +19,7 @@ const sourceNodes = async (gatsbyApi, options) => {
             createNode,
             createNodeId,
             getNode,
+            cache,
         });
         await (0, getPages_1.getPages)({
             token,
@@ -30,6 +31,7 @@ const sourceNodes = async (gatsbyApi, options) => {
             createNodeId,
             createParentChildLink,
             getNode,
+            cache,
         });
     }
     catch (e) {

package/dist/util/processor.d.ts

@@ -1,6 +1,6 @@
 import { Actions, GatsbyCache, Reporter } from "gatsby";
 import { BaseContentBlock } from "notion-types";
-export declare const processor: (blocks: BaseContentBlock[], actions: Actions, getCache: (this: void, id: string) => GatsbyCache, createNodeId: (this: void, input: string) => string, reporter: Reporter) => Promise<[string | null, {
+export declare const processor: (blocks: BaseContentBlock[], actions: Actions, getCache: (this: void, id: string) => GatsbyCache, createNodeId: (this: void, input: string) => string, reporter: Reporter, cache: GatsbyCache) => Promise<[string | null, {
     type: string;
     hash: string;
     title: string;

package/dist/util/processor.js

@@ -4,13 +4,13 @@ exports.processor = void 0;
 const gatsby_source_filesystem_1 = require("gatsby-source-filesystem");
 const metadataProcessor_1 = require("./metadataProcessor");
 const tableOfContent_1 = require("./tableOfContent");
-const processor = async (blocks, actions, getCache, createNodeId, reporter) => {
-    const { thumbnail, tableOfContents, updatedBlocks, rawText } = await processBlocksForContent(blocks, actions, getCache, createNodeId, reporter);
+const processor = async (blocks, actions, getCache, createNodeId, reporter, cache) => {
+    const { thumbnail, tableOfContents, updatedBlocks, rawText } = await processBlocksForContent(blocks, actions, getCache, createNodeId, reporter, cache);
     await (0, metadataProcessor_1.processMetadata)(blocks, actions, createNodeId, reporter);
     return [thumbnail, tableOfContents, updatedBlocks, rawText];
 };
 exports.processor = processor;
-const processBlocksForContent = async (blocks, actions, getCache, createNodeId, reporter) => {
+const processBlocksForContent = async (blocks, actions, getCache, createNodeId, reporter, cache) => {
     const tableOfContents = [];
     let thumbnail = null;
     let rawText = "";
@@ -21,7 +21,7 @@ const processBlocksForContent = async (blocks, actions, getCache, createNodeId,
             rawText += plainText + " ";
         }
         if (isImageBlock(block)) {
-            const updatedBlock = await processImageBlock(block, actions, getCache, createNodeId, reporter);
+            const updatedBlock = await processImageBlock(block, actions, getCache, createNodeId, reporter, cache);
             if (!thumbnail && updatedBlock?.image?.fileId) {
                 thumbnail = updatedBlock.image.fileId;
             }
@@ -58,7 +58,7 @@ const extractPlainText = (block) => {
 const isImageBlock = (block) => {
     return block.type === "image" && "image" in block;
 };
-const processImageBlock = async (block, actions, getCache, createNodeId, reporter) => {
+const processImageBlock = async (block, actions, getCache, createNodeId, reporter, cache) => {
     const { createNode } = actions;
     if (block.type === "image" && "image" in block) {
         const imageSourceType = block.image.type;
@@ -67,6 +67,19 @@ const processImageBlock = async (block, actions, getCache, createNodeId, reporte
             : block.image?.file?.url;
         if (!imageUrl)
             return null;
+        const cacheKey = `${imageUrl}-post-image`;
+        const cachedFileNodeId = await cache.get(cacheKey);
+        if (cachedFileNodeId) {
+            reporter.info(`[CACHE HIT] Image already processed: ${imageUrl}`);
+            const updatedBlock = {
+                ...block,
+                image: {
+                    fileId: cachedFileNodeId,
+                    caption: block.image.caption,
+                },
+            };
+            return updatedBlock;
+        }
         try {
             const fileNode = await (0, gatsby_source_filesystem_1.createRemoteFileNode)({
                 url: imageUrl,
@@ -84,6 +97,7 @@ const processImageBlock = async (block, actions, getCache, createNodeId, reporte
                     },
                 };
                 reporter.info(`[SUCCESS] Image processed: ${fileNode.id}`);
+                await cache.set(cacheKey, fileNode.id);
                 return updatedBlock;
             }
         }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gatsby-source-notion-churnotion",
   "description": "Gatsby plugin that can connect with One Notion Database RECURSIVELY using official API",
-  "version": "1.0.74",
+  "version": "1.0.76",
   "skipLibCheck": true,
   "license": "0BSD",
   "main": "./dist/gatsby-node.js",
@@ -41,7 +41,6 @@
     "gatsby-source-filesystem": "^5.14.0",
     "gatsby-transformer-json": "^5.14.0",
     "gatsby-transformer-sharp": "^5.14.0",
-    "mecab-ya": "^0.1.1",
     "metascraper": "^5.45.25",
     "metascraper-description": "^5.45.25",
     "metascraper-image": "^5.45.27",