gatsby-source-notion-churnotion 1.0.73 → 1.0.75

package/dist/api/getBooks.d.ts

@@ -1,2 +1,2 @@
 import { IGetBooksParams } from "../types";
-export declare const getBooks: ({ bookDatabaseId, reporter, createNode, createNodeId, getNode, }: IGetBooksParams) => Promise<void>;
+export declare const getBooks: ({ bookDatabaseId, reporter, createNode, createNodeId, getNode, cache, }: IGetBooksParams) => Promise<void>;

package/dist/api/getBooks.js

@@ -7,16 +7,26 @@ exports.getBooks = void 0;
 const crypto_1 = __importDefault(require("crypto"));
 const constants_1 = require("../constants");
 const fetchData_1 = require("../util/fetchData");
-const getBooks = async ({ bookDatabaseId, reporter, createNode, createNodeId, getNode, }) => {
+const getBooks = async ({ bookDatabaseId, reporter, createNode, createNodeId, getNode, cache, }) => {
     const databaseUrl = `databases/${bookDatabaseId}/query`;
-    const body = {};
-    const result = await (0, fetchData_1.fetchPostWithRetry)(databaseUrl, body);
+    const cacheKey = `booksDatabase-${bookDatabaseId}`;
+    let result = await cache.get(cacheKey);
+    if (!result) {
+        const body = {};
+        result = await (0, fetchData_1.fetchPostWithRetry)(databaseUrl, body);
+        await cache.set(cacheKey, result);
+    }
     if (result?.results?.length) {
         reporter.info(`[SUCCESS] total BOOK pages > ${result.results.length}`);
     }
     for (const page of result.results) {
         reporter.info(`[CHECK] BOOK page: ${page.id}`);
         const nodeId = createNodeId(`${page.id}-book`);
+        const cachedNode = await cache.get(nodeId);
+        if (cachedNode) {
+            reporter.info(`[CACHE HIT] Skipping already created node: ${nodeId}`);
+            continue;
+        }
         const slug = page.properties?.slug?.rich_text?.[0]?.plain_text || `unnamed-slug`;
         const categoryId = page.properties?.category?.relation?.[0]?.id || null;
         let book_category = null;
@@ -43,6 +53,7 @@ const getBooks = async ({ bookDatabaseId, reporter, createNode, createNodeId, ge
         };
         reporter.info(`[DEBUG] Book ${bookNode.book_name} has book_category: ${bookNode.book_category}`);
         createNode(bookNode);
+        await cache.set(nodeId, bookNode);
     }
 };
 exports.getBooks = getBooks;

package/dist/api/getPages.d.ts

@@ -1,2 +1,2 @@
 import { IGetPagesParams } from "../types";
-export declare const getPages: ({ databaseId, reporter, getCache, actions, createNode, createNodeId, createParentChildLink, getNode, }: IGetPagesParams) => Promise<void>;
+export declare const getPages: ({ databaseId, reporter, getCache, actions, createNode, createNodeId, createParentChildLink, getNode, cache, }: IGetPagesParams) => Promise<void>;

package/dist/api/getPages.js

@@ -7,9 +7,9 @@ exports.getPages = void 0;
 const crypto_1 = __importDefault(require("crypto"));
 const constants_1 = require("../constants");
 const fetchData_1 = require("../util/fetchData");
-const slugify_1 = require("../util/slugify");
 const processor_1 = require("../util/processor");
-const getPages = async ({ databaseId, reporter, getCache, actions, createNode, createNodeId, createParentChildLink, getNode, }) => {
+const slugify_1 = require("../util/slugify");
+const getPages = async ({ databaseId, reporter, getCache, actions, createNode, createNodeId, createParentChildLink, getNode, cache, }) => {
     /**
      * 데이터베이스 내에 페이지들을 읽어서 재귀적으로 추가하는 서브 메서드드
      * @param databaseId 데이터베이스 아이디
@@ -139,7 +139,7 @@ const getPages = async ({ databaseId, reporter, getCache, actions, createNode, c
                             });
                         }
                         const bookId = page.properties?.book?.relation?.[0]?.id || null;
-                        const [imageNode, tableOfContents, updatedBlocks, rawText] = await (0, processor_1.processor)(pageData.results, actions, getCache, createNodeId, reporter);
+                        const [imageNode, tableOfContents, updatedBlocks, rawText] = await (0, processor_1.processor)(pageData.results, actions, getCache, createNodeId, reporter, cache);
                         const postNode = {
                             id: nodeId,
                             category: parentCategoryId,

package/dist/gatsby-node.d.ts

@@ -2,4 +2,3 @@ export type { IPluginOptions } from "./types";
 export { onPluginInit } from "./onPluginInit";
 export { sourceNodes } from "./source-nodes";
 export { createSchemaCustomization } from "./createSchemaCustomization";
-export { onPostBootstrap } from "./onPostBootstrap";

package/dist/gatsby-node.js

@@ -1,15 +1,10 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.onPostBootstrap = exports.createSchemaCustomization = exports.sourceNodes = exports.onPluginInit = void 0;
-require("@babel/register")({
-    extensions: [".js", ".ts"],
-    ignore: [/node_modules\/(?!kiwi-nlp)/],
-});
+exports.createSchemaCustomization = exports.sourceNodes = exports.onPluginInit = void 0;
 var onPluginInit_1 = require("./onPluginInit");
 Object.defineProperty(exports, "onPluginInit", { enumerable: true, get: function () { return onPluginInit_1.onPluginInit; } });
 var source_nodes_1 = require("./source-nodes");
 Object.defineProperty(exports, "sourceNodes", { enumerable: true, get: function () { return source_nodes_1.sourceNodes; } });
 var createSchemaCustomization_1 = require("./createSchemaCustomization");
 Object.defineProperty(exports, "createSchemaCustomization", { enumerable: true, get: function () { return createSchemaCustomization_1.createSchemaCustomization; } });
-var onPostBootstrap_1 = require("./onPostBootstrap");
-Object.defineProperty(exports, "onPostBootstrap", { enumerable: true, get: function () { return onPostBootstrap_1.onPostBootstrap; } });
+// export { onPostBootstrap } from "./onPostBootstrap";

package/dist/onPostBootstrap.js

@@ -1,37 +1,4 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || (function () {
-    var ownKeys = function(o) {
-        ownKeys = Object.getOwnPropertyNames || function (o) {
-            var ar = [];
-            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
-            return ar;
-        };
-        return ownKeys(o);
-    };
-    return function (mod) {
-        if (mod && mod.__esModule) return mod;
-        var result = {};
-        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
-        __setModuleDefault(result, mod);
-        return result;
-    };
-})();
 var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
@@ -88,24 +55,15 @@ const getTextFromRawText = async (doc) => {
         .replace(/http[^ ]+/g, "")
         .replace(/[\#\!\(\)\*\_\[\]\|\=\>\+\`\:\-]/g, "");
 };
-const getSpaceSeparatedDoc = async (doc, kiwi) => {
-    return kiwi.tokenize(doc).map((tokenInfo) => tokenInfo.str);
+const getSpaceSeparatedDoc = async (doc) => {
+    return "";
 };
-const onPostBootstrap = async ({ actions, getNode, getNodesByType, createNodeId, reporter, cache, }) => {
-    const { KiwiBuilder } = await Promise.resolve().then(() => __importStar(require("kiwi-nlp")));
-    const builder = await KiwiBuilder.create("/dist/kiwi-wasm.wasm");
-    const kiwi = await builder.build({
-        modelFiles: {
-            "combiningRule.txt": "/dist/model/combiningRule.txt",
-            "default.dict": "/dist/model/default.dict",
-            "extract.mdl": "/dist/model/extract.mdl",
-            "multi.dict": "/dist/model/multi.dict",
-            "sj.knlm": "/dist/model/sj.knlm",
-            "sj.morph": "/dist/model/sj.morph",
-            "skipbigram.mdl": "/dist/model/skipbigram.mdl",
-            "typo.dict": "/dist/model/typo.dict",
-        },
-    });
+const onPostBootstrap = async ({ actions, getNode, getNodesByType, createNodeId, reporter, cache }, options) => {
+    const { etriToken } = options;
+    if (etriToken === "") {
+        reporter.info("[Related Post] Skip getting related post logic.");
+        return;
+    }
     const nodes = getNodesByType(constants_1.NODE_TYPE.Post);
     const docs = nodes.map((node) => ({
         id: node.id,
@@ -121,7 +79,7 @@ const onPostBootstrap = async ({ actions, getNode, getNodesByType, createNodeId,
                 tfidf.addDocument(cached_ssd);
             }
             else {
-                const ssd = await getSpaceSeparatedDoc(await getTextFromRawText(doc.text), kiwi);
+                const ssd = await getSpaceSeparatedDoc(await getTextFromRawText(doc.text));
                 tfidf.addDocument(ssd);
                 await cache.set(key, ssd);
             }

package/dist/source-nodes.js

@@ -4,7 +4,7 @@ exports.sourceNodes = void 0;
 const getPages_1 = require("./api/getPages");
 const getBooks_1 = require("./api/getBooks");
 const sourceNodes = async (gatsbyApi, options) => {
-    const { actions, reporter, createNodeId, getNode, getCache } = gatsbyApi;
+    const { actions, reporter, createNodeId, getNode, getCache, cache } = gatsbyApi;
     const { createNode, createParentChildLink } = actions;
     const { token, databaseId, bookDatabaseId } = options;
     if (!token || !databaseId) {
@@ -19,6 +19,7 @@ const sourceNodes = async (gatsbyApi, options) => {
             createNode,
             createNodeId,
             getNode,
+            cache,
         });
         await (0, getPages_1.getPages)({
             token,
@@ -30,6 +31,7 @@ const sourceNodes = async (gatsbyApi, options) => {
             createNodeId,
             createParentChildLink,
             getNode,
+            cache,
         });
     }
     catch (e) {

package/dist/util/etriApi.d.ts

@@ -0,0 +1 @@
+export declare const etriApi: (apiKey: string, text: string, analysisCode?: string) => Promise<any>;

package/dist/util/etriApi.js

@@ -0,0 +1,31 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.etriApi = void 0;
+const etriApi = async (apiKey, text, analysisCode = "morp") => {
+    try {
+        const response = await fetch("http://aiopen.etri.re.kr:8000/WiseNLU", {
+            method: "POST",
+            headers: {
+                Authorization: apiKey,
+                "Content-Type": "application/json",
+            },
+            body: JSON.stringify({
+                request_id: "reserved field",
+                argument: {
+                    analysis_code: analysisCode,
+                    text: text,
+                },
+            }),
+        });
+        if (!response.ok) {
+            throw new Error(`HTTP error! status: ${response.status}`);
+        }
+        const result = await response.json();
+        return result;
+    }
+    catch (error) {
+        console.error("Error calling ETRI API:", error);
+        throw error;
+    }
+};
+exports.etriApi = etriApi;

package/dist/util/processor.d.ts

@@ -1,6 +1,6 @@
 import { Actions, GatsbyCache, Reporter } from "gatsby";
 import { BaseContentBlock } from "notion-types";
-export declare const processor: (blocks: BaseContentBlock[], actions: Actions, getCache: (this: void, id: string) => GatsbyCache, createNodeId: (this: void, input: string) => string, reporter: Reporter) => Promise<[string | null, {
+export declare const processor: (blocks: BaseContentBlock[], actions: Actions, getCache: (this: void, id: string) => GatsbyCache, createNodeId: (this: void, input: string) => string, reporter: Reporter, cache: GatsbyCache) => Promise<[string | null, {
     type: string;
     hash: string;
     title: string;

package/dist/util/processor.js

@@ -4,13 +4,13 @@ exports.processor = void 0;
 const gatsby_source_filesystem_1 = require("gatsby-source-filesystem");
 const metadataProcessor_1 = require("./metadataProcessor");
 const tableOfContent_1 = require("./tableOfContent");
-const processor = async (blocks, actions, getCache, createNodeId, reporter) => {
-    const { thumbnail, tableOfContents, updatedBlocks, rawText } = await processBlocksForContent(blocks, actions, getCache, createNodeId, reporter);
+const processor = async (blocks, actions, getCache, createNodeId, reporter, cache) => {
+    const { thumbnail, tableOfContents, updatedBlocks, rawText } = await processBlocksForContent(blocks, actions, getCache, createNodeId, reporter, cache);
     await (0, metadataProcessor_1.processMetadata)(blocks, actions, createNodeId, reporter);
     return [thumbnail, tableOfContents, updatedBlocks, rawText];
 };
 exports.processor = processor;
-const processBlocksForContent = async (blocks, actions, getCache, createNodeId, reporter) => {
+const processBlocksForContent = async (blocks, actions, getCache, createNodeId, reporter, cache) => {
     const tableOfContents = [];
     let thumbnail = null;
     let rawText = "";
@@ -21,7 +21,7 @@ const processBlocksForContent = async (blocks, actions, getCache, createNodeId,
             rawText += plainText + " ";
         }
         if (isImageBlock(block)) {
-            const updatedBlock = await processImageBlock(block, actions, getCache, createNodeId, reporter);
+            const updatedBlock = await processImageBlock(block, actions, getCache, createNodeId, reporter, cache);
             if (!thumbnail && updatedBlock?.image?.fileId) {
                 thumbnail = updatedBlock.image.fileId;
             }
@@ -58,7 +58,7 @@ const extractPlainText = (block) => {
 const isImageBlock = (block) => {
     return block.type === "image" && "image" in block;
 };
-const processImageBlock = async (block, actions, getCache, createNodeId, reporter) => {
+const processImageBlock = async (block, actions, getCache, createNodeId, reporter, cache) => {
     const { createNode } = actions;
     if (block.type === "image" && "image" in block) {
         const imageSourceType = block.image.type;
@@ -67,6 +67,19 @@ const processImageBlock = async (block, actions, getCache, createNodeId, reporte
             : block.image?.file?.url;
         if (!imageUrl)
             return null;
+        const cacheKey = `${imageUrl}-post-image`;
+        const cachedFileNodeId = await cache.get(cacheKey);
+        if (cachedFileNodeId) {
+            reporter.info(`[CACHE HIT] Image already processed: ${imageUrl}`);
+            const updatedBlock = {
+                ...block,
+                image: {
+                    fileId: cachedFileNodeId,
+                    caption: block.image.caption,
+                },
+            };
+            return updatedBlock;
+        }
         try {
             const fileNode = await (0, gatsby_source_filesystem_1.createRemoteFileNode)({
                 url: imageUrl,
@@ -84,6 +97,7 @@ const processImageBlock = async (block, actions, getCache, createNodeId, reporte
                     },
                 };
                 reporter.info(`[SUCCESS] Image processed: ${fileNode.id}`);
+                await cache.set(cacheKey, fileNode.id);
                 return updatedBlock;
             }
         }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gatsby-source-notion-churnotion",
   "description": "Gatsby plugin that can connect with One Notion Database RECURSIVELY using official API",
-  "version": "1.0.73",
+  "version": "1.0.75",
   "skipLibCheck": true,
   "license": "0BSD",
   "main": "./dist/gatsby-node.js",
@@ -41,7 +41,6 @@
     "gatsby-source-filesystem": "^5.14.0",
     "gatsby-transformer-json": "^5.14.0",
     "gatsby-transformer-sharp": "^5.14.0",
-    "kiwi-nlp": "^0.20.3",
     "metascraper": "^5.45.25",
     "metascraper-description": "^5.45.25",
     "metascraper-image": "^5.45.27",
@@ -54,9 +53,6 @@
     "typescript": "^5.7.2"
   },
   "devDependencies": {
-    "@babel/preset-env": "^7.26.0",
-    "@babel/preset-typescript": "^7.26.0",
-    "@babel/register": "^7.25.9",
     "gatsby": "^5.14.0"
   },
   "peerDependencies": {