gatsby-source-notion-churnotion 1.0.65 → 1.0.67

package/dist/api/getPages.js

@@ -139,7 +139,7 @@ const getPages = async ({ databaseId, reporter, getCache, actions, createNode, c
                             });
                         }
                         const bookId = page.properties?.book?.relation?.[0]?.id || null;
-                        const [imageNode, tableOfContents, updatedBlocks] = await (0, processor_1.processor)(pageData.results, actions, getCache, createNodeId, reporter);
+                        const [imageNode, tableOfContents, updatedBlocks, rawText] = await (0, processor_1.processor)(pageData.results, actions, getCache, createNodeId, reporter);
                         const postNode = {
                             id: nodeId,
                             category: parentCategoryId,
@@ -167,6 +167,7 @@ const getPages = async ({ databaseId, reporter, getCache, actions, createNode, c
                             parent: null,
                             url: `${constants_1.COMMON_URI}/${constants_1.POST_URI}${parentCategoryUrl}/${slug}`,
                             thumbnail: imageNode,
+                            rawText,
                         };
                         await createNode(postNode);
                         // book과 post 부모-자식 관계 설정

package/dist/constants.d.ts

@@ -7,6 +7,7 @@ export declare const NODE_TYPE: {
     Tag: string;
     Book: string;
     Metadata: string;
+    RelatedPost: string;
 };
 export declare const COMMON_URI = "blog";
 export declare const POST_URI = "post";

package/dist/constants.js

@@ -11,6 +11,7 @@ exports.NODE_TYPE = {
     Tag: `NTag`,
     Book: `NBook`,
     Metadata: `NMetadata`,
+    RelatedPost: `RelatedPost`,
 };
 ///////////////url resource///////////////////////
 exports.COMMON_URI = `blog`;

package/dist/createSchemaCustomization.js

@@ -22,6 +22,7 @@ const createSchemaCustomization = ({ actions }) => {
             category_list: [${constants_1.NODE_TYPE.Category}]
             url: String!
             thumbnail: File @link(by: "id", from: "thumbnail")
+            rawText: String!
         }
         
         type ${constants_1.NODE_TYPE.Tag} implements Node {
@@ -61,6 +62,10 @@ const createSchemaCustomization = ({ actions }) => {
             image: String,
             url: String,
         }
+
+        type ${constants_1.NODE_TYPE.RelatedPost} implements Node {
+            posts: [${constants_1.NODE_TYPE.RelatedPost}]
+        }
     `);
 };
 exports.createSchemaCustomization = createSchemaCustomization;

package/dist/gatsby-node.d.ts

@@ -1,4 +1,4 @@
 export type { IPluginOptions } from "./types";
-export { onPluginInit } from "./on-plugin-init";
+export { onPluginInit } from "./onPluginInit";
 export { sourceNodes } from "./source-nodes";
 export { createSchemaCustomization } from "./createSchemaCustomization";

package/dist/gatsby-node.js

@@ -1,8 +1,8 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.createSchemaCustomization = exports.sourceNodes = exports.onPluginInit = void 0;
-var on_plugin_init_1 = require("./on-plugin-init");
-Object.defineProperty(exports, "onPluginInit", { enumerable: true, get: function () { return on_plugin_init_1.onPluginInit; } });
+var onPluginInit_1 = require("./onPluginInit");
+Object.defineProperty(exports, "onPluginInit", { enumerable: true, get: function () { return onPluginInit_1.onPluginInit; } });
 var source_nodes_1 = require("./source-nodes");
 Object.defineProperty(exports, "sourceNodes", { enumerable: true, get: function () { return source_nodes_1.sourceNodes; } });
 var createSchemaCustomization_1 = require("./createSchemaCustomization");

package/dist/{on-plugin-init.js → onPluginInit.js}

@@ -2,6 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.onPluginInit = void 0;
 const onPluginInit = ({ reporter }) => {
-    reporter.info(`Example plugin loaded...`);
+    reporter.info(`Churnotion plugin loaded`);
 };
 exports.onPluginInit = onPluginInit;

package/dist/onPostBootstrap.js

@@ -0,0 +1,113 @@
+"use strict";
+// import { GatsbyNode } from "gatsby";
+// import { TfIdf, TfIdfTerm } from "natural";
+// import { NODE_TYPE } from "./constants";
+// import crypto from "crypto";
+// const md5 = (str: string): string => {
+//   const md5 = crypto.createHash("md5");
+//   return md5.update(str, "binary").digest("hex");
+// };
+// const getSpaceSeparatedDoc: {
+//     [key: string]: (doc: string) => Promise<string[]>;
+//   } = {
+//     en: async (doc) => {
+//       return doc.toLowerCase().split(' ');
+//     },
+//     ja: async (doc) => {
+//       if (kuromoji_tokenizer === null)
+//         kuromoji_tokenizer = await getKuromojiTokenizer();
+//       return kuromoji_tokenizer
+//         .tokenize(doc)
+//         .filter(
+//           (x) =>
+//             x.pos === '名詞' &&
+//             ['一般', '固有名詞'].indexOf(x.pos_detail_1) !== -1
+//         )
+//         .map((x) => (x.basic_form !== '*' ? x.basic_form : x.surface_form));
+//     },
+//   };
+// export const onPostBootstrap: GatsbyNode["onPostBootstrap"] = async ({
+//   actions,
+//   getNode,
+//   getNodesByType,
+//   createNodeId,
+//   reporter,
+//   cache,
+// }) => {
+//   const nodes = getNodesByType(NODE_TYPE.Post);
+//   const docs: Record<string, string>[] = nodes.map((node) => ({
+//     id: node.id,
+//     text: node.rawText as string,
+//   }));
+//   const tfidf = new TfIdf();
+//   for (let doc of docs) {
+//     const key = `${md5(doc.text)}-related-post`;
+//     const cached_ssd = await cache.get(key);
+//     if (cached_ssd !== undefined) {
+//       tfidf.addDocument(cached_ssd);
+//       continue;
+//     }
+//     const ssd = await getSpaceSeparatedDoc[option.doc_lang](
+//       getTextFromMarkdown(doc.text)
+//     );
+//     tfidf.addDocument(ssd);
+//     await cache.set(key, ssd);
+//   }
+//   // generate bow vectors
+//   type Term = TfIdfTerm & {
+//     tf: number;
+//     idf: number;
+//   };
+//   //// extract keywords from each document
+//   const doc_terms = docs.map((_, i) =>
+//     (tfidf.listTerms(i) as Term[])
+//       .map((x) => ({ ...x, tfidf: (x as Term).tf * (x as Term).idf }))
+//       .sort((x, y) => y.tfidf - x.tfidf)
+//   );
+//   // DEBUG: print terms
+//   // doc_terms.forEach((x, i) =>
+//   //  console.log(
+//   //    docs[i].id,
+//   //    x.map((x) => x.term)
+//   //  )
+//   //);
+//   const all_keywords = new Set<string>();
+//   const tfidf_map_for_each_doc: Map<string, number>[] = [];
+//   doc_terms.forEach((x, i) => {
+//     tfidf_map_for_each_doc[i] = new Map<string, number>();
+//     x.slice(0, option.each_bow_size).forEach((x) => {
+//       all_keywords.add(x.term);
+//       tfidf_map_for_each_doc[i].set(x.term, x.tfidf);
+//     });
+//   });
+//   //// generate vectors
+//   const bow_vectors = new Map<string, BowVector>();
+//   docs.forEach((x, i) => {
+//     if (bow_vectors === null) return;
+//     bow_vectors.set(
+//       x.id,
+//       Array.from(all_keywords)
+//         .map((x) => tfidf_map_for_each_doc[i].get(x))
+//         .map((x) => (x === undefined ? 0 : x))
+//     );
+//   });
+//   reporter.info(
+//     `[related-posts] bow vectors generated, dimention: ${all_keywords.size}`
+//   );
+//   // create related nodes
+//   nodes.forEach((node) => {
+//     const related_nodes = getRelatedPosts(node.id, bow_vectors)
+//       .slice(1)
+//       .map((id) => getNode(id));
+//     const digest = `${node.id} >>> related${option.target_node}s`;
+//     actions.createNode({
+//       id: createNodeId(digest),
+//       parent: node.id,
+//       internal: {
+//         type: `related${option.target_node}s`,
+//         contentDigest: digest,
+//       },
+//       posts: related_nodes,
+//     });
+//   });
+// };

package/dist/util/processor.d.ts

@@ -4,4 +4,4 @@ export declare const processor: (blocks: BaseContentBlock[], actions: Actions, g
     type: string;
     hash: string;
     title: string;
-}[], BaseContentBlock[]]>;
+}[], BaseContentBlock[], string]>;

package/dist/util/processor.js

@@ -5,22 +5,51 @@ const gatsby_source_filesystem_1 = require("gatsby-source-filesystem");
 const metadataProcessor_1 = require("./metadataProcessor");
 const tableOfContent_1 = require("./tableOfContent");
 const processor = async (blocks, actions, getCache, createNodeId, reporter) => {
-    const { thumbnail, tableOfContents, updatedBlocks } = await processBlocksForContent(blocks, actions, getCache, createNodeId, reporter);
+    const { thumbnail, tableOfContents, updatedBlocks, rawText } = await processBlocksForContent(blocks, actions, getCache, createNodeId, reporter);
     await (0, metadataProcessor_1.processMetadata)(blocks, actions, createNodeId, reporter);
-    return [thumbnail, tableOfContents, updatedBlocks];
+    return [thumbnail, tableOfContents, updatedBlocks, rawText];
 };
 exports.processor = processor;
 const processBlocksForContent = async (blocks, actions, getCache, createNodeId, reporter) => {
     const tableOfContents = [];
     let thumbnail = null;
+    let rawText = "";
     const updatedBlocks = (await Promise.all(blocks.map(async (block) => {
         await (0, tableOfContent_1.processTableOfContents)(block, tableOfContents);
+        const plainText = extractPlainText(block);
+        if (plainText) {
+            rawText += plainText + " ";
+        }
         if (isImageBlock(block)) {
-            return await processImageBlock(block, actions, getCache, createNodeId, reporter);
+            const updatedBlock = await processImageBlock(block, actions, getCache, createNodeId, reporter);
+            if (!thumbnail && updatedBlock?.image?.fileId) {
+                thumbnail = updatedBlock.image.fileId;
+            }
+            return updatedBlock;
         }
         return block;
     }))).filter((block) => block !== null);
-    return { thumbnail, tableOfContents, updatedBlocks };
+    return { thumbnail, tableOfContents, updatedBlocks, rawText };
+};
+const isTextContentBlock = (block) => {
+    return [
+        "paragraph",
+        "heading_1",
+        "heading_2",
+        "heading_3",
+        "quote",
+        "bulleted_list_item",
+        "numbered_list_item",
+        "callout",
+        "code",
+    ].includes(block.type);
+};
+const extractPlainText = (block) => {
+    if (isTextContentBlock(block)) {
+        const richTextArray = block[block.type]?.rich_text || [];
+        return richTextArray.map((text) => text.plain_text).join(" ");
+    }
+    return null;
 };
 const isImageBlock = (block) => {
     return block.type === "image" && "image" in block;

package/dist/util/relatedPost.d.ts

@@ -0,0 +1,2 @@
+import { GatsbyNode } from "gatsby";
+export declare const onPostBootstrap: GatsbyNode[`onPostBootstrap`];

package/dist/util/relatedPost.js

@@ -1 +1,34 @@
 "use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.onPostBootstrap = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+const natural_1 = require("natural");
+const constants_1 = require("../constants");
+const md5 = (str) => {
+    const md5 = crypto_1.default.createHash("md5");
+    return md5.update(str, "binary").digest("hex");
+};
+const getSpaceSeparatedDoc = async (doc) => { };
+const onPostBootstrap = async ({ actions, getNode, getNodesByType, createNodeId, reporter, cache, }) => {
+    const nodes = getNodesByType(constants_1.NODE_TYPE.Post);
+    const docs = nodes.map((node) => ({
+        id: node.id,
+        text: node.rawText,
+    }));
+    const tfidf = new natural_1.TfIdf();
+    docs.map(async (doc) => {
+        if (doc.text) {
+            const key = `${md5(doc.text)}-doc`;
+            const cached_ssd = await cache.get(key);
+            if (cached_ssd !== undefined) {
+                tfidf.addDocument(cached_ssd);
+            }
+            else {
+            }
+        }
+    });
+};
+exports.onPostBootstrap = onPostBootstrap;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gatsby-source-notion-churnotion",
   "description": "Gatsby plugin that can connect with One Notion Database RECURSIVELY using official API",
-  "version": "1.0.65",
+  "version": "1.0.67",
   "skipLibCheck": true,
   "license": "0BSD",
   "main": "./dist/gatsby-node.js",
@@ -40,11 +40,13 @@
     "gatsby-source-filesystem": "^5.14.0",
     "gatsby-transformer-json": "^5.14.0",
     "gatsby-transformer-sharp": "^5.14.0",
+    "kiwi-nlp": "^0.20.3",
     "metascraper": "^5.45.25",
     "metascraper-description": "^5.45.25",
     "metascraper-image": "^5.45.27",
     "metascraper-title": "^5.45.25",
     "metascraper-url": "^5.45.25",
+    "natural": "^8.0.1",
     "node-fetch": "^3.3.2",
     "notion-to-md": "^3.1.1",
     "notion-types": "^7.1.5",