gatsby-source-notion-churnotion 1.0.66 → 1.0.68

package/dist/gatsby-node.d.ts

@@ -2,3 +2,4 @@ export type { IPluginOptions } from "./types";
 export { onPluginInit } from "./onPluginInit";
 export { sourceNodes } from "./source-nodes";
 export { createSchemaCustomization } from "./createSchemaCustomization";
+export { onPostBootstrap } from "./onPostBootstrap";

package/dist/gatsby-node.js

@@ -1,9 +1,11 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.createSchemaCustomization = exports.sourceNodes = exports.onPluginInit = void 0;
+exports.onPostBootstrap = exports.createSchemaCustomization = exports.sourceNodes = exports.onPluginInit = void 0;
 var onPluginInit_1 = require("./onPluginInit");
 Object.defineProperty(exports, "onPluginInit", { enumerable: true, get: function () { return onPluginInit_1.onPluginInit; } });
 var source_nodes_1 = require("./source-nodes");
 Object.defineProperty(exports, "sourceNodes", { enumerable: true, get: function () { return source_nodes_1.sourceNodes; } });
 var createSchemaCustomization_1 = require("./createSchemaCustomization");
 Object.defineProperty(exports, "createSchemaCustomization", { enumerable: true, get: function () { return createSchemaCustomization_1.createSchemaCustomization; } });
+var onPostBootstrap_1 = require("./onPostBootstrap");
+Object.defineProperty(exports, "onPostBootstrap", { enumerable: true, get: function () { return onPostBootstrap_1.onPostBootstrap; } });

package/dist/onPluginInit.js

@@ -2,6 +2,6 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.onPluginInit = void 0;
 const onPluginInit = ({ reporter }) => {
-    reporter.info(`Churnotion plugin loaded...`);
+    reporter.info(`Churnotion plugin loaded`);
 };
 exports.onPluginInit = onPluginInit;

package/dist/onPostBootstrap.d.ts

@@ -0,0 +1,2 @@
+import { GatsbyNode } from "gatsby";
+export declare const onPostBootstrap: GatsbyNode[`onPostBootstrap`];

package/dist/onPostBootstrap.js

@@ -1,113 +1,135 @@
 "use strict";
-// import { GatsbyNode } from "gatsby";
-// import { TfIdf, TfIdfTerm } from "natural";
-// import { NODE_TYPE } from "./constants";
-// import crypto from "crypto";
-// const md5 = (str: string): string => {
-//   const md5 = crypto.createHash("md5");
-//   return md5.update(str, "binary").digest("hex");
-// };
-// const getSpaceSeparatedDoc: {
-//     [key: string]: (doc: string) => Promise<string[]>;
-//   } = {
-//     en: async (doc) => {
-//       return doc.toLowerCase().split(' ');
-//     },
-//     ja: async (doc) => {
-//       if (kuromoji_tokenizer === null)
-//         kuromoji_tokenizer = await getKuromojiTokenizer();
-//       return kuromoji_tokenizer
-//         .tokenize(doc)
-//         .filter(
-//           (x) =>
-//             x.pos === '名詞' &&
-//             ['一般', '固有名詞'].indexOf(x.pos_detail_1) !== -1
-//         )
-//         .map((x) => (x.basic_form !== '*' ? x.basic_form : x.surface_form));
-//     },
-//   };
-// export const onPostBootstrap: GatsbyNode["onPostBootstrap"] = async ({
-//   actions,
-//   getNode,
-//   getNodesByType,
-//   createNodeId,
-//   reporter,
-//   cache,
-// }) => {
-//   const nodes = getNodesByType(NODE_TYPE.Post);
-//   const docs: Record<string, string>[] = nodes.map((node) => ({
-//     id: node.id,
-//     text: node.rawText as string,
-//   }));
-//   const tfidf = new TfIdf();
-//   for (let doc of docs) {
-//     const key = `${md5(doc.text)}-related-post`;
-//     const cached_ssd = await cache.get(key);
-//     if (cached_ssd !== undefined) {
-//       tfidf.addDocument(cached_ssd);
-//       continue;
-//     }
-//     const ssd = await getSpaceSeparatedDoc[option.doc_lang](
-//       getTextFromMarkdown(doc.text)
-//     );
-//     tfidf.addDocument(ssd);
-//     await cache.set(key, ssd);
-//   }
-//   // generate bow vectors
-//   type Term = TfIdfTerm & {
-//     tf: number;
-//     idf: number;
-//   };
-//   //// extract keywords from each document
-//   const doc_terms = docs.map((_, i) =>
-//     (tfidf.listTerms(i) as Term[])
-//       .map((x) => ({ ...x, tfidf: (x as Term).tf * (x as Term).idf }))
-//       .sort((x, y) => y.tfidf - x.tfidf)
-//   );
-//   // DEBUG: print terms
-//   // doc_terms.forEach((x, i) =>
-//   //  console.log(
-//   //    docs[i].id,
-//   //    x.map((x) => x.term)
-//   //  )
-//   //);
-//   const all_keywords = new Set<string>();
-//   const tfidf_map_for_each_doc: Map<string, number>[] = [];
-//   doc_terms.forEach((x, i) => {
-//     tfidf_map_for_each_doc[i] = new Map<string, number>();
-//     x.slice(0, option.each_bow_size).forEach((x) => {
-//       all_keywords.add(x.term);
-//       tfidf_map_for_each_doc[i].set(x.term, x.tfidf);
-//     });
-//   });
-//   //// generate vectors
-//   const bow_vectors = new Map<string, BowVector>();
-//   docs.forEach((x, i) => {
-//     if (bow_vectors === null) return;
-//     bow_vectors.set(
-//       x.id,
-//       Array.from(all_keywords)
-//         .map((x) => tfidf_map_for_each_doc[i].get(x))
-//         .map((x) => (x === undefined ? 0 : x))
-//     );
-//   });
-//   reporter.info(
-//     `[related-posts] bow vectors generated, dimention: ${all_keywords.size}`
-//   );
-//   // create related nodes
-//   nodes.forEach((node) => {
-//     const related_nodes = getRelatedPosts(node.id, bow_vectors)
-//       .slice(1)
-//       .map((id) => getNode(id));
-//     const digest = `${node.id} >>> related${option.target_node}s`;
-//     actions.createNode({
-//       id: createNodeId(digest),
-//       parent: node.id,
-//       internal: {
-//         type: `related${option.target_node}s`,
-//         contentDigest: digest,
-//       },
-//       posts: related_nodes,
-//     });
-//   });
-// };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.onPostBootstrap = void 0;
+const crypto_1 = __importDefault(require("crypto"));
+const kiwi_nlp_1 = require("kiwi-nlp");
+const natural_1 = require("natural");
+const constants_1 = require("./constants");
+const computeCosineSimilarity = require("compute-cosine-similarity");
+const vector_similarity_memo = new Map();
+const md5 = (str) => {
+    const md5 = crypto_1.default.createHash("md5");
+    return md5.update(str, "binary").digest("hex");
+};
+const getMemorizedVectorSimilarity = (v1, v2) => {
+    const id = v1.id < v2.id ? `${v1.id} ${v2.id}` : `${v2.id} ${v1.id}`;
+    const memorized_similarity = vector_similarity_memo.get(id);
+    if (memorized_similarity !== undefined)
+        return memorized_similarity;
+    const similarity = calcVectorSimilarity(v1.vector, v2.vector);
+    vector_similarity_memo.set(id, similarity);
+    return similarity;
+};
+const calcVectorSimilarity = (v1, v2) => {
+    if (v1.length !== v2.length)
+        throw new Error("Both vector's size must be equal");
+    return computeCosineSimilarity(v1, v2);
+};
+const getRelatedPosts = (id, bow_vectors) => {
+    const vector = bow_vectors.get(id);
+    if (vector === undefined)
+        return [];
+    const vector_node = {
+        id,
+        vector,
+    };
+    return Array.from(bow_vectors.entries())
+        .sort((x, y) => {
+        const vector_x = {
+            id: x[0],
+            vector: x[1],
+        };
+        const vector_y = {
+            id: y[0],
+            vector: y[1],
+        };
+        return (getMemorizedVectorSimilarity(vector_y, vector_node) -
+            getMemorizedVectorSimilarity(vector_x, vector_node));
+    })
+        .map((x) => x[0]);
+};
+const getTextFromRawText = async (doc) => {
+    return doc
+        .replace(/http[^ ]+/g, "")
+        .replace(/[\#\!\(\)\*\_\[\]\|\=\>\+\`\:\-]/g, "");
+};
+const getSpaceSeparatedDoc = async (doc, kiwi) => {
+    return kiwi.tokenize(doc).map((tokenInfo) => tokenInfo.str);
+};
+const onPostBootstrap = async ({ actions, getNode, getNodesByType, createNodeId, reporter, cache, }) => {
+    const builder = await kiwi_nlp_1.KiwiBuilder.create("/dist/kiwi-wasm.wasm");
+    const kiwi = await builder.build({
+        modelFiles: {
+            "combiningRule.txt": "/dist/model/combiningRule.txt",
+            "default.dict": "/dist/model/default.dict",
+            "extract.mdl": "/dist/model/extract.mdl",
+            "multi.dict": "/dist/model/multi.dict",
+            "sj.knlm": "/dist/model/sj.knlm",
+            "sj.morph": "/dist/model/sj.morph",
+            "skipbigram.mdl": "/dist/model/skipbigram.mdl",
+            "typo.dict": "/dist/model/typo.dict",
+        },
+    });
+    const nodes = getNodesByType(constants_1.NODE_TYPE.Post);
+    const docs = nodes.map((node) => ({
+        id: node.id,
+        text: node.rawText,
+    }));
+    const tfidf = new natural_1.TfIdf();
+    // tfidf
+    docs.map(async (doc) => {
+        if (doc.text) {
+            const key = `${md5(doc.text)}-doc`;
+            const cached_ssd = await cache.get(key);
+            if (cached_ssd !== undefined) {
+                tfidf.addDocument(cached_ssd);
+            }
+            else {
+                const ssd = await getSpaceSeparatedDoc(await getTextFromRawText(doc.text), kiwi);
+                tfidf.addDocument(ssd);
+                await cache.set(key, ssd);
+            }
+        }
+    });
+    //
+    const doc_terms = docs.map((_, i) => tfidf.listTerms(i)
+        .map((x) => ({ ...x, tfidf: x.tf * x.idf }))
+        .sort((x, y) => y.tfidf - x.tfidf));
+    const all_keywords = new Set();
+    const tfidf_map_for_each_doc = [];
+    doc_terms.forEach((x, i) => {
+        tfidf_map_for_each_doc[i] = new Map();
+        x.slice(0, 30).forEach((x) => {
+            all_keywords.add(x.term);
+            tfidf_map_for_each_doc[i].set(x.term, x.tfidf);
+        });
+    });
+    const bow_vectors = new Map();
+    docs.forEach((x, i) => {
+        if (bow_vectors === null)
+            return;
+        bow_vectors.set(x.id, Array.from(all_keywords)
+            .map((x) => tfidf_map_for_each_doc[i].get(x))
+            .map((x) => (x === undefined ? 0 : x)));
+    });
+    reporter.info(`[related-posts] bow vectors generated, dimention: ${all_keywords.size}`);
+    nodes.forEach((node) => {
+        const related_nodes = getRelatedPosts(node.id, bow_vectors)
+            .slice(1)
+            .map((id) => getNode(id));
+        const digest = `${node.id} - ${constants_1.NODE_TYPE.RelatedPost}`;
+        actions.createNode({
+            id: createNodeId(digest),
+            parent: node.id,
+            internal: {
+                type: `related${constants_1.NODE_TYPE.RelatedPost}s`,
+                contentDigest: digest,
+            },
+            posts: related_nodes,
+        });
+    });
+};
+exports.onPostBootstrap = onPostBootstrap;

package/dist/util/processor.js

@@ -40,12 +40,18 @@ const isTextContentBlock = (block) => {
         "quote",
         "bulleted_list_item",
         "numbered_list_item",
+        "callout",
+        "code",
     ].includes(block.type);
 };
 const extractPlainText = (block) => {
     if (isTextContentBlock(block)) {
         const richTextArray = block[block.type]?.rich_text || [];
-        return richTextArray.map((text) => text.plain_text).join(" ");
+        return richTextArray
+            .map((text) => block.type === "code" // code의 \n 제거
+            ? text.plain_text.replace(/\\n/g, "")
+            : text.plain_text)
+            .join(" ");
     }
     return null;
 };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "gatsby-source-notion-churnotion",
   "description": "Gatsby plugin that can connect with One Notion Database RECURSIVELY using official API",
-  "version": "1.0.66",
+  "version": "1.0.68",
   "skipLibCheck": true,
   "license": "0BSD",
   "main": "./dist/gatsby-node.js",
@@ -36,10 +36,12 @@
     "@notionhq/client": "^2.2.15",
     "@types/node": "^22.10.2",
     "axios": "^1.7.9",
+    "compute-cosine-similarity": "^1.1.0",
     "gatsby-plugin-sharp": "^5.14.0",
     "gatsby-source-filesystem": "^5.14.0",
     "gatsby-transformer-json": "^5.14.0",
     "gatsby-transformer-sharp": "^5.14.0",
+    "kiwi-nlp": "^0.20.3",
     "metascraper": "^5.45.25",
     "metascraper-description": "^5.45.25",
     "metascraper-image": "^5.45.27",

package/dist/util/relatedPost.js

@@ -1 +0,0 @@
-"use strict";