noblog 0.0.10 → 0.0.12

package/dist/generate.js

@@ -1,13 +1,79 @@
 #!/usr/bin/env node
 import * as path from 'path';
 import * as fs from 'fs/promises';
-import { NotionToMarkdown } from 'notion-to-md';
-import 'node-fetch';
-import 'markdown-table';
+import fetch from 'node-fetch';
+import { markdownTable } from 'markdown-table';
 import 'dotenv/config';
 import retry from 'async-retry';
 import { Client, APIResponseError } from '@notionhq/client';
 
+const inlineEquation = (text) => {
+    return `$${text}$`;
+};
+const link = (text, href) => {
+    return `[${text}](${href})`;
+};
+const codeBlock = (text, language) => {
+    if (language === "plain text")
+        language = "text";
+    return `\`\`\`${language}
+${text}
+\`\`\``;
+};
+const equation = (text) => {
+    return `$$
+${text}
+$$`;
+};
+const heading1 = (text) => {
+    return `# ${text}`;
+};
+const heading2 = (text) => {
+    return `## ${text}`;
+};
+const heading3 = (text) => {
+    return `### ${text}`;
+};
+const quote = (text) => {
+    // the replace is done to handle multiple lines
+    return `> ${text.replace(/\n/g, "  \n> ")}`;
+};
+const callout = (text, icon) => {
+    let emoji;
+    if (icon?.type === "emoji") {
+        emoji = icon.emoji;
+    }
+    // the replace is done to handle multiple lines
+    return `> ${emoji ? emoji + " " : ""}${text.replace(/\n/g, "  \n> ")}`;
+};
+const bullet = (text, count) => {
+    let renderText = text.trim();
+    return count ? `${count}. ${renderText}` : `- ${renderText}`;
+};
+const todo = (text, checked) => {
+    return checked ? `- [x] ${text}` : `- [ ] ${text}`;
+};
+const image = async (alt, href, convertToBase64 = false) => {
+    // In case the user does not want to convert the images to Base64
+    // or the image is already base64
+    if (!convertToBase64 || href.startsWith("data:")) {
+        if (href.startsWith("data:")) {
+            // Extract base64 data, i.e. the string after 'data:mime/type;base64,'
+            const base64 = href.split(",").pop();
+            // This overrides incorrect data: string format to png
+            // so that browsers can correctly render the data
+            return `![${alt}](data:image/png;base64,${base64})`;
+        }
+        return `![${alt}](${href})`;
+    }
+    else {
+        // Otherwise, download the image and convert it to base64
+        const res = await fetch(href);
+        const buf = await res.arrayBuffer();
+        const base64 = Buffer.from(buf).toString("base64");
+        return `![${alt}](data:image/png;base64,${base64})`;
+    }
+};
 const addTabSpace = (text, n = 0) => {
     const tab = "	";
     for (let i = 0; i < n; i++) {
@@ -20,6 +86,9 @@ const addTabSpace = (text, n = 0) => {
     }
     return text;
 };
+const divider = () => {
+    return "---";
+};
 const toggle = (summary, children) => {
     if (!summary)
         return children || "";
@@ -28,9 +97,12 @@ const toggle = (summary, children) => {
 ${children || ""}
 </details>\n\n`;
 };
+const table = (cells) => {
+    return markdownTable(cells);
+};
 
-const NOTION_API_SECRET = process.env.NOTION_API_SECRET ?? '';
-const NOBLOG_DATABASE_ID = process.env.NOBLOG_DATABASE_ID ?? '';
+const NOTION_API_SECRET = process.env.NOTION_API_SECRET ?? "";
+const NOBLOG_DATABASE_ID = process.env.NOBLOG_DATABASE_ID ?? "";
 const client = new Client({
     auth: NOTION_API_SECRET,
 });
@@ -39,15 +111,19 @@ async function GetAllPosts() {
     const params = {
         database_id: NOBLOG_DATABASE_ID,
         filter: {
-            and: [{
-                    property: 'publish',
-                    checkbox: { equals: true, },
-                }]
+            and: [
+                {
+                    property: "publish",
+                    checkbox: { equals: true },
+                },
+            ],
         },
-        sorts: [{
-                property: 'date',
-                direction: 'descending',
-            }],
+        sorts: [
+            {
+                property: "date",
+                direction: "descending",
+            },
+        ],
         page_size: 100,
     };
     let results = [];
@@ -64,12 +140,12 @@ async function GetAllPosts() {
                 }
                 throw error;
             }
-        }, { retries: numberOfRetry, });
+        }, { retries: numberOfRetry });
         results = results.concat(res.results);
         if (!res.has_more) {
             break;
         }
-        params['start_cursor'] = res.next_cursor;
+        params["start_cursor"] = res.next_cursor;
     }
     return results;
 }
@@ -86,18 +162,48 @@ async function GetPageMeta(page_id) {
             }
             throw error;
         }
-    }, { retries: numberOfRetry, });
+    }, { retries: numberOfRetry });
+    return result;
+}
+async function GetBlockChildren(block_id, totalPage) {
+    let result = [];
+    let pageCount = 0;
+    let start_cursor = undefined;
+    do {
+        const response = (await client.blocks.children.list({
+            start_cursor: start_cursor,
+            block_id: block_id,
+        }));
+        result.push(...response.results);
+        start_cursor = response?.next_cursor;
+        pageCount += 1;
+    } while (start_cursor != null &&
+        (totalPage == null || pageCount < totalPage));
+    ModifyNumberedListObject(result);
     return result;
 }
+function ModifyNumberedListObject(blocks) {
+    let numberedListIndex = 0;
+    for (const block of blocks) {
+        if ("type" in block && block.type === "numbered_list_item") {
+            // add numbers
+            // @ts-ignore
+            block.numbered_list_item.number = ++numberedListIndex;
+        }
+        else {
+            numberedListIndex = 0;
+        }
+    }
+}
 
-class Noblog extends NotionToMarkdown {
+class Noblog {
     Posts = [];
     CurrPage = "";
     SavePath = "./src/pages/posts/";
     ChildPath = "nob_children/";
     MdCollection = {};
+    CustomTransformers = {};
     constructor(posts, save_path, child_path) {
-        super({ notionClient: client });
         this.Posts = posts;
         if (child_path != undefined) {
             this.ChildPath = child_path;
@@ -105,21 +211,21 @@ class Noblog extends NotionToMarkdown {
         if (save_path != undefined) {
             this.SavePath = save_path;
         }
-        this.setCustomTransformer("child_page", async (block) => {
+        this.CustomTransformers["child_page"] = async (block) => {
             const page = block;
             if (!page?.id) {
                 return false;
             }
             let sub_path;
-            // if (curr_page.id in this.Post) and (page.id in this.Post)
-            // if (curr_page.id not in this.Post) and (page.id not in this.Post)
+            // if (curr_page.id in this.Post) and (page.id in this.Post) ||
+            //    (curr_page.id not in this.Post) and (page.id not in this.Post)
             //    * path should be "../pageid"
             // if (curr_page.id in this.Post) and (page.id not in this.Post)
             //    * path should be "../save_path/pageid"
             // if (curr_page.id not in this.Post) and (page.id in this.Post)
             //    * path should be "../../pageid"
-            const parent_surface = (this.Posts.filter(post => post.id === this.CurrPage).length > 0);
-            const child_surface = (this.Posts.filter(post => post.id === page.id).length > 0);
+            const parent_surface = this.Posts.filter((post) => post.id === this.CurrPage).length > 0;
+            const child_surface = this.Posts.filter((post) => post.id === page.id).length > 0;
             if (parent_surface === child_surface) {
                 sub_path = "";
             }
@@ -131,7 +237,7 @@ class Noblog extends NotionToMarkdown {
             }
             const full_path = path.join("../", sub_path, page.id);
             return `＆[${page.child_page.title}](${full_path})`;
-        });
+        };
     }
     async Collect(recur = true) {
         const save_dir = this.SavePath;
@@ -142,7 +248,7 @@ class Noblog extends NotionToMarkdown {
         try {
             for (const pageid of Object.keys(this.MdCollection)) {
                 let fpath = sub_dir;
-                if (this.Posts.find(page => page.id === pageid))
+                if (this.Posts.find((page) => page.id === pageid))
                     fpath = save_dir;
                 await this.SaveJelly(pageid + ".md", fpath, this.MdCollection[pageid]);
             }
@@ -153,7 +259,7 @@ class Noblog extends NotionToMarkdown {
                 console.log(`failed: consider run verbose with v=true - ${error.message}`);
             }
             else {
-                console.log('failed: an unknown error occurred');
+                console.log("failed: an unknown error occurred");
             }
         }
     }
@@ -172,7 +278,8 @@ class Noblog extends NotionToMarkdown {
                 block.type !== "numbered_list_item") {
                 line_wrap = "\n";
             }
-            jelly.content += (line_wrap + `${addTabSpace(block.parent, 0)}\n` + line_wrap);
+            jelly.content +=
+                line_wrap + `${addTabSpace(block.parent, 0)}\n` + line_wrap;
             if (block.type === "child_page") {
                 jelly.children = [...jelly.children, block.blockId];
                 if (recursive)
@@ -192,23 +299,326 @@ class Noblog extends NotionToMarkdown {
         if (this.MdCollection.hasOwnProperty(page_id)) {
             return this.MdCollection[page_id];
         }
-        const blocks = await this.pageToMarkdown(page_id);
+        const blocks = await this.PageToMarkdown(page_id);
         const jelly = await this.FromBlocks(blocks, recursive);
         const meta = await GetPageMeta(page_id);
         const astro_meta = await this.AssembleAstroFrontmatter(meta);
         jelly.content = astro_meta + jelly.content;
         this.MdCollection[page_id] = jelly;
-        // restore the prev page info 
+        // restore the prev page info
         this.CurrPage = prev_page;
         return jelly;
     }
+    async PageToMarkdown(id, totalPage) {
+        if (!client) {
+            throw new Error("notion client is not provided, for more details check out https://github.com/souvikinator/notion-to-md");
+        }
+        const blocks = await GetBlockChildren(id, totalPage);
+        const parsedData = await this.BlocksToMarkdown(blocks);
+        return parsedData;
+    }
+    async BlocksToMarkdown(blocks, totalPage, mdBlocks) {
+        if (!client) {
+            throw new Error("notion client is not provided, for more details check out https://github.com/souvikinator/notion-to-md");
+        }
+        if (!blocks)
+            return mdBlocks;
+        for (let i = 0; i < blocks.length; i++) {
+            let block = blocks[i];
+            if (
+            // @ts-ignore
+            block.type === "unsupported" ||
+                // @ts-ignore
+                (block.type === "child_page" && !this.config.parseChildPages)) {
+                continue;
+            }
+            if ("has_children" in block && block.has_children) {
+                const block_id = block.type == "synced_block" &&
+                    block.synced_block?.synced_from?.block_id
+                    ? block.synced_block.synced_from.block_id
+                    : block.id;
+                // Get children of this block.
+                let child_blocks = await GetBlockChildren(block_id, totalPage);
+                // Recursively call BlocksToMarkdown to get children of this block.
+                // check for custom transformer before parsing child
+                if (!(block.type in this.CustomTransformers) &&
+                    !this.CustomTransformers[block.type]) {
+                    let l = mdBlocks.length;
+                    if (mdBlocks !== undefined) {
+                        await this.BlocksToMarkdown(child_blocks, totalPage, mdBlocks[l - 1]?.children);
+                    }
+                }
+                // Push this block to mdBlocks.
+                mdBlocks.push({
+                    type: block.type,
+                    blockId: block.id,
+                    parent: await this.BlockToMarkdown(block),
+                    children: [],
+                });
+                continue;
+            }
+            let tmp = await this.BlockToMarkdown(block);
+            mdBlocks.push({
+                // @ts-ignore
+                type: block.type,
+                blockId: block.id,
+                parent: tmp,
+                children: [],
+            });
+        }
+        return mdBlocks;
+    }
+    async BlockToMarkdown(block) {
+        if (typeof block !== "object" || !("type" in block))
+            return "";
+        let parsedData = "";
+        const { type } = block;
+        if (type in this.CustomTransformers && !!this.CustomTransformers[type]) {
+            const transformer = this.CustomTransformers[type];
+            const customTransformerValue = transformer(block);
+            if (typeof customTransformerValue === "string")
+                return customTransformerValue;
+        }
+        switch (type) {
+            case "image":
+                let blockContent = block.image;
+                let image_title = "image";
+                const image_caption_plain = blockContent.caption
+                    .map((item) => item.plain_text)
+                    .join("");
+                const image_type = blockContent.type;
+                let link$1 = "";
+                if (image_type === "external") {
+                    link$1 = blockContent.external.url;
+                }
+                if (image_type === "file") {
+                    link$1 = blockContent.file.url;
+                }
+                // image caption with high priority
+                if (image_caption_plain.trim().length > 0) {
+                    image_title = image_caption_plain;
+                }
+                else if (image_type === "file" || image_type === "external") {
+                    const matches = link$1.match(/[^\/\\&\?]+\.\w{3,4}(?=([\?&].*$|$))/);
+                    image_title = matches ? matches[0] : image_title;
+                }
+                return await image(image_title, link$1, false);
+            case "divider":
+                return divider();
+            case "equation":
+                return equation(block.equation.expression);
+            case "video":
+            case "file":
+            case "pdf":
+                {
+                    let blockContent;
+                    let title = type;
+                    if (type === "video")
+                        blockContent = block.video;
+                    if (type === "file")
+                        blockContent = block.file;
+                    if (type === "pdf")
+                        blockContent = block.pdf;
+                    const caption = blockContent?.caption
+                        .map((item) => item.plain_text)
+                        .join("");
+                    if (blockContent) {
+                        const file_type = blockContent.type;
+                        let link$1 = "";
+                        if (file_type === "external")
+                            link$1 = blockContent.external.url;
+                        if (file_type === "file")
+                            link$1 = blockContent.file.url;
+                        if (caption && caption.trim().length > 0) {
+                            title = caption;
+                        }
+                        else if (link$1) {
+                            const matches = link$1.match(/[^\/\\&\?]+\.\w{3,4}(?=([\?&].*$|$))/);
+                            title = matches ? matches[0] : type;
+                        }
+                        return link(title, link$1);
+                    }
+                }
+                break;
+            case "bookmark":
+            case "embed":
+            case "link_preview":
+            case "link_to_page":
+                {
+                    let blockContent;
+                    let title = type;
+                    if (type === "bookmark")
+                        blockContent = block.bookmark;
+                    if (type === "embed")
+                        blockContent = block.embed;
+                    if (type === "link_preview")
+                        blockContent = block.link_preview;
+                    if (type === "link_to_page" &&
+                        block.link_to_page.type === "page_id") {
+                        blockContent = {
+                            url: `https://www.notion.so/${block.link_to_page.page_id}`,
+                        };
+                    }
+                    if (blockContent)
+                        return link(title, blockContent.url);
+                }
+                break;
+            case "child_page": {
+                let pageTitle = block.child_page.title;
+                return heading2(pageTitle);
+            }
+            case "child_database": {
+                let pageTitle = block.child_database.title || `child_database`;
+                return heading2(pageTitle);
+            }
+            case "table": {
+                const { id, has_children } = block;
+                let tableArr = [];
+                if (has_children) {
+                    const tableRows = await GetBlockChildren(id, 100);
+                    let rowsPromise = tableRows?.map(async (row) => {
+                        const { type } = row;
+                        const cells = row[type]["cells"];
+                        /**
+                         * this is more like a hack since matching the type text was
+                         * difficult. So converting each cell to paragraph type to
+                         * reuse the BlockToMarkdown function
+                         */
+                        let cellStringPromise = cells.map(async (cell) => await this.BlockToMarkdown({
+                            type: "paragraph",
+                            paragraph: { rich_text: cell },
+                        }));
+                        const cellStringArr = await Promise.all(cellStringPromise);
+                        tableArr.push(cellStringArr);
+                    });
+                    await Promise.all(rowsPromise || []);
+                }
+                return table(tableArr);
+            }
+            // Rest of the types:
+            //   "paragraph" "heading_1" "heading_2" "heading_3"
+            //   "bulleted_list_item" "numbered_list_item"
+            //   "quote" "to_do" "template" "synced_block"
+            //   "child_page" "child_database" "code" "callout" "breadcrumb"
+            //   "table_of_contents" "link_to_page" "audio" "unsupported"
+            default: {
+                // @ts-ignore
+                let blockContent = block[type].text || block[type].rich_text || [];
+                let bucket = [];
+                let code_range = [];
+                let bold_range = [];
+                let italic_range = [];
+                let underline_range = [];
+                let strikethrough_range = [];
+                const push_or_merge = (arr, idx) => {
+                    if (arr.length == 0) {
+                        return;
+                    }
+                    // @ts-ignore
+                    if (idx === arr[arr.length - 1][1]) {
+                        // @ts-ignore
+                        arr[arr.length - 1][1] += 1;
+                    }
+                    else {
+                        arr.push([idx, idx + 1]);
+                    }
+                };
+                for (let i = 0; i < blockContent.length; i++) {
+                    let content = blockContent[i];
+                    if (content.type === "equation") {
+                        bucket.push(inlineEquation(content.equation.expression));
+                        continue;
+                    }
+                    let plain_text = content.plain_text;
+                    if (content["href"])
+                        plain_text = link(plain_text, content["href"]);
+                    bucket.push(plain_text);
+                    const annotations = content.annotations;
+                    if (annotations.code)
+                        push_or_merge(code_range, i);
+                    if (annotations.bold)
+                        push_or_merge(bold_range, i);
+                    if (annotations.italic)
+                        push_or_merge(italic_range, i);
+                    if (annotations.underline)
+                        push_or_merge(underline_range, i);
+                    if (annotations.strikethrough)
+                        push_or_merge(strikethrough_range, i);
+                }
+                let annoed_bucket = [];
+                for (let range of code_range)
+                    annoed_bucket.push({ range: range, type: "code" });
+                for (let range of bold_range)
+                    annoed_bucket.push({ range: range, type: "bold" });
+                for (let range of italic_range)
+                    annoed_bucket.push({ range: range, type: "italic" });
+                for (let range of underline_range)
+                    annoed_bucket.push({ range: range, type: "underline" });
+                for (let range of strikethrough_range)
+                    annoed_bucket.push({ range: range, type: "strikethrough" });
+                annoed_bucket.sort(
+                // @ts-ignore
+                (a, b) => a.range[1] - a.range[0] - (b.range[1] - b.range[0]));
+                for (let annoed of annoed_bucket) {
+                    annoed.type;
+                    let [from, to] = annoed.range;
+                    // @ts-ignore
+                    bucket[from] = annotation + bucket[from];
+                    // @ts-ignore
+                    bucket[to - 1] = bucket[to - 1] + annotation;
+                }
+                bucket.forEach((x) => (parsedData += x));
+            }
+        }
+        switch (type) {
+            case "code":
+                parsedData = codeBlock(parsedData, block[type].language);
+                break;
+            case "heading_1":
+                parsedData = heading1(parsedData);
+                break;
+            case "heading_2":
+                parsedData = heading2(parsedData);
+                break;
+            case "heading_3":
+                parsedData = heading3(parsedData);
+                break;
+            case "quote":
+                parsedData = quote(parsedData);
+                break;
+            case "callout":
+                const { id, has_children } = block;
+                let callout_string = "";
+                if (!has_children) {
+                    return callout(parsedData, block[type].icon);
+                }
+                const callout_children_object = await GetBlockChildren(id, 100);
+                const callout_children = await this.BlocksToMarkdown(callout_children_object);
+                callout_string += `${parsedData}\n`;
+                callout_children.map((child) => {
+                    callout_string += `${child.parent}\n\n`;
+                });
+                parsedData = callout(callout_string.trim(), block[type].icon);
+                break;
+            case "bulleted_list_item":
+                parsedData = bullet(parsedData);
+                break;
+            case "numbered_list_item":
+                parsedData = bullet(parsedData, block.numbered_list_item.number);
+                break;
+            case "to_do":
+                parsedData = todo(parsedData, block.to_do.checked);
+                break;
+        }
+        return parsedData;
+    }
     async SaveJelly(fname, dir, jelly) {
         if (jelly === undefined)
             return `failed: ${fname} has empty jelly: `;
         try {
             await fs.mkdir(dir, { recursive: true });
             const filePath = path.join(dir, fname);
-            await fs.writeFile(filePath, jelly.content, 'utf-8');
+            await fs.writeFile(filePath, jelly.content, "utf-8");
             return `success: ${fname} saved to ${filePath}`;
         }
         catch (error) {
@@ -216,25 +626,40 @@ class Noblog extends NotionToMarkdown {
                 return `failed: ${fname} - ${error.message}`;
             }
             else {
-                return 'failed: an unknown error occurred';
+                return "failed: an unknown error occurred";
             }
         }
     }
     async AssembleAstroFrontmatter(page) {
         let frontmatter = "";
         // default layout
-        const ischild = this.Posts.filter(post => post.id === page.id).length > 0 ? "" : "../";
-        frontmatter += "layout: " + ischild + "../../layouts/MarkdownPostLayout.astro" + "\n";
+        const ischild = this.Posts.filter((post) => post.id === page.id).length > 0 ? "" : "../";
+        frontmatter +=
+            "layout: " + ischild + "../../layouts/MarkdownPostLayout.astro" + "\n";
         // get title
-        frontmatter += "title: " + JSON.stringify(page.properties.title?.title?.[0]?.plain_text ?? "") + "\n";
+        frontmatter +=
+            "title: " +
+                JSON.stringify(page.properties.title?.title?.[0]?.plain_text ?? "") +
+                "\n";
         // get tags
-        frontmatter += "tags: " + JSON.stringify((page.properties.tags?.multi_select ?? []).map(tag => tag.name)) + "\n";
+        frontmatter +=
+            "tags: " +
+                JSON.stringify((page.properties.tags?.multi_select ?? []).map((tag) => tag.name)) +
+                "\n";
         // get date
-        frontmatter += "pubDate: " + (page.properties.date?.date?.start ?? new Date().toISOString().split("T")[0]) + "\n";
+        frontmatter +=
+            "pubDate: " +
+                (page.properties.date?.date?.start ??
+                    new Date().toISOString().split("T")[0]) +
+                "\n";
         // get archived
-        frontmatter += "archived: " + (page.properties.archived ? "true" : "false") + "\n";
+        frontmatter +=
+            "archived: " + (page.properties.archived ? "true" : "false") + "\n";
         // get description
-        frontmatter += "description: " + JSON.stringify(page.properties.description?.rich_text?.[0]?.plain_text ?? "") + "\n";
+        frontmatter +=
+            "description: " +
+                JSON.stringify(page.properties.description?.rich_text?.[0]?.plain_text ?? "") +
+                "\n";
         return "---\n" + frontmatter + "---\n";
     }
 }