noblog 0.0.1-2.alpha

package/dist/generate.js

@@ -0,0 +1,691 @@
+#!/usr/bin/env node
+import * as path from 'path';
+import * as fs from 'fs/promises';
+import fetch from 'node-fetch';
+import { markdownTable } from 'markdown-table';
+import 'dotenv/config';
+import retry from 'async-retry';
+import { Client, APIResponseError } from '@notionhq/client';
+
+const inlineEquation = (text) => {
+    return `$${text}$`;
+};
+const link = (text, href) => {
+    return `[${text}](${href})`;
+};
+const codeBlock = (text, language) => {
+    if (language === "plain text")
+        language = "text";
+    return `\`\`\`${language}
+${text}
+\`\`\``;
+};
+const equation = (text) => {
+    return `$$
+${text}
+$$`;
+};
+const heading1 = (text) => {
+    return `# ${text}`;
+};
+const heading2 = (text) => {
+    return `## ${text}`;
+};
+const heading3 = (text) => {
+    return `### ${text}`;
+};
+const quote = (text) => {
+    // the replace is done to handle multiple lines
+    return `> ${text.replace(/\n/g, "  \n> ")}`;
+};
+const callout = (text, icon) => {
+    let emoji;
+    if (icon?.type === "emoji") {
+        emoji = icon.emoji;
+    }
+    // the replace is done to handle multiple lines
+    return `> ${emoji ? emoji + " " : ""}${text.replace(/\n/g, "  \n> ")}`;
+};
+const bullet = (text, count) => {
+    let renderText = text.trim();
+    return count ? `${count}. ${renderText}` : `- ${renderText}`;
+};
+const todo = (text, checked) => {
+    return checked ? `- [x] ${text}` : `- [ ] ${text}`;
+};
+const image = async (alt, href, convertToBase64 = false) => {
+    // In case the user does not want to convert the images to Base64
+    // or the image is already base64
+    if (!convertToBase64 || href.startsWith("data:")) {
+        if (href.startsWith("data:")) {
+            // Extract base64 data, i.e. the string after 'data:mime/type;base64,'
+            const base64 = href.split(",").pop();
+            // This overrides incorrect data: string format to png
+            // so that browsers can correctly render the data
+            return `![${alt}](data:image/png;base64,${base64})`;
+        }
+        return `![${alt}](${href})`;
+    }
+    else {
+        // Otherwise, download the image and convert it to base64
+        const res = await fetch(href);
+        const buf = await res.arrayBuffer();
+        const base64 = Buffer.from(buf).toString("base64");
+        return `![${alt}](data:image/png;base64,${base64})`;
+    }
+};
+const addTabSpace = (text, n = 0) => {
+    const tab = "	";
+    for (let i = 0; i < n; i++) {
+        if (text.includes("\n")) {
+            const multiLineText = text.split(/(?:^|\n)/).join(`\n${tab}`);
+            text = tab + multiLineText;
+        }
+        else
+            text = tab + text;
+    }
+    return text;
+};
+const divider = () => {
+    return "---";
+};
+const toggle = (summary, children) => {
+    if (!summary)
+        return children || "";
+    return `<details>
+<summary>${summary}</summary>
+${children || ""}
+</details>\n\n`;
+};
+const table = (cells) => {
+    return markdownTable(cells);
+};
+
+const NOTION_API_SECRET = process.env.NOTION_API_SECRET ?? "";
+const NOBLOG_DATABASE_ID = process.env.NOBLOG_DATABASE_ID ?? "";
+const client = new Client({
+    auth: NOTION_API_SECRET,
+});
+const numberOfRetry = 2;
+async function GetAllPosts() {
+    const params = {
+        database_id: NOBLOG_DATABASE_ID,
+        filter: {
+            and: [
+                {
+                    property: "publish",
+                    checkbox: { equals: true },
+                },
+            ],
+        },
+        sorts: [
+            {
+                property: "date",
+                direction: "descending",
+            },
+        ],
+        page_size: 100,
+    };
+    let results = [];
+    while (true) {
+        const res = await retry(async (bail) => {
+            try {
+                return (await client.databases.query(params));
+            }
+            catch (error) {
+                if (error instanceof APIResponseError) {
+                    if (error.status && error.status >= 400 && error.status < 500) {
+                        bail(error);
+                    }
+                }
+                throw error;
+            }
+        }, { retries: numberOfRetry });
+        results = results.concat(res.results);
+        if (!res.has_more) {
+            break;
+        }
+        params["start_cursor"] = res.next_cursor;
+    }
+    return results;
+}
+async function GetPageMeta(page_id) {
+    const result = await retry(async (bail) => {
+        try {
+            return (await client.pages.retrieve({ page_id: page_id }));
+        }
+        catch (error) {
+            if (error instanceof APIResponseError) {
+                if (error.status && error.status >= 400 && error.status < 500) {
+                    bail(error);
+                }
+            }
+            throw error;
+        }
+    }, { retries: numberOfRetry });
+    return result;
+}
+async function GetBlockChildren(block_id, totalPage) {
+    let result = [];
+    let pageCount = 0;
+    let start_cursor = undefined;
+    do {
+        const response = (await client.blocks.children.list({
+            start_cursor: start_cursor,
+            block_id: block_id,
+        }));
+        result.push(...response.results);
+        start_cursor = response?.next_cursor;
+        pageCount += 1;
+    } while (start_cursor != null &&
+        (totalPage == null || pageCount < totalPage));
+    ModifyNumberedListObject(result);
+    return result;
+}
+function ModifyNumberedListObject(blocks) {
+    let numberedListIndex = 0;
+    for (const block of blocks) {
+        if ("type" in block && block.type === "numbered_list_item") {
+            // add numbers
+            // @ts-ignore
+            block.numbered_list_item.number = ++numberedListIndex;
+        }
+        else {
+            numberedListIndex = 0;
+        }
+    }
+}
+
+class Noblog {
+    Posts = [];
+    CurrPage = "";
+    SavePath = "./src/pages/posts/";
+    ChildPath = "nob_children/";
+    MdCollection = {};
+    CustomTransformers = {};
+    constructor(posts, save_path, child_path) {
+        this.Posts = posts;
+        if (child_path != undefined) {
+            this.ChildPath = child_path;
+        }
+        if (save_path != undefined) {
+            this.SavePath = save_path;
+        }
+        this.CustomTransformers["child_page"] = async (block) => {
+            const page = block;
+            if (!page?.id) {
+                return false;
+            }
+            let sub_path;
+            // if (curr_page.id in this.Post) and (page.id in this.Post) ||
+            //    (curr_page.id not in this.Post) and (page.id not in this.Post)
+            //    * path should be "../pageid"
+            // if (curr_page.id in this.Post) and (page.id not in this.Post)
+            //    * path should be "../save_path/pageid"
+            // if (curr_page.id not in this.Post) and (page.id in this.Post)
+            //    * path should be "../../pageid"
+            const parent_surface = this.Posts.filter((post) => post.id === this.CurrPage).length > 0;
+            const child_surface = this.Posts.filter((post) => post.id === page.id).length > 0;
+            if (parent_surface === child_surface) {
+                sub_path = "";
+            }
+            else if (parent_surface) {
+                sub_path = this.ChildPath;
+            }
+            else {
+                sub_path = "..";
+            }
+            const full_path = path.join("../", sub_path, page.id);
+            return `＆[${page.child_page.title}](${full_path})`;
+        };
+    }
+    async Collect(recur = true) {
+        const save_dir = this.SavePath;
+        const sub_dir = path.join(save_dir, this.ChildPath);
+        for (let page of this.Posts) {
+            await this.FromPageid(page.id, recur);
+        }
+        try {
+            for (const pageid of Object.keys(this.MdCollection)) {
+                let fpath = sub_dir;
+                if (this.Posts.find((page) => page.id === pageid))
+                    fpath = save_dir;
+                await this.SaveJelly(pageid + ".md", fpath, this.MdCollection[pageid]);
+            }
+            console.log(`success: finish dump all files to "${save_dir}"`);
+        }
+        catch (error) {
+            if (error instanceof Error) {
+                console.log(`failed: consider run verbose with v=true - ${error.message}`);
+            }
+            else {
+                console.log("failed: an unknown error occurred");
+            }
+        }
+    }
+    async FromBlocks(blocks, recursive) {
+        let jelly = { content: "", children: [] };
+        for (const block of blocks) {
+            const subjelly = await this.FromBlocks(block.children, recursive);
+            if (block.type === "toggle") {
+                jelly.content += toggle(block.parent, subjelly.content);
+                jelly.children = [...jelly.children, ...subjelly.children];
+                continue;
+            }
+            let line_wrap = "";
+            if (block.type !== "to_do" &&
+                block.type !== "bulleted_list_item" &&
+                block.type !== "numbered_list_item") {
+                line_wrap = "\n";
+            }
+            jelly.content +=
+                line_wrap + `${addTabSpace(block.parent, 0)}\n` + line_wrap;
+            if (block.type === "child_page") {
+                jelly.children = [...jelly.children, block.blockId];
+                if (recursive)
+                    await this.FromPageid(block.blockId, recursive);
+            }
+            else {
+                jelly.content += subjelly.content;
+                jelly.children = [...jelly.children, ...subjelly.children];
+            }
+        }
+        return jelly;
+    }
+    async FromPageid(page_id, recursive = true) {
+        // preserve the curr page info
+        const prev_page = this.CurrPage;
+        this.CurrPage = page_id;
+        if (this.MdCollection.hasOwnProperty(page_id)) {
+            return this.MdCollection[page_id];
+        }
+        const blocks = await this.PageToMarkdown(page_id);
+        const jelly = await this.FromBlocks(blocks, recursive);
+        const meta = await GetPageMeta(page_id);
+        const astro_meta = await this.AssembleAstroFrontmatter(meta);
+        jelly.content = astro_meta + jelly.content;
+        this.MdCollection[page_id] = jelly;
+        // restore the prev page info
+        this.CurrPage = prev_page;
+        return jelly;
+    }
+    async PageToMarkdown(id, totalPage) {
+        if (!client) {
+            throw new Error("notion client is not provided, for more details check out https://github.com/souvikinator/notion-to-md");
+        }
+        const blocks = await GetBlockChildren(id, totalPage);
+        const parsedData = await this.BlocksToMarkdown(blocks);
+        return parsedData;
+    }
+    async BlocksToMarkdown(blocks, totalPage, mdBlocks) {
+        if (!client) {
+            throw new Error("notion client is not provided, for more details check out https://github.com/souvikinator/notion-to-md");
+        }
+        if (!blocks)
+            return mdBlocks;
+        for (let i = 0; i < blocks.length; i++) {
+            let block = blocks[i];
+            if (
+            // @ts-ignore
+            block.type === "unsupported" ||
+                // @ts-ignore
+                (block.type === "child_page" && !this.config.parseChildPages)) {
+                continue;
+            }
+            if ("has_children" in block && block.has_children) {
+                const block_id = block.type == "synced_block" &&
+                    block.synced_block?.synced_from?.block_id
+                    ? block.synced_block.synced_from.block_id
+                    : block.id;
+                // Get children of this block.
+                let child_blocks = await GetBlockChildren(block_id, totalPage);
+                // Recursively call BlocksToMarkdown to get children of this block.
+                // check for custom transformer before parsing child
+                if (!(block.type in this.CustomTransformers) &&
+                    !this.CustomTransformers[block.type]) {
+                    let l = mdBlocks.length;
+                    if (mdBlocks !== undefined) {
+                        await this.BlocksToMarkdown(child_blocks, totalPage, mdBlocks[l - 1]?.children);
+                    }
+                }
+                // Push this block to mdBlocks.
+                mdBlocks.push({
+                    type: block.type,
+                    blockId: block.id,
+                    parent: await this.BlockToMarkdown(block),
+                    children: [],
+                });
+                continue;
+            }
+            let tmp = await this.BlockToMarkdown(block);
+            mdBlocks.push({
+                // @ts-ignore
+                type: block.type,
+                blockId: block.id,
+                parent: tmp,
+                children: [],
+            });
+        }
+        return mdBlocks;
+    }
+    async BlockToMarkdown(block) {
+        if (typeof block !== "object" || !("type" in block))
+            return "";
+        let parsedData = "";
+        const { type } = block;
+        if (type in this.CustomTransformers && !!this.CustomTransformers[type]) {
+            const transformer = this.CustomTransformers[type];
+            const customTransformerValue = transformer(block);
+            if (typeof customTransformerValue === "string")
+                return customTransformerValue;
+        }
+        switch (type) {
+            case "image":
+                let blockContent = block.image;
+                let image_title = "image";
+                const image_caption_plain = blockContent.caption
+                    .map((item) => item.plain_text)
+                    .join("");
+                const image_type = blockContent.type;
+                let link$1 = "";
+                if (image_type === "external") {
+                    link$1 = blockContent.external.url;
+                }
+                if (image_type === "file") {
+                    link$1 = blockContent.file.url;
+                }
+                // image caption with high priority
+                if (image_caption_plain.trim().length > 0) {
+                    image_title = image_caption_plain;
+                }
+                else if (image_type === "file" || image_type === "external") {
+                    const matches = link$1.match(/[^\/\\&\?]+\.\w{3,4}(?=([\?&].*$|$))/);
+                    image_title = matches ? matches[0] : image_title;
+                }
+                return await image(image_title, link$1, false);
+            case "divider":
+                return divider();
+            case "equation":
+                return equation(block.equation.expression);
+            case "video":
+            case "file":
+            case "pdf":
+                {
+                    let blockContent;
+                    let title = type;
+                    if (type === "video")
+                        blockContent = block.video;
+                    if (type === "file")
+                        blockContent = block.file;
+                    if (type === "pdf")
+                        blockContent = block.pdf;
+                    const caption = blockContent?.caption
+                        .map((item) => item.plain_text)
+                        .join("");
+                    if (blockContent) {
+                        const file_type = blockContent.type;
+                        let link$1 = "";
+                        if (file_type === "external")
+                            link$1 = blockContent.external.url;
+                        if (file_type === "file")
+                            link$1 = blockContent.file.url;
+                        if (caption && caption.trim().length > 0) {
+                            title = caption;
+                        }
+                        else if (link$1) {
+                            const matches = link$1.match(/[^\/\\&\?]+\.\w{3,4}(?=([\?&].*$|$))/);
+                            title = matches ? matches[0] : type;
+                        }
+                        return link(title, link$1);
+                    }
+                }
+                break;
+            case "bookmark":
+            case "embed":
+            case "link_preview":
+            case "link_to_page":
+                {
+                    let blockContent;
+                    let title = type;
+                    if (type === "bookmark")
+                        blockContent = block.bookmark;
+                    if (type === "embed")
+                        blockContent = block.embed;
+                    if (type === "link_preview")
+                        blockContent = block.link_preview;
+                    if (type === "link_to_page" &&
+                        block.link_to_page.type === "page_id") {
+                        blockContent = {
+                            url: `https://www.notion.so/${block.link_to_page.page_id}`,
+                        };
+                    }
+                    if (blockContent)
+                        return link(title, blockContent.url);
+                }
+                break;
+            case "child_page": {
+                let pageTitle = block.child_page.title;
+                return heading2(pageTitle);
+            }
+            case "child_database": {
+                let pageTitle = block.child_database.title || `child_database`;
+                return heading2(pageTitle);
+            }
+            case "table": {
+                const { id, has_children } = block;
+                let tableArr = [];
+                if (has_children) {
+                    const tableRows = await GetBlockChildren(id, 100);
+                    let rowsPromise = tableRows?.map(async (row) => {
+                        const { type } = row;
+                        const cells = row[type]["cells"];
+                        /**
+                         * this is more like a hack since matching the type text was
+                         * difficult. So converting each cell to paragraph type to
+                         * reuse the BlockToMarkdown function
+                         */
+                        let cellStringPromise = cells.map(async (cell) => await this.BlockToMarkdown({
+                            type: "paragraph",
+                            paragraph: { rich_text: cell },
+                        }));
+                        const cellStringArr = await Promise.all(cellStringPromise);
+                        tableArr.push(cellStringArr);
+                    });
+                    await Promise.all(rowsPromise || []);
+                }
+                return table(tableArr);
+            }
+            // Rest of the types:
+            //   "paragraph" "heading_1" "heading_2" "heading_3"
+            //   "bulleted_list_item" "numbered_list_item"
+            //   "quote" "to_do" "template" "synced_block"
+            //   "child_page" "child_database" "code" "callout" "breadcrumb"
+            //   "table_of_contents" "link_to_page" "audio" "unsupported"
+            default: {
+                // @ts-ignore
+                let blockContent = block[type].text || block[type].rich_text || [];
+                let bucket = [];
+                let code_range = [];
+                let bold_range = [];
+                let italic_range = [];
+                let underline_range = [];
+                let strikethrough_range = [];
+                const push_or_merge = (arr, idx) => {
+                    if (arr.length == 0) {
+                        return;
+                    }
+                    // @ts-ignore
+                    if (idx === arr[arr.length - 1][1]) {
+                        // @ts-ignore
+                        arr[arr.length - 1][1] += 1;
+                    }
+                    else {
+                        arr.push([idx, idx + 1]);
+                    }
+                };
+                for (let i = 0; i < blockContent.length; i++) {
+                    let content = blockContent[i];
+                    if (content.type === "equation") {
+                        bucket.push(inlineEquation(content.equation.expression));
+                        continue;
+                    }
+                    let plain_text = content.plain_text;
+                    if (content["href"])
+                        plain_text = link(plain_text, content["href"]);
+                    bucket.push(plain_text);
+                    const annotations = content.annotations;
+                    if (annotations.code)
+                        push_or_merge(code_range, i);
+                    if (annotations.bold)
+                        push_or_merge(bold_range, i);
+                    if (annotations.italic)
+                        push_or_merge(italic_range, i);
+                    if (annotations.underline)
+                        push_or_merge(underline_range, i);
+                    if (annotations.strikethrough)
+                        push_or_merge(strikethrough_range, i);
+                }
+                let annoed_bucket = [];
+                for (let range of code_range)
+                    annoed_bucket.push({ range: range, type: "code" });
+                for (let range of bold_range)
+                    annoed_bucket.push({ range: range, type: "bold" });
+                for (let range of italic_range)
+                    annoed_bucket.push({ range: range, type: "italic" });
+                for (let range of underline_range)
+                    annoed_bucket.push({ range: range, type: "underline" });
+                for (let range of strikethrough_range)
+                    annoed_bucket.push({ range: range, type: "strikethrough" });
+                annoed_bucket.sort(
+                // @ts-ignore
+                (a, b) => a.range[1] - a.range[0] - (b.range[1] - b.range[0]));
+                for (let annoed of annoed_bucket) {
+                    annoed.type;
+                    let [from, to] = annoed.range;
+                    // @ts-ignore
+                    bucket[from] = annotation + bucket[from];
+                    // @ts-ignore
+                    bucket[to - 1] = bucket[to - 1] + annotation;
+                }
+                bucket.forEach((x) => (parsedData += x));
+            }
+        }
+        switch (type) {
+            case "code":
+                parsedData = codeBlock(parsedData, block[type].language);
+                break;
+            case "heading_1":
+                parsedData = heading1(parsedData);
+                break;
+            case "heading_2":
+                parsedData = heading2(parsedData);
+                break;
+            case "heading_3":
+                parsedData = heading3(parsedData);
+                break;
+            case "quote":
+                parsedData = quote(parsedData);
+                break;
+            case "callout":
+                const { id, has_children } = block;
+                let callout_string = "";
+                if (!has_children) {
+                    return callout(parsedData, block[type].icon);
+                }
+                const callout_children_object = await GetBlockChildren(id, 100);
+                const callout_children = await this.BlocksToMarkdown(callout_children_object);
+                callout_string += `${parsedData}\n`;
+                callout_children.map((child) => {
+                    callout_string += `${child.parent}\n\n`;
+                });
+                parsedData = callout(callout_string.trim(), block[type].icon);
+                break;
+            case "bulleted_list_item":
+                parsedData = bullet(parsedData);
+                break;
+            case "numbered_list_item":
+                parsedData = bullet(parsedData, block.numbered_list_item.number);
+                break;
+            case "to_do":
+                parsedData = todo(parsedData, block.to_do.checked);
+                break;
+        }
+        return parsedData;
+    }
+    async SaveJelly(fname, dir, jelly) {
+        if (jelly === undefined)
+            return `failed: ${fname} has empty jelly: `;
+        try {
+            await fs.mkdir(dir, { recursive: true });
+            const filePath = path.join(dir, fname);
+            await fs.writeFile(filePath, jelly.content, "utf-8");
+            return `success: ${fname} saved to ${filePath}`;
+        }
+        catch (error) {
+            if (error instanceof Error) {
+                return `failed: ${fname} - ${error.message}`;
+            }
+            else {
+                return "failed: an unknown error occurred";
+            }
+        }
+    }
+    async AssembleAstroFrontmatter(page) {
+        let frontmatter = "";
+        // default layout
+        const ischild = this.Posts.filter((post) => post.id === page.id).length > 0 ? "" : "../";
+        frontmatter +=
+            "layout: " + ischild + "../../layouts/MarkdownPostLayout.astro" + "\n";
+        // get title
+        frontmatter +=
+            "title: " +
+                JSON.stringify(page.properties.title?.title?.[0]?.plain_text ?? "") +
+                "\n";
+        // get tags
+        frontmatter +=
+            "tags: " +
+                JSON.stringify((page.properties.tags?.multi_select ?? []).map((tag) => tag.name)) +
+                "\n";
+        // get date
+        frontmatter +=
+            "pubDate: " +
+                (page.properties.date?.date?.start ??
+                    new Date().toISOString().split("T")[0]) +
+                "\n";
+        // get archived
+        frontmatter +=
+            "archived: " + (page.properties.archived ? "true" : "false") + "\n";
+        // get description
+        frontmatter +=
+            "description: " +
+                JSON.stringify(page.properties.description?.rich_text?.[0]?.plain_text ?? "") +
+                "\n";
+        return "---\n" + frontmatter + "---\n";
+    }
+}
+
+function help() {
+    console.log(`\n
+Usage: noblog [SAVE_PATH] [SUB_PATH]
+
+Example:
+  noblog
+  noblog src/pages/posts/ nob_children/
+
+Available <option>:
+  -v  Print more messages for debugging.\n`);
+}
+async function main() {
+    const argv = process.argv.slice(2);
+    if (argv.length > 2) {
+        console.error("Error: invallid arg.");
+        help();
+    }
+    const save_dir = argv[0] ?? "./src/pages/posts/";
+    const sub_dir = argv[1] ?? "nob_children/";
+    const pages = await GetAllPosts();
+    const nob = new Noblog(pages, save_dir, sub_dir);
+    nob.Collect(true);
+}
+main();
+//# sourceMappingURL=generate.js.map