@sillsdev/docu-notion 0.12.0

package/dist/transform.js

@@ -0,0 +1,195 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getMarkdownFromNotionBlocks = exports.getMarkdownForPage = void 0;
+const chalk_1 = __importDefault(require("chalk"));
+const log_1 = require("./log");
+function getMarkdownForPage(config, context, page) {
+    return __awaiter(this, void 0, void 0, function* () {
+        (0, log_1.info)(`Reading & converting page ${page.layoutContext}/${page.nameOrTitle} (${chalk_1.default.blue(page.hasExplicitSlug
+            ? page.slug
+            : page.foundDirectlyInOutline
+                ? "Descendant of Outline, not Database"
+                : "NO SLUG")})`);
+        const blocks = yield context.getBlockChildren(page.pageId);
+        (0, log_1.logDebugFn)("markdown from page", () => JSON.stringify(blocks, null, 2));
+        const body = yield getMarkdownFromNotionBlocks(context, config, blocks);
+        const frontmatter = getFrontMatter(page); // todo should be a plugin
+        return `${frontmatter}\n${body}`;
+    });
+}
+exports.getMarkdownForPage = getMarkdownForPage;
+// this is split off from getMarkdownForPage so that unit tests can provide the block contents
+function getMarkdownFromNotionBlocks(context, config, blocks) {
+    return __awaiter(this, void 0, void 0, function* () {
+        // changes to the blocks we get from notion API
+        doNotionBlockTransforms(blocks, config);
+        // overrides for the default notion-to-markdown conversions
+        registerNotionToMarkdownCustomTransforms(config, context);
+        // the main conversion to markdown, using the notion-to-md library
+        let markdown = yield doNotionToMarkdown(context, blocks); // ?
+        // corrections to links after they are converted to markdown,
+        // with access to all the pages we've seen
+        markdown = doLinkFixes(context, markdown, config);
+        //console.log("markdown after link fixes", markdown);
+        // simple regex-based tweaks. These are usually related to docusaurus
+        const { imports, body } = yield doTransformsOnMarkdown(config, markdown);
+        // console.log("markdown after regex fixes", markdown);
+        // console.log("body after regex", body);
+        return `${imports}\n${body}`;
+    });
+}
+exports.getMarkdownFromNotionBlocks = getMarkdownFromNotionBlocks;
+// operations on notion blocks before they are converted to markdown
+function doNotionBlockTransforms(blocks, config) {
+    for (const block of blocks) {
+        config.plugins.forEach(plugin => {
+            if (plugin.notionBlockModifications) {
+                plugin.notionBlockModifications.forEach(transform => {
+                    (0, log_1.logDebug)("transforming block with plugin", plugin.name);
+                    transform.modify(block);
+                });
+            }
+        });
+    }
+}
+function doTransformsOnMarkdown(config, input) {
+    var _a;
+    return __awaiter(this, void 0, void 0, function* () {
+        const regexMods = config.plugins
+            .filter(plugin => !!plugin.regexMarkdownModifications)
+            .map(plugin => {
+            const mods = plugin.regexMarkdownModifications;
+            // stick the name of the plugin into each mode for logging
+            const modsWithNames = mods.map(m => (Object.assign({ name: plugin.name }, m)));
+            return modsWithNames;
+        })
+            .flat();
+        // regex that matches markdown code blocks
+        const codeBlocks = /```.*\n[\s\S]*?\n```/;
+        let body = input;
+        //console.log("body before regex: " + body);
+        let match;
+        const imports = new Set();
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        for (const mod of regexMods) {
+            let replacement = undefined;
+            // regex.exec is stateful, so we don't want to mess up the plugin's use of its own regex, so we clone it.
+            // we also add the "g" flag to make sure we get all matches
+            const regex = new RegExp(`${codeBlocks.source}|(${mod.regex.source})`, "g");
+            let count = 0;
+            while ((match = regex.exec(input)) !== null) {
+                if (match[0]) {
+                    const original = match[0];
+                    if (original.startsWith("```") && original.endsWith("```")) {
+                        continue; // code block
+                    }
+                    if (mod.getReplacement) {
+                        replacement = yield mod.getReplacement(original);
+                    }
+                    else if (mod.replacementPattern) {
+                        console.log(`mod.replacementPattern.replace("$1", ${match[2]}`);
+                        replacement = mod.replacementPattern.replace("$1", match[2]);
+                    }
+                    if (replacement !== undefined) {
+                        (0, log_1.verbose)(`[${mod.name}] ${original} --> ${replacement}`);
+                        const precedingPart = body.substring(0, match.index); // ?
+                        const partStartingFromThisMatch = body.substring(match.index); // ?
+                        body =
+                            precedingPart +
+                                partStartingFromThisMatch.replace(original, replacement);
+                        // add any library imports
+                        (_a = mod.imports) === null || _a === void 0 ? void 0 : _a.forEach(imp => imports.add(imp));
+                    }
+                }
+            }
+        }
+        (0, log_1.logDebug)("doTransformsOnMarkdown", "body after regex: " + body);
+        const uniqueImports = [...new Set(imports)];
+        return { body, imports: [...uniqueImports].join("\n") };
+    });
+}
+function doNotionToMarkdown(docunotionContext, blocks) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const mdBlocks = yield docunotionContext.notionToMarkdown.blocksToMarkdown(blocks);
+        let markdown = docunotionContext.notionToMarkdown.toMarkdownString(mdBlocks);
+        return markdown;
+    });
+}
+// corrections to links after they are converted to markdown
+// Note: from notion (or notion-md?) we get slightly different hrefs depending on whether the links is "inline"
+// (has some other text that's been turned into a link) or "raw".
+// Raw links come in without a leading slash, e.g. [link_to_page](4a6de8c0-b90b-444b-8a7b-d534d6ec71a4)
+// Inline links come in with a leading slash, e.g. [pointer to the introduction](/4a6de8c0b90b444b8a7bd534d6ec71a4)
+function doLinkFixes(context, markdown, config) {
+    const linkRegExp = /\[.*\]\([^\)]*\)/g;
+    (0, log_1.logDebug)("markdown before link fixes", markdown);
+    let match;
+    // since we're going to make changes to the markdown,
+    // we need to keep track of where we are in the string as we search
+    const markdownToSearch = markdown;
+    // The key to understanding this `while` is that linkRegExp actually has state, and
+    // it gives you a new one each time. https://stackoverflow.com/a/1520853/723299
+    while ((match = linkRegExp.exec(markdownToSearch)) !== null) {
+        const originalLinkMarkdown = match[0];
+        (0, log_1.verbose)(`Checking to see if a plugin wants to modify "${originalLinkMarkdown}" `);
+        // We only use the first plugin that matches and makes a change to the link.
+        // Enhance: we could take the time to see if multiple plugins match, and
+        // and point this out in verbose logging mode.
+        config.plugins.some(plugin => {
+            if (!plugin.linkModifier)
+                return false;
+            if (plugin.linkModifier.match.exec(originalLinkMarkdown) === null) {
+                return false;
+            }
+            const newMarkdown = plugin.linkModifier.convert(context, originalLinkMarkdown);
+            if (newMarkdown !== originalLinkMarkdown) {
+                markdown = markdown.replace(originalLinkMarkdown, newMarkdown);
+                (0, log_1.verbose)(`plugin "${plugin.name}" transformed link: ${originalLinkMarkdown}-->${newMarkdown}`);
+                return true; // the first plugin that matches and does something wins
+            }
+            else {
+                (0, log_1.verbose)(`plugin "${plugin.name}" did not change this url`);
+                return false;
+            }
+        });
+    }
+    return markdown;
+}
+// overrides for the conversions that notion-to-md does
+function registerNotionToMarkdownCustomTransforms(config, docunotionContext) {
+    config.plugins.forEach(plugin => {
+        if (plugin.notionToMarkdownTransforms) {
+            plugin.notionToMarkdownTransforms.forEach(transform => {
+                (0, log_1.logDebug)("registering custom transform", `${plugin.name} for ${transform.type}`);
+                docunotionContext.notionToMarkdown.setCustomTransformer(transform.type, (block) => {
+                    (0, log_1.logDebug)("notion to MD conversion of ", `${transform.type} with plugin: ${plugin.name}`);
+                    return transform.getStringFromBlock(docunotionContext, block);
+                });
+            });
+        }
+    });
+}
+// enhance:make this built-in plugin so that it can be overridden
+function getFrontMatter(page) {
+    var _a;
+    let frontmatter = "---\n";
+    frontmatter += `title: ${page.nameOrTitle.replaceAll(":", "-")}\n`; // I have not found a way to escape colons
+    frontmatter += `sidebar_position: ${page.order}\n`;
+    frontmatter += `slug: ${(_a = page.slug) !== null && _a !== void 0 ? _a : ""}\n`;
+    if (page.keywords)
+        frontmatter += `keywords: [${page.keywords}]\n`;
+    frontmatter += "---\n";
+    return frontmatter;
+}

package/dist/types.d.ts

@@ -0,0 +1,8 @@
+import { BlockObjectResponse } from "@notionhq/client/build/src/api-endpoints";
+export type NotionBlock = BlockObjectResponse;
+export type ICounts = {
+    output_normally: number;
+    skipped_because_empty: number;
+    skipped_because_status: number;
+    skipped_because_level_cannot_have_content: number;
+};

package/dist/types.js

@@ -0,0 +1,2 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });

package/package.json

@@ -0,0 +1,96 @@
+{
+  "scripts": {
+    "test": "jest",
+    "build": "yarn test && tsc && cp ./src/css/*.css dist/",
+    "build-only": "tsc && cp ./src/css/*.css dist/",
+    "clean": "rimraf ./dist/",
+    "semantic-release": "semantic-release",
+    "typecheck": "tsc --noEmit",
+    "notion-download": "node dist/index.js",
+    "cmdhelp": "ts-node src/index.ts",
+    "// note that we're not using ts-node at the moment because of ": "https://github.com/Codex-/cosmiconfig-typescript-loader/issues/70",
+    "ts": "yarn tsc && rimraf ./docs/ && cross-var node dist/index.js",
+    "// test out with a private sample notion db": "",
+    "pull-test-tagged": "yarn ts -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID  --log-level debug --status-tag test",
+    "pull-sample-site": "yarn ts -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE  --log-level debug",
+    "// test with a semi-stable/public site:": "",
+    "pull-sample": "yarn ts -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE -m ./sample --locales en,es,fr,de --log-level verbose",
+    "pull-sample-with-paths": "yarn ts -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE -m ./sample --img-output-path ./sample_img"
+  },
+  "//file-type": "have to use this version before they switched to ESM, which gives a compile error related to require()",
+  "//node-fetch@2.6.6file-type": "have to use this version before they switched to ESM, which gives a compile error related to require()",
+  "//chalk@4": "also ESM related problem",
+  "//notion-client@4": "also ESM related problem",
+  "dependencies": {
+    "@notionhq/client": "2.2.3",
+    "chalk": "^4.1.2",
+    "commander": "^9.2.0",
+    "cosmiconfig": "^8.0.0",
+    "cosmiconfig-typescript-loader": "^4.3.0",
+    "file-type": "16.5.1",
+    "fs-extra": "^10.1.0",
+    "limiter": "^2.1.0",
+    "markdown-table": "^2.0.0",
+    "node-fetch": "2.6.6",
+    "notion-client": "^4",
+    "notion-to-md": "^2.5.5",
+    "path": "^0.12.7",
+    "sanitize-filename": "^1.6.3"
+  },
+  "devDependencies": {
+    "@types/fs-extra": "^9.0.13",
+    "@types/jest": "^28.1.6",
+    "@types/markdown-table": "^2.0.0",
+    "@types/node": "^12.20.11",
+    "@typescript-eslint/eslint-plugin": "^4.22.0",
+    "@typescript-eslint/parser": "^4.22.0",
+    "cross-var": "^1.1.0",
+    "cz-conventional-changelog": "^3.3.0",
+    "eslint": "^7.25.0",
+    "eslint-config-prettier": "^8.3.0",
+    "eslint-plugin-node": "^11.1.0",
+    "eslint-plugin-prettier": "^3.4.0",
+    "jest": "^28.1.3",
+    "lint-staged": "^10.5.4",
+    "prettier": "^2.2.1",
+    "rimraf": "^4.1.2",
+    "semantic-release": "^19.0.2",
+    "ts-jest": "^28.0.7",
+    "ts-node": "^10.2.1",
+    "typescript": "^4.6.4"
+  },
+  "name": "@sillsdev/docu-notion",
+  "description": "Download Notion pages as markdown and image files, preserving hierarchy and enabling workflow properties. Works with Docusaurus.",
+  "license": "MIT",
+  "author": {
+    "name": "SIL Language Software Development",
+    "email": "sillsdev@users.noreply.github.com",
+    "url": "https://github.com/sillsdev"
+  },
+  "keywords": [
+    "docusaurus",
+    "notion",
+    "markdown",
+    "documentation"
+  ],
+  "bugs": {
+    "url": "https://github.com/sillsdev/docu-notion/issues"
+  },
+  "homepage": "https://github.com/sillsdev/docu-notion#readme",
+  "packageManager": "yarn@3.4.1",
+  "main": "./dist/index.js",
+  "bin": "dist/index.js",
+  "files": [
+    "dist/**/*"
+  ],
+  "release": {
+    "branches": [
+      "main",
+      "release"
+    ]
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "version": "0.12.0"
+}