@sillsdev/docu-notion 0.13.2-alpha.2 → 0.13.2-alpha.4

package/README.md

@@ -31,7 +31,7 @@ Currently, docu-notion expects that each page has only one of the following: sub
 
 ## 6. Pull your pages
 
-First, determine the id of your root page by clicking "Share" and looking at the the url it gives you. E.g.
+First, determine the id of your root page by clicking "Share" and looking at the url it gives you. E.g.
 https://www.notion.so/hattonjohn/My-Docs-0456aa5842946bdbea3a4f37c97a0e5
 means that the id is "0456aa5842946PRETEND4f37c97a0e5".
 
@@ -133,3 +133,16 @@ Options:
 # Plugins
 
 If your project needs some processing that docu-notion doesn't already provide, you can provide a plugin that does it. See the [plugin readme](src/plugins/README.md).
+
+# Callouts ➜ Admonitions
+
+To map Notion callouts to Docusaurus admonitions, ensure the icon is for the type you want.
+
+- ℹ️ ➜ note
+- 📝➜ note
+- 💡➜ tip
+- ❗➜ info
+- ⚠️➜ caution
+- 🔥➜ danger
+
+The default admonition type, if no matching icon is found, is "note".

package/dist/config/default.docunotion.config.js

@@ -10,6 +10,7 @@ const EscapeHtmlBlockModifier_1 = require("../plugins/EscapeHtmlBlockModifier");
 const HeadingTransformer_1 = require("../plugins/HeadingTransformer");
 const NumberedListTransformer_1 = require("../plugins/NumberedListTransformer");
 const TableTransformer_1 = require("../plugins/TableTransformer");
+const VideoTransformer_1 = require("../plugins/VideoTransformer");
 const externalLinks_1 = require("../plugins/externalLinks");
 const defaultConfig = {
     plugins: [
@@ -24,14 +25,13 @@ const defaultConfig = {
         CalloutTransformer_1.standardCalloutTransformer,
         TableTransformer_1.standardTableTransformer,
         NumberedListTransformer_1.standardNumberedListTransformer,
+        VideoTransformer_1.standardVideoTransformer,
         // Link modifiers, which are special because they can read metadata from all the pages in order to figure out the correct url
         internalLinks_1.standardInternalLinkConversion,
         externalLinks_1.standardExternalLinkConversion,
         // Regexps plus javascript `import`s that operate on the Markdown output
         embedTweaks_1.imgurGifEmbed,
         embedTweaks_1.gifEmbed,
-        embedTweaks_1.youtubeEmbed,
-        embedTweaks_1.vimeoEmbed,
     ],
 };
 exports.default = defaultConfig;

package/dist/plugins/ColumnTransformer.js

@@ -11,6 +11,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.standardColumnTransformer = void 0;
 const notion_client_1 = require("notion-client");
+const pull_1 = require("../pull");
 exports.standardColumnTransformer = {
     name: "standardColumnTransformer",
     notionToMarkdownTransforms: [
@@ -49,8 +50,10 @@ function getColumnWidth(block) {
     return __awaiter(this, void 0, void 0, function* () {
         const unofficialNotionClient = new notion_client_1.NotionAPI();
         const blockId = block.id;
-        // Yes, it is odd to call 'getPage' for a block, but that's how we access the format info.
-        const recordMap = yield unofficialNotionClient.getPage(blockId);
+        const recordMap = yield (0, pull_1.executeWithRateLimitAndRetries)(`unofficialNotionClient.getPage(${blockId}) in getColumnWidth()`, () => {
+            // Yes, it is odd to call 'getPage' for a block, but that's how we access the format info.
+            return unofficialNotionClient.getPage(blockId);
+        });
         const blockResult = recordMap.block[blockId];
         // ENHANCE: could we use https://github.com/NotionX/react-notion-x/tree/master/packages/notion-types
         // to get away from "any", which might be particularly helpful in the future

package/dist/plugins/VideoTransformer.d.ts

@@ -0,0 +1,2 @@
+import { IPlugin } from "./pluginTypes";
+export declare const standardVideoTransformer: IPlugin;

package/dist/plugins/VideoTransformer.js

@@ -0,0 +1,25 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.standardVideoTransformer = void 0;
+const log_1 = require("../log");
+exports.standardVideoTransformer = {
+    name: "video",
+    notionToMarkdownTransforms: [
+        {
+            type: "video",
+            getStringFromBlock: (context, block) => {
+                const video = block.video;
+                if (video.type === "external") {
+                    if (!context.imports)
+                        context.imports = [];
+                    context.imports.push(`import ReactPlayer from "react-player";`);
+                    return `<ReactPlayer controls url="${video.external.url}" />`;
+                }
+                else {
+                    (0, log_1.warning)(`[standardVideoTransformer] Found Notion "video" block with type ${video.type}. The best docu-notion can do for now is ignore it.`);
+                }
+                return "";
+            },
+        },
+    ],
+};

package/dist/plugins/VideoTransformer.spec.d.ts

@@ -0,0 +1 @@
+export {};

package/dist/plugins/VideoTransformer.spec.js

@@ -0,0 +1,88 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const log_1 = require("../log");
+const VideoTransformer_1 = require("./VideoTransformer");
+const pluginTestRun_1 = require("./pluginTestRun");
+test("youtube embedded", () => __awaiter(void 0, void 0, void 0, function* () {
+    const config = { plugins: [VideoTransformer_1.standardVideoTransformer] };
+    const result = yield (0, pluginTestRun_1.blocksToMarkdown)(config, [
+        {
+            object: "block",
+            type: "video",
+            video: {
+                caption: [
+                    {
+                        type: "text",
+                        text: {
+                            content: "A video about editing in Notion",
+                            link: null,
+                        },
+                        plain_text: "A video about editing in Notion",
+                        href: null,
+                    },
+                ],
+                type: "external",
+                external: { url: "https://www.youtube.com/watch?v=FXIrojSK3Jo" },
+            },
+        },
+    ]);
+    expect(result).toContain(`import ReactPlayer from "react-player";`);
+    expect(result).toContain(`<ReactPlayer controls url="https://www.youtube.com/watch?v=FXIrojSK3Jo" />`);
+}));
+test("vimeo embedded", () => __awaiter(void 0, void 0, void 0, function* () {
+    (0, log_1.setLogLevel)("verbose");
+    const config = { plugins: [VideoTransformer_1.standardVideoTransformer] };
+    const result = yield (0, pluginTestRun_1.blocksToMarkdown)(config, [
+        {
+            object: "block",
+            type: "video",
+            video: {
+                caption: [],
+                type: "external",
+                external: { url: "https://vimeo.com/4613611xx" },
+            },
+        },
+    ]);
+    expect(result).toContain(`import ReactPlayer from "react-player";`);
+    expect(result).toContain(`<ReactPlayer controls url="https://vimeo.com/4613611xx" />`);
+}));
+test("video link, not embedded", () => __awaiter(void 0, void 0, void 0, function* () {
+    (0, log_1.setLogLevel)("verbose");
+    const config = { plugins: [VideoTransformer_1.standardVideoTransformer] };
+    const result = yield (0, pluginTestRun_1.blocksToMarkdown)(config, [
+        {
+            object: "block",
+            type: "paragraph",
+            paragraph: {
+                rich_text: [
+                    {
+                        type: "text",
+                        text: {
+                            content: "https://vimeo.com/4613611xx",
+                            link: {
+                                url: "https://vimeo.com/4613611xx",
+                            },
+                        },
+                        annotations: {
+                            code: false,
+                        },
+                        plain_text: "https://vimeo.com/4613611xx",
+                        href: "https://vimeo.com/4613611xx",
+                    },
+                ],
+                color: "default",
+            },
+        },
+    ]);
+    expect(result).toContain("[https://vimeo.com/4613611xx](https://vimeo.com/4613611xx)");
+    expect(result).not.toContain(`import`);
+}));

package/dist/plugins/embedTweaks.d.ts

@@ -1,5 +1,3 @@
 import { IPlugin } from "./pluginTypes";
 export declare const gifEmbed: IPlugin;
 export declare const imgurGifEmbed: IPlugin;
-export declare const youtubeEmbed: IPlugin;
-export declare const vimeoEmbed: IPlugin;

package/dist/plugins/embedTweaks.js

@@ -1,6 +1,6 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.vimeoEmbed = exports.youtubeEmbed = exports.imgurGifEmbed = exports.gifEmbed = void 0;
+exports.imgurGifEmbed = exports.gifEmbed = void 0;
 exports.gifEmbed = {
     name: "gif",
     regexMarkdownModifications: [
@@ -22,25 +22,3 @@ exports.imgurGifEmbed = {
         },
     ],
 };
-exports.youtubeEmbed = {
-    name: "youtube",
-    regexMarkdownModifications: [
-        {
-            regex: /\[.*\]\((.*youtube\.com\/watch.*)\)/,
-            imports: [`import ReactPlayer from "react-player";`],
-            replacementPattern: `<ReactPlayer controls url="$1" />`,
-        },
-    ],
-};
-exports.vimeoEmbed = {
-    name: "vimeo",
-    regexMarkdownModifications: [
-        {
-            regex: /\[.*\]\((https:\/\/.*vimeo.*)\)/,
-            // we use to have the following, but the above should handle both the player an not-player urls.
-            //regex: /\[.*\]\((.*player\.vimeo.*)\)/gm, // player.vimeo
-            imports: [`import ReactPlayer from "react-player";`],
-            replacementPattern: `<ReactPlayer controls url="$1" />`,
-        },
-    ],
-};

package/dist/plugins/embedTweaks.spec.js

@@ -12,53 +12,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const log_1 = require("../log");
 const pluginTestRun_1 = require("./pluginTestRun");
 const embedTweaks_1 = require("./embedTweaks");
-test("youtube", () => __awaiter(void 0, void 0, void 0, function* () {
-    const config = { plugins: [embedTweaks_1.youtubeEmbed] };
-    const result = yield (0, pluginTestRun_1.blocksToMarkdown)(config, [
-        {
-            object: "block",
-            id: "e6ddd1d4-36d4-4925-94c1-5dff4662c1f3",
-            has_children: false,
-            archived: false,
-            type: "video",
-            video: {
-                caption: [
-                    {
-                        type: "text",
-                        text: {
-                            content: "A video about editing in Notion",
-                            link: null,
-                        },
-                        plain_text: "A video about editing in Notion",
-                        href: null,
-                    },
-                ],
-                type: "external",
-                external: { url: "https://www.youtube.com/watch?v=FXIrojSK3Jo" },
-            },
-        },
-    ]);
-    expect(result).toContain(`import ReactPlayer from "react-player";`);
-    expect(result).toContain(`<ReactPlayer controls url="https://www.youtube.com/watch?v=FXIrojSK3Jo" />`);
-}));
-test("vimeo", () => __awaiter(void 0, void 0, void 0, function* () {
-    (0, log_1.setLogLevel)("verbose");
-    const config = { plugins: [embedTweaks_1.vimeoEmbed] };
-    const result = yield (0, pluginTestRun_1.blocksToMarkdown)(config, [
-        {
-            object: "block",
-            id: "39ff83a3-2fb5-4411-a715-960656a177ff",
-            type: "video",
-            video: {
-                caption: [],
-                type: "external",
-                external: { url: "https://vimeo.com/4613611xx" },
-            },
-        },
-    ]);
-    expect(result).toContain(`import ReactPlayer from "react-player";`);
-    expect(result).toContain(`<ReactPlayer controls url="https://vimeo.com/4613611xx" />`);
-}));
 test("imgur", () => __awaiter(void 0, void 0, void 0, function* () {
     (0, log_1.setLogLevel)("verbose");
     const config = { plugins: [embedTweaks_1.imgurGifEmbed] };

package/dist/plugins/externalLinks.spec.js

@@ -23,6 +23,19 @@ test("links turned into bookmarks", () => __awaiter(void 0, void 0, void 0, func
     });
     expect(results.trim()).toBe("[https://github.com](https://github.com)");
 }));
+test("video links turned into bookmarks", () => __awaiter(void 0, void 0, void 0, function* () {
+    (0, log_1.setLogLevel)("debug");
+    const results = yield getMarkdown({
+        object: "block",
+        type: "bookmark",
+        bookmark: {
+            caption: [],
+            url: "https://vimeo.com/4613611xx",
+        },
+    });
+    expect(results).toContain("[https://vimeo.com/4613611xx](https://vimeo.com/4613611xx)");
+    expect(results).not.toContain(`import`);
+}));
 test("external link inside callout", () => __awaiter(void 0, void 0, void 0, function* () {
     const results = yield getMarkdown({
         type: "callout",

package/dist/plugins/pluginTypes.d.ts

@@ -40,5 +40,6 @@ export type IDocuNotionContext = {
     convertNotionLinkToLocalDocusaurusLink: (url: string) => string | undefined;
     pages: NotionPage[];
     counts: ICounts;
+    imports?: string[];
 };
 export {};

package/dist/pull.d.ts

@@ -9,4 +9,5 @@ export type DocuNotionOptions = {
     statusTag: string;
 };
 export declare function notionPull(options: DocuNotionOptions): Promise<void>;
+export declare function executeWithRateLimitAndRetries<T>(label: string, asyncFunction: () => Promise<T>): Promise<T>;
 export declare function initNotionClient(notionToken: string): Client;

package/dist/pull.js

@@ -32,7 +32,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.initNotionClient = exports.notionPull = void 0;
+exports.initNotionClient = exports.executeWithRateLimitAndRetries = exports.notionPull = void 0;
 const fs = __importStar(require("fs-extra"));
 const notion_to_md_1 = require("notion-to-md");
 const HierarchicalNamedLayoutStrategy_1 = require("./HierarchicalNamedLayoutStrategy");
@@ -188,12 +188,45 @@ const notionLimiter = new limiter_1.RateLimiter({
 let notionClient;
 function getPageMetadata(id) {
     return __awaiter(this, void 0, void 0, function* () {
-        yield rateLimit();
-        return yield notionClient.pages.retrieve({
-            page_id: id,
+        return yield executeWithRateLimitAndRetries(`pages.retrieve(${id})`, () => {
+            return notionClient.pages.retrieve({
+                page_id: id,
+            });
         });
     });
 }
+// While everything works fine locally, on Github Actions we are getting a lot of timeouts, so
+// we're trying this extra retry-able wrapper.
+function executeWithRateLimitAndRetries(label, asyncFunction) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield rateLimit();
+        const kRetries = 10;
+        let lastException = undefined;
+        for (let i = 0; i < kRetries; i++) {
+            try {
+                return yield asyncFunction();
+            }
+            catch (e) {
+                lastException = e;
+                if ((e === null || e === void 0 ? void 0 : e.code) === "notionhq_client_request_timeout" ||
+                    e.message.includes("timeout") ||
+                    e.message.includes("Timeout") ||
+                    e.message.includes("limit") ||
+                    e.message.includes("Limit")) {
+                    const secondsToWait = i + 1;
+                    (0, log_1.info)(`While doing "${label}", got error "${e.message}". Will retry after  ${secondsToWait}s...`);
+                    yield new Promise(resolve => setTimeout(resolve, 1000 * secondsToWait));
+                }
+                else {
+                    throw e;
+                }
+            }
+        }
+        (0, log_1.error)(`Error: could not complete "${label}" after ${kRetries} retries.`);
+        throw lastException;
+    });
+}
+exports.executeWithRateLimitAndRetries = executeWithRateLimitAndRetries;
 function rateLimit() {
     return __awaiter(this, void 0, void 0, function* () {
         if (notionLimiter.getTokensRemaining() < 1) {
@@ -213,10 +246,11 @@ function getBlockChildren(id) {
         // Note: there is a now a collectPaginatedAPI() in the notion client, so
         // we could switch to using that (I don't know if it does rate limiting?)
         do {
-            yield rateLimit();
-            const response = yield notionClient.blocks.children.list({
-                start_cursor: start_cursor,
-                block_id: id,
+            const response = yield executeWithRateLimitAndRetries(`getBlockChildren(${id})`, () => {
+                return notionClient.blocks.children.list({
+                    start_cursor: start_cursor,
+                    block_id: id,
+                });
             });
             if (!overallResult) {
                 overallResult = response;

package/dist/transform.js

@@ -44,9 +44,12 @@ function getMarkdownFromNotionBlocks(context, config, blocks) {
         markdown = doLinkFixes(context, markdown, config);
         //console.log("markdown after link fixes", markdown);
         // simple regex-based tweaks. These are usually related to docusaurus
-        const { imports, body } = yield doTransformsOnMarkdown(context, config, markdown);
+        const body = yield doTransformsOnMarkdown(context, config, markdown);
         // console.log("markdown after regex fixes", markdown);
         // console.log("body after regex", body);
+        const uniqueImports = [...new Set(context.imports)];
+        const imports = uniqueImports.join("\n");
+        context.imports = []; // reset for next page
         return `${imports}\n${body}`;
     });
 }
@@ -65,7 +68,6 @@ function doNotionBlockTransforms(blocks, config) {
     }
 }
 function doTransformsOnMarkdown(context, config, input) {
-    var _a;
     return __awaiter(this, void 0, void 0, function* () {
         const regexMods = config.plugins
             .filter(plugin => !!plugin.regexMarkdownModifications)
@@ -81,7 +83,6 @@ function doTransformsOnMarkdown(context, config, input) {
         let body = input;
         //console.log("body before regex: " + body);
         let match;
-        const imports = new Set();
         // eslint-disable-next-line @typescript-eslint/no-unused-vars
         for (const mod of regexMods) {
             let replacement = undefined;
@@ -103,7 +104,6 @@ function doTransformsOnMarkdown(context, config, input) {
                         replacement = yield mod.getReplacement(context, matchAsThePluginWouldExpectIt);
                     }
                     else if (mod.replacementPattern) {
-                        console.log(`mod.replacementPattern.replace("$1", ${match[2]}`);
                         replacement = mod.replacementPattern.replace("$1", match[2]);
                     }
                     if (replacement !== undefined) {
@@ -114,14 +114,15 @@ function doTransformsOnMarkdown(context, config, input) {
                             precedingPart +
                                 partStartingFromThisMatch.replace(original, replacement);
                         // add any library imports
-                        (_a = mod.imports) === null || _a === void 0 ? void 0 : _a.forEach(imp => imports.add(imp));
+                        if (!context.imports)
+                            context.imports = [];
+                        context.imports.push(...(mod.imports || []));
                     }
                 }
             }
         }
         (0, log_1.logDebug)("doTransformsOnMarkdown", "body after regex: " + body);
-        const uniqueImports = [...new Set(imports)];
-        return { body, imports: [...uniqueImports].join("\n") };
+        return body;
     });
 }
 function doNotionToMarkdown(docunotionContext, blocks) {

package/package.json

@@ -11,6 +11,7 @@
     "// typescript check": "",
     "tsc": "tsc",
     "// test out with a private sample notion db": "",
+    "large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE  --locales en,fr",
     "pull-test-tagged": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID  --log-level debug --status-tag test",
     "pull-sample-site": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE  --log-level debug",
     "// test with a semi-stable/public site:": "",
@@ -89,5 +90,5 @@
   "volta": {
     "node": "18.16.0"
   },
-  "version": "0.13.2-alpha.2"
+  "version": "0.13.2-alpha.4"
 }