@sillsdev/docu-notion 0.13.1 → 0.13.2-alpha.3

package/README.md

@@ -38,22 +38,22 @@ means that the id is "0456aa5842946PRETEND4f37c97a0e5".
 Determine where you want the markdown files and images to land. The following works well for Docusaurus instances:
 
 ```
-npx docu-notion -n secret_PRETEND123456789PRETEND123456789PRETEND6789 -r 0456aa5842946PRETEND4f37c97a0e5"
+npx @sillsdev/docu-notion -n secret_PRETEND123456789PRETEND123456789PRETEND6789 -r 0456aa5842946PRETEND4f37c97a0e5"
 ```
 
 Likely, you will want to store these codes in your environment variables and then use them like this:
 
 ```
 (windows)
-npx docu-notion -n %MY_NOTION_TOKEN% -r %MY_NOTION_DOCS_ROOT_PAGE_ID%
+npx @sillsdev/docu-notion -n %MY_NOTION_TOKEN% -r %MY_NOTION_DOCS_ROOT_PAGE_ID%
 ```
 
 ```
 (linux / mac)
-npx docu-notion -n $MY_NOTION_TOKEN -r $MY_NOTION_DOCS_ROOT_PAGE_ID
+npx @sillsdev/docu-notion -n $MY_NOTION_TOKEN -r $MY_NOTION_DOCS_ROOT_PAGE_ID
 ```
 
-NOTE: In the above, we are using `npx` to use the latest `docu-notion`. A more conservative approach would be to `npm i cross-var docu-notion` and then create a script in your package.json like this:
+NOTE: In the above, we are using `npx` to use the latest `docu-notion`. A more conservative approach would be to `npm i cross-var @sillsdev/docu-notion` and then create a script in your package.json like this:
 
 ```
  "scripts": {

package/dist/plugins/ColumnTransformer.js

@@ -11,6 +11,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.standardColumnTransformer = void 0;
 const notion_client_1 = require("notion-client");
+const pull_1 = require("../pull");
 exports.standardColumnTransformer = {
     name: "standardColumnTransformer",
     notionToMarkdownTransforms: [
@@ -49,8 +50,10 @@ function getColumnWidth(block) {
     return __awaiter(this, void 0, void 0, function* () {
         const unofficialNotionClient = new notion_client_1.NotionAPI();
         const blockId = block.id;
-        // Yes, it is odd to call 'getPage' for a block, but that's how we access the format info.
-        const recordMap = yield unofficialNotionClient.getPage(blockId);
+        const recordMap = yield (0, pull_1.executeWithRateLimitAndRetries)(`unofficialNotionClient.getPage(${blockId}) in getColumnWidth()`, () => {
+            // Yes, it is odd to call 'getPage' for a block, but that's how we access the format info.
+            return unofficialNotionClient.getPage(blockId);
+        });
         const blockResult = recordMap.block[blockId];
         // ENHANCE: could we use https://github.com/NotionX/react-notion-x/tree/master/packages/notion-types
         // to get away from "any", which might be particularly helpful in the future

package/dist/plugins/internalLinks.spec.js

@@ -346,6 +346,34 @@ test("does not interfere with mailto links", () => __awaiter(void 0, void 0, voi
     });
     expect(results.trim()).toBe(`[mailme](mailto:foo@example.com)`);
 }));
+test("does not interfere with https links", () => __awaiter(void 0, void 0, void 0, function* () {
+    const results = yield getMarkdown({
+        type: "paragraph",
+        paragraph: {
+            rich_text: [
+                {
+                    type: "text",
+                    text: {
+                        content: "google",
+                        link: { url: `https://www.google.com` },
+                    },
+                    annotations: {
+                        bold: false,
+                        italic: false,
+                        strikethrough: false,
+                        underline: false,
+                        code: false,
+                        color: "default",
+                    },
+                    plain_text: "google",
+                    href: `https://www.google.com`,
+                },
+            ],
+            color: "default",
+        },
+    });
+    expect(results.trim()).toBe(`[google](https://www.google.com)`);
+}));
 test("links to other notion pages that are not in this site give PROBLEM LINK", () => __awaiter(void 0, void 0, void 0, function* () {
     const results = yield getMarkdown({
         type: "paragraph",

package/dist/pull.d.ts

@@ -9,4 +9,5 @@ export type DocuNotionOptions = {
     statusTag: string;
 };
 export declare function notionPull(options: DocuNotionOptions): Promise<void>;
+export declare function executeWithRateLimitAndRetries<T>(label: string, asyncFunction: () => Promise<T>): Promise<T>;
 export declare function initNotionClient(notionToken: string): Client;

package/dist/pull.js

@@ -32,7 +32,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.initNotionClient = exports.notionPull = void 0;
+exports.initNotionClient = exports.executeWithRateLimitAndRetries = exports.notionPull = void 0;
 const fs = __importStar(require("fs-extra"));
 const notion_to_md_1 = require("notion-to-md");
 const HierarchicalNamedLayoutStrategy_1 = require("./HierarchicalNamedLayoutStrategy");
@@ -188,12 +188,45 @@ const notionLimiter = new limiter_1.RateLimiter({
 let notionClient;
 function getPageMetadata(id) {
     return __awaiter(this, void 0, void 0, function* () {
-        yield rateLimit();
-        return yield notionClient.pages.retrieve({
-            page_id: id,
+        return yield executeWithRateLimitAndRetries(`pages.retrieve(${id})`, () => {
+            return notionClient.pages.retrieve({
+                page_id: id,
+            });
         });
     });
 }
+// While everything works fine locally, on Github Actions we are getting a lot of timeouts, so
+// we're trying this extra retry-able wrapper.
+function executeWithRateLimitAndRetries(label, asyncFunction) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield rateLimit();
+        const kRetries = 10;
+        let lastException = undefined;
+        for (let i = 0; i < kRetries; i++) {
+            try {
+                return yield asyncFunction();
+            }
+            catch (e) {
+                lastException = e;
+                if ((e === null || e === void 0 ? void 0 : e.code) === "notionhq_client_request_timeout" ||
+                    e.message.includes("timeout") ||
+                    e.message.includes("Timeout") ||
+                    e.message.includes("limit") ||
+                    e.message.includes("Limit")) {
+                    const secondsToWait = i + 1;
+                    (0, log_1.info)(`While doing "${label}", got error "${e.message}". Will retry after  ${secondsToWait}s...`);
+                    yield new Promise(resolve => setTimeout(resolve, 1000 * secondsToWait));
+                }
+                else {
+                    throw e;
+                }
+            }
+        }
+        (0, log_1.error)(`Error: could not complete "${label}" after ${kRetries} retries.`);
+        throw lastException;
+    });
+}
+exports.executeWithRateLimitAndRetries = executeWithRateLimitAndRetries;
 function rateLimit() {
     return __awaiter(this, void 0, void 0, function* () {
         if (notionLimiter.getTokensRemaining() < 1) {
@@ -213,10 +246,11 @@ function getBlockChildren(id) {
         // Note: there is a now a collectPaginatedAPI() in the notion client, so
         // we could switch to using that (I don't know if it does rate limiting?)
         do {
-            yield rateLimit();
-            const response = yield notionClient.blocks.children.list({
-                start_cursor: start_cursor,
-                block_id: id,
+            const response = yield executeWithRateLimitAndRetries(`getBlockChildren(${id})`, () => {
+                return notionClient.blocks.children.list({
+                    start_cursor: start_cursor,
+                    block_id: id,
+                });
             });
             if (!overallResult) {
                 overallResult = response;

package/package.json

@@ -11,6 +11,7 @@
     "// typescript check": "",
     "tsc": "tsc",
     "// test out with a private sample notion db": "",
+    "large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE  --locales en,fr",
     "pull-test-tagged": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID  --log-level debug --status-tag test",
     "pull-sample-site": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE  --log-level debug",
     "// test with a semi-stable/public site:": "",
@@ -89,5 +90,5 @@
   "volta": {
     "node": "18.16.0"
   },
-  "version": "0.13.1"
+  "version": "0.13.2-alpha.3"
 }