@sillsdev/docu-notion 0.13.2-alpha.2 → 0.13.2-alpha.3

package/dist/plugins/ColumnTransformer.js

@@ -11,6 +11,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.standardColumnTransformer = void 0;
 const notion_client_1 = require("notion-client");
+const pull_1 = require("../pull");
 exports.standardColumnTransformer = {
     name: "standardColumnTransformer",
     notionToMarkdownTransforms: [
@@ -49,8 +50,10 @@ function getColumnWidth(block) {
     return __awaiter(this, void 0, void 0, function* () {
         const unofficialNotionClient = new notion_client_1.NotionAPI();
         const blockId = block.id;
-        // Yes, it is odd to call 'getPage' for a block, but that's how we access the format info.
-        const recordMap = yield unofficialNotionClient.getPage(blockId);
+        const recordMap = yield (0, pull_1.executeWithRateLimitAndRetries)(`unofficialNotionClient.getPage(${blockId}) in getColumnWidth()`, () => {
+            // Yes, it is odd to call 'getPage' for a block, but that's how we access the format info.
+            return unofficialNotionClient.getPage(blockId);
+        });
         const blockResult = recordMap.block[blockId];
         // ENHANCE: could we use https://github.com/NotionX/react-notion-x/tree/master/packages/notion-types
         // to get away from "any", which might be particularly helpful in the future

package/dist/pull.d.ts

@@ -9,4 +9,5 @@ export type DocuNotionOptions = {
     statusTag: string;
 };
 export declare function notionPull(options: DocuNotionOptions): Promise<void>;
+export declare function executeWithRateLimitAndRetries<T>(label: string, asyncFunction: () => Promise<T>): Promise<T>;
 export declare function initNotionClient(notionToken: string): Client;

package/dist/pull.js

@@ -32,7 +32,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.initNotionClient = exports.notionPull = void 0;
+exports.initNotionClient = exports.executeWithRateLimitAndRetries = exports.notionPull = void 0;
 const fs = __importStar(require("fs-extra"));
 const notion_to_md_1 = require("notion-to-md");
 const HierarchicalNamedLayoutStrategy_1 = require("./HierarchicalNamedLayoutStrategy");
@@ -188,12 +188,45 @@ const notionLimiter = new limiter_1.RateLimiter({
 let notionClient;
 function getPageMetadata(id) {
     return __awaiter(this, void 0, void 0, function* () {
-        yield rateLimit();
-        return yield notionClient.pages.retrieve({
-            page_id: id,
+        return yield executeWithRateLimitAndRetries(`pages.retrieve(${id})`, () => {
+            return notionClient.pages.retrieve({
+                page_id: id,
+            });
         });
     });
 }
+// While everything works fine locally, on Github Actions we are getting a lot of timeouts, so
+// we're trying this extra retry-able wrapper.
+function executeWithRateLimitAndRetries(label, asyncFunction) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield rateLimit();
+        const kRetries = 10;
+        let lastException = undefined;
+        for (let i = 0; i < kRetries; i++) {
+            try {
+                return yield asyncFunction();
+            }
+            catch (e) {
+                lastException = e;
+                if ((e === null || e === void 0 ? void 0 : e.code) === "notionhq_client_request_timeout" ||
+                    e.message.includes("timeout") ||
+                    e.message.includes("Timeout") ||
+                    e.message.includes("limit") ||
+                    e.message.includes("Limit")) {
+                    const secondsToWait = i + 1;
+                    (0, log_1.info)(`While doing "${label}", got error "${e.message}". Will retry after  ${secondsToWait}s...`);
+                    yield new Promise(resolve => setTimeout(resolve, 1000 * secondsToWait));
+                }
+                else {
+                    throw e;
+                }
+            }
+        }
+        (0, log_1.error)(`Error: could not complete "${label}" after ${kRetries} retries.`);
+        throw lastException;
+    });
+}
+exports.executeWithRateLimitAndRetries = executeWithRateLimitAndRetries;
 function rateLimit() {
     return __awaiter(this, void 0, void 0, function* () {
         if (notionLimiter.getTokensRemaining() < 1) {
@@ -213,10 +246,11 @@ function getBlockChildren(id) {
         // Note: there is a now a collectPaginatedAPI() in the notion client, so
         // we could switch to using that (I don't know if it does rate limiting?)
         do {
-            yield rateLimit();
-            const response = yield notionClient.blocks.children.list({
-                start_cursor: start_cursor,
-                block_id: id,
+            const response = yield executeWithRateLimitAndRetries(`getBlockChildren(${id})`, () => {
+                return notionClient.blocks.children.list({
+                    start_cursor: start_cursor,
+                    block_id: id,
+                });
             });
             if (!overallResult) {
                 overallResult = response;

package/package.json

@@ -11,6 +11,7 @@
     "// typescript check": "",
     "tsc": "tsc",
     "// test out with a private sample notion db": "",
+    "large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE  --locales en,fr",
     "pull-test-tagged": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID  --log-level debug --status-tag test",
     "pull-sample-site": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE  --log-level debug",
     "// test with a semi-stable/public site:": "",
@@ -89,5 +90,5 @@
   "volta": {
     "node": "18.16.0"
   },
-  "version": "0.13.2-alpha.2"
+  "version": "0.13.2-alpha.3"
 }