@sillsdev/docu-notion 0.17.0-alpha.2 → 0.17.0-alpha.3

package/dist/plugins/ColumnListTransformer.js

@@ -18,8 +18,13 @@ function notionColumnListToMarkdown(notionToMarkdown, getBlockChildren, block) {
         if (!has_children)
             return "";
         const column_list_children = yield getBlockChildren(id);
-        const column_list_promise = column_list_children.map((column) => __awaiter(this, void 0, void 0, function* () { return yield notionToMarkdown.blockToMarkdown(column); }));
-        const columns = yield Promise.all(column_list_promise);
+        const columns = [];
+        for (const column of column_list_children) {
+            // Keep column rendering sequential. A column block can trigger more Notion
+            // reads downstream, so Promise.all() here would turn one page into a burst
+            // of concurrent API requests during stage 2.
+            columns.push(yield notionToMarkdown.blockToMarkdown(column));
+        }
         return `<div class='notion-row'>\n${columns.join("\n\n")}\n</div>`;
     });
 }

package/dist/plugins/ColumnTransformer.js

@@ -29,7 +29,13 @@ function notionColumnToMarkdown(notionToMarkdown, getBlockChildren, block) {
         if (!has_children)
             return "";
         const columnChildren = yield getBlockChildren(id);
-        const childrenMdBlocksArray = yield Promise.all(columnChildren.map((child) => __awaiter(this, void 0, void 0, function* () { return yield notionToMarkdown.blocksToMarkdown([child]); })));
+        const childrenMdBlocksArray = [];
+        for (const child of columnChildren) {
+            // Intentionally serialize these subtree conversions. notion-to-md will fetch
+            // nested block children during blocksToMarkdown(), and parallelizing sibling
+            // columns creates bursts that can exceed Notion's per-integration rate limit.
+            childrenMdBlocksArray.push(yield notionToMarkdown.blocksToMarkdown([child]));
+        }
         const childrenMarkdown = childrenMdBlocksArray.map(mdBlockArray => notionToMarkdown.toMarkdownString(mdBlockArray).parent);
         const columnWidth = yield getColumnWidth(block);
         return (`<div class='notion-column' style={{width: '${columnWidth}'}}>\n\n${childrenMarkdown.join("\n")}\n</div>` +

package/dist/pull.js

@@ -89,9 +89,7 @@ function notionPull(options) {
         (0, log_1.info)("Connecting to Notion...");
         // Do a  quick test to see if we can connect to the root so that we can give a better error than just a generic "could not find page" one.
         try {
-            yield executeWithRateLimitAndRetries("retrieving root page", () => __awaiter(this, void 0, void 0, function* () {
-                yield notionClient.pages.retrieve({ page_id: options.rootPage });
-            }));
+            yield notionClient.pages.retrieve({ page_id: options.rootPage });
         }
         catch (e) {
             (0, log_1.error)(`docu-notion could not retrieve the root page from Notion. \r\na) Check that the root page id really is "${options.rootPage}".\r\nb) Check that your Notion API token (the "Integration Secret") is correct. It starts with "${optionsForLogging.notionToken}".\r\nc) Check that your root page includes your "integration" in its "connections".\r\nThis internal error message may help:\r\n    ${e.message}`);
@@ -223,13 +221,31 @@ const notionLimiter = new limiter_1.RateLimiter({
 let notionClient;
 function getPageMetadata(id) {
     return __awaiter(this, void 0, void 0, function* () {
-        return yield executeWithRateLimitAndRetries(`pages.retrieve(${id})`, () => {
-            return notionClient.pages.retrieve({
-                page_id: id,
-            });
+        return yield notionClient.pages.retrieve({
+            page_id: id,
         });
     });
 }
+function isRetryableNotionError(error) {
+    const message = String((error === null || error === void 0 ? void 0 : error.message) || "");
+    return ((error === null || error === void 0 ? void 0 : error.code) === "notionhq_client_request_timeout" ||
+        (error === null || error === void 0 ? void 0 : error.code) === "notionhq_client_response_error" ||
+        (error === null || error === void 0 ? void 0 : error.code) === "service_unavailable" ||
+        (error === null || error === void 0 ? void 0 : error.code) === client_1.APIErrorCode.RateLimited ||
+        message.includes("timeout") ||
+        message.includes("Timeout") ||
+        message.includes("limit") ||
+        message.includes("Limit"));
+}
+function getRetryDelayMilliseconds(error, retryIndex) {
+    var _a, _b;
+    const retryAfterHeader = (_b = (_a = error === null || error === void 0 ? void 0 : error.headers) === null || _a === void 0 ? void 0 : _a.get) === null || _b === void 0 ? void 0 : _b.call(_a, "retry-after");
+    const retryAfterSeconds = Number.parseInt(retryAfterHeader || "", 10);
+    if (Number.isFinite(retryAfterSeconds) && retryAfterSeconds > 0) {
+        return retryAfterSeconds * 1000;
+    }
+    return (retryIndex + 1) * 1000;
+}
 // While everything works fine locally, on Github Actions we are getting a lot of timeouts, so
 // we're trying this extra retry-able wrapper.
 function executeWithRateLimitAndRetries(label, asyncFunction) {
@@ -243,16 +259,10 @@ function executeWithRateLimitAndRetries(label, asyncFunction) {
             }
             catch (e) {
                 lastException = e;
-                if ((e === null || e === void 0 ? void 0 : e.code) === "notionhq_client_request_timeout" ||
-                    e.message.includes("timeout") ||
-                    e.message.includes("Timeout") ||
-                    e.message.includes("limit") ||
-                    e.message.includes("Limit") ||
-                    (e === null || e === void 0 ? void 0 : e.code) === "notionhq_client_response_error" ||
-                    (e === null || e === void 0 ? void 0 : e.code) === "service_unavailable") {
-                    const secondsToWait = i + 1;
-                    (0, log_1.warning)(`While doing "${label}", got error "${e.message}". Will retry after ${secondsToWait}s...`);
-                    yield new Promise(resolve => setTimeout(resolve, 1000 * secondsToWait));
+                if (isRetryableNotionError(e)) {
+                    const millisecondsToWait = getRetryDelayMilliseconds(e, i);
+                    (0, log_1.warning)(`While doing "${label}", got error "${e.message}". Will retry after ${millisecondsToWait / 1000}s...`);
+                    yield new Promise(resolve => setTimeout(resolve, millisecondsToWait));
                 }
                 else {
                     throw e;
@@ -282,11 +292,9 @@ function getBlockChildren(id) {
         // Note: there is a now a collectPaginatedAPI() in the notion client, so
         // we could switch to using that (I don't know if it does rate limiting?)
         do {
-            const response = yield executeWithRateLimitAndRetries(`getBlockChildren(${id})`, () => {
-                return notionClient.blocks.children.list({
-                    start_cursor: start_cursor,
-                    block_id: id,
-                });
+            const response = yield notionClient.blocks.children.list({
+                start_cursor: start_cursor,
+                block_id: id,
             });
             if (!overallResult) {
                 overallResult = response;
@@ -309,6 +317,10 @@ function initNotionClient(notionToken) {
     notionClient = new client_1.Client({
         auth: notionToken,
     });
+    const originalRequest = notionClient.request.bind(notionClient);
+    notionClient.request = (args) => __awaiter(this, void 0, void 0, function* () {
+        return yield executeWithRateLimitAndRetries(`${args.method.toUpperCase()} ${args.path}`, () => originalRequest(args));
+    });
     return notionClient;
 }
 function fromPageId(context, pageId, order, foundDirectlyInOutline) {

package/dist/transform.js

@@ -16,7 +16,6 @@ exports.getMarkdownForPage = getMarkdownForPage;
 exports.getMarkdownFromNotionBlocks = getMarkdownFromNotionBlocks;
 const chalk_1 = __importDefault(require("chalk"));
 const log_1 = require("./log");
-const pull_1 = require("./pull");
 function getMarkdownForPage(config, context, page) {
     return __awaiter(this, void 0, void 0, function* () {
         (0, log_1.info)(`Reading & converting page ${page.layoutContext}/${page.nameOrTitle} (${chalk_1.default.blue(page.hasExplicitSlug
@@ -127,17 +126,10 @@ function doTransformsOnMarkdown(context, config, input) {
 }
 function doNotionToMarkdown(docunotionContext, blocks) {
     return __awaiter(this, void 0, void 0, function* () {
-        let mdBlocks;
-        yield (0, pull_1.executeWithRateLimitAndRetries)("notionToMarkdown.blocksToMarkdown", () => __awaiter(this, void 0, void 0, function* () {
-            mdBlocks = yield docunotionContext.notionToMarkdown.blocksToMarkdown(
-            // We need to provide a copy of blocks.
-            // Calling blocksToMarkdown can modify the values in the blocks. If it does, and then
-            // we have to retry, we end up retrying with the modified values, which
-            // causes various issues (like using the transformed image url instead of the original one).
-            // Note, currently, we don't do anything else with blocks after this.
-            // If that changes, we'll need to figure out a more sophisticated approach.
-            JSON.parse(JSON.stringify(blocks)));
-        }));
+        const mdBlocks = yield docunotionContext.notionToMarkdown.blocksToMarkdown(
+        // We need to provide a copy of blocks.
+        // Calling blocksToMarkdown can modify the values in the blocks.
+        JSON.parse(JSON.stringify(blocks)));
         const markdown = docunotionContext.notionToMarkdown.toMarkdownString(mdBlocks).parent || "";
         return markdown;
     });

package/package.json

@@ -92,5 +92,5 @@
   "volta": {
     "node": "22.21.0"
   },
-  "version": "0.17.0-alpha.2"
+  "version": "0.17.0-alpha.3"
 }