@sillsdev/docu-notion 0.13.2 → 0.13.4

package/README.md

@@ -31,29 +31,29 @@ Currently, docu-notion expects that each page has only one of the following: sub
 
 ## 6. Pull your pages
 
-First, determine the id of your root page by clicking "Share" and looking at the the url it gives you. E.g.
+First, determine the id of your root page by clicking "Share" and looking at the url it gives you. E.g.
 https://www.notion.so/hattonjohn/My-Docs-0456aa5842946bdbea3a4f37c97a0e5
 means that the id is "0456aa5842946PRETEND4f37c97a0e5".
 
 Determine where you want the markdown files and images to land. The following works well for Docusaurus instances:
 
 ```
-npx docu-notion -n secret_PRETEND123456789PRETEND123456789PRETEND6789 -r 0456aa5842946PRETEND4f37c97a0e5"
+npx @sillsdev/docu-notion -n secret_PRETEND123456789PRETEND123456789PRETEND6789 -r 0456aa5842946PRETEND4f37c97a0e5"
 ```
 
 Likely, you will want to store these codes in your environment variables and then use them like this:
 
 ```
 (windows)
-npx docu-notion -n %MY_NOTION_TOKEN% -r %MY_NOTION_DOCS_ROOT_PAGE_ID%
+npx @sillsdev/docu-notion -n %MY_NOTION_TOKEN% -r %MY_NOTION_DOCS_ROOT_PAGE_ID%
 ```
 
 ```
 (linux / mac)
-npx docu-notion -n $MY_NOTION_TOKEN -r $MY_NOTION_DOCS_ROOT_PAGE_ID
+npx @sillsdev/docu-notion -n $MY_NOTION_TOKEN -r $MY_NOTION_DOCS_ROOT_PAGE_ID
 ```
 
-NOTE: In the above, we are using `npx` to use the latest `docu-notion`. A more conservative approach would be to `npm i cross-var docu-notion` and then create a script in your package.json like this:
+NOTE: In the above, we are using `npx` to use the latest `docu-notion`. A more conservative approach would be to `npm i cross-var @sillsdev/docu-notion` and then create a script in your package.json like this:
 
 ```
  "scripts": {
@@ -133,3 +133,16 @@ Options:
 # Plugins
 
 If your project needs some processing that docu-notion doesn't already provide, you can provide a plugin that does it. See the [plugin readme](src/plugins/README.md).
+
+# Callouts ➜ Admonitions
+
+To map Notion callouts to Docusaurus admonitions, ensure the icon is for the type you want.
+
+- ℹ️ ➜ note
+- 📝➜ note
+- 💡➜ tip
+- ❗➜ info
+- ⚠️➜ caution
+- 🔥➜ danger
+
+The default admonition type, if no matching icon is found, is "note".

package/dist/MakeImagePersistencePlan.d.ts

@@ -1,2 +1,3 @@
 import { ImageSet } from "./images";
 export declare function makeImagePersistencePlan(imageSet: ImageSet, imageOutputRootPath: string, imagePrefix: string): void;
+export declare function hashOfString(s: string): number;

package/dist/MakeImagePersistencePlan.js

@@ -23,21 +23,21 @@ var __importStar = (this && this.__importStar) || function (mod) {
     return result;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.makeImagePersistencePlan = void 0;
+exports.hashOfString = exports.makeImagePersistencePlan = void 0;
 const Path = __importStar(require("path"));
 const log_1 = require("./log");
 const process_1 = require("process");
 function makeImagePersistencePlan(imageSet, imageOutputRootPath, imagePrefix) {
-    var _a;
+    var _a, _b;
     if ((_a = imageSet.fileType) === null || _a === void 0 ? void 0 : _a.ext) {
         // Since most images come from pasting screenshots, there isn't normally a filename. That's fine, we just make a hash of the url
         // Images that are stored by notion come to us with a complex url that changes over time, so we pick out the UUID that doesn't change. Example:
         //    https://s3.us-west-2.amazonaws.com/secure.notion-static.com/d1058f46-4d2f-4292-8388-4ad393383439/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20220516%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20220516T233630Z&X-Amz-Expires=3600&X-Amz-Signature=f215704094fcc884d37073b0b108cf6d1c9da9b7d57a898da38bc30c30b4c4b5&X-Amz-SignedHeaders=host&x-id=GetObject
-        let thingToHash = imageSet.primaryUrl;
-        const m = /.*secure\.notion-static\.com\/(.*)\//gm.exec(imageSet.primaryUrl);
-        if (m && m.length > 1) {
-            thingToHash = m[1];
-        }
+        // But around Sept 2023, they changed the url to be something like:
+        //    https://prod-files-secure.s3.us-west-2.amazonaws.com/d9a2b712-cf69-4bd6-9d65-87a4ceeacca2/d1bcdc8c-b065-4e40-9a11-392aabeb220e/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20230915%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20230915T161258Z&X-Amz-Expires=3600&X-Amz-Signature=28fca48e65fba86d539c3c4b7676fce1fa0857aa194f7b33dd4a468ecca6ab24&X-Amz-SignedHeaders=host&x-id=GetObject
+        // The thing we want is the last UUID before the ?
+        const urlBeforeQuery = imageSet.primaryUrl.split("?")[0];
+        const thingToHash = (_b = findLastUuid(urlBeforeQuery)) !== null && _b !== void 0 ? _b : urlBeforeQuery;
         const hash = hashOfString(thingToHash);
         imageSet.outputFileName = `${hash}.${imageSet.fileType.ext}`;
         imageSet.primaryFileOutputPath = Path.posix.join((imageOutputRootPath === null || imageOutputRootPath === void 0 ? void 0 : imageOutputRootPath.length) > 0
@@ -58,9 +58,18 @@ function makeImagePersistencePlan(imageSet, imageOutputRootPath, imagePrefix) {
     }
 }
 exports.makeImagePersistencePlan = makeImagePersistencePlan;
+function findLastUuid(url) {
+    // Regex for a UUID surrounded by slashes
+    const uuidPattern = /(?<=\/)[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}(?=\/)/gi;
+    // Find all UUIDs
+    const uuids = url.match(uuidPattern);
+    // Return the last UUID if any exist, else return null
+    return uuids ? uuids[uuids.length - 1].trim() : null;
+}
 function hashOfString(s) {
     let hash = 0;
     for (let i = 0; i < s.length; ++i)
         hash = Math.imul(31, hash) + s.charCodeAt(i);
     return Math.abs(hash);
 }
+exports.hashOfString = hashOfString;

package/dist/makeImagePersistencePlan.spec.js

@@ -9,9 +9,10 @@ test("primary file with explicit file output path and prefix", () => {
         fileType: { ext: "png", mime: "image/png" },
     };
     (0, MakeImagePersistencePlan_1.makeImagePersistencePlan)(imageSet, "./static/notion_imgs", "/notion_imgs");
-    expect(imageSet.outputFileName).toBe("463556435.png");
-    expect(imageSet.primaryFileOutputPath).toBe("static/notion_imgs/463556435.png");
-    expect(imageSet.filePathToUseInMarkdown).toBe("/notion_imgs/463556435.png");
+    const expectedHash = (0, MakeImagePersistencePlan_1.hashOfString)("https://s3.us-west-2.amazonaws.com/primaryImage");
+    expect(imageSet.outputFileName).toBe(`${expectedHash}.png`);
+    expect(imageSet.primaryFileOutputPath).toBe(`static/notion_imgs/${expectedHash}.png`);
+    expect(imageSet.filePathToUseInMarkdown).toBe(`/notion_imgs/${expectedHash}.png`);
 });
 test("primary file with defaults for image output path and prefix", () => {
     const imageSet = {
@@ -21,10 +22,31 @@ test("primary file with defaults for image output path and prefix", () => {
         fileType: { ext: "png", mime: "image/png" },
     };
     (0, MakeImagePersistencePlan_1.makeImagePersistencePlan)(imageSet, "", "");
-    expect(imageSet.outputFileName).toBe("463556435.png");
+    const expectedHash = (0, MakeImagePersistencePlan_1.hashOfString)("https://s3.us-west-2.amazonaws.com/primaryImage");
+    expect(imageSet.outputFileName).toBe(`${expectedHash}.png`);
     // the default behavior is to put the image next to the markdown file
-    expect(imageSet.primaryFileOutputPath).toBe("/pathToParentSomewhere/463556435.png");
-    expect(imageSet.filePathToUseInMarkdown).toBe("./463556435.png");
+    expect(imageSet.primaryFileOutputPath).toBe(`/pathToParentSomewhere/${expectedHash}.png`);
+    expect(imageSet.filePathToUseInMarkdown).toBe(`./${expectedHash}.png`);
+});
+test("properly extract UUID from old-style notion image url", () => {
+    const imageSet = {
+        primaryUrl: "https://s3.us-west-2.amazonaws.com/secure.notion-static.com/e1058f46-4d2f-4292-8388-4ad393383439/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20220516%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20220516T233630Z&X-Amz-Expires=3600&X-Amz-Signature=f215704094fcc884d37073b0b108cf6d1c9da9b7d57a898da38bc30c30b4c4b5&X-Amz-SignedHeaders=host&x-id=GetObject",
+        localizedUrls: [],
+        fileType: { ext: "png", mime: "image/png" },
+    };
+    (0, MakeImagePersistencePlan_1.makeImagePersistencePlan)(imageSet, "./static/notion_imgs", "/notion_imgs");
+    const expectedHash = (0, MakeImagePersistencePlan_1.hashOfString)("e1058f46-4d2f-4292-8388-4ad393383439");
+    expect(imageSet.outputFileName).toBe(`${expectedHash}.png`);
+});
+test("properly extract UUID from new-style (Sept 2023) notion image url", () => {
+    const imageSet = {
+        primaryUrl: "https://prod-files-secure.s3.us-west-2.amazonaws.com/d9a2b712-cf69-4bd6-9d65-87a4ceeacca2/d1bcdc8c-b065-4e40-9a11-392aabeb220e/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20230915%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20230915T161258Z&X-Amz-Expires=3600&X-Amz-Signature=28fca48e65fba86d539c3c4b7676fce1fa0857aa194f7b33dd4a468ecca6ab24&X-Amz-SignedHeaders=host&x-id=GetObject",
+        localizedUrls: [],
+        fileType: { ext: "png", mime: "image/png" },
+    };
+    (0, MakeImagePersistencePlan_1.makeImagePersistencePlan)(imageSet, "./static/notion_imgs", "/notion_imgs");
+    const expectedHash = (0, MakeImagePersistencePlan_1.hashOfString)("d1bcdc8c-b065-4e40-9a11-392aabeb220e");
+    expect(imageSet.outputFileName).toBe(`${expectedHash}.png`);
 });
 // In order to make image fallback work with other languages, we have to have
 // a file for each image, in each Docusaurus language directory. This is true

package/dist/plugins/ColumnTransformer.js

@@ -11,6 +11,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.standardColumnTransformer = void 0;
 const notion_client_1 = require("notion-client");
+const pull_1 = require("../pull");
 exports.standardColumnTransformer = {
     name: "standardColumnTransformer",
     notionToMarkdownTransforms: [
@@ -49,8 +50,10 @@ function getColumnWidth(block) {
     return __awaiter(this, void 0, void 0, function* () {
         const unofficialNotionClient = new notion_client_1.NotionAPI();
         const blockId = block.id;
-        // Yes, it is odd to call 'getPage' for a block, but that's how we access the format info.
-        const recordMap = yield unofficialNotionClient.getPage(blockId);
+        const recordMap = yield (0, pull_1.executeWithRateLimitAndRetries)(`unofficialNotionClient.getPage(${blockId}) in getColumnWidth()`, () => {
+            // Yes, it is odd to call 'getPage' for a block, but that's how we access the format info.
+            return unofficialNotionClient.getPage(blockId);
+        });
         const blockResult = recordMap.block[blockId];
         // ENHANCE: could we use https://github.com/NotionX/react-notion-x/tree/master/packages/notion-types
         // to get away from "any", which might be particularly helpful in the future

package/dist/plugins/internalLinks.spec.js

@@ -346,6 +346,34 @@ test("does not interfere with mailto links", () => __awaiter(void 0, void 0, voi
     });
     expect(results.trim()).toBe(`[mailme](mailto:foo@example.com)`);
 }));
+test("does not interfere with https links", () => __awaiter(void 0, void 0, void 0, function* () {
+    const results = yield getMarkdown({
+        type: "paragraph",
+        paragraph: {
+            rich_text: [
+                {
+                    type: "text",
+                    text: {
+                        content: "google",
+                        link: { url: `https://www.google.com` },
+                    },
+                    annotations: {
+                        bold: false,
+                        italic: false,
+                        strikethrough: false,
+                        underline: false,
+                        code: false,
+                        color: "default",
+                    },
+                    plain_text: "google",
+                    href: `https://www.google.com`,
+                },
+            ],
+            color: "default",
+        },
+    });
+    expect(results.trim()).toBe(`[google](https://www.google.com)`);
+}));
 test("links to other notion pages that are not in this site give PROBLEM LINK", () => __awaiter(void 0, void 0, void 0, function* () {
     const results = yield getMarkdown({
         type: "paragraph",

package/dist/pull.d.ts

@@ -9,4 +9,5 @@ export type DocuNotionOptions = {
     statusTag: string;
 };
 export declare function notionPull(options: DocuNotionOptions): Promise<void>;
+export declare function executeWithRateLimitAndRetries<T>(label: string, asyncFunction: () => Promise<T>): Promise<T>;
 export declare function initNotionClient(notionToken: string): Client;

package/dist/pull.js

@@ -32,7 +32,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.initNotionClient = exports.notionPull = void 0;
+exports.initNotionClient = exports.executeWithRateLimitAndRetries = exports.notionPull = void 0;
 const fs = __importStar(require("fs-extra"));
 const notion_to_md_1 = require("notion-to-md");
 const HierarchicalNamedLayoutStrategy_1 = require("./HierarchicalNamedLayoutStrategy");
@@ -188,12 +188,45 @@ const notionLimiter = new limiter_1.RateLimiter({
 let notionClient;
 function getPageMetadata(id) {
     return __awaiter(this, void 0, void 0, function* () {
-        yield rateLimit();
-        return yield notionClient.pages.retrieve({
-            page_id: id,
+        return yield executeWithRateLimitAndRetries(`pages.retrieve(${id})`, () => {
+            return notionClient.pages.retrieve({
+                page_id: id,
+            });
         });
     });
 }
+// While everything works fine locally, on Github Actions we are getting a lot of timeouts, so
+// we're trying this extra retry-able wrapper.
+function executeWithRateLimitAndRetries(label, asyncFunction) {
+    return __awaiter(this, void 0, void 0, function* () {
+        yield rateLimit();
+        const kRetries = 10;
+        let lastException = undefined;
+        for (let i = 0; i < kRetries; i++) {
+            try {
+                return yield asyncFunction();
+            }
+            catch (e) {
+                lastException = e;
+                if ((e === null || e === void 0 ? void 0 : e.code) === "notionhq_client_request_timeout" ||
+                    e.message.includes("timeout") ||
+                    e.message.includes("Timeout") ||
+                    e.message.includes("limit") ||
+                    e.message.includes("Limit")) {
+                    const secondsToWait = i + 1;
+                    (0, log_1.info)(`While doing "${label}", got error "${e.message}". Will retry after  ${secondsToWait}s...`);
+                    yield new Promise(resolve => setTimeout(resolve, 1000 * secondsToWait));
+                }
+                else {
+                    throw e;
+                }
+            }
+        }
+        (0, log_1.error)(`Error: could not complete "${label}" after ${kRetries} retries.`);
+        throw lastException;
+    });
+}
+exports.executeWithRateLimitAndRetries = executeWithRateLimitAndRetries;
 function rateLimit() {
     return __awaiter(this, void 0, void 0, function* () {
         if (notionLimiter.getTokensRemaining() < 1) {
@@ -213,10 +246,11 @@ function getBlockChildren(id) {
         // Note: there is a now a collectPaginatedAPI() in the notion client, so
         // we could switch to using that (I don't know if it does rate limiting?)
         do {
-            yield rateLimit();
-            const response = yield notionClient.blocks.children.list({
-                start_cursor: start_cursor,
-                block_id: id,
+            const response = yield executeWithRateLimitAndRetries(`getBlockChildren(${id})`, () => {
+                return notionClient.blocks.children.list({
+                    start_cursor: start_cursor,
+                    block_id: id,
+                });
             });
             if (!overallResult) {
                 overallResult = response;

package/dist/transform.js

@@ -103,7 +103,6 @@ function doTransformsOnMarkdown(context, config, input) {
                         replacement = yield mod.getReplacement(context, matchAsThePluginWouldExpectIt);
                     }
                     else if (mod.replacementPattern) {
-                        console.log(`mod.replacementPattern.replace("$1", ${match[2]}`);
                         replacement = mod.replacementPattern.replace("$1", match[2]);
                     }
                     if (replacement !== undefined) {

package/package.json

@@ -11,6 +11,7 @@
     "// typescript check": "",
     "tsc": "tsc",
     "// test out with a private sample notion db": "",
+    "large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE  --locales en,fr",
     "pull-test-tagged": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID  --log-level debug --status-tag test",
     "pull-sample-site": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE  --log-level debug",
     "// test with a semi-stable/public site:": "",
@@ -89,5 +90,5 @@
   "volta": {
     "node": "18.16.0"
   },
-  "version": "0.13.2"
+  "version": "0.13.4"
 }