@sillsdev/docu-notion 0.14.0-alpha.11 → 0.14.0-alpha.13

package/README.md

@@ -148,3 +148,12 @@ To map Notion callouts to Docusaurus admonitions, ensure the icon is for the typ
 - 🔥➜ danger
 
 The default admonition type, if no matching icon is found, is "note".
+
+# Known Workarounds
+
+### Start a numbered list at a number other than 1
+In Notion, make sure the block is "Text," not "Numbered List".
+- But make sure the number does NOT have a space in front of it. This can/will cause issues with sub-list items.
+- One way to get Notion to let you do this:
+    - Create a numbered list item where the text duplicates the number you want. Convert that numbered list item to "Text."
+    - i.e. Type "1. 1. Item one." Notion makes the first "1." into a number in a list. When you convert back to "Text," you're left with plain text "1. Item one."

package/dist/MakeImagePersistencePlan.js

@@ -29,33 +29,36 @@ const log_1 = require("./log");
 const process_1 = require("process");
 function makeImagePersistencePlan(imageSet, imageOutputRootPath, imagePrefix) {
     var _a, _b;
-    if ((_a = imageSet.fileType) === null || _a === void 0 ? void 0 : _a.ext) {
-        // Since most images come from pasting screenshots, there isn't normally a filename. That's fine, we just make a hash of the url
-        // Images that are stored by notion come to us with a complex url that changes over time, so we pick out the UUID that doesn't change. Example:
-        //    https://s3.us-west-2.amazonaws.com/secure.notion-static.com/d1058f46-4d2f-4292-8388-4ad393383439/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20220516%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20220516T233630Z&X-Amz-Expires=3600&X-Amz-Signature=f215704094fcc884d37073b0b108cf6d1c9da9b7d57a898da38bc30c30b4c4b5&X-Amz-SignedHeaders=host&x-id=GetObject
-        // But around Sept 2023, they changed the url to be something like:
-        //    https://prod-files-secure.s3.us-west-2.amazonaws.com/d9a2b712-cf69-4bd6-9d65-87a4ceeacca2/d1bcdc8c-b065-4e40-9a11-392aabeb220e/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20230915%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20230915T161258Z&X-Amz-Expires=3600&X-Amz-Signature=28fca48e65fba86d539c3c4b7676fce1fa0857aa194f7b33dd4a468ecca6ab24&X-Amz-SignedHeaders=host&x-id=GetObject
-        // The thing we want is the last UUID before the ?
-        const urlBeforeQuery = imageSet.primaryUrl.split("?")[0];
-        const thingToHash = (_b = findLastUuid(urlBeforeQuery)) !== null && _b !== void 0 ? _b : urlBeforeQuery;
-        const hash = hashOfString(thingToHash);
-        imageSet.outputFileName = `${hash}.${imageSet.fileType.ext}`;
-        imageSet.primaryFileOutputPath = Path.posix.join((imageOutputRootPath === null || imageOutputRootPath === void 0 ? void 0 : imageOutputRootPath.length) > 0
-            ? imageOutputRootPath
-            : imageSet.pathToParentDocument, imageSet.outputFileName);
-        if (imageOutputRootPath && imageSet.localizedUrls.length) {
-            (0, log_1.error)("imageOutputPath was declared, but one or more localizedUrls were found too. If you are going to localize screenshots, then you can't declare an imageOutputPath.");
+    const urlBeforeQuery = imageSet.primaryUrl.split("?")[0];
+    let imageFileExtension = (_a = imageSet.fileType) === null || _a === void 0 ? void 0 : _a.ext;
+    if (!imageFileExtension) {
+        // Try to get the extension from the url
+        imageFileExtension = urlBeforeQuery.split(".").pop();
+        if (!imageFileExtension) {
+            (0, log_1.error)(`Something wrong with the filetype extension on the blob we got from ${imageSet.primaryUrl}`);
             (0, process_1.exit)(1);
         }
-        imageSet.filePathToUseInMarkdown =
-            ((imagePrefix === null || imagePrefix === void 0 ? void 0 : imagePrefix.length) > 0 ? imagePrefix : ".") +
-                "/" +
-                imageSet.outputFileName;
     }
-    else {
-        (0, log_1.error)(`Something wrong with the filetype extension on the blob we got from ${imageSet.primaryUrl}`);
+    // Since most images come from pasting screenshots, there isn't normally a filename. That's fine, we just make a hash of the url
+    // Images that are stored by notion come to us with a complex url that changes over time, so we pick out the UUID that doesn't change. Example:
+    //    https://s3.us-west-2.amazonaws.com/secure.notion-static.com/d1058f46-4d2f-4292-8388-4ad393383439/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20220516%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20220516T233630Z&X-Amz-Expires=3600&X-Amz-Signature=f215704094fcc884d37073b0b108cf6d1c9da9b7d57a898da38bc30c30b4c4b5&X-Amz-SignedHeaders=host&x-id=GetObject
+    // But around Sept 2023, they changed the url to be something like:
+    //    https://prod-files-secure.s3.us-west-2.amazonaws.com/d9a2b712-cf69-4bd6-9d65-87a4ceeacca2/d1bcdc8c-b065-4e40-9a11-392aabeb220e/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20230915%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20230915T161258Z&X-Amz-Expires=3600&X-Amz-Signature=28fca48e65fba86d539c3c4b7676fce1fa0857aa194f7b33dd4a468ecca6ab24&X-Amz-SignedHeaders=host&x-id=GetObject
+    // The thing we want is the last UUID before the ?
+    const thingToHash = (_b = findLastUuid(urlBeforeQuery)) !== null && _b !== void 0 ? _b : urlBeforeQuery;
+    const hash = hashOfString(thingToHash);
+    imageSet.outputFileName = `${hash}.${imageFileExtension}`;
+    imageSet.primaryFileOutputPath = Path.posix.join((imageOutputRootPath === null || imageOutputRootPath === void 0 ? void 0 : imageOutputRootPath.length) > 0
+        ? imageOutputRootPath
+        : imageSet.pathToParentDocument, imageSet.outputFileName);
+    if (imageOutputRootPath && imageSet.localizedUrls.length) {
+        (0, log_1.error)("imageOutputPath was declared, but one or more localizedUrls were found too. If you are going to localize screenshots, then you can't declare an imageOutputPath.");
         (0, process_1.exit)(1);
     }
+    imageSet.filePathToUseInMarkdown =
+        ((imagePrefix === null || imagePrefix === void 0 ? void 0 : imagePrefix.length) > 0 ? imagePrefix : ".") +
+            "/" +
+            imageSet.outputFileName;
 }
 exports.makeImagePersistencePlan = makeImagePersistencePlan;
 function findLastUuid(url) {

package/dist/images.js

@@ -38,7 +38,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.cleanupOldImages = exports.parseImageBlock = exports.markdownToMDImageTransformer = exports.standardImageTransformer = exports.initImageHandling = void 0;
 const fs = __importStar(require("fs-extra"));
 const file_type_1 = __importDefault(require("file-type"));
-const node_fetch_1 = __importDefault(require("node-fetch"));
+const axios_1 = __importDefault(require("axios"));
 const Path = __importStar(require("path"));
 const MakeImagePersistencePlan_1 = require("./MakeImagePersistencePlan");
 const log_1 = require("./log");
@@ -131,9 +131,12 @@ function processImageBlock(imageBlock, pathToParentDocument, relativePathToThisP
 }
 function readPrimaryImage(imageSet) {
     return __awaiter(this, void 0, void 0, function* () {
-        const response = yield (0, node_fetch_1.default)(imageSet.primaryUrl);
-        const arrayBuffer = yield response.arrayBuffer();
-        imageSet.primaryBuffer = Buffer.from(arrayBuffer);
+        // In Mar 2024, we started having a problem getting a particular gif from imgur using
+        // node-fetch. Switching to axios resolved it. I don't know why.
+        const response = yield axios_1.default.get(imageSet.primaryUrl, {
+            responseType: "arraybuffer",
+        });
+        imageSet.primaryBuffer = Buffer.from(response.data, "utf-8");
         imageSet.fileType = yield file_type_1.default.fromBuffer(imageSet.primaryBuffer);
     });
 }
@@ -145,7 +148,7 @@ function saveImage(imageSet) {
             // if we have a urls for the localized screenshot, download it
             if ((localizedImage === null || localizedImage === void 0 ? void 0 : localizedImage.url.length) > 0) {
                 (0, log_1.verbose)(`Retrieving ${localizedImage.iso632Code} version...`);
-                const response = yield (0, node_fetch_1.default)(localizedImage.url);
+                const response = yield fetch(localizedImage.url);
                 const arrayBuffer = yield response.arrayBuffer();
                 buffer = Buffer.from(arrayBuffer);
             }

package/dist/makeImagePersistencePlan.spec.js

@@ -28,6 +28,16 @@ test("primary file with defaults for image output path and prefix", () => {
     expect(imageSet.primaryFileOutputPath).toBe(`/pathToParentSomewhere/${expectedHash}.png`);
     expect(imageSet.filePathToUseInMarkdown).toBe(`./${expectedHash}.png`);
 });
+test("falls back to getting file extension from url if not in fileType", () => {
+    const imageSet = {
+        primaryUrl: "https://s3.us-west-2.amazonaws.com/primaryImage.png",
+        localizedUrls: [],
+        pathToParentDocument: "/pathToParentSomewhere/",
+    };
+    (0, MakeImagePersistencePlan_1.makeImagePersistencePlan)(imageSet, "", "");
+    const expectedHash = (0, MakeImagePersistencePlan_1.hashOfString)("https://s3.us-west-2.amazonaws.com/primaryImage.png");
+    expect(imageSet.outputFileName).toBe(`${expectedHash}.png`);
+});
 test("properly extract UUID from old-style notion image url", () => {
     const imageSet = {
         primaryUrl: "https://s3.us-west-2.amazonaws.com/secure.notion-static.com/e1058f46-4d2f-4292-8388-4ad393383439/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20220516%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20220516T233630Z&X-Amz-Expires=3600&X-Amz-Signature=f215704094fcc884d37073b0b108cf6d1c9da9b7d57a898da38bc30c30b4c4b5&X-Amz-SignedHeaders=host&x-id=GetObject",

package/package.json

@@ -12,7 +12,7 @@
     "tsc": "tsc",
     "// test out with a private sample notion db": "",
     "large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE  --locales en,fr   --log-level debug",
-    "pull-test-tagged": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID  --log-level debug --status-tag test",
+    "pull-test-tagged": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID  --log-level info --status-tag test",
     "pull-test-css": "npm run ts -- --css-output-directory ./test/css -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID  --log-level debug --status-tag test",
     "pull-sample-site": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE  --log-level debug",
     "// test with a semi-stable/public site:": "",
@@ -20,26 +20,25 @@
     "pull-sample-with-paths": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE -m ./sample --img-output-path ./sample_img"
   },
   "//file-type": "have to use this version before they switched to ESM, which gives a compile error related to require()",
-  "//node-fetch@2.6.6file-type": "have to use this version before they switched to ESM, which gives a compile error related to require()",
   "//chalk@4": "also ESM related problem",
   "//notion-client@4": "also ESM related problem",
   "//note: ts-node": "really is a runtime dependency",
   "dependencies": {
     "@notionhq/client": "2.2.3",
+    "axios": "^1.6.8",
     "chalk": "^4.1.2",
     "commander": "^9.2.0",
     "cosmiconfig": "^8.0.0",
     "cosmiconfig-typescript-loader": "^4.3.0",
-    "file-type": "16.5.1",
+    "file-type": "16.5.3",
     "fs-extra": "^10.1.0",
     "limiter": "^2.1.0",
     "markdown-table": "^2.0.0",
-    "node-fetch": "2.6.6",
     "notion-client": "^4",
     "notion-to-md": "3.1.1",
     "path": "^0.12.7",
-    "ts-node": "^10.2.1",
-    "sanitize-filename": "^1.6.3"
+    "sanitize-filename": "^1.6.3",
+    "ts-node": "^10.2.1"
   },
   "devDependencies": {
     "@types/fs-extra": "^9.0.13",
@@ -91,5 +90,5 @@
   "volta": {
     "node": "18.16.0"
   },
-  "version": "0.14.0-alpha.11"
+  "version": "0.14.0-alpha.13"
 }