@sillsdev/docu-notion 0.14.0-alpha.1 → 0.14.0-alpha.10

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -10,6 +10,8 @@ Example Site: https://sillsdev.github.io/docu-notion-sample-site/
10
10
 
11
11
  First, prepare your markdown-based static file system like [Docusaurus](https://docusaurus.io/). For a shortcut with github actions, search, and deployment to github pages, you can just copy [this template](https://github.com/sillsdev/docu-notion-sample-site).
12
12
 
13
+ If you do not use the above sample, you will need to manually add [notion-styles.css](src/css/notion-styles.css) to your site. This enables multi-column layouts and another other special styling that we need to make the output of docu-notion look right.
14
+
13
15
  ## 2. In Notion, duplicate the docu-notion template
14
16
 
15
17
  Go to [this template page](https://hattonjohn.notion.site/Documentation-Template-Docusaurus-0e998b32da3c47edad0f62a25b49818c). Duplicate it into your own workspace.
@@ -17,13 +19,13 @@ You can name it anything you like, e.g. "Documentation Root".
17
19
 
18
20
  ## 3. Create a Notion Integration
19
21
 
20
- In order for docu-notion to read your site via Notion's API, you need to create what Notion calls an "integration". Follow [these instructions](https://developers.notion.com/docs/getting-started) to make an integration and get your token. Limit your integration to "READ" access.
22
+ In order for docu-notion to read your site via Notion's API, you need to create what Notion calls an "integration". Follow [these instructions](https://developers.notion.com/docs/getting-started) to make an integration and get your token. Remember to limit your integration to "READ" access.
21
23
 
22
- ## 4. "Invite" your Notion Integration to read you page
24
+ ## 4. Connect your Integration
23
25
 
24
- In Notion, click "Share" on the root of your documentation and "invite" your integration to it.
26
+ Go to the page that will be the root of your site. This page should have, as direct children, your "Outline" (required) and "Database" (optional) pages. Follow [these instructions](https://developers.notion.com/docs/create-a-notion-integration#give-your-integration-page-permissions).
25
27
 
26
- ![image](https://user-images.githubusercontent.com/8448/168930238-1dcf46df-a690-4839-bf4c-c63157f104d8.png)
28
+ <img width="318" alt="image" src="https://github.com/sillsdev/docu-notion/assets/8448/810c6dca-f9ab-4370-93b4-dc1479332af7">
27
29
 
28
30
  ## 5. Add your pages under your Outline page.
29
31
 
@@ -35,7 +37,7 @@ First, determine the id of your root page by clicking "Share" and looking at the
35
37
  https://www.notion.so/hattonjohn/My-Docs-0456aa5842946bdbea3a4f37c97a0e5
36
38
  means that the id is "0456aa5842946PRETEND4f37c97a0e5".
37
39
 
38
- Determine where you want the markdown files and images to land. The following works well for Docusaurus instances:
40
+ Try it out:
39
41
 
40
42
  ```
41
43
  npx @sillsdev/docu-notion -n secret_PRETEND123456789PRETEND123456789PRETEND6789 -r 0456aa5842946PRETEND4f37c97a0e5"
@@ -1,2 +1,3 @@
1
1
  import { ImageSet } from "./images";
2
2
  export declare function makeImagePersistencePlan(imageSet: ImageSet, imageOutputRootPath: string, imagePrefix: string): void;
3
+ export declare function hashOfString(s: string): number;
@@ -23,21 +23,21 @@ var __importStar = (this && this.__importStar) || function (mod) {
23
23
  return result;
24
24
  };
25
25
  Object.defineProperty(exports, "__esModule", { value: true });
26
- exports.makeImagePersistencePlan = void 0;
26
+ exports.hashOfString = exports.makeImagePersistencePlan = void 0;
27
27
  const Path = __importStar(require("path"));
28
28
  const log_1 = require("./log");
29
29
  const process_1 = require("process");
30
30
  function makeImagePersistencePlan(imageSet, imageOutputRootPath, imagePrefix) {
31
- var _a;
31
+ var _a, _b;
32
32
  if ((_a = imageSet.fileType) === null || _a === void 0 ? void 0 : _a.ext) {
33
33
  // Since most images come from pasting screenshots, there isn't normally a filename. That's fine, we just make a hash of the url
34
34
  // Images that are stored by notion come to us with a complex url that changes over time, so we pick out the UUID that doesn't change. Example:
35
35
  // https://s3.us-west-2.amazonaws.com/secure.notion-static.com/d1058f46-4d2f-4292-8388-4ad393383439/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20220516%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20220516T233630Z&X-Amz-Expires=3600&X-Amz-Signature=f215704094fcc884d37073b0b108cf6d1c9da9b7d57a898da38bc30c30b4c4b5&X-Amz-SignedHeaders=host&x-id=GetObject
36
- let thingToHash = imageSet.primaryUrl;
37
- const m = /.*secure\.notion-static\.com\/(.*)\//gm.exec(imageSet.primaryUrl);
38
- if (m && m.length > 1) {
39
- thingToHash = m[1];
40
- }
36
+ // But around Sept 2023, they changed the url to be something like:
37
+ // https://prod-files-secure.s3.us-west-2.amazonaws.com/d9a2b712-cf69-4bd6-9d65-87a4ceeacca2/d1bcdc8c-b065-4e40-9a11-392aabeb220e/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20230915%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20230915T161258Z&X-Amz-Expires=3600&X-Amz-Signature=28fca48e65fba86d539c3c4b7676fce1fa0857aa194f7b33dd4a468ecca6ab24&X-Amz-SignedHeaders=host&x-id=GetObject
38
+ // The thing we want is the last UUID before the ?
39
+ const urlBeforeQuery = imageSet.primaryUrl.split("?")[0];
40
+ const thingToHash = (_b = findLastUuid(urlBeforeQuery)) !== null && _b !== void 0 ? _b : urlBeforeQuery;
41
41
  const hash = hashOfString(thingToHash);
42
42
  imageSet.outputFileName = `${hash}.${imageSet.fileType.ext}`;
43
43
  imageSet.primaryFileOutputPath = Path.posix.join((imageOutputRootPath === null || imageOutputRootPath === void 0 ? void 0 : imageOutputRootPath.length) > 0
@@ -58,9 +58,18 @@ function makeImagePersistencePlan(imageSet, imageOutputRootPath, imagePrefix) {
58
58
  }
59
59
  }
60
60
  exports.makeImagePersistencePlan = makeImagePersistencePlan;
61
+ function findLastUuid(url) {
62
+ // Regex for a UUID surrounded by slashes
63
+ const uuidPattern = /(?<=\/)[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}(?=\/)/gi;
64
+ // Find all UUIDs
65
+ const uuids = url.match(uuidPattern);
66
+ // Return the last UUID if any exist, else return null
67
+ return uuids ? uuids[uuids.length - 1].trim() : null;
68
+ }
61
69
  function hashOfString(s) {
62
70
  let hash = 0;
63
71
  for (let i = 0; i < s.length; ++i)
64
72
  hash = Math.imul(31, hash) + s.charCodeAt(i);
65
73
  return Math.abs(hash);
66
74
  }
75
+ exports.hashOfString = hashOfString;
@@ -8,7 +8,6 @@ const ColumnListTransformer_1 = require("../plugins/ColumnListTransformer");
8
8
  const ColumnTransformer_1 = require("../plugins/ColumnTransformer");
9
9
  const EscapeHtmlBlockModifier_1 = require("../plugins/EscapeHtmlBlockModifier");
10
10
  const HeadingTransformer_1 = require("../plugins/HeadingTransformer");
11
- const NumberedListTransformer_1 = require("../plugins/NumberedListTransformer");
12
11
  const TableTransformer_1 = require("../plugins/TableTransformer");
13
12
  const VideoTransformer_1 = require("../plugins/VideoTransformer");
14
13
  const externalLinks_1 = require("../plugins/externalLinks");
@@ -24,7 +23,6 @@ const defaultConfig = {
24
23
  images_1.standardImageTransformer,
25
24
  CalloutTransformer_1.standardCalloutTransformer,
26
25
  TableTransformer_1.standardTableTransformer,
27
- NumberedListTransformer_1.standardNumberedListTransformer,
28
26
  VideoTransformer_1.standardVideoTransformer,
29
27
  // Link modifiers, which are special because they can read metadata from all the pages in order to figure out the correct url
30
28
  internalLinks_1.standardInternalLinkConversion,
@@ -1,58 +1,60 @@
1
- /* Note, I haven't figure out how a Docusaurus app can actually include this, yet.
2
- So currently this has to be duplicated by the client.
3
- */
4
-
5
- /* Copied from
6
- https://github1s.com/NotionX/react-notion-x/blob/master/packages/react-notion-x/src/styles.css#L934
7
- and
8
- https://github1s.com/NotionX/react-notion-x/blob/master/packages/react-notion-x/src/styles.css#L1063
9
- */
10
- .notion-column {
11
- display: flex;
12
- flex-direction: column;
13
- padding-top: 12px;
14
- padding-bottom: 12px;
15
- }
16
-
17
- .notion-column > *:first-child {
18
- margin-top: 0;
19
- margin-left: 0;
20
- margin-right: 0;
21
- }
22
-
23
- .notion-column > *:last-child {
24
- margin-left: 0;
25
- margin-right: 0;
26
- margin-bottom: 0;
27
- }
28
-
29
- .notion-row {
30
- display: flex;
31
- overflow: hidden;
32
- width: 100%;
33
- max-width: 100%;
34
- }
35
-
36
- @media (max-width: 640px) {
37
- .notion-row {
38
- flex-direction: column;
39
- }
40
-
41
- .notion-row .notion-column {
42
- width: 100% !important;
43
- }
44
-
45
- .notion-row .notion-spacer {
46
- display: none;
47
- }
48
- }
49
-
50
- .notion-spacer {
51
- /* This matches the value in ColumnTransformer.ts */
52
- width: calc(min(32px, 4vw));
53
- }
54
-
55
- .notion-spacer:last-child {
56
- display: none;
57
- }
58
- /* End copied from NotionX */
1
+ /* This should be added to the docusaurus.config.js in order to show some notion things correctly.
2
+ See the option: --css-output-directory
3
+ See the docusaurus docs: https://docusaurus.io/docs/styling-layout
4
+ See the use in the docu-notion-sample-site: https://github.com/sillsdev/docu-notion-sample-site/blob/main/docusaurus.config.js
5
+ */
6
+
7
+ /* Copied from
8
+ https://github1s.com/NotionX/react-notion-x/blob/master/packages/react-notion-x/src/styles.css#L934
9
+ and
10
+ https://github1s.com/NotionX/react-notion-x/blob/master/packages/react-notion-x/src/styles.css#L1063
11
+ */
12
+ .notion-column {
13
+ display: flex;
14
+ flex-direction: column;
15
+ padding-top: 12px;
16
+ padding-bottom: 12px;
17
+ }
18
+
19
+ .notion-column > *:first-child {
20
+ margin-top: 0;
21
+ margin-left: 0;
22
+ margin-right: 0;
23
+ }
24
+
25
+ .notion-column > *:last-child {
26
+ margin-left: 0;
27
+ margin-right: 0;
28
+ margin-bottom: 0;
29
+ }
30
+
31
+ .notion-row {
32
+ display: flex;
33
+ overflow: hidden;
34
+ width: 100%;
35
+ max-width: 100%;
36
+ }
37
+
38
+ @media (max-width: 640px) {
39
+ .notion-row {
40
+ flex-direction: column;
41
+ }
42
+
43
+ .notion-row .notion-column {
44
+ width: 100% !important;
45
+ }
46
+
47
+ .notion-row .notion-spacer {
48
+ display: none;
49
+ }
50
+ }
51
+
52
+ .notion-spacer {
53
+ /* This matches the value in ColumnTransformer.ts */
54
+ width: calc(min(32px, 4vw));
55
+ }
56
+
57
+ .notion-spacer:last-child {
58
+ display: none;
59
+ }
60
+ /* End copied from NotionX */
@@ -0,0 +1,97 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ var __importDefault = (this && this.__importDefault) || function (mod) {
12
+ return (mod && mod.__esModule) ? mod : { "default": mod };
13
+ };
14
+ Object.defineProperty(exports, "__esModule", { value: true });
15
+ const notion_to_md_1 = require("notion-to-md");
16
+ const HierarchicalNamedLayoutStrategy_1 = require("./HierarchicalNamedLayoutStrategy");
17
+ const transform_1 = require("./transform");
18
+ const internalLinks_1 = require("./plugins/internalLinks");
19
+ const pull_1 = require("./pull");
20
+ const default_docunotion_config_1 = __importDefault(require("./config/default.docunotion.config"));
21
+ test("Latex Rendering", () => __awaiter(void 0, void 0, void 0, function* () {
22
+ const pages = new Array();
23
+ const counts = {
24
+ output_normally: 0,
25
+ skipped_because_empty: 0,
26
+ skipped_because_status: 0,
27
+ skipped_because_level_cannot_have_content: 0,
28
+ };
29
+ const notionClient = (0, pull_1.initNotionClient)("");
30
+ const layoutStrategy = new HierarchicalNamedLayoutStrategy_1.HierarchicalNamedLayoutStrategy();
31
+ const config = default_docunotion_config_1.default;
32
+ const context = {
33
+ getBlockChildren: (id) => {
34
+ return new Promise(resolve => resolve(new Array()));
35
+ },
36
+ directoryContainingMarkdown: "",
37
+ relativeFilePathToFolderContainingPage: "",
38
+ layoutStrategy: layoutStrategy,
39
+ notionToMarkdown: new notion_to_md_1.NotionToMarkdown({ notionClient }),
40
+ options: {
41
+ notionToken: "",
42
+ rootPage: "",
43
+ locales: [""],
44
+ markdownOutputPath: "",
45
+ imgOutputPath: "",
46
+ imgPrefixInMarkdown: "",
47
+ statusTag: "",
48
+ },
49
+ pages: pages,
50
+ counts: counts,
51
+ imports: [],
52
+ convertNotionLinkToLocalDocusaurusLink: (url) => (0, internalLinks_1.convertInternalUrl)(context, url),
53
+ };
54
+ const blocks = [
55
+ {
56
+ object: "block",
57
+ id: "169e1c47-6706-4518-adca-73086b2738ac",
58
+ parent: {
59
+ type: "page_id",
60
+ page_id: "2acc11a4-82a9-4759-b429-fa011c164888",
61
+ },
62
+ created_time: "2023-08-18T15:51:00.000Z",
63
+ last_edited_time: "2023-08-18T15:51:00.000Z",
64
+ created_by: {
65
+ object: "user",
66
+ id: "af5c163e-82b1-49d1-9f1c-539907bb9fb9",
67
+ },
68
+ last_edited_by: {
69
+ object: "user",
70
+ id: "af5c163e-82b1-49d1-9f1c-539907bb9fb9",
71
+ },
72
+ has_children: false,
73
+ archived: false,
74
+ type: "paragraph",
75
+ paragraph: {
76
+ rich_text: [
77
+ {
78
+ type: "equation",
79
+ equation: { expression: "x" },
80
+ annotations: {
81
+ bold: false,
82
+ italic: false,
83
+ strikethrough: false,
84
+ underline: false,
85
+ code: false,
86
+ color: "default",
87
+ },
88
+ plain_text: "x",
89
+ href: null,
90
+ },
91
+ ],
92
+ color: "default",
93
+ },
94
+ },
95
+ ];
96
+ expect(yield (0, transform_1.getMarkdownFromNotionBlocks)(context, config, blocks)).toContain("$x$");
97
+ }));
@@ -9,9 +9,10 @@ test("primary file with explicit file output path and prefix", () => {
9
9
  fileType: { ext: "png", mime: "image/png" },
10
10
  };
11
11
  (0, MakeImagePersistencePlan_1.makeImagePersistencePlan)(imageSet, "./static/notion_imgs", "/notion_imgs");
12
- expect(imageSet.outputFileName).toBe("463556435.png");
13
- expect(imageSet.primaryFileOutputPath).toBe("static/notion_imgs/463556435.png");
14
- expect(imageSet.filePathToUseInMarkdown).toBe("/notion_imgs/463556435.png");
12
+ const expectedHash = (0, MakeImagePersistencePlan_1.hashOfString)("https://s3.us-west-2.amazonaws.com/primaryImage");
13
+ expect(imageSet.outputFileName).toBe(`${expectedHash}.png`);
14
+ expect(imageSet.primaryFileOutputPath).toBe(`static/notion_imgs/${expectedHash}.png`);
15
+ expect(imageSet.filePathToUseInMarkdown).toBe(`/notion_imgs/${expectedHash}.png`);
15
16
  });
16
17
  test("primary file with defaults for image output path and prefix", () => {
17
18
  const imageSet = {
@@ -21,10 +22,31 @@ test("primary file with defaults for image output path and prefix", () => {
21
22
  fileType: { ext: "png", mime: "image/png" },
22
23
  };
23
24
  (0, MakeImagePersistencePlan_1.makeImagePersistencePlan)(imageSet, "", "");
24
- expect(imageSet.outputFileName).toBe("463556435.png");
25
+ const expectedHash = (0, MakeImagePersistencePlan_1.hashOfString)("https://s3.us-west-2.amazonaws.com/primaryImage");
26
+ expect(imageSet.outputFileName).toBe(`${expectedHash}.png`);
25
27
  // the default behavior is to put the image next to the markdown file
26
- expect(imageSet.primaryFileOutputPath).toBe("/pathToParentSomewhere/463556435.png");
27
- expect(imageSet.filePathToUseInMarkdown).toBe("./463556435.png");
28
+ expect(imageSet.primaryFileOutputPath).toBe(`/pathToParentSomewhere/${expectedHash}.png`);
29
+ expect(imageSet.filePathToUseInMarkdown).toBe(`./${expectedHash}.png`);
30
+ });
31
+ test("properly extract UUID from old-style notion image url", () => {
32
+ const imageSet = {
33
+ primaryUrl: "https://s3.us-west-2.amazonaws.com/secure.notion-static.com/e1058f46-4d2f-4292-8388-4ad393383439/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20220516%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20220516T233630Z&X-Amz-Expires=3600&X-Amz-Signature=f215704094fcc884d37073b0b108cf6d1c9da9b7d57a898da38bc30c30b4c4b5&X-Amz-SignedHeaders=host&x-id=GetObject",
34
+ localizedUrls: [],
35
+ fileType: { ext: "png", mime: "image/png" },
36
+ };
37
+ (0, MakeImagePersistencePlan_1.makeImagePersistencePlan)(imageSet, "./static/notion_imgs", "/notion_imgs");
38
+ const expectedHash = (0, MakeImagePersistencePlan_1.hashOfString)("e1058f46-4d2f-4292-8388-4ad393383439");
39
+ expect(imageSet.outputFileName).toBe(`${expectedHash}.png`);
40
+ });
41
+ test("properly extract UUID from new-style (Sept 2023) notion image url", () => {
42
+ const imageSet = {
43
+ primaryUrl: "https://prod-files-secure.s3.us-west-2.amazonaws.com/d9a2b712-cf69-4bd6-9d65-87a4ceeacca2/d1bcdc8c-b065-4e40-9a11-392aabeb220e/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20230915%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20230915T161258Z&X-Amz-Expires=3600&X-Amz-Signature=28fca48e65fba86d539c3c4b7676fce1fa0857aa194f7b33dd4a468ecca6ab24&X-Amz-SignedHeaders=host&x-id=GetObject",
44
+ localizedUrls: [],
45
+ fileType: { ext: "png", mime: "image/png" },
46
+ };
47
+ (0, MakeImagePersistencePlan_1.makeImagePersistencePlan)(imageSet, "./static/notion_imgs", "/notion_imgs");
48
+ const expectedHash = (0, MakeImagePersistencePlan_1.hashOfString)("d1bcdc8c-b065-4e40-9a11-392aabeb220e");
49
+ expect(imageSet.outputFileName).toBe(`${expectedHash}.png`);
28
50
  });
29
51
  // In order to make image fallback work with other languages, we have to have
30
52
  // a file for each image, in each Docusaurus language directory. This is true
@@ -28,16 +28,14 @@ function notionColumnToMarkdown(notionToMarkdown, getBlockChildren, block) {
28
28
  const { id, has_children } = block; // "any" because the notion api type system is complex with a union that don't know how to help TS to cope with
29
29
  if (!has_children)
30
30
  return "";
31
- const children = yield getBlockChildren(id);
32
- const childrenPromises = children.map((column) => __awaiter(this, void 0, void 0, function* () { return yield notionToMarkdown.blockToMarkdown(column); }));
33
- const childrenStrings = yield Promise.all(childrenPromises);
31
+ const columnChildren = yield getBlockChildren(id);
32
+ const childrenMdBlocksArray = yield Promise.all(columnChildren.map((child) => __awaiter(this, void 0, void 0, function* () { return yield notionToMarkdown.blocksToMarkdown([child]); })));
33
+ const childrenMarkdown = childrenMdBlocksArray.map(mdBlockArray => notionToMarkdown.toMarkdownString(mdBlockArray).parent);
34
34
  const columnWidth = yield getColumnWidth(block);
35
- // note: it would look better in the markup with \n, but that
36
- // causes notion-to-md to give us ":::A" instead of \n for some reason.
37
- return (`<div class='notion-column' style={{width: '${columnWidth}'}}>\n\n${childrenStrings.join("\n\n")}\n\n</div>` +
35
+ return (`<div class='notion-column' style={{width: '${columnWidth}'}}>\n\n${childrenMarkdown.join("\n")}\n</div>` +
38
36
  // Spacer between columns. CSS takes care of hiding this for the last column
39
37
  // and when the screen is too narrow for multiple columns.
40
- `<div className='notion-spacer' />`);
38
+ `<div className='notion-spacer'></div>`);
41
39
  });
42
40
  }
43
41
  // The official API doesn't give us access to the format information, including column_ratio.
@@ -0,0 +1 @@
1
+ export {};
@@ -0,0 +1,118 @@
1
+ "use strict";
2
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
+ return new (P || (P = Promise))(function (resolve, reject) {
5
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
9
+ });
10
+ };
11
+ Object.defineProperty(exports, "__esModule", { value: true });
12
+ const pluginTestRun_1 = require("./pluginTestRun");
13
+ const ColumnTransformer_1 = require("./ColumnTransformer");
14
+ // Even though we can set up most tests with our own children
15
+ // so that we aren't relying on real data from Notion,
16
+ // we can't prevent the notion-to-md library from making an API call
17
+ // every time it processes a block with has_children:true.
18
+ // So for these tests with children, we need any valid API key.
19
+ const runTestsWhichRequireAnyValidApiKey = !!process.env.DOCU_NOTION_INTEGRATION_TOKEN;
20
+ // To test grandchildren, we can't get around notion-to-md making an API call
21
+ // to get real children. So we need a specific notion record.
22
+ // For that reason, we don't try to run these tests unless the user changes this flag.
23
+ // But it is an important test; grandchildren in columns were broken.
24
+ // See https://github.com/sillsdev/docu-notion/issues/70.
25
+ const runManualTestsWhichRequireSpecificNotionRecords = false;
26
+ const columnBlock = {
27
+ object: "block",
28
+ id: "e6d2d7b7-b1ed-464a-86d2-bb5f6be78a03",
29
+ has_children: true,
30
+ type: "column",
31
+ column: {},
32
+ };
33
+ function getResults(children) {
34
+ return __awaiter(this, void 0, void 0, function* () {
35
+ return yield (0, pluginTestRun_1.blocksToMarkdown)({ plugins: [ColumnTransformer_1.standardColumnTransformer] }, [columnBlock], undefined, children, process.env.DOCU_NOTION_INTEGRATION_TOKEN);
36
+ });
37
+ }
38
+ const columnWrapperStart = "<div class='notion-column' style=\\{\\{width: '.*?'\\}\\}>\\n\\n";
39
+ const columnWrapperEnd = "\\n\\n<\\/div><div className='notion-spacer'><\\/div>";
40
+ if (runTestsWhichRequireAnyValidApiKey) {
41
+ columnBlock.has_children = true;
42
+ test("requires API key - column with paragraph", () => __awaiter(void 0, void 0, void 0, function* () {
43
+ const results = yield getResults([getTestParagraphBlock()]);
44
+ expect(results).toMatch(new RegExp(`${columnWrapperStart}\\s*?my paragraph\\s*?${columnWrapperEnd}`));
45
+ }), 20000);
46
+ test("requires API key - column with two paragraphs", () => __awaiter(void 0, void 0, void 0, function* () {
47
+ const results = yield getResults([
48
+ getTestParagraphBlock(1),
49
+ getTestParagraphBlock(2),
50
+ ]);
51
+ expect(results).toMatch(new RegExp(`${columnWrapperStart}\\s*?my paragraph 1\\s+?my paragraph 2\\s*?${columnWrapperEnd}`));
52
+ }), 20000);
53
+ test("requires API key - column with numbered list", () => __awaiter(void 0, void 0, void 0, function* () {
54
+ const results = yield getResults([
55
+ getNumberedListItemBlock(1),
56
+ getNumberedListItemBlock(2),
57
+ ]);
58
+ expect(results).toMatch(new RegExp(`${columnWrapperStart}\\s*?1\\. Numbered list item 1\\s+?2\\. Numbered list item 2\\s*?${columnWrapperEnd}`, "s"));
59
+ }), 20000);
60
+ if (runManualTestsWhichRequireSpecificNotionRecords) {
61
+ test("manual test - requires specific notion record and API key - column with numbered list with sublist", () => __awaiter(void 0, void 0, void 0, function* () {
62
+ const realNumberedListBlock = getNumberedListItemBlock(1);
63
+ realNumberedListBlock.id = "ca08d14b-9b70-4f6f-9d17-9fd74b57afeb";
64
+ realNumberedListBlock.has_children = true;
65
+ const results = yield getResults([realNumberedListBlock]);
66
+ expect(results).toMatch(new RegExp(`${columnWrapperStart}\\s*?1\\. Numbered list item 1\\s+?- unordered sub-bullet\\s*?${columnWrapperEnd}`, "s"));
67
+ }), 20000);
68
+ }
69
+ }
70
+ else {
71
+ // This test prevents an error when runTestsWhichRequireAnyValidApiKey is false
72
+ // due to having a test suite with no tests.
73
+ test("no column transformer tests were run because there is no API key provided", () => {
74
+ expect(true).toBe(true);
75
+ });
76
+ }
77
+ function getNumberedListItemBlock(identifier) {
78
+ const content = identifier
79
+ ? `Numbered list item ${identifier}`
80
+ : `Numbered list item`;
81
+ return {
82
+ object: "block",
83
+ type: "numbered_list_item",
84
+ numbered_list_item: {
85
+ rich_text: [
86
+ {
87
+ type: "text",
88
+ text: { content: content },
89
+ annotations: {
90
+ code: false,
91
+ },
92
+ plain_text: content,
93
+ },
94
+ ],
95
+ },
96
+ };
97
+ }
98
+ function getTestParagraphBlock(identifier) {
99
+ const content = identifier ? `my paragraph ${identifier}` : `my paragraph`;
100
+ return {
101
+ object: "block",
102
+ type: "paragraph",
103
+ paragraph: {
104
+ rich_text: [
105
+ {
106
+ type: "text",
107
+ text: {
108
+ content: content,
109
+ },
110
+ annotations: {
111
+ code: false,
112
+ },
113
+ plain_text: content,
114
+ },
115
+ ],
116
+ },
117
+ };
118
+ }
@@ -7,7 +7,7 @@ exports.gifEmbed = {
7
7
  {
8
8
  // I once saw a gif coming from Notion that wasn't a full
9
9
  // url, which wouldn't work, hence the "http" requirement
10
- regex: /\[.*\]\((http.*(\.(gif|GIF)))\)/,
10
+ regex: /\[.*?\]\((http.*?(\.(gif|GIF)))\)/,
11
11
  replacementPattern: `![]($1)`,
12
12
  },
13
13
  ],
@@ -16,7 +16,7 @@ exports.imgurGifEmbed = {
16
16
  name: "imgur",
17
17
  regexMarkdownModifications: [
18
18
  {
19
- regex: /\[.*\]\((.*imgur\.com\/.*)\)/,
19
+ regex: /\[.*?\]\((.*?imgur\.com\/.*?)\)/,
20
20
  // imgur links to gifs need a .gif at the end, but the url they give you doesn't have one.
21
21
  replacementPattern: `![]($1.gif)`,
22
22
  },
@@ -14,7 +14,6 @@ const pluginTestRun_1 = require("./pluginTestRun");
14
14
  const CalloutTransformer_1 = require("./CalloutTransformer");
15
15
  const externalLinks_1 = require("./externalLinks");
16
16
  const internalLinks_1 = require("./internalLinks");
17
- const NumberedListTransformer_1 = require("./NumberedListTransformer");
18
17
  test("urls that show up as raw text get left that way", () => __awaiter(void 0, void 0, void 0, function* () {
19
18
  const results = yield getMarkdown({
20
19
  type: "paragraph",
@@ -523,7 +522,6 @@ function getMarkdown(block, targetPage) {
523
522
  const config = {
524
523
  plugins: [
525
524
  CalloutTransformer_1.standardCalloutTransformer,
526
- NumberedListTransformer_1.standardNumberedListTransformer,
527
525
  internalLinks_1.standardInternalLinkConversion,
528
526
  externalLinks_1.standardExternalLinkConversion,
529
527
  ],
@@ -1,7 +1,7 @@
1
1
  import { NotionPage } from "../NotionPage";
2
2
  import { IDocuNotionConfig } from "../config/configuration";
3
3
  import { NotionBlock } from "../types";
4
- export declare function blocksToMarkdown(config: IDocuNotionConfig, blocks: NotionBlock[], pages?: NotionPage[]): Promise<string>;
4
+ export declare function blocksToMarkdown(config: IDocuNotionConfig, blocks: NotionBlock[], pages?: NotionPage[], children?: NotionBlock[], validApiKey?: string): Promise<string>;
5
5
  export declare function makeSamplePageObject(options: {
6
6
  slug?: string;
7
7
  name?: string;
@@ -16,9 +16,17 @@ const HierarchicalNamedLayoutStrategy_1 = require("../HierarchicalNamedLayoutStr
16
16
  const NotionPage_1 = require("../NotionPage");
17
17
  const transform_1 = require("../transform");
18
18
  const internalLinks_1 = require("./internalLinks");
19
- function blocksToMarkdown(config, blocks, pages) {
19
+ const pull_1 = require("../pull");
20
+ function blocksToMarkdown(config, blocks, pages,
21
+ // Notes on children:
22
+ // - These children will apply to each block in blocks. (could enhance but not needed yet)
23
+ // - If you are passing in children, it is probably because your parent block has has_children=true.
24
+ // In that case, notion-to-md will make an API call... you'll need to set any validApiKey.
25
+ children, validApiKey) {
20
26
  return __awaiter(this, void 0, void 0, function* () {
21
- const notionClient = new client_1.Client({ auth: "unused" });
27
+ const notionClient = new client_1.Client({
28
+ auth: validApiKey || "unused",
29
+ });
22
30
  const notionToMD = new notion_to_md_1.NotionToMarkdown({
23
31
  notionClient,
24
32
  });
@@ -28,12 +36,12 @@ function blocksToMarkdown(config, blocks, pages) {
28
36
  // }
29
37
  const docunotionContext = {
30
38
  notionToMarkdown: notionToMD,
31
- // TODO when does this actually need to do get some children?
32
- // We can add a children argument to this method, but for the tests
33
- // I have so far, it's not needed.
34
39
  getBlockChildren: (id) => {
40
+ // We call numberChildrenIfNumberedList here because the real getBlockChildren does
41
+ if (children)
42
+ (0, pull_1.numberChildrenIfNumberedList)(children);
35
43
  return new Promise((resolve, reject) => {
36
- resolve([]);
44
+ resolve(children !== null && children !== void 0 ? children : []);
37
45
  });
38
46
  },
39
47
  convertNotionLinkToLocalDocusaurusLink: (url) => {
package/dist/pull.d.ts CHANGED
@@ -1,4 +1,5 @@
1
1
  import { Client } from "@notionhq/client";
2
+ import { ListBlockChildrenResponseResults } from "notion-to-md/build/types";
2
3
  export type DocuNotionOptions = {
3
4
  notionToken: string;
4
5
  rootPage: string;
@@ -11,3 +12,4 @@ export type DocuNotionOptions = {
11
12
  export declare function notionPull(options: DocuNotionOptions): Promise<void>;
12
13
  export declare function executeWithRateLimitAndRetries<T>(label: string, asyncFunction: () => Promise<T>): Promise<T>;
13
14
  export declare function initNotionClient(notionToken: string): Client;
15
+ export declare function numberChildrenIfNumberedList(blocks: ListBlockChildrenResponseResults): void;
package/dist/pull.js CHANGED
@@ -32,7 +32,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
32
32
  });
33
33
  };
34
34
  Object.defineProperty(exports, "__esModule", { value: true });
35
- exports.initNotionClient = exports.executeWithRateLimitAndRetries = exports.notionPull = void 0;
35
+ exports.numberChildrenIfNumberedList = exports.initNotionClient = exports.executeWithRateLimitAndRetries = exports.notionPull = void 0;
36
36
  const fs = __importStar(require("fs-extra"));
37
37
  const notion_to_md_1 = require("notion-to-md");
38
38
  const HierarchicalNamedLayoutStrategy_1 = require("./HierarchicalNamedLayoutStrategy");
@@ -62,7 +62,7 @@ function notionPull(options) {
62
62
  const optionsForLogging = Object.assign({}, options);
63
63
  // Just show the first few letters of the notion token, which start with "secret" anyhow.
64
64
  optionsForLogging.notionToken =
65
- optionsForLogging.notionToken.substring(0, 3) + "...";
65
+ optionsForLogging.notionToken.substring(0, 10) + "...";
66
66
  const config = yield (0, configuration_1.loadConfigAsync)();
67
67
  (0, log_1.verbose)(`Options:${JSON.stringify(optionsForLogging, null, 2)}`);
68
68
  yield (0, images_1.initImageHandling)(options.imgPrefixInMarkdown || options.imgOutputPath || "", options.imgOutputPath || "", options.locales);
@@ -73,6 +73,16 @@ function notionPull(options) {
73
73
  layoutStrategy.setRootDirectoryForMarkdown(options.markdownOutputPath.replace(/\/+$/, "") // trim any trailing slash
74
74
  );
75
75
  (0, log_1.info)("Connecting to Notion...");
76
+ // Do a quick test to see if we can connect to the root so that we can give a better error than just a generic "could not find page" one.
77
+ try {
78
+ yield executeWithRateLimitAndRetries("retrieving root page", () => __awaiter(this, void 0, void 0, function* () {
79
+ yield notionClient.pages.retrieve({ page_id: options.rootPage });
80
+ }));
81
+ }
82
+ catch (e) {
83
+ (0, log_1.error)(`docu-notion could not retrieve the root page from Notion. \r\na) Check that the root page id really is "${options.rootPage}".\r\nb) Check that your Notion API token (the "Integration Secret") is correct. It starts with "${optionsForLogging.notionToken}".\r\nc) Check that your root page includes your "integration" in its "connections".\r\nThis internal error message may help:\r\n ${e.message}`);
84
+ (0, process_1.exit)(1);
85
+ }
76
86
  (0, log_1.group)("Stage 1: walk children of the page named 'Outline', looking for pages...");
77
87
  yield getPagesRecursively(options, "", options.rootPage, 0, true);
78
88
  (0, log_1.logDebug)("getPagesRecursively", JSON.stringify(pages, null, 2));
@@ -213,9 +223,11 @@ function executeWithRateLimitAndRetries(label, asyncFunction) {
213
223
  e.message.includes("timeout") ||
214
224
  e.message.includes("Timeout") ||
215
225
  e.message.includes("limit") ||
216
- e.message.includes("Limit")) {
226
+ e.message.includes("Limit") ||
227
+ (e === null || e === void 0 ? void 0 : e.code) === "notionhq_client_response_error" ||
228
+ (e === null || e === void 0 ? void 0 : e.code) === "service_unavailable") {
217
229
  const secondsToWait = i + 1;
218
- (0, log_1.info)(`While doing "${label}", got error "${e.message}". Will retry after ${secondsToWait}s...`);
230
+ (0, log_1.warning)(`While doing "${label}", got error "${e.message}". Will retry after ${secondsToWait}s...`);
219
231
  yield new Promise(resolve => setTimeout(resolve, 1000 * secondsToWait));
220
232
  }
221
233
  else {
@@ -265,7 +277,9 @@ function getBlockChildren(id) {
265
277
  (0, log_1.error)(`The Notion API returned some blocks that were not full blocks. docu-notion does not handle this yet. Please report it.`);
266
278
  (0, process_1.exit)(1);
267
279
  }
268
- return (_b = overallResult === null || overallResult === void 0 ? void 0 : overallResult.results) !== null && _b !== void 0 ? _b : [];
280
+ const result = (_b = overallResult === null || overallResult === void 0 ? void 0 : overallResult.results) !== null && _b !== void 0 ? _b : [];
281
+ numberChildrenIfNumberedList(result);
282
+ return result;
269
283
  });
270
284
  }
271
285
  function initNotionClient(notionToken) {
@@ -288,3 +302,20 @@ function fromPageId(context, pageId, order, foundDirectlyInOutline) {
288
302
  });
289
303
  });
290
304
  }
305
+ // This function is copied (and renamed from modifyNumberedListObject) from notion-to-md.
306
+ // They always run it on the results of their getBlockChildren.
307
+ // When we use our own getBlockChildren, we need to run it too.
308
+ function numberChildrenIfNumberedList(blocks) {
309
+ let numberedListIndex = 0;
310
+ for (const block of blocks) {
311
+ if ("type" in block && block.type === "numbered_list_item") {
312
+ // add numbers
313
+ // @ts-ignore
314
+ block.numbered_list_item.number = ++numberedListIndex;
315
+ }
316
+ else {
317
+ numberedListIndex = 0;
318
+ }
319
+ }
320
+ }
321
+ exports.numberChildrenIfNumberedList = numberChildrenIfNumberedList;
package/dist/run.d.ts CHANGED
@@ -1 +1 @@
1
- export declare function run(): void;
1
+ export declare function run(): Promise<void>;
package/dist/run.js CHANGED
@@ -1,33 +1,86 @@
1
1
  "use strict";
2
+ var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
3
+ if (k2 === undefined) k2 = k;
4
+ var desc = Object.getOwnPropertyDescriptor(m, k);
5
+ if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
6
+ desc = { enumerable: true, get: function() { return m[k]; } };
7
+ }
8
+ Object.defineProperty(o, k2, desc);
9
+ }) : (function(o, m, k, k2) {
10
+ if (k2 === undefined) k2 = k;
11
+ o[k2] = m[k];
12
+ }));
13
+ var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
14
+ Object.defineProperty(o, "default", { enumerable: true, value: v });
15
+ }) : function(o, v) {
16
+ o["default"] = v;
17
+ });
18
+ var __importStar = (this && this.__importStar) || function (mod) {
19
+ if (mod && mod.__esModule) return mod;
20
+ var result = {};
21
+ if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
22
+ __setModuleDefault(result, mod);
23
+ return result;
24
+ };
25
+ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
26
+ function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
27
+ return new (P || (P = Promise))(function (resolve, reject) {
28
+ function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
29
+ function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
30
+ function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
31
+ step((generator = generator.apply(thisArg, _arguments || [])).next());
32
+ });
33
+ };
34
+ var __importDefault = (this && this.__importDefault) || function (mod) {
35
+ return (mod && mod.__esModule) ? mod : { "default": mod };
36
+ };
2
37
  Object.defineProperty(exports, "__esModule", { value: true });
3
38
  exports.run = void 0;
39
+ const fs = __importStar(require("fs-extra"));
4
40
  const commander_1 = require("commander");
5
41
  const log_1 = require("./log");
6
42
  const pull_1 = require("./pull");
43
+ const path_1 = __importDefault(require("path"));
7
44
  function run() {
8
- const pkg = require("../package.json");
9
- // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
10
- console.log(`docu-notion version ${pkg.version}`);
11
- commander_1.program.name("docu-notion").description("");
12
- commander_1.program.usage("-n <token> -r <root> [options]");
13
- commander_1.program
14
- .requiredOption("-n, --notion-token <string>", "notion api token, which looks like secret_3bc1b50XFYb15123RHF243x43450XFY33250XFYa343")
15
- .requiredOption("-r, --root-page <string>", "The 31 character ID of the page which is the root of your docs page in notion. The code will look like 9120ec9960244ead80fa2ef4bc1bba25. This page must have a child page named 'Outline'")
16
- .option("-m, --markdown-output-path <string>", "Root of the hierarchy for md files. WARNING: node-pull-mdx will delete files from this directory. Note also that if it finds localized images, it will create an i18n/ directory as a sibling.", "./docs")
17
- .option("-t, --status-tag <string>", "Database pages without a Notion page property 'status' matching this will be ignored. Use '*' to ignore status altogether.", "Publish")
18
- .option("--locales <codes>", "Comma-separated list of iso 639-2 codes, the same list as in docusaurus.config.js, minus the primary (i.e. 'en'). This is needed for image localization.", parseLocales, [])
19
- .addOption(new commander_1.Option("-l, --log-level <level>", "Log level").choices([
20
- "info",
21
- "verbose",
22
- "debug",
23
- ]))
24
- .option("-i, --img-output-path <string>", "Path to directory where images will be stored. If this is not included, images will be placed in the same directory as the document that uses them, which then allows for localization of screenshots.")
25
- .option("-p, --img-prefix-in-markdown <string>", "When referencing an image from markdown, prefix with this path instead of the full img-output-path. Should be used only in conjunction with --img-output-path.");
26
- commander_1.program.showHelpAfterError();
27
- commander_1.program.parse();
28
- (0, log_1.setLogLevel)(commander_1.program.opts().logLevel);
29
- console.log(JSON.stringify(commander_1.program.opts));
30
- (0, pull_1.notionPull)(commander_1.program.opts()).then(() => console.log("docu-notion Finished."));
45
+ return __awaiter(this, void 0, void 0, function* () {
46
+ const pkg = require("../package.json");
47
+ // eslint-disable-next-line @typescript-eslint/restrict-template-expressions
48
+ console.log(`docu-notion version ${pkg.version}`);
49
+ commander_1.program.name("docu-notion").description("");
50
+ commander_1.program.usage("-n <token> -r <root> [options]");
51
+ commander_1.program
52
+ .requiredOption("-n, --notion-token <string>", "notion api token, which looks like secret_3bc1b50XFYb15123RHF243x43450XFY33250XFYa343")
53
+ .requiredOption("-r, --root-page <string>", "The 31 character ID of the page which is the root of your docs page in notion. The code will look like 9120ec9960244ead80fa2ef4bc1bba25. This page must have a child page named 'Outline'")
54
+ .option("-m, --markdown-output-path <string>", "Root of the hierarchy for md files. WARNING: docu-notion will delete files from this directory. Note also that if it finds localized images, it will create an i18n/ directory as a sibling.", "./docs")
55
+ .option("--css-output-directory <string>", "docu-notion has a docu-notion-styles.css file that you will need to use to get things like notion columns to look right. This option specifies where that file should be copied to.", "./css")
56
+ .option("-t, --status-tag <string>", "Database pages without a Notion page property 'status' matching this will be ignored. Use '*' to ignore status altogether.", "Publish")
57
+ .option("--locales <codes>", "Comma-separated list of iso 639-2 codes, the same list as in docusaurus.config.js, minus the primary (i.e. 'en'). This is needed for image localization.", parseLocales, [])
58
+ .addOption(new commander_1.Option("-l, --log-level <level>", "Log level").choices([
59
+ "info",
60
+ "verbose",
61
+ "debug",
62
+ ]))
63
+ .option("-i, --img-output-path <string>", "Path to directory where images will be stored. If this is not included, images will be placed in the same directory as the document that uses them, which then allows for localization of screenshots.")
64
+ .option("-p, --img-prefix-in-markdown <string>", "When referencing an image from markdown, prefix with this path instead of the full img-output-path. Should be used only in conjunction with --img-output-path.");
65
+ commander_1.program.showHelpAfterError();
66
+ commander_1.program.parse();
67
+ (0, log_1.setLogLevel)(commander_1.program.opts().logLevel);
68
+ console.log(JSON.stringify(commander_1.program.opts()));
69
+ // copy in the this version of the css needed to make columns (and maybe other things?) work
70
+ let pathToCss = "";
71
+ try {
72
+ pathToCss = require.resolve("@sillsdev/docu-notion/dist/docu-notion-styles.css");
73
+ }
74
+ catch (e) {
75
+ // when testing from the docu-notion project itself:
76
+ pathToCss = "./src/css/docu-notion-styles.css";
77
+ }
78
+ // make any missing parts of the path exist
79
+ fs.ensureDirSync(commander_1.program.opts().cssOutputDirectory);
80
+ fs.copyFileSync(pathToCss, path_1.default.join(commander_1.program.opts().cssOutputDirectory, "docu-notion-styles.css"));
81
+ // pull and convert
82
+ yield (0, pull_1.notionPull)(commander_1.program.opts()).then(() => console.log("docu-notion Finished."));
83
+ });
31
84
  }
32
85
  exports.run = run;
33
86
  function parseLocales(value) {
package/dist/transform.js CHANGED
@@ -15,6 +15,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
15
15
  exports.getMarkdownFromNotionBlocks = exports.getMarkdownForPage = void 0;
16
16
  const chalk_1 = __importDefault(require("chalk"));
17
17
  const log_1 = require("./log");
18
+ const pull_1 = require("./pull");
18
19
  function getMarkdownForPage(config, context, page) {
19
20
  return __awaiter(this, void 0, void 0, function* () {
20
21
  (0, log_1.info)(`Reading & converting page ${page.layoutContext}/${page.nameOrTitle} (${chalk_1.default.blue(page.hasExplicitSlug
@@ -127,8 +128,18 @@ function doTransformsOnMarkdown(context, config, input) {
127
128
  }
128
129
  function doNotionToMarkdown(docunotionContext, blocks) {
129
130
  return __awaiter(this, void 0, void 0, function* () {
130
- const mdBlocks = yield docunotionContext.notionToMarkdown.blocksToMarkdown(blocks);
131
- const markdown = docunotionContext.notionToMarkdown.toMarkdownString(mdBlocks);
131
+ let mdBlocks;
132
+ yield (0, pull_1.executeWithRateLimitAndRetries)("notionToMarkdown.blocksToMarkdown", () => __awaiter(this, void 0, void 0, function* () {
133
+ mdBlocks = yield docunotionContext.notionToMarkdown.blocksToMarkdown(
134
+ // We need to provide a copy of blocks.
135
+ // Calling blocksToMarkdown can modify the values in the blocks. If it does, and then
136
+ // we have to retry, we end up retrying with the modified values, which
137
+ // causes various issues (like using the transformed image url instead of the original one).
138
+ // Note, currently, we don't do anything else with blocks after this.
139
+ // If that changes, we'll need to figure out a more sophisticated approach.
140
+ JSON.parse(JSON.stringify(blocks)));
141
+ }));
142
+ const markdown = docunotionContext.notionToMarkdown.toMarkdownString(mdBlocks).parent || "";
132
143
  return markdown;
133
144
  });
134
145
  }
package/package.json CHANGED
@@ -11,8 +11,9 @@
11
11
  "// typescript check": "",
12
12
  "tsc": "tsc",
13
13
  "// test out with a private sample notion db": "",
14
- "large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE --locales en,fr",
14
+ "large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE --locales en,fr --log-level debug",
15
15
  "pull-test-tagged": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID --log-level debug --status-tag test",
16
+ "pull-test-css": "npm run ts -- --css-output-directory ./test/css -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID --log-level debug --status-tag test",
16
17
  "pull-sample-site": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE --log-level debug",
17
18
  "// test with a semi-stable/public site:": "",
18
19
  "pull-sample": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE -m ./sample --locales en,es,fr,de --log-level verbose",
@@ -35,7 +36,7 @@
35
36
  "markdown-table": "^2.0.0",
36
37
  "node-fetch": "2.6.6",
37
38
  "notion-client": "^4",
38
- "notion-to-md": "2.5.5",
39
+ "notion-to-md": "3.1.1",
39
40
  "path": "^0.12.7",
40
41
  "ts-node": "^10.2.1",
41
42
  "sanitize-filename": "^1.6.3"
@@ -90,5 +91,5 @@
90
91
  "volta": {
91
92
  "node": "18.16.0"
92
93
  },
93
- "version": "0.14.0-alpha.1"
94
+ "version": "0.14.0-alpha.10"
94
95
  }
@@ -1,2 +0,0 @@
1
- import { IPlugin } from "./pluginTypes";
2
- export declare const standardNumberedListTransformer: IPlugin;
@@ -1,55 +0,0 @@
1
- "use strict";
2
- Object.defineProperty(exports, "__esModule", { value: true });
3
- exports.standardNumberedListTransformer = void 0;
4
- // This is mostly what notion-to-markdown would normally do with a block of type
5
- // numbered_list_item. A patch is documented at the end.
6
- function numberedListTransformer(notionToMarkdown, block) {
7
- var _a, _b, _c;
8
- //console.log("got numbered list block " + JSON.stringify(block));
9
- // In this case typescript is not able to index the types properly, hence ignoring the error
10
- // @ts-ignore
11
- const blockContent =
12
- // @ts-ignore
13
- ((_a = block.numbered_list_item) === null || _a === void 0 ? void 0 : _a.text) || ((_b = block.numbered_list_item) === null || _b === void 0 ? void 0 : _b.rich_text) || [];
14
- let parsedData = "";
15
- blockContent.map((content) => {
16
- const annotations = content.annotations;
17
- let plain_text = content.plain_text;
18
- plain_text = notionToMarkdown.annotatePlainText(plain_text, annotations);
19
- if (content["href"]) {
20
- plain_text = `[${plain_text}](${content["href"]})`;
21
- }
22
- parsedData += plain_text;
23
- });
24
- // There is code in notion-to-md which attempts to set an incrementing number
25
- // on each of these. Somehow it fails; in my testing, block.numbered_list_item never
26
- // has a field 'number'. But we don't actually need incrementing numbers;
27
- // markdown will do the numbering if we just make something that looks like
28
- // a member of a numbered list by starting with number followed by period and space.
29
- // I'm keeping the original code in case notion-to-md gets fixed and there is actually
30
- // some reason to use incrementing numbers (it would at least make the markdown more
31
- // human-readable); but this at least works.
32
- // A problem is that in notion, a numbered list may continue after some intermediate
33
- // content. To achieve this in markdown, we'd need to indent the intermediate content
34
- // by a tab. Not only is it difficult to do this, but there appears to be no way to
35
- // know whether we should. The data we get from notion doesn't include the item number,
36
- // and its parent is the page rather than a particular list. So there is no way I can
37
- // see to distinguish a list continuation from a new list. The code here will leave
38
- // it up to markdown to decide whether to start a new list; I believe it will do so
39
- // if it sees any intervening lines that are not list items.
40
- let num = (_c = block.numbered_list_item) === null || _c === void 0 ? void 0 : _c.number;
41
- //console.log("got number " + num?.toString());
42
- if (!num) {
43
- num = 1;
44
- }
45
- return Promise.resolve(`${num}. ${parsedData.trim()}`);
46
- }
47
- exports.standardNumberedListTransformer = {
48
- name: "standardNumberedListTransformer",
49
- notionToMarkdownTransforms: [
50
- {
51
- type: "numbered_list_item",
52
- getStringFromBlock: (context, block) => numberedListTransformer(context.notionToMarkdown, block),
53
- },
54
- ],
55
- };
@@ -1,86 +0,0 @@
1
- "use strict";
2
- var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
3
- function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
4
- return new (P || (P = Promise))(function (resolve, reject) {
5
- function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
6
- function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
7
- function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
8
- step((generator = generator.apply(thisArg, _arguments || [])).next());
9
- });
10
- };
11
- Object.defineProperty(exports, "__esModule", { value: true });
12
- const pluginTestRun_1 = require("./pluginTestRun");
13
- const NumberedListTransformer_1 = require("./NumberedListTransformer");
14
- let block;
15
- beforeEach(() => {
16
- block = {
17
- has_children: false,
18
- archived: false,
19
- type: "callout",
20
- callout: {
21
- rich_text: [
22
- {
23
- type: "text",
24
- text: { content: "This is information callout", link: null },
25
- annotations: {
26
- bold: false,
27
- italic: false,
28
- strikethrough: false,
29
- underline: false,
30
- code: false,
31
- color: "default",
32
- },
33
- plain_text: "This is the callout",
34
- href: null,
35
- },
36
- ],
37
- icon: { type: "emoji", emoji: "ℹ️" },
38
- color: "gray_background",
39
- },
40
- };
41
- });
42
- test("external link inside numbered list, italic preserved", () => __awaiter(void 0, void 0, void 0, function* () {
43
- const config = { plugins: [NumberedListTransformer_1.standardNumberedListTransformer] };
44
- const results = yield (0, pluginTestRun_1.blocksToMarkdown)(config, [
45
- {
46
- type: "numbered_list_item",
47
- numbered_list_item: {
48
- rich_text: [
49
- {
50
- type: "text",
51
- text: { content: "link ", link: null },
52
- annotations: {
53
- bold: false,
54
- italic: false,
55
- strikethrough: false,
56
- underline: false,
57
- code: false,
58
- color: "default",
59
- },
60
- plain_text: "link ",
61
- href: null,
62
- },
63
- {
64
- type: "text",
65
- text: {
66
- content: "github",
67
- link: { url: "https://github.com" },
68
- },
69
- annotations: {
70
- bold: false,
71
- italic: true,
72
- strikethrough: false,
73
- underline: false,
74
- code: false,
75
- color: "default",
76
- },
77
- plain_text: "github",
78
- href: "https://github.com",
79
- },
80
- ],
81
- color: "default",
82
- },
83
- },
84
- ]);
85
- expect(results.trim()).toBe(`1. link [_github_](https://github.com)`);
86
- }));