@sillsdev/docu-notion 0.14.0-alpha.1 → 0.14.0-alpha.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -5
- package/dist/MakeImagePersistencePlan.d.ts +1 -0
- package/dist/MakeImagePersistencePlan.js +16 -7
- package/dist/config/default.docunotion.config.js +0 -2
- package/dist/{notion-styles.css → docu-notion-styles.css} +60 -58
- package/dist/latex.spec.js +97 -0
- package/dist/makeImagePersistencePlan.spec.js +28 -6
- package/dist/plugins/ColumnTransformer.js +5 -7
- package/dist/plugins/ColumnTransformer.spec.d.ts +1 -0
- package/dist/plugins/ColumnTransformer.spec.js +118 -0
- package/dist/plugins/embedTweaks.js +2 -2
- package/dist/plugins/internalLinks.spec.js +0 -2
- package/dist/plugins/pluginTestRun.d.ts +1 -1
- package/dist/plugins/pluginTestRun.js +14 -6
- package/dist/pull.d.ts +2 -0
- package/dist/pull.js +36 -5
- package/dist/run.d.ts +1 -1
- package/dist/run.js +76 -23
- package/dist/transform.js +13 -2
- package/package.json +4 -3
- package/dist/plugins/NumberedListTransformer.d.ts +0 -2
- package/dist/plugins/NumberedListTransformer.js +0 -55
- package/dist/plugins/NumberedListTransformer.spec.js +0 -86
- /package/dist/{plugins/NumberedListTransformer.spec.d.ts → latex.spec.d.ts} +0 -0
package/README.md
CHANGED
|
@@ -10,6 +10,8 @@ Example Site: https://sillsdev.github.io/docu-notion-sample-site/
|
|
|
10
10
|
|
|
11
11
|
First, prepare your markdown-based static file system like [Docusaurus](https://docusaurus.io/). For a shortcut with github actions, search, and deployment to github pages, you can just copy [this template](https://github.com/sillsdev/docu-notion-sample-site).
|
|
12
12
|
|
|
13
|
+
If you do not use the above sample, you will need to manually tell your `docusaurus.config.js` about `docu-notion-styles.css`. See [Styling and Layout](https://docusaurus.io/docs/styling-layout). This stylesheet enables various Notion things to look right, for example multi-column layouts. By default, docu-notion will copy this file to the `css/` directory. There is an option to change that location if you want.
|
|
14
|
+
|
|
13
15
|
## 2. In Notion, duplicate the docu-notion template
|
|
14
16
|
|
|
15
17
|
Go to [this template page](https://hattonjohn.notion.site/Documentation-Template-Docusaurus-0e998b32da3c47edad0f62a25b49818c). Duplicate it into your own workspace.
|
|
@@ -17,13 +19,13 @@ You can name it anything you like, e.g. "Documentation Root".
|
|
|
17
19
|
|
|
18
20
|
## 3. Create a Notion Integration
|
|
19
21
|
|
|
20
|
-
In order for docu-notion to read your site via Notion's API, you need to create what Notion calls an "integration". Follow [these instructions](https://developers.notion.com/docs/getting-started) to make an integration and get your token.
|
|
22
|
+
In order for docu-notion to read your site via Notion's API, you need to create what Notion calls an "integration". Follow [these instructions](https://developers.notion.com/docs/getting-started) to make an integration and get your token. Remember to limit your integration to "READ" access.
|
|
21
23
|
|
|
22
|
-
## 4.
|
|
24
|
+
## 4. Connect your Integration
|
|
23
25
|
|
|
24
|
-
|
|
26
|
+
Go to the page that will be the root of your site. This page should have, as direct children, your "Outline" (required) and "Database" (optional) pages. Follow [these instructions](https://developers.notion.com/docs/create-a-notion-integration#give-your-integration-page-permissions).
|
|
25
27
|
|
|
26
|
-
|
|
28
|
+
<img width="318" alt="image" src="https://github.com/sillsdev/docu-notion/assets/8448/810c6dca-f9ab-4370-93b4-dc1479332af7">
|
|
27
29
|
|
|
28
30
|
## 5. Add your pages under your Outline page.
|
|
29
31
|
|
|
@@ -35,7 +37,7 @@ First, determine the id of your root page by clicking "Share" and looking at the
|
|
|
35
37
|
https://www.notion.so/hattonjohn/My-Docs-0456aa5842946bdbea3a4f37c97a0e5
|
|
36
38
|
means that the id is "0456aa5842946PRETEND4f37c97a0e5".
|
|
37
39
|
|
|
38
|
-
|
|
40
|
+
Try it out:
|
|
39
41
|
|
|
40
42
|
```
|
|
41
43
|
npx @sillsdev/docu-notion -n secret_PRETEND123456789PRETEND123456789PRETEND6789 -r 0456aa5842946PRETEND4f37c97a0e5"
|
|
@@ -23,21 +23,21 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.makeImagePersistencePlan = void 0;
|
|
26
|
+
exports.hashOfString = exports.makeImagePersistencePlan = void 0;
|
|
27
27
|
const Path = __importStar(require("path"));
|
|
28
28
|
const log_1 = require("./log");
|
|
29
29
|
const process_1 = require("process");
|
|
30
30
|
function makeImagePersistencePlan(imageSet, imageOutputRootPath, imagePrefix) {
|
|
31
|
-
var _a;
|
|
31
|
+
var _a, _b;
|
|
32
32
|
if ((_a = imageSet.fileType) === null || _a === void 0 ? void 0 : _a.ext) {
|
|
33
33
|
// Since most images come from pasting screenshots, there isn't normally a filename. That's fine, we just make a hash of the url
|
|
34
34
|
// Images that are stored by notion come to us with a complex url that changes over time, so we pick out the UUID that doesn't change. Example:
|
|
35
35
|
// https://s3.us-west-2.amazonaws.com/secure.notion-static.com/d1058f46-4d2f-4292-8388-4ad393383439/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20220516%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20220516T233630Z&X-Amz-Expires=3600&X-Amz-Signature=f215704094fcc884d37073b0b108cf6d1c9da9b7d57a898da38bc30c30b4c4b5&X-Amz-SignedHeaders=host&x-id=GetObject
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
36
|
+
// But around Sept 2023, they changed the url to be something like:
|
|
37
|
+
// https://prod-files-secure.s3.us-west-2.amazonaws.com/d9a2b712-cf69-4bd6-9d65-87a4ceeacca2/d1bcdc8c-b065-4e40-9a11-392aabeb220e/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20230915%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20230915T161258Z&X-Amz-Expires=3600&X-Amz-Signature=28fca48e65fba86d539c3c4b7676fce1fa0857aa194f7b33dd4a468ecca6ab24&X-Amz-SignedHeaders=host&x-id=GetObject
|
|
38
|
+
// The thing we want is the last UUID before the ?
|
|
39
|
+
const urlBeforeQuery = imageSet.primaryUrl.split("?")[0];
|
|
40
|
+
const thingToHash = (_b = findLastUuid(urlBeforeQuery)) !== null && _b !== void 0 ? _b : urlBeforeQuery;
|
|
41
41
|
const hash = hashOfString(thingToHash);
|
|
42
42
|
imageSet.outputFileName = `${hash}.${imageSet.fileType.ext}`;
|
|
43
43
|
imageSet.primaryFileOutputPath = Path.posix.join((imageOutputRootPath === null || imageOutputRootPath === void 0 ? void 0 : imageOutputRootPath.length) > 0
|
|
@@ -58,9 +58,18 @@ function makeImagePersistencePlan(imageSet, imageOutputRootPath, imagePrefix) {
|
|
|
58
58
|
}
|
|
59
59
|
}
|
|
60
60
|
exports.makeImagePersistencePlan = makeImagePersistencePlan;
|
|
61
|
+
function findLastUuid(url) {
|
|
62
|
+
// Regex for a UUID surrounded by slashes
|
|
63
|
+
const uuidPattern = /(?<=\/)[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}(?=\/)/gi;
|
|
64
|
+
// Find all UUIDs
|
|
65
|
+
const uuids = url.match(uuidPattern);
|
|
66
|
+
// Return the last UUID if any exist, else return null
|
|
67
|
+
return uuids ? uuids[uuids.length - 1].trim() : null;
|
|
68
|
+
}
|
|
61
69
|
function hashOfString(s) {
|
|
62
70
|
let hash = 0;
|
|
63
71
|
for (let i = 0; i < s.length; ++i)
|
|
64
72
|
hash = Math.imul(31, hash) + s.charCodeAt(i);
|
|
65
73
|
return Math.abs(hash);
|
|
66
74
|
}
|
|
75
|
+
exports.hashOfString = hashOfString;
|
|
@@ -8,7 +8,6 @@ const ColumnListTransformer_1 = require("../plugins/ColumnListTransformer");
|
|
|
8
8
|
const ColumnTransformer_1 = require("../plugins/ColumnTransformer");
|
|
9
9
|
const EscapeHtmlBlockModifier_1 = require("../plugins/EscapeHtmlBlockModifier");
|
|
10
10
|
const HeadingTransformer_1 = require("../plugins/HeadingTransformer");
|
|
11
|
-
const NumberedListTransformer_1 = require("../plugins/NumberedListTransformer");
|
|
12
11
|
const TableTransformer_1 = require("../plugins/TableTransformer");
|
|
13
12
|
const VideoTransformer_1 = require("../plugins/VideoTransformer");
|
|
14
13
|
const externalLinks_1 = require("../plugins/externalLinks");
|
|
@@ -24,7 +23,6 @@ const defaultConfig = {
|
|
|
24
23
|
images_1.standardImageTransformer,
|
|
25
24
|
CalloutTransformer_1.standardCalloutTransformer,
|
|
26
25
|
TableTransformer_1.standardTableTransformer,
|
|
27
|
-
NumberedListTransformer_1.standardNumberedListTransformer,
|
|
28
26
|
VideoTransformer_1.standardVideoTransformer,
|
|
29
27
|
// Link modifiers, which are special because they can read metadata from all the pages in order to figure out the correct url
|
|
30
28
|
internalLinks_1.standardInternalLinkConversion,
|
|
@@ -1,58 +1,60 @@
|
|
|
1
|
-
/*
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
https://github1s.com/NotionX/react-notion-x/blob/master/packages/react-notion-x/src/styles.css#
|
|
9
|
-
|
|
10
|
-
.notion-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
margin-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
margin-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
1
|
+
/* This should be added to the docusaurus.config.js in order to show some notion things correctly.
|
|
2
|
+
See the option: --css-output-directory
|
|
3
|
+
See the docusaurus docs: https://docusaurus.io/docs/styling-layout
|
|
4
|
+
See the use in the docu-notion-sample-site: https://github.com/sillsdev/docu-notion-sample-site/blob/main/docusaurus.config.js
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
/* Copied from
|
|
8
|
+
https://github1s.com/NotionX/react-notion-x/blob/master/packages/react-notion-x/src/styles.css#L934
|
|
9
|
+
and
|
|
10
|
+
https://github1s.com/NotionX/react-notion-x/blob/master/packages/react-notion-x/src/styles.css#L1063
|
|
11
|
+
*/
|
|
12
|
+
.notion-column {
|
|
13
|
+
display: flex;
|
|
14
|
+
flex-direction: column;
|
|
15
|
+
padding-top: 12px;
|
|
16
|
+
padding-bottom: 12px;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
.notion-column > *:first-child {
|
|
20
|
+
margin-top: 0;
|
|
21
|
+
margin-left: 0;
|
|
22
|
+
margin-right: 0;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
.notion-column > *:last-child {
|
|
26
|
+
margin-left: 0;
|
|
27
|
+
margin-right: 0;
|
|
28
|
+
margin-bottom: 0;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
.notion-row {
|
|
32
|
+
display: flex;
|
|
33
|
+
overflow: hidden;
|
|
34
|
+
width: 100%;
|
|
35
|
+
max-width: 100%;
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
@media (max-width: 640px) {
|
|
39
|
+
.notion-row {
|
|
40
|
+
flex-direction: column;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
.notion-row .notion-column {
|
|
44
|
+
width: 100% !important;
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
.notion-row .notion-spacer {
|
|
48
|
+
display: none;
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
.notion-spacer {
|
|
53
|
+
/* This matches the value in ColumnTransformer.ts */
|
|
54
|
+
width: calc(min(32px, 4vw));
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
.notion-spacer:last-child {
|
|
58
|
+
display: none;
|
|
59
|
+
}
|
|
60
|
+
/* End copied from NotionX */
|
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
12
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
13
|
+
};
|
|
14
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
15
|
+
const notion_to_md_1 = require("notion-to-md");
|
|
16
|
+
const HierarchicalNamedLayoutStrategy_1 = require("./HierarchicalNamedLayoutStrategy");
|
|
17
|
+
const transform_1 = require("./transform");
|
|
18
|
+
const internalLinks_1 = require("./plugins/internalLinks");
|
|
19
|
+
const pull_1 = require("./pull");
|
|
20
|
+
const default_docunotion_config_1 = __importDefault(require("./config/default.docunotion.config"));
|
|
21
|
+
test("Latex Rendering", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
22
|
+
const pages = new Array();
|
|
23
|
+
const counts = {
|
|
24
|
+
output_normally: 0,
|
|
25
|
+
skipped_because_empty: 0,
|
|
26
|
+
skipped_because_status: 0,
|
|
27
|
+
skipped_because_level_cannot_have_content: 0,
|
|
28
|
+
};
|
|
29
|
+
const notionClient = (0, pull_1.initNotionClient)("");
|
|
30
|
+
const layoutStrategy = new HierarchicalNamedLayoutStrategy_1.HierarchicalNamedLayoutStrategy();
|
|
31
|
+
const config = default_docunotion_config_1.default;
|
|
32
|
+
const context = {
|
|
33
|
+
getBlockChildren: (id) => {
|
|
34
|
+
return new Promise(resolve => resolve(new Array()));
|
|
35
|
+
},
|
|
36
|
+
directoryContainingMarkdown: "",
|
|
37
|
+
relativeFilePathToFolderContainingPage: "",
|
|
38
|
+
layoutStrategy: layoutStrategy,
|
|
39
|
+
notionToMarkdown: new notion_to_md_1.NotionToMarkdown({ notionClient }),
|
|
40
|
+
options: {
|
|
41
|
+
notionToken: "",
|
|
42
|
+
rootPage: "",
|
|
43
|
+
locales: [""],
|
|
44
|
+
markdownOutputPath: "",
|
|
45
|
+
imgOutputPath: "",
|
|
46
|
+
imgPrefixInMarkdown: "",
|
|
47
|
+
statusTag: "",
|
|
48
|
+
},
|
|
49
|
+
pages: pages,
|
|
50
|
+
counts: counts,
|
|
51
|
+
imports: [],
|
|
52
|
+
convertNotionLinkToLocalDocusaurusLink: (url) => (0, internalLinks_1.convertInternalUrl)(context, url),
|
|
53
|
+
};
|
|
54
|
+
const blocks = [
|
|
55
|
+
{
|
|
56
|
+
object: "block",
|
|
57
|
+
id: "169e1c47-6706-4518-adca-73086b2738ac",
|
|
58
|
+
parent: {
|
|
59
|
+
type: "page_id",
|
|
60
|
+
page_id: "2acc11a4-82a9-4759-b429-fa011c164888",
|
|
61
|
+
},
|
|
62
|
+
created_time: "2023-08-18T15:51:00.000Z",
|
|
63
|
+
last_edited_time: "2023-08-18T15:51:00.000Z",
|
|
64
|
+
created_by: {
|
|
65
|
+
object: "user",
|
|
66
|
+
id: "af5c163e-82b1-49d1-9f1c-539907bb9fb9",
|
|
67
|
+
},
|
|
68
|
+
last_edited_by: {
|
|
69
|
+
object: "user",
|
|
70
|
+
id: "af5c163e-82b1-49d1-9f1c-539907bb9fb9",
|
|
71
|
+
},
|
|
72
|
+
has_children: false,
|
|
73
|
+
archived: false,
|
|
74
|
+
type: "paragraph",
|
|
75
|
+
paragraph: {
|
|
76
|
+
rich_text: [
|
|
77
|
+
{
|
|
78
|
+
type: "equation",
|
|
79
|
+
equation: { expression: "x" },
|
|
80
|
+
annotations: {
|
|
81
|
+
bold: false,
|
|
82
|
+
italic: false,
|
|
83
|
+
strikethrough: false,
|
|
84
|
+
underline: false,
|
|
85
|
+
code: false,
|
|
86
|
+
color: "default",
|
|
87
|
+
},
|
|
88
|
+
plain_text: "x",
|
|
89
|
+
href: null,
|
|
90
|
+
},
|
|
91
|
+
],
|
|
92
|
+
color: "default",
|
|
93
|
+
},
|
|
94
|
+
},
|
|
95
|
+
];
|
|
96
|
+
expect(yield (0, transform_1.getMarkdownFromNotionBlocks)(context, config, blocks)).toContain("$x$");
|
|
97
|
+
}));
|
|
@@ -9,9 +9,10 @@ test("primary file with explicit file output path and prefix", () => {
|
|
|
9
9
|
fileType: { ext: "png", mime: "image/png" },
|
|
10
10
|
};
|
|
11
11
|
(0, MakeImagePersistencePlan_1.makeImagePersistencePlan)(imageSet, "./static/notion_imgs", "/notion_imgs");
|
|
12
|
-
|
|
13
|
-
expect(imageSet.
|
|
14
|
-
expect(imageSet.
|
|
12
|
+
const expectedHash = (0, MakeImagePersistencePlan_1.hashOfString)("https://s3.us-west-2.amazonaws.com/primaryImage");
|
|
13
|
+
expect(imageSet.outputFileName).toBe(`${expectedHash}.png`);
|
|
14
|
+
expect(imageSet.primaryFileOutputPath).toBe(`static/notion_imgs/${expectedHash}.png`);
|
|
15
|
+
expect(imageSet.filePathToUseInMarkdown).toBe(`/notion_imgs/${expectedHash}.png`);
|
|
15
16
|
});
|
|
16
17
|
test("primary file with defaults for image output path and prefix", () => {
|
|
17
18
|
const imageSet = {
|
|
@@ -21,10 +22,31 @@ test("primary file with defaults for image output path and prefix", () => {
|
|
|
21
22
|
fileType: { ext: "png", mime: "image/png" },
|
|
22
23
|
};
|
|
23
24
|
(0, MakeImagePersistencePlan_1.makeImagePersistencePlan)(imageSet, "", "");
|
|
24
|
-
|
|
25
|
+
const expectedHash = (0, MakeImagePersistencePlan_1.hashOfString)("https://s3.us-west-2.amazonaws.com/primaryImage");
|
|
26
|
+
expect(imageSet.outputFileName).toBe(`${expectedHash}.png`);
|
|
25
27
|
// the default behavior is to put the image next to the markdown file
|
|
26
|
-
expect(imageSet.primaryFileOutputPath).toBe(
|
|
27
|
-
expect(imageSet.filePathToUseInMarkdown).toBe(
|
|
28
|
+
expect(imageSet.primaryFileOutputPath).toBe(`/pathToParentSomewhere/${expectedHash}.png`);
|
|
29
|
+
expect(imageSet.filePathToUseInMarkdown).toBe(`./${expectedHash}.png`);
|
|
30
|
+
});
|
|
31
|
+
test("properly extract UUID from old-style notion image url", () => {
|
|
32
|
+
const imageSet = {
|
|
33
|
+
primaryUrl: "https://s3.us-west-2.amazonaws.com/secure.notion-static.com/e1058f46-4d2f-4292-8388-4ad393383439/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20220516%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20220516T233630Z&X-Amz-Expires=3600&X-Amz-Signature=f215704094fcc884d37073b0b108cf6d1c9da9b7d57a898da38bc30c30b4c4b5&X-Amz-SignedHeaders=host&x-id=GetObject",
|
|
34
|
+
localizedUrls: [],
|
|
35
|
+
fileType: { ext: "png", mime: "image/png" },
|
|
36
|
+
};
|
|
37
|
+
(0, MakeImagePersistencePlan_1.makeImagePersistencePlan)(imageSet, "./static/notion_imgs", "/notion_imgs");
|
|
38
|
+
const expectedHash = (0, MakeImagePersistencePlan_1.hashOfString)("e1058f46-4d2f-4292-8388-4ad393383439");
|
|
39
|
+
expect(imageSet.outputFileName).toBe(`${expectedHash}.png`);
|
|
40
|
+
});
|
|
41
|
+
test("properly extract UUID from new-style (Sept 2023) notion image url", () => {
|
|
42
|
+
const imageSet = {
|
|
43
|
+
primaryUrl: "https://prod-files-secure.s3.us-west-2.amazonaws.com/d9a2b712-cf69-4bd6-9d65-87a4ceeacca2/d1bcdc8c-b065-4e40-9a11-392aabeb220e/Untitled.png?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIAT73L2G45EIPT3X45%2F20230915%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Date=20230915T161258Z&X-Amz-Expires=3600&X-Amz-Signature=28fca48e65fba86d539c3c4b7676fce1fa0857aa194f7b33dd4a468ecca6ab24&X-Amz-SignedHeaders=host&x-id=GetObject",
|
|
44
|
+
localizedUrls: [],
|
|
45
|
+
fileType: { ext: "png", mime: "image/png" },
|
|
46
|
+
};
|
|
47
|
+
(0, MakeImagePersistencePlan_1.makeImagePersistencePlan)(imageSet, "./static/notion_imgs", "/notion_imgs");
|
|
48
|
+
const expectedHash = (0, MakeImagePersistencePlan_1.hashOfString)("d1bcdc8c-b065-4e40-9a11-392aabeb220e");
|
|
49
|
+
expect(imageSet.outputFileName).toBe(`${expectedHash}.png`);
|
|
28
50
|
});
|
|
29
51
|
// In order to make image fallback work with other languages, we have to have
|
|
30
52
|
// a file for each image, in each Docusaurus language directory. This is true
|
|
@@ -28,16 +28,14 @@ function notionColumnToMarkdown(notionToMarkdown, getBlockChildren, block) {
|
|
|
28
28
|
const { id, has_children } = block; // "any" because the notion api type system is complex with a union that don't know how to help TS to cope with
|
|
29
29
|
if (!has_children)
|
|
30
30
|
return "";
|
|
31
|
-
const
|
|
32
|
-
const
|
|
33
|
-
const
|
|
31
|
+
const columnChildren = yield getBlockChildren(id);
|
|
32
|
+
const childrenMdBlocksArray = yield Promise.all(columnChildren.map((child) => __awaiter(this, void 0, void 0, function* () { return yield notionToMarkdown.blocksToMarkdown([child]); })));
|
|
33
|
+
const childrenMarkdown = childrenMdBlocksArray.map(mdBlockArray => notionToMarkdown.toMarkdownString(mdBlockArray).parent);
|
|
34
34
|
const columnWidth = yield getColumnWidth(block);
|
|
35
|
-
|
|
36
|
-
// causes notion-to-md to give us ":::A" instead of \n for some reason.
|
|
37
|
-
return (`<div class='notion-column' style={{width: '${columnWidth}'}}>\n\n${childrenStrings.join("\n\n")}\n\n</div>` +
|
|
35
|
+
return (`<div class='notion-column' style={{width: '${columnWidth}'}}>\n\n${childrenMarkdown.join("\n")}\n</div>` +
|
|
38
36
|
// Spacer between columns. CSS takes care of hiding this for the last column
|
|
39
37
|
// and when the screen is too narrow for multiple columns.
|
|
40
|
-
`<div className='notion-spacer'
|
|
38
|
+
`<div className='notion-spacer'></div>`);
|
|
41
39
|
});
|
|
42
40
|
}
|
|
43
41
|
// The official API doesn't give us access to the format information, including column_ratio.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
const pluginTestRun_1 = require("./pluginTestRun");
|
|
13
|
+
const ColumnTransformer_1 = require("./ColumnTransformer");
|
|
14
|
+
// Even though we can set up most tests with our own children
|
|
15
|
+
// so that we aren't relying on real data from Notion,
|
|
16
|
+
// we can't prevent the notion-to-md library from making an API call
|
|
17
|
+
// every time it processes a block with has_children:true.
|
|
18
|
+
// So for these tests with children, we need any valid API key.
|
|
19
|
+
const runTestsWhichRequireAnyValidApiKey = !!process.env.DOCU_NOTION_INTEGRATION_TOKEN;
|
|
20
|
+
// To test grandchildren, we can't get around notion-to-md making an API call
|
|
21
|
+
// to get real children. So we need a specific notion record.
|
|
22
|
+
// For that reason, we don't try to run these tests unless the user changes this flag.
|
|
23
|
+
// But it is an important test; grandchildren in columns were broken.
|
|
24
|
+
// See https://github.com/sillsdev/docu-notion/issues/70.
|
|
25
|
+
const runManualTestsWhichRequireSpecificNotionRecords = false;
|
|
26
|
+
const columnBlock = {
|
|
27
|
+
object: "block",
|
|
28
|
+
id: "e6d2d7b7-b1ed-464a-86d2-bb5f6be78a03",
|
|
29
|
+
has_children: true,
|
|
30
|
+
type: "column",
|
|
31
|
+
column: {},
|
|
32
|
+
};
|
|
33
|
+
function getResults(children) {
|
|
34
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
35
|
+
return yield (0, pluginTestRun_1.blocksToMarkdown)({ plugins: [ColumnTransformer_1.standardColumnTransformer] }, [columnBlock], undefined, children, process.env.DOCU_NOTION_INTEGRATION_TOKEN);
|
|
36
|
+
});
|
|
37
|
+
}
|
|
38
|
+
const columnWrapperStart = "<div class='notion-column' style=\\{\\{width: '.*?'\\}\\}>\\n\\n";
|
|
39
|
+
const columnWrapperEnd = "\\n\\n<\\/div><div className='notion-spacer'><\\/div>";
|
|
40
|
+
if (runTestsWhichRequireAnyValidApiKey) {
|
|
41
|
+
columnBlock.has_children = true;
|
|
42
|
+
test("requires API key - column with paragraph", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
43
|
+
const results = yield getResults([getTestParagraphBlock()]);
|
|
44
|
+
expect(results).toMatch(new RegExp(`${columnWrapperStart}\\s*?my paragraph\\s*?${columnWrapperEnd}`));
|
|
45
|
+
}), 20000);
|
|
46
|
+
test("requires API key - column with two paragraphs", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
47
|
+
const results = yield getResults([
|
|
48
|
+
getTestParagraphBlock(1),
|
|
49
|
+
getTestParagraphBlock(2),
|
|
50
|
+
]);
|
|
51
|
+
expect(results).toMatch(new RegExp(`${columnWrapperStart}\\s*?my paragraph 1\\s+?my paragraph 2\\s*?${columnWrapperEnd}`));
|
|
52
|
+
}), 20000);
|
|
53
|
+
test("requires API key - column with numbered list", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
54
|
+
const results = yield getResults([
|
|
55
|
+
getNumberedListItemBlock(1),
|
|
56
|
+
getNumberedListItemBlock(2),
|
|
57
|
+
]);
|
|
58
|
+
expect(results).toMatch(new RegExp(`${columnWrapperStart}\\s*?1\\. Numbered list item 1\\s+?2\\. Numbered list item 2\\s*?${columnWrapperEnd}`, "s"));
|
|
59
|
+
}), 20000);
|
|
60
|
+
if (runManualTestsWhichRequireSpecificNotionRecords) {
|
|
61
|
+
test("manual test - requires specific notion record and API key - column with numbered list with sublist", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
62
|
+
const realNumberedListBlock = getNumberedListItemBlock(1);
|
|
63
|
+
realNumberedListBlock.id = "ca08d14b-9b70-4f6f-9d17-9fd74b57afeb";
|
|
64
|
+
realNumberedListBlock.has_children = true;
|
|
65
|
+
const results = yield getResults([realNumberedListBlock]);
|
|
66
|
+
expect(results).toMatch(new RegExp(`${columnWrapperStart}\\s*?1\\. Numbered list item 1\\s+?- unordered sub-bullet\\s*?${columnWrapperEnd}`, "s"));
|
|
67
|
+
}), 20000);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
else {
|
|
71
|
+
// This test prevents an error when runTestsWhichRequireAnyValidApiKey is false
|
|
72
|
+
// due to having a test suite with no tests.
|
|
73
|
+
test("no column transformer tests were run because there is no API key provided", () => {
|
|
74
|
+
expect(true).toBe(true);
|
|
75
|
+
});
|
|
76
|
+
}
|
|
77
|
+
function getNumberedListItemBlock(identifier) {
|
|
78
|
+
const content = identifier
|
|
79
|
+
? `Numbered list item ${identifier}`
|
|
80
|
+
: `Numbered list item`;
|
|
81
|
+
return {
|
|
82
|
+
object: "block",
|
|
83
|
+
type: "numbered_list_item",
|
|
84
|
+
numbered_list_item: {
|
|
85
|
+
rich_text: [
|
|
86
|
+
{
|
|
87
|
+
type: "text",
|
|
88
|
+
text: { content: content },
|
|
89
|
+
annotations: {
|
|
90
|
+
code: false,
|
|
91
|
+
},
|
|
92
|
+
plain_text: content,
|
|
93
|
+
},
|
|
94
|
+
],
|
|
95
|
+
},
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
function getTestParagraphBlock(identifier) {
|
|
99
|
+
const content = identifier ? `my paragraph ${identifier}` : `my paragraph`;
|
|
100
|
+
return {
|
|
101
|
+
object: "block",
|
|
102
|
+
type: "paragraph",
|
|
103
|
+
paragraph: {
|
|
104
|
+
rich_text: [
|
|
105
|
+
{
|
|
106
|
+
type: "text",
|
|
107
|
+
text: {
|
|
108
|
+
content: content,
|
|
109
|
+
},
|
|
110
|
+
annotations: {
|
|
111
|
+
code: false,
|
|
112
|
+
},
|
|
113
|
+
plain_text: content,
|
|
114
|
+
},
|
|
115
|
+
],
|
|
116
|
+
},
|
|
117
|
+
};
|
|
118
|
+
}
|
|
@@ -7,7 +7,7 @@ exports.gifEmbed = {
|
|
|
7
7
|
{
|
|
8
8
|
// I once saw a gif coming from Notion that wasn't a full
|
|
9
9
|
// url, which wouldn't work, hence the "http" requirement
|
|
10
|
-
regex: /\[
|
|
10
|
+
regex: /\[.*?\]\((http.*?(\.(gif|GIF)))\)/,
|
|
11
11
|
replacementPattern: ``,
|
|
12
12
|
},
|
|
13
13
|
],
|
|
@@ -16,7 +16,7 @@ exports.imgurGifEmbed = {
|
|
|
16
16
|
name: "imgur",
|
|
17
17
|
regexMarkdownModifications: [
|
|
18
18
|
{
|
|
19
|
-
regex: /\[
|
|
19
|
+
regex: /\[.*?\]\((.*?imgur\.com\/.*?)\)/,
|
|
20
20
|
// imgur links to gifs need a .gif at the end, but the url they give you doesn't have one.
|
|
21
21
|
replacementPattern: ``,
|
|
22
22
|
},
|
|
@@ -14,7 +14,6 @@ const pluginTestRun_1 = require("./pluginTestRun");
|
|
|
14
14
|
const CalloutTransformer_1 = require("./CalloutTransformer");
|
|
15
15
|
const externalLinks_1 = require("./externalLinks");
|
|
16
16
|
const internalLinks_1 = require("./internalLinks");
|
|
17
|
-
const NumberedListTransformer_1 = require("./NumberedListTransformer");
|
|
18
17
|
test("urls that show up as raw text get left that way", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
19
18
|
const results = yield getMarkdown({
|
|
20
19
|
type: "paragraph",
|
|
@@ -523,7 +522,6 @@ function getMarkdown(block, targetPage) {
|
|
|
523
522
|
const config = {
|
|
524
523
|
plugins: [
|
|
525
524
|
CalloutTransformer_1.standardCalloutTransformer,
|
|
526
|
-
NumberedListTransformer_1.standardNumberedListTransformer,
|
|
527
525
|
internalLinks_1.standardInternalLinkConversion,
|
|
528
526
|
externalLinks_1.standardExternalLinkConversion,
|
|
529
527
|
],
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { NotionPage } from "../NotionPage";
|
|
2
2
|
import { IDocuNotionConfig } from "../config/configuration";
|
|
3
3
|
import { NotionBlock } from "../types";
|
|
4
|
-
export declare function blocksToMarkdown(config: IDocuNotionConfig, blocks: NotionBlock[], pages?: NotionPage[]): Promise<string>;
|
|
4
|
+
export declare function blocksToMarkdown(config: IDocuNotionConfig, blocks: NotionBlock[], pages?: NotionPage[], children?: NotionBlock[], validApiKey?: string): Promise<string>;
|
|
5
5
|
export declare function makeSamplePageObject(options: {
|
|
6
6
|
slug?: string;
|
|
7
7
|
name?: string;
|
|
@@ -16,9 +16,17 @@ const HierarchicalNamedLayoutStrategy_1 = require("../HierarchicalNamedLayoutStr
|
|
|
16
16
|
const NotionPage_1 = require("../NotionPage");
|
|
17
17
|
const transform_1 = require("../transform");
|
|
18
18
|
const internalLinks_1 = require("./internalLinks");
|
|
19
|
-
|
|
19
|
+
const pull_1 = require("../pull");
|
|
20
|
+
function blocksToMarkdown(config, blocks, pages,
|
|
21
|
+
// Notes on children:
|
|
22
|
+
// - These children will apply to each block in blocks. (could enhance but not needed yet)
|
|
23
|
+
// - If you are passing in children, it is probably because your parent block has has_children=true.
|
|
24
|
+
// In that case, notion-to-md will make an API call... you'll need to set any validApiKey.
|
|
25
|
+
children, validApiKey) {
|
|
20
26
|
return __awaiter(this, void 0, void 0, function* () {
|
|
21
|
-
const notionClient = new client_1.Client({
|
|
27
|
+
const notionClient = new client_1.Client({
|
|
28
|
+
auth: validApiKey || "unused",
|
|
29
|
+
});
|
|
22
30
|
const notionToMD = new notion_to_md_1.NotionToMarkdown({
|
|
23
31
|
notionClient,
|
|
24
32
|
});
|
|
@@ -28,12 +36,12 @@ function blocksToMarkdown(config, blocks, pages) {
|
|
|
28
36
|
// }
|
|
29
37
|
const docunotionContext = {
|
|
30
38
|
notionToMarkdown: notionToMD,
|
|
31
|
-
// TODO when does this actually need to do get some children?
|
|
32
|
-
// We can add a children argument to this method, but for the tests
|
|
33
|
-
// I have so far, it's not needed.
|
|
34
39
|
getBlockChildren: (id) => {
|
|
40
|
+
// We call numberChildrenIfNumberedList here because the real getBlockChildren does
|
|
41
|
+
if (children)
|
|
42
|
+
(0, pull_1.numberChildrenIfNumberedList)(children);
|
|
35
43
|
return new Promise((resolve, reject) => {
|
|
36
|
-
resolve([]);
|
|
44
|
+
resolve(children !== null && children !== void 0 ? children : []);
|
|
37
45
|
});
|
|
38
46
|
},
|
|
39
47
|
convertNotionLinkToLocalDocusaurusLink: (url) => {
|
package/dist/pull.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { Client } from "@notionhq/client";
|
|
2
|
+
import { ListBlockChildrenResponseResults } from "notion-to-md/build/types";
|
|
2
3
|
export type DocuNotionOptions = {
|
|
3
4
|
notionToken: string;
|
|
4
5
|
rootPage: string;
|
|
@@ -11,3 +12,4 @@ export type DocuNotionOptions = {
|
|
|
11
12
|
export declare function notionPull(options: DocuNotionOptions): Promise<void>;
|
|
12
13
|
export declare function executeWithRateLimitAndRetries<T>(label: string, asyncFunction: () => Promise<T>): Promise<T>;
|
|
13
14
|
export declare function initNotionClient(notionToken: string): Client;
|
|
15
|
+
export declare function numberChildrenIfNumberedList(blocks: ListBlockChildrenResponseResults): void;
|
package/dist/pull.js
CHANGED
|
@@ -32,7 +32,7 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
|
|
|
32
32
|
});
|
|
33
33
|
};
|
|
34
34
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
35
|
-
exports.initNotionClient = exports.executeWithRateLimitAndRetries = exports.notionPull = void 0;
|
|
35
|
+
exports.numberChildrenIfNumberedList = exports.initNotionClient = exports.executeWithRateLimitAndRetries = exports.notionPull = void 0;
|
|
36
36
|
const fs = __importStar(require("fs-extra"));
|
|
37
37
|
const notion_to_md_1 = require("notion-to-md");
|
|
38
38
|
const HierarchicalNamedLayoutStrategy_1 = require("./HierarchicalNamedLayoutStrategy");
|
|
@@ -62,7 +62,7 @@ function notionPull(options) {
|
|
|
62
62
|
const optionsForLogging = Object.assign({}, options);
|
|
63
63
|
// Just show the first few letters of the notion token, which start with "secret" anyhow.
|
|
64
64
|
optionsForLogging.notionToken =
|
|
65
|
-
optionsForLogging.notionToken.substring(0,
|
|
65
|
+
optionsForLogging.notionToken.substring(0, 10) + "...";
|
|
66
66
|
const config = yield (0, configuration_1.loadConfigAsync)();
|
|
67
67
|
(0, log_1.verbose)(`Options:${JSON.stringify(optionsForLogging, null, 2)}`);
|
|
68
68
|
yield (0, images_1.initImageHandling)(options.imgPrefixInMarkdown || options.imgOutputPath || "", options.imgOutputPath || "", options.locales);
|
|
@@ -73,6 +73,16 @@ function notionPull(options) {
|
|
|
73
73
|
layoutStrategy.setRootDirectoryForMarkdown(options.markdownOutputPath.replace(/\/+$/, "") // trim any trailing slash
|
|
74
74
|
);
|
|
75
75
|
(0, log_1.info)("Connecting to Notion...");
|
|
76
|
+
// Do a quick test to see if we can connect to the root so that we can give a better error than just a generic "could not find page" one.
|
|
77
|
+
try {
|
|
78
|
+
yield executeWithRateLimitAndRetries("retrieving root page", () => __awaiter(this, void 0, void 0, function* () {
|
|
79
|
+
yield notionClient.pages.retrieve({ page_id: options.rootPage });
|
|
80
|
+
}));
|
|
81
|
+
}
|
|
82
|
+
catch (e) {
|
|
83
|
+
(0, log_1.error)(`docu-notion could not retrieve the root page from Notion. \r\na) Check that the root page id really is "${options.rootPage}".\r\nb) Check that your Notion API token (the "Integration Secret") is correct. It starts with "${optionsForLogging.notionToken}".\r\nc) Check that your root page includes your "integration" in its "connections".\r\nThis internal error message may help:\r\n ${e.message}`);
|
|
84
|
+
(0, process_1.exit)(1);
|
|
85
|
+
}
|
|
76
86
|
(0, log_1.group)("Stage 1: walk children of the page named 'Outline', looking for pages...");
|
|
77
87
|
yield getPagesRecursively(options, "", options.rootPage, 0, true);
|
|
78
88
|
(0, log_1.logDebug)("getPagesRecursively", JSON.stringify(pages, null, 2));
|
|
@@ -213,9 +223,11 @@ function executeWithRateLimitAndRetries(label, asyncFunction) {
|
|
|
213
223
|
e.message.includes("timeout") ||
|
|
214
224
|
e.message.includes("Timeout") ||
|
|
215
225
|
e.message.includes("limit") ||
|
|
216
|
-
e.message.includes("Limit")
|
|
226
|
+
e.message.includes("Limit") ||
|
|
227
|
+
(e === null || e === void 0 ? void 0 : e.code) === "notionhq_client_response_error" ||
|
|
228
|
+
(e === null || e === void 0 ? void 0 : e.code) === "service_unavailable") {
|
|
217
229
|
const secondsToWait = i + 1;
|
|
218
|
-
(0, log_1.
|
|
230
|
+
(0, log_1.warning)(`While doing "${label}", got error "${e.message}". Will retry after ${secondsToWait}s...`);
|
|
219
231
|
yield new Promise(resolve => setTimeout(resolve, 1000 * secondsToWait));
|
|
220
232
|
}
|
|
221
233
|
else {
|
|
@@ -265,7 +277,9 @@ function getBlockChildren(id) {
|
|
|
265
277
|
(0, log_1.error)(`The Notion API returned some blocks that were not full blocks. docu-notion does not handle this yet. Please report it.`);
|
|
266
278
|
(0, process_1.exit)(1);
|
|
267
279
|
}
|
|
268
|
-
|
|
280
|
+
const result = (_b = overallResult === null || overallResult === void 0 ? void 0 : overallResult.results) !== null && _b !== void 0 ? _b : [];
|
|
281
|
+
numberChildrenIfNumberedList(result);
|
|
282
|
+
return result;
|
|
269
283
|
});
|
|
270
284
|
}
|
|
271
285
|
function initNotionClient(notionToken) {
|
|
@@ -288,3 +302,20 @@ function fromPageId(context, pageId, order, foundDirectlyInOutline) {
|
|
|
288
302
|
});
|
|
289
303
|
});
|
|
290
304
|
}
|
|
305
|
+
// This function is copied (and renamed from modifyNumberedListObject) from notion-to-md.
|
|
306
|
+
// They always run it on the results of their getBlockChildren.
|
|
307
|
+
// When we use our own getBlockChildren, we need to run it too.
|
|
308
|
+
function numberChildrenIfNumberedList(blocks) {
|
|
309
|
+
let numberedListIndex = 0;
|
|
310
|
+
for (const block of blocks) {
|
|
311
|
+
if ("type" in block && block.type === "numbered_list_item") {
|
|
312
|
+
// add numbers
|
|
313
|
+
// @ts-ignore
|
|
314
|
+
block.numbered_list_item.number = ++numberedListIndex;
|
|
315
|
+
}
|
|
316
|
+
else {
|
|
317
|
+
numberedListIndex = 0;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
320
|
+
}
|
|
321
|
+
exports.numberChildrenIfNumberedList = numberChildrenIfNumberedList;
|
package/dist/run.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare function run(): void
|
|
1
|
+
export declare function run(): Promise<void>;
|
package/dist/run.js
CHANGED
|
@@ -1,33 +1,86 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
26
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
27
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
28
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
29
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
30
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
31
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
32
|
+
});
|
|
33
|
+
};
|
|
34
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
35
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
36
|
+
};
|
|
2
37
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
38
|
exports.run = void 0;
|
|
39
|
+
const fs = __importStar(require("fs-extra"));
|
|
4
40
|
const commander_1 = require("commander");
|
|
5
41
|
const log_1 = require("./log");
|
|
6
42
|
const pull_1 = require("./pull");
|
|
43
|
+
const path_1 = __importDefault(require("path"));
|
|
7
44
|
function run() {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
.
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
45
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
46
|
+
const pkg = require("../package.json");
|
|
47
|
+
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
|
|
48
|
+
console.log(`docu-notion version ${pkg.version}`);
|
|
49
|
+
commander_1.program.name("docu-notion").description("");
|
|
50
|
+
commander_1.program.usage("-n <token> -r <root> [options]");
|
|
51
|
+
commander_1.program
|
|
52
|
+
.requiredOption("-n, --notion-token <string>", "notion api token, which looks like secret_3bc1b50XFYb15123RHF243x43450XFY33250XFYa343")
|
|
53
|
+
.requiredOption("-r, --root-page <string>", "The 31 character ID of the page which is the root of your docs page in notion. The code will look like 9120ec9960244ead80fa2ef4bc1bba25. This page must have a child page named 'Outline'")
|
|
54
|
+
.option("-m, --markdown-output-path <string>", "Root of the hierarchy for md files. WARNING: docu-notion will delete files from this directory. Note also that if it finds localized images, it will create an i18n/ directory as a sibling.", "./docs")
|
|
55
|
+
.option("--css-output-directory <string>", "docu-notion has a docu-notion-styles.css file that you will need to use to get things like notion columns to look right. This option specifies where that file should be copied to.", "./css")
|
|
56
|
+
.option("-t, --status-tag <string>", "Database pages without a Notion page property 'status' matching this will be ignored. Use '*' to ignore status altogether.", "Publish")
|
|
57
|
+
.option("--locales <codes>", "Comma-separated list of iso 639-2 codes, the same list as in docusaurus.config.js, minus the primary (i.e. 'en'). This is needed for image localization.", parseLocales, [])
|
|
58
|
+
.addOption(new commander_1.Option("-l, --log-level <level>", "Log level").choices([
|
|
59
|
+
"info",
|
|
60
|
+
"verbose",
|
|
61
|
+
"debug",
|
|
62
|
+
]))
|
|
63
|
+
.option("-i, --img-output-path <string>", "Path to directory where images will be stored. If this is not included, images will be placed in the same directory as the document that uses them, which then allows for localization of screenshots.")
|
|
64
|
+
.option("-p, --img-prefix-in-markdown <string>", "When referencing an image from markdown, prefix with this path instead of the full img-output-path. Should be used only in conjunction with --img-output-path.");
|
|
65
|
+
commander_1.program.showHelpAfterError();
|
|
66
|
+
commander_1.program.parse();
|
|
67
|
+
(0, log_1.setLogLevel)(commander_1.program.opts().logLevel);
|
|
68
|
+
console.log(JSON.stringify(commander_1.program.opts()));
|
|
69
|
+
// copy in the this version of the css needed to make columns (and maybe other things?) work
|
|
70
|
+
let pathToCss = "";
|
|
71
|
+
try {
|
|
72
|
+
pathToCss = require.resolve("@sillsdev/docu-notion/dist/docu-notion-styles.css");
|
|
73
|
+
}
|
|
74
|
+
catch (e) {
|
|
75
|
+
// when testing from the docu-notion project itself:
|
|
76
|
+
pathToCss = "./src/css/docu-notion-styles.css";
|
|
77
|
+
}
|
|
78
|
+
// make any missing parts of the path exist
|
|
79
|
+
fs.ensureDirSync(commander_1.program.opts().cssOutputDirectory);
|
|
80
|
+
fs.copyFileSync(pathToCss, path_1.default.join(commander_1.program.opts().cssOutputDirectory, "docu-notion-styles.css"));
|
|
81
|
+
// pull and convert
|
|
82
|
+
yield (0, pull_1.notionPull)(commander_1.program.opts()).then(() => console.log("docu-notion Finished."));
|
|
83
|
+
});
|
|
31
84
|
}
|
|
32
85
|
exports.run = run;
|
|
33
86
|
function parseLocales(value) {
|
package/dist/transform.js
CHANGED
|
@@ -15,6 +15,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
|
15
15
|
exports.getMarkdownFromNotionBlocks = exports.getMarkdownForPage = void 0;
|
|
16
16
|
const chalk_1 = __importDefault(require("chalk"));
|
|
17
17
|
const log_1 = require("./log");
|
|
18
|
+
const pull_1 = require("./pull");
|
|
18
19
|
function getMarkdownForPage(config, context, page) {
|
|
19
20
|
return __awaiter(this, void 0, void 0, function* () {
|
|
20
21
|
(0, log_1.info)(`Reading & converting page ${page.layoutContext}/${page.nameOrTitle} (${chalk_1.default.blue(page.hasExplicitSlug
|
|
@@ -127,8 +128,18 @@ function doTransformsOnMarkdown(context, config, input) {
|
|
|
127
128
|
}
|
|
128
129
|
function doNotionToMarkdown(docunotionContext, blocks) {
|
|
129
130
|
return __awaiter(this, void 0, void 0, function* () {
|
|
130
|
-
|
|
131
|
-
|
|
131
|
+
let mdBlocks;
|
|
132
|
+
yield (0, pull_1.executeWithRateLimitAndRetries)("notionToMarkdown.blocksToMarkdown", () => __awaiter(this, void 0, void 0, function* () {
|
|
133
|
+
mdBlocks = yield docunotionContext.notionToMarkdown.blocksToMarkdown(
|
|
134
|
+
// We need to provide a copy of blocks.
|
|
135
|
+
// Calling blocksToMarkdown can modify the values in the blocks. If it does, and then
|
|
136
|
+
// we have to retry, we end up retrying with the modified values, which
|
|
137
|
+
// causes various issues (like using the transformed image url instead of the original one).
|
|
138
|
+
// Note, currently, we don't do anything else with blocks after this.
|
|
139
|
+
// If that changes, we'll need to figure out a more sophisticated approach.
|
|
140
|
+
JSON.parse(JSON.stringify(blocks)));
|
|
141
|
+
}));
|
|
142
|
+
const markdown = docunotionContext.notionToMarkdown.toMarkdownString(mdBlocks).parent || "";
|
|
132
143
|
return markdown;
|
|
133
144
|
});
|
|
134
145
|
}
|
package/package.json
CHANGED
|
@@ -11,8 +11,9 @@
|
|
|
11
11
|
"// typescript check": "",
|
|
12
12
|
"tsc": "tsc",
|
|
13
13
|
"// test out with a private sample notion db": "",
|
|
14
|
-
"large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE --locales en,fr",
|
|
14
|
+
"large-site-test": "npm run ts -- -n $SIL_BLOOM_DOCS_NOTION_TOKEN -r $SIL_BLOOM_DOCS_NOTION_ROOT_PAGE --locales en,fr --log-level debug",
|
|
15
15
|
"pull-test-tagged": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID --log-level debug --status-tag test",
|
|
16
|
+
"pull-test-css": "npm run ts -- --css-output-directory ./test/css -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_TEST_ROOT_PAGE_ID --log-level debug --status-tag test",
|
|
16
17
|
"pull-sample-site": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE --log-level debug",
|
|
17
18
|
"// test with a semi-stable/public site:": "",
|
|
18
19
|
"pull-sample": "npm run ts -- -n $DOCU_NOTION_INTEGRATION_TOKEN -r $DOCU_NOTION_SAMPLE_ROOT_PAGE -m ./sample --locales en,es,fr,de --log-level verbose",
|
|
@@ -35,7 +36,7 @@
|
|
|
35
36
|
"markdown-table": "^2.0.0",
|
|
36
37
|
"node-fetch": "2.6.6",
|
|
37
38
|
"notion-client": "^4",
|
|
38
|
-
"notion-to-md": "
|
|
39
|
+
"notion-to-md": "3.1.1",
|
|
39
40
|
"path": "^0.12.7",
|
|
40
41
|
"ts-node": "^10.2.1",
|
|
41
42
|
"sanitize-filename": "^1.6.3"
|
|
@@ -90,5 +91,5 @@
|
|
|
90
91
|
"volta": {
|
|
91
92
|
"node": "18.16.0"
|
|
92
93
|
},
|
|
93
|
-
"version": "0.14.0-alpha.
|
|
94
|
+
"version": "0.14.0-alpha.11"
|
|
94
95
|
}
|
|
@@ -1,55 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.standardNumberedListTransformer = void 0;
|
|
4
|
-
// This is mostly what notion-to-markdown would normally do with a block of type
|
|
5
|
-
// numbered_list_item. A patch is documented at the end.
|
|
6
|
-
function numberedListTransformer(notionToMarkdown, block) {
|
|
7
|
-
var _a, _b, _c;
|
|
8
|
-
//console.log("got numbered list block " + JSON.stringify(block));
|
|
9
|
-
// In this case typescript is not able to index the types properly, hence ignoring the error
|
|
10
|
-
// @ts-ignore
|
|
11
|
-
const blockContent =
|
|
12
|
-
// @ts-ignore
|
|
13
|
-
((_a = block.numbered_list_item) === null || _a === void 0 ? void 0 : _a.text) || ((_b = block.numbered_list_item) === null || _b === void 0 ? void 0 : _b.rich_text) || [];
|
|
14
|
-
let parsedData = "";
|
|
15
|
-
blockContent.map((content) => {
|
|
16
|
-
const annotations = content.annotations;
|
|
17
|
-
let plain_text = content.plain_text;
|
|
18
|
-
plain_text = notionToMarkdown.annotatePlainText(plain_text, annotations);
|
|
19
|
-
if (content["href"]) {
|
|
20
|
-
plain_text = `[${plain_text}](${content["href"]})`;
|
|
21
|
-
}
|
|
22
|
-
parsedData += plain_text;
|
|
23
|
-
});
|
|
24
|
-
// There is code in notion-to-md which attempts to set an incrementing number
|
|
25
|
-
// on each of these. Somehow it fails; in my testing, block.numbered_list_item never
|
|
26
|
-
// has a field 'number'. But we don't actually need incrementing numbers;
|
|
27
|
-
// markdown will do the numbering if we just make something that looks like
|
|
28
|
-
// a member of a numbered list by starting with number followed by period and space.
|
|
29
|
-
// I'm keeping the original code in case notion-to-md gets fixed and there is actually
|
|
30
|
-
// some reason to use incrementing numbers (it would at least make the markdown more
|
|
31
|
-
// human-readable); but this at least works.
|
|
32
|
-
// A problem is that in notion, a numbered list may continue after some intermediate
|
|
33
|
-
// content. To achieve this in markdown, we'd need to indent the intermediate content
|
|
34
|
-
// by a tab. Not only is it difficult to do this, but there appears to be no way to
|
|
35
|
-
// know whether we should. The data we get from notion doesn't include the item number,
|
|
36
|
-
// and its parent is the page rather than a particular list. So there is no way I can
|
|
37
|
-
// see to distinguish a list continuation from a new list. The code here will leave
|
|
38
|
-
// it up to markdown to decide whether to start a new list; I believe it will do so
|
|
39
|
-
// if it sees any intervening lines that are not list items.
|
|
40
|
-
let num = (_c = block.numbered_list_item) === null || _c === void 0 ? void 0 : _c.number;
|
|
41
|
-
//console.log("got number " + num?.toString());
|
|
42
|
-
if (!num) {
|
|
43
|
-
num = 1;
|
|
44
|
-
}
|
|
45
|
-
return Promise.resolve(`${num}. ${parsedData.trim()}`);
|
|
46
|
-
}
|
|
47
|
-
exports.standardNumberedListTransformer = {
|
|
48
|
-
name: "standardNumberedListTransformer",
|
|
49
|
-
notionToMarkdownTransforms: [
|
|
50
|
-
{
|
|
51
|
-
type: "numbered_list_item",
|
|
52
|
-
getStringFromBlock: (context, block) => numberedListTransformer(context.notionToMarkdown, block),
|
|
53
|
-
},
|
|
54
|
-
],
|
|
55
|
-
};
|
|
@@ -1,86 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
-
});
|
|
10
|
-
};
|
|
11
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
-
const pluginTestRun_1 = require("./pluginTestRun");
|
|
13
|
-
const NumberedListTransformer_1 = require("./NumberedListTransformer");
|
|
14
|
-
let block;
|
|
15
|
-
beforeEach(() => {
|
|
16
|
-
block = {
|
|
17
|
-
has_children: false,
|
|
18
|
-
archived: false,
|
|
19
|
-
type: "callout",
|
|
20
|
-
callout: {
|
|
21
|
-
rich_text: [
|
|
22
|
-
{
|
|
23
|
-
type: "text",
|
|
24
|
-
text: { content: "This is information callout", link: null },
|
|
25
|
-
annotations: {
|
|
26
|
-
bold: false,
|
|
27
|
-
italic: false,
|
|
28
|
-
strikethrough: false,
|
|
29
|
-
underline: false,
|
|
30
|
-
code: false,
|
|
31
|
-
color: "default",
|
|
32
|
-
},
|
|
33
|
-
plain_text: "This is the callout",
|
|
34
|
-
href: null,
|
|
35
|
-
},
|
|
36
|
-
],
|
|
37
|
-
icon: { type: "emoji", emoji: "ℹ️" },
|
|
38
|
-
color: "gray_background",
|
|
39
|
-
},
|
|
40
|
-
};
|
|
41
|
-
});
|
|
42
|
-
test("external link inside numbered list, italic preserved", () => __awaiter(void 0, void 0, void 0, function* () {
|
|
43
|
-
const config = { plugins: [NumberedListTransformer_1.standardNumberedListTransformer] };
|
|
44
|
-
const results = yield (0, pluginTestRun_1.blocksToMarkdown)(config, [
|
|
45
|
-
{
|
|
46
|
-
type: "numbered_list_item",
|
|
47
|
-
numbered_list_item: {
|
|
48
|
-
rich_text: [
|
|
49
|
-
{
|
|
50
|
-
type: "text",
|
|
51
|
-
text: { content: "link ", link: null },
|
|
52
|
-
annotations: {
|
|
53
|
-
bold: false,
|
|
54
|
-
italic: false,
|
|
55
|
-
strikethrough: false,
|
|
56
|
-
underline: false,
|
|
57
|
-
code: false,
|
|
58
|
-
color: "default",
|
|
59
|
-
},
|
|
60
|
-
plain_text: "link ",
|
|
61
|
-
href: null,
|
|
62
|
-
},
|
|
63
|
-
{
|
|
64
|
-
type: "text",
|
|
65
|
-
text: {
|
|
66
|
-
content: "github",
|
|
67
|
-
link: { url: "https://github.com" },
|
|
68
|
-
},
|
|
69
|
-
annotations: {
|
|
70
|
-
bold: false,
|
|
71
|
-
italic: true,
|
|
72
|
-
strikethrough: false,
|
|
73
|
-
underline: false,
|
|
74
|
-
code: false,
|
|
75
|
-
color: "default",
|
|
76
|
-
},
|
|
77
|
-
plain_text: "github",
|
|
78
|
-
href: "https://github.com",
|
|
79
|
-
},
|
|
80
|
-
],
|
|
81
|
-
color: "default",
|
|
82
|
-
},
|
|
83
|
-
},
|
|
84
|
-
]);
|
|
85
|
-
expect(results.trim()).toBe(`1. link [_github_](https://github.com)`);
|
|
86
|
-
}));
|
|
File without changes
|