@sillsdev/docu-notion 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +135 -0
- package/dist/FlatGuidLayoutStrategy.d.ts +6 -0
- package/dist/FlatGuidLayoutStrategy.js +25 -0
- package/dist/HierarchicalNamedLayoutStrategy.d.ts +7 -0
- package/dist/HierarchicalNamedLayoutStrategy.js +80 -0
- package/dist/LayoutStrategy.d.ts +12 -0
- package/dist/LayoutStrategy.js +83 -0
- package/dist/MakeImagePersistencePlan.d.ts +2 -0
- package/dist/MakeImagePersistencePlan.js +66 -0
- package/dist/NotionImage-CaptionReading.spec.d.ts +1 -0
- package/dist/NotionImage-CaptionReading.spec.js +233 -0
- package/dist/NotionPage.d.ts +44 -0
- package/dist/NotionPage.js +194 -0
- package/dist/config/configuration.d.ts +5 -0
- package/dist/config/configuration.js +86 -0
- package/dist/config/default.docunotion.config.d.ts +3 -0
- package/dist/config/default.docunotion.config.js +37 -0
- package/dist/images.d.ts +24 -0
- package/dist/images.js +230 -0
- package/dist/index.d.ts +7 -0
- package/dist/index.js +37 -0
- package/dist/log.d.ts +11 -0
- package/dist/log.js +61 -0
- package/dist/makeImagePersistencePlan.spec.d.ts +1 -0
- package/dist/makeImagePersistencePlan.spec.js +35 -0
- package/dist/notion-styles.css +58 -0
- package/dist/plugins/CalloutTransformer.d.ts +24 -0
- package/dist/plugins/CalloutTransformer.js +88 -0
- package/dist/plugins/CalloutTransformer.spec.d.ts +1 -0
- package/dist/plugins/CalloutTransformer.spec.js +199 -0
- package/dist/plugins/ColumnListTransformer.d.ts +2 -0
- package/dist/plugins/ColumnListTransformer.js +34 -0
- package/dist/plugins/ColumnTransformer.d.ts +2 -0
- package/dist/plugins/ColumnTransformer.js +67 -0
- package/dist/plugins/EscapeHtmlBlockModifier.d.ts +2 -0
- package/dist/plugins/EscapeHtmlBlockModifier.js +41 -0
- package/dist/plugins/EscapeHtmlBlockModifier.spec.d.ts +1 -0
- package/dist/plugins/EscapeHtmlBlockModifier.spec.js +130 -0
- package/dist/plugins/HeadingTranformer.spec.d.ts +1 -0
- package/dist/plugins/HeadingTranformer.spec.js +46 -0
- package/dist/plugins/HeadingTransformer.d.ts +2 -0
- package/dist/plugins/HeadingTransformer.js +63 -0
- package/dist/plugins/NumberedListTransformer.d.ts +2 -0
- package/dist/plugins/NumberedListTransformer.js +55 -0
- package/dist/plugins/NumberedListTransformer.spec.d.ts +1 -0
- package/dist/plugins/NumberedListTransformer.spec.js +86 -0
- package/dist/plugins/TableTransformer.d.ts +5 -0
- package/dist/plugins/TableTransformer.js +70 -0
- package/dist/plugins/embedTweaks.d.ts +5 -0
- package/dist/plugins/embedTweaks.js +46 -0
- package/dist/plugins/embedTweaks.spec.d.ts +1 -0
- package/dist/plugins/embedTweaks.spec.js +230 -0
- package/dist/plugins/externalLinks.d.ts +2 -0
- package/dist/plugins/externalLinks.js +26 -0
- package/dist/plugins/externalLinks.spec.d.ts +1 -0
- package/dist/plugins/externalLinks.spec.js +132 -0
- package/dist/plugins/internalLinks.d.ts +6 -0
- package/dist/plugins/internalLinks.js +78 -0
- package/dist/plugins/internalLinks.spec.d.ts +1 -0
- package/dist/plugins/internalLinks.spec.js +442 -0
- package/dist/plugins/pluginTestRun.d.ts +10 -0
- package/dist/plugins/pluginTestRun.js +248 -0
- package/dist/plugins/pluginTypes.d.ts +42 -0
- package/dist/plugins/pluginTypes.js +2 -0
- package/dist/pull.d.ts +12 -0
- package/dist/pull.js +253 -0
- package/dist/run.d.ts +1 -0
- package/dist/run.js +35 -0
- package/dist/transform.d.ts +6 -0
- package/dist/transform.js +195 -0
- package/dist/types.d.ts +8 -0
- package/dist/types.js +2 -0
- package/package.json +96 -0
|
@@ -0,0 +1,248 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
3
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
4
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
5
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
6
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
7
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
8
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
9
|
+
});
|
|
10
|
+
};
|
|
11
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
12
|
+
exports.oneBlockToMarkdown = exports.makeSamplePageObject = exports.blocksToMarkdown = void 0;
|
|
13
|
+
const client_1 = require("@notionhq/client");
|
|
14
|
+
const notion_to_md_1 = require("notion-to-md");
|
|
15
|
+
const HierarchicalNamedLayoutStrategy_1 = require("../HierarchicalNamedLayoutStrategy");
|
|
16
|
+
const NotionPage_1 = require("../NotionPage");
|
|
17
|
+
const transform_1 = require("../transform");
|
|
18
|
+
function blocksToMarkdown(config, blocks, pages) {
|
|
19
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
20
|
+
const notionClient = new client_1.Client({ auth: "unused" });
|
|
21
|
+
const notionToMD = new notion_to_md_1.NotionToMarkdown({
|
|
22
|
+
notionClient,
|
|
23
|
+
});
|
|
24
|
+
// if (pages && pages.length) {
|
|
25
|
+
// console.log(pages[0]);
|
|
26
|
+
// console.log(pages[0].matchesLinkId);
|
|
27
|
+
// }
|
|
28
|
+
const docunotionContext = {
|
|
29
|
+
notionToMarkdown: notionToMD,
|
|
30
|
+
// TODO when does this actually need to do get some children?
|
|
31
|
+
// We can add a children argument to this method, but for the tests
|
|
32
|
+
// I have so far, it's not needed.
|
|
33
|
+
getBlockChildren: (id) => {
|
|
34
|
+
return new Promise((resolve, reject) => {
|
|
35
|
+
resolve([]);
|
|
36
|
+
});
|
|
37
|
+
},
|
|
38
|
+
//TODO might be needed for some tests, e.g. the image transformer...
|
|
39
|
+
directoryContainingMarkdown: "not yet",
|
|
40
|
+
relativeFilePathToFolderContainingPage: "not yet",
|
|
41
|
+
layoutStrategy: new HierarchicalNamedLayoutStrategy_1.HierarchicalNamedLayoutStrategy(),
|
|
42
|
+
options: {
|
|
43
|
+
notionToken: "",
|
|
44
|
+
rootPage: "",
|
|
45
|
+
locales: [],
|
|
46
|
+
markdownOutputPath: "",
|
|
47
|
+
imgOutputPath: "",
|
|
48
|
+
imgPrefixInMarkdown: "",
|
|
49
|
+
statusTag: "",
|
|
50
|
+
},
|
|
51
|
+
pages: pages !== null && pages !== void 0 ? pages : [],
|
|
52
|
+
counts: {
|
|
53
|
+
output_normally: 0,
|
|
54
|
+
skipped_because_empty: 0,
|
|
55
|
+
skipped_because_status: 0,
|
|
56
|
+
skipped_because_level_cannot_have_content: 0,
|
|
57
|
+
},
|
|
58
|
+
// enhance: this needs more thinking, how we want to do logging in tests
|
|
59
|
+
// one thing is to avoid a situation where we break people's tests that
|
|
60
|
+
// have come to rely on logs that we later tweak in some way.
|
|
61
|
+
// log: {
|
|
62
|
+
// error: (s: string) => {
|
|
63
|
+
// error(s);
|
|
64
|
+
// },
|
|
65
|
+
// warning: (s: string) => {
|
|
66
|
+
// warning(s);
|
|
67
|
+
// },
|
|
68
|
+
// info: (s: string) => {
|
|
69
|
+
// // info(s);
|
|
70
|
+
// },
|
|
71
|
+
// verbose: (s: string) => {
|
|
72
|
+
// // verbose(s);
|
|
73
|
+
// },
|
|
74
|
+
// debug: (s: string) => {
|
|
75
|
+
// // logDebug("Testrun-TODO", s);
|
|
76
|
+
// },
|
|
77
|
+
// },
|
|
78
|
+
};
|
|
79
|
+
if (pages && pages.length) {
|
|
80
|
+
console.log(pages[0].matchesLinkId);
|
|
81
|
+
console.log(docunotionContext.pages[0].matchesLinkId);
|
|
82
|
+
}
|
|
83
|
+
const r = yield (0, transform_1.getMarkdownFromNotionBlocks)(docunotionContext, config, blocks);
|
|
84
|
+
//console.log("blocksToMarkdown", r);
|
|
85
|
+
return r;
|
|
86
|
+
});
|
|
87
|
+
}
|
|
88
|
+
exports.blocksToMarkdown = blocksToMarkdown;
|
|
89
|
+
// This is used for things like testing links to other pages and frontmatter creation,
|
|
90
|
+
// when just testing what happens to individual blocks is not enough.
|
|
91
|
+
// after getting this, you can make changes to it, then pass it to blocksToMarkdown
|
|
92
|
+
function makeSamplePageObject(options) {
|
|
93
|
+
let slugObject = {
|
|
94
|
+
Slug: {
|
|
95
|
+
id: "%7D%3D~K",
|
|
96
|
+
type: "rich_text",
|
|
97
|
+
rich_text: [],
|
|
98
|
+
},
|
|
99
|
+
};
|
|
100
|
+
if (options.slug)
|
|
101
|
+
slugObject = {
|
|
102
|
+
id: "%7D%3D~K",
|
|
103
|
+
type: "rich_text",
|
|
104
|
+
rich_text: [
|
|
105
|
+
{
|
|
106
|
+
type: "text",
|
|
107
|
+
text: {
|
|
108
|
+
content: options.slug,
|
|
109
|
+
link: null,
|
|
110
|
+
},
|
|
111
|
+
annotations: {
|
|
112
|
+
bold: false,
|
|
113
|
+
italic: false,
|
|
114
|
+
strikethrough: false,
|
|
115
|
+
underline: false,
|
|
116
|
+
code: false,
|
|
117
|
+
color: "default",
|
|
118
|
+
},
|
|
119
|
+
plain_text: options.slug,
|
|
120
|
+
href: null,
|
|
121
|
+
},
|
|
122
|
+
],
|
|
123
|
+
};
|
|
124
|
+
const id = options.id || "4a6de8c0-b90b-444b-8a7b-d534d6ec71a4";
|
|
125
|
+
const m = {
|
|
126
|
+
object: "page",
|
|
127
|
+
id: id,
|
|
128
|
+
created_time: "2022-08-08T21:07:00.000Z",
|
|
129
|
+
last_edited_time: "2023-01-03T14:38:00.000Z",
|
|
130
|
+
created_by: {
|
|
131
|
+
object: "user",
|
|
132
|
+
id: "11fb7f16-0560-4aee-ab88-ed75a850cfc4",
|
|
133
|
+
},
|
|
134
|
+
last_edited_by: {
|
|
135
|
+
object: "user",
|
|
136
|
+
id: "11fb7f16-0560-4aee-ab88-ed75a850cfc4",
|
|
137
|
+
},
|
|
138
|
+
cover: null,
|
|
139
|
+
icon: null,
|
|
140
|
+
parent: {
|
|
141
|
+
type: "database_id",
|
|
142
|
+
database_id: "c13f520c-06ad-41e4-a021-bdc2841ab24a",
|
|
143
|
+
},
|
|
144
|
+
archived: false,
|
|
145
|
+
properties: {
|
|
146
|
+
Keywords: {
|
|
147
|
+
id: "%3F%7DLZ",
|
|
148
|
+
type: "rich_text",
|
|
149
|
+
rich_text: [],
|
|
150
|
+
},
|
|
151
|
+
Property: {
|
|
152
|
+
id: "GmKE",
|
|
153
|
+
type: "rich_text",
|
|
154
|
+
rich_text: [],
|
|
155
|
+
},
|
|
156
|
+
Label: {
|
|
157
|
+
id: "Phor",
|
|
158
|
+
type: "multi_select",
|
|
159
|
+
multi_select: [],
|
|
160
|
+
},
|
|
161
|
+
Status: {
|
|
162
|
+
id: "oB~%3D",
|
|
163
|
+
type: "select",
|
|
164
|
+
select: {
|
|
165
|
+
id: "1",
|
|
166
|
+
name: "Ready For Review",
|
|
167
|
+
color: "red",
|
|
168
|
+
},
|
|
169
|
+
},
|
|
170
|
+
Authors: {
|
|
171
|
+
id: "tA%3BF",
|
|
172
|
+
type: "multi_select",
|
|
173
|
+
multi_select: [],
|
|
174
|
+
},
|
|
175
|
+
Slug: slugObject,
|
|
176
|
+
Name: {
|
|
177
|
+
id: "title",
|
|
178
|
+
type: "title",
|
|
179
|
+
title: [
|
|
180
|
+
{
|
|
181
|
+
type: "text",
|
|
182
|
+
text: {
|
|
183
|
+
content: options.name || "Hello World",
|
|
184
|
+
link: null,
|
|
185
|
+
},
|
|
186
|
+
annotations: {
|
|
187
|
+
bold: false,
|
|
188
|
+
italic: false,
|
|
189
|
+
strikethrough: false,
|
|
190
|
+
underline: false,
|
|
191
|
+
code: false,
|
|
192
|
+
color: "default",
|
|
193
|
+
},
|
|
194
|
+
plain_text: options.name || "Hello World",
|
|
195
|
+
href: null,
|
|
196
|
+
},
|
|
197
|
+
],
|
|
198
|
+
},
|
|
199
|
+
},
|
|
200
|
+
url: `https://www.notion.so/Hello-World-${id}`,
|
|
201
|
+
};
|
|
202
|
+
const p = new NotionPage_1.NotionPage({
|
|
203
|
+
layoutContext: "/Second-Level/Third-Level",
|
|
204
|
+
pageId: id,
|
|
205
|
+
order: 0,
|
|
206
|
+
metadata: m,
|
|
207
|
+
foundDirectlyInOutline: false,
|
|
208
|
+
});
|
|
209
|
+
console.log(p.matchesLinkId);
|
|
210
|
+
return p;
|
|
211
|
+
}
|
|
212
|
+
exports.makeSamplePageObject = makeSamplePageObject;
|
|
213
|
+
function oneBlockToMarkdown(config, block, targetPage) {
|
|
214
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
215
|
+
// just in case someone expects these other properties that aren't normally relevant,
|
|
216
|
+
// we merge the given block properties into an actual, full block
|
|
217
|
+
const fullBlock = Object.assign({
|
|
218
|
+
object: "block",
|
|
219
|
+
id: "937e77e5-f058-4316-9805-a538e7b4082d",
|
|
220
|
+
parent: {
|
|
221
|
+
type: "page_id",
|
|
222
|
+
page_id: "d20d8391-b365-42cb-8821-cf3c5382c6ed",
|
|
223
|
+
},
|
|
224
|
+
created_time: "2023-01-13T16:33:00.000Z",
|
|
225
|
+
last_edited_time: "2023-01-13T16:33:00.000Z",
|
|
226
|
+
created_by: {
|
|
227
|
+
object: "user",
|
|
228
|
+
id: "11fb7f16-0560-4aee-ab88-ed75a850cfc4",
|
|
229
|
+
},
|
|
230
|
+
last_edited_by: {
|
|
231
|
+
object: "user",
|
|
232
|
+
id: "11fb7f16-0560-4aee-ab88-ed75a850cfc4",
|
|
233
|
+
},
|
|
234
|
+
has_children: false,
|
|
235
|
+
archived: false,
|
|
236
|
+
}, block);
|
|
237
|
+
const dummyPage1 = makeSamplePageObject({
|
|
238
|
+
slug: "dummy1",
|
|
239
|
+
name: "Dummy1",
|
|
240
|
+
});
|
|
241
|
+
const dummyPage2 = makeSamplePageObject({
|
|
242
|
+
slug: "dummy2",
|
|
243
|
+
name: "Dummy2",
|
|
244
|
+
});
|
|
245
|
+
return yield blocksToMarkdown(config, [fullBlock], targetPage ? [dummyPage1, targetPage, dummyPage2] : undefined);
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
exports.oneBlockToMarkdown = oneBlockToMarkdown;
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { ListBlockChildrenResponseResult } from "notion-to-md/build/types";
|
|
2
|
+
import { NotionPage } from "../NotionPage";
|
|
3
|
+
import { NotionToMarkdown } from "notion-to-md";
|
|
4
|
+
import { DocuNotionOptions } from "../pull";
|
|
5
|
+
import { LayoutStrategy } from "../LayoutStrategy";
|
|
6
|
+
import { ICounts, NotionBlock } from "../index";
|
|
7
|
+
type linkConversionFunction = (context: IDocuNotionContext, markdownLink: string) => string;
|
|
8
|
+
export type IPlugin = {
|
|
9
|
+
name: string;
|
|
10
|
+
notionBlockModifications?: {
|
|
11
|
+
modify: (block: NotionBlock) => void;
|
|
12
|
+
}[];
|
|
13
|
+
notionToMarkdownTransforms?: {
|
|
14
|
+
type: string;
|
|
15
|
+
getStringFromBlock: (context: IDocuNotionContext, block: NotionBlock) => string | Promise<string>;
|
|
16
|
+
}[];
|
|
17
|
+
linkModifier?: {
|
|
18
|
+
match: RegExp;
|
|
19
|
+
convert: linkConversionFunction;
|
|
20
|
+
};
|
|
21
|
+
regexMarkdownModifications?: IRegexMarkdownModification[];
|
|
22
|
+
init?(plugin: IPlugin): Promise<void>;
|
|
23
|
+
};
|
|
24
|
+
export type IRegexMarkdownModification = {
|
|
25
|
+
regex: RegExp;
|
|
26
|
+
replacementPattern?: string;
|
|
27
|
+
getReplacement?(s: string): Promise<string>;
|
|
28
|
+
imports?: string[];
|
|
29
|
+
};
|
|
30
|
+
export type ICustomNotionToMarkdownConversion = (block: ListBlockChildrenResponseResult, context: IDocuNotionContext) => () => Promise<string>;
|
|
31
|
+
export type IGetBlockChildrenFn = (id: string) => Promise<NotionBlock[]>;
|
|
32
|
+
export type IDocuNotionContext = {
|
|
33
|
+
layoutStrategy: LayoutStrategy;
|
|
34
|
+
options: DocuNotionOptions;
|
|
35
|
+
getBlockChildren: IGetBlockChildrenFn;
|
|
36
|
+
notionToMarkdown: NotionToMarkdown;
|
|
37
|
+
directoryContainingMarkdown: string;
|
|
38
|
+
relativeFilePathToFolderContainingPage: string;
|
|
39
|
+
pages: NotionPage[];
|
|
40
|
+
counts: ICounts;
|
|
41
|
+
};
|
|
42
|
+
export {};
|
package/dist/pull.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { Client } from "@notionhq/client";
|
|
2
|
+
export type DocuNotionOptions = {
|
|
3
|
+
notionToken: string;
|
|
4
|
+
rootPage: string;
|
|
5
|
+
locales: string[];
|
|
6
|
+
markdownOutputPath: string;
|
|
7
|
+
imgOutputPath: string;
|
|
8
|
+
imgPrefixInMarkdown: string;
|
|
9
|
+
statusTag: string;
|
|
10
|
+
};
|
|
11
|
+
export declare function notionPull(options: DocuNotionOptions): Promise<void>;
|
|
12
|
+
export declare function initNotionClient(notionToken: string): Client;
|
package/dist/pull.js
ADDED
|
@@ -0,0 +1,253 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
+
if (mod && mod.__esModule) return mod;
|
|
20
|
+
var result = {};
|
|
21
|
+
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
+
__setModuleDefault(result, mod);
|
|
23
|
+
return result;
|
|
24
|
+
};
|
|
25
|
+
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
26
|
+
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
27
|
+
return new (P || (P = Promise))(function (resolve, reject) {
|
|
28
|
+
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
29
|
+
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
30
|
+
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
31
|
+
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
32
|
+
});
|
|
33
|
+
};
|
|
34
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
35
|
+
exports.initNotionClient = exports.notionPull = void 0;
|
|
36
|
+
const fs = __importStar(require("fs-extra"));
|
|
37
|
+
const notion_to_md_1 = require("notion-to-md");
|
|
38
|
+
const HierarchicalNamedLayoutStrategy_1 = require("./HierarchicalNamedLayoutStrategy");
|
|
39
|
+
const NotionPage_1 = require("./NotionPage");
|
|
40
|
+
const images_1 = require("./images");
|
|
41
|
+
const Path = __importStar(require("path"));
|
|
42
|
+
const log_1 = require("./log");
|
|
43
|
+
const transform_1 = require("./transform");
|
|
44
|
+
const limiter_1 = require("limiter");
|
|
45
|
+
const client_1 = require("@notionhq/client");
|
|
46
|
+
const process_1 = require("process");
|
|
47
|
+
const configuration_1 = require("./config/configuration");
|
|
48
|
+
let layoutStrategy;
|
|
49
|
+
let notionToMarkdown;
|
|
50
|
+
const pages = new Array();
|
|
51
|
+
const counts = {
|
|
52
|
+
output_normally: 0,
|
|
53
|
+
skipped_because_empty: 0,
|
|
54
|
+
skipped_because_status: 0,
|
|
55
|
+
skipped_because_level_cannot_have_content: 0,
|
|
56
|
+
};
|
|
57
|
+
function notionPull(options) {
|
|
58
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
59
|
+
// It's helpful when troubleshooting CI secrets and environment variables to see what options actually made it to docu-notion.
|
|
60
|
+
// eslint-disable-next-line @typescript-eslint/no-unsafe-call
|
|
61
|
+
const optionsForLogging = Object.assign({}, options);
|
|
62
|
+
// Just show the first few letters of the notion token, which start with "secret" anyhow.
|
|
63
|
+
optionsForLogging.notionToken =
|
|
64
|
+
optionsForLogging.notionToken.substring(0, 3) + "...";
|
|
65
|
+
const config = yield (0, configuration_1.loadConfigAsync)();
|
|
66
|
+
(0, log_1.verbose)(`Options:${JSON.stringify(optionsForLogging, null, 2)}`);
|
|
67
|
+
yield (0, images_1.initImageHandling)(options.imgPrefixInMarkdown || options.imgOutputPath || "", options.imgOutputPath || "", options.locales);
|
|
68
|
+
const notionClient = initNotionClient(options.notionToken);
|
|
69
|
+
notionToMarkdown = new notion_to_md_1.NotionToMarkdown({ notionClient });
|
|
70
|
+
layoutStrategy = new HierarchicalNamedLayoutStrategy_1.HierarchicalNamedLayoutStrategy();
|
|
71
|
+
yield fs.mkdir(options.markdownOutputPath, { recursive: true });
|
|
72
|
+
layoutStrategy.setRootDirectoryForMarkdown(options.markdownOutputPath.replace(/\/+$/, "") // trim any trailing slash
|
|
73
|
+
);
|
|
74
|
+
(0, log_1.info)("Connecting to Notion...");
|
|
75
|
+
(0, log_1.group)("Stage 1: walk children of the page named 'Outline', looking for pages...");
|
|
76
|
+
yield getPagesRecursively(options, "", options.rootPage, 0, true);
|
|
77
|
+
(0, log_1.logDebug)("getPagesRecursively", JSON.stringify(pages, null, 2));
|
|
78
|
+
(0, log_1.info)(`Found ${pages.length} pages`);
|
|
79
|
+
(0, log_1.endGroup)();
|
|
80
|
+
(0, log_1.group)(`Stage 2: convert ${pages.length} Notion pages to markdown and save locally...`);
|
|
81
|
+
yield outputPages(options, config, pages);
|
|
82
|
+
(0, log_1.endGroup)();
|
|
83
|
+
(0, log_1.group)("Stage 3: clean up old files & images...");
|
|
84
|
+
yield layoutStrategy.cleanupOldFiles();
|
|
85
|
+
yield (0, images_1.cleanupOldImages)();
|
|
86
|
+
(0, log_1.endGroup)();
|
|
87
|
+
});
|
|
88
|
+
}
|
|
89
|
+
exports.notionPull = notionPull;
|
|
90
|
+
function outputPages(options, config, pages) {
|
|
91
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
92
|
+
const context = {
|
|
93
|
+
getBlockChildren: getBlockChildren,
|
|
94
|
+
directoryContainingMarkdown: "",
|
|
95
|
+
relativeFilePathToFolderContainingPage: "",
|
|
96
|
+
layoutStrategy: layoutStrategy,
|
|
97
|
+
notionToMarkdown: notionToMarkdown,
|
|
98
|
+
options: options,
|
|
99
|
+
pages: pages,
|
|
100
|
+
counts: counts, // review will this get copied or pointed to?
|
|
101
|
+
};
|
|
102
|
+
for (const page of pages) {
|
|
103
|
+
layoutStrategy.pageWasSeen(page);
|
|
104
|
+
const mdPath = layoutStrategy.getPathForPage(page, ".md");
|
|
105
|
+
// most plugins should not write to disk, but those handling image files need these paths
|
|
106
|
+
context.directoryContainingMarkdown = Path.dirname(mdPath);
|
|
107
|
+
// TODO: This needs clarifying: getLinkPathForPage() is about urls, but
|
|
108
|
+
// downstream images.ts is using it as a file system path
|
|
109
|
+
context.relativeFilePathToFolderContainingPage = Path.dirname(layoutStrategy.getLinkPathForPage(page));
|
|
110
|
+
if (page.type === NotionPage_1.PageType.DatabasePage &&
|
|
111
|
+
context.options.statusTag != "*" &&
|
|
112
|
+
page.status !== context.options.statusTag) {
|
|
113
|
+
(0, log_1.verbose)(`Skipping page because status is not '${context.options.statusTag}': ${page.nameOrTitle}`);
|
|
114
|
+
++context.counts.skipped_because_status;
|
|
115
|
+
}
|
|
116
|
+
else {
|
|
117
|
+
const markdown = yield (0, transform_1.getMarkdownForPage)(config, context, page);
|
|
118
|
+
writePage(page, markdown);
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
(0, log_1.info)(`Finished processing ${pages.length} pages`);
|
|
122
|
+
(0, log_1.info)(JSON.stringify(counts));
|
|
123
|
+
});
|
|
124
|
+
}
|
|
125
|
+
// This walks the "Outline" page and creates a list of all the nodes that will
|
|
126
|
+
// be in the sidebar, including the directories, the pages that are linked to
|
|
127
|
+
// that are parented in from the "Database", and any pages we find in the
|
|
128
|
+
// outline that contain content (which we call "Simple" pages). Later, we can
|
|
129
|
+
// then step through this list creating the files we need, and, crucially, be
|
|
130
|
+
// able to figure out what the url will be for any links between content pages.
|
|
131
|
+
function getPagesRecursively(options, incomingContext, pageIdOfThisParent, orderOfThisParent, rootLevel) {
|
|
132
|
+
var _a;
|
|
133
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
134
|
+
const pageInTheOutline = yield fromPageId(incomingContext, pageIdOfThisParent, orderOfThisParent, true);
|
|
135
|
+
(0, log_1.info)(`Looking for children and links from ${incomingContext}/${pageInTheOutline.nameOrTitle}`);
|
|
136
|
+
const r = yield getBlockChildren(pageInTheOutline.pageId);
|
|
137
|
+
const pageInfo = yield pageInTheOutline.getContentInfo(r);
|
|
138
|
+
if (!rootLevel &&
|
|
139
|
+
pageInfo.hasParagraphs &&
|
|
140
|
+
pageInfo.childPageIdsAndOrder.length) {
|
|
141
|
+
(0, log_1.error)(`Skipping "${pageInTheOutline.nameOrTitle}" and its children. docu-notion does not support pages that are both levels and have content at the same time.`);
|
|
142
|
+
++counts.skipped_because_level_cannot_have_content;
|
|
143
|
+
return;
|
|
144
|
+
}
|
|
145
|
+
if (!rootLevel && pageInfo.hasParagraphs) {
|
|
146
|
+
pages.push(pageInTheOutline);
|
|
147
|
+
// The best practice is to keep content pages in the "database" (e.g. kanban board), but we do allow people to make pages in the outline directly.
|
|
148
|
+
// So how can we tell the difference between a page that is supposed to be content and one that is meant to form the sidebar? If it
|
|
149
|
+
// has only links, then it's a page for forming the sidebar. If it has contents and no links, then it's a content page. But what if
|
|
150
|
+
// it has both? Well then we assume it's a content page.
|
|
151
|
+
if ((_a = pageInfo.linksPageIdsAndOrder) === null || _a === void 0 ? void 0 : _a.length) {
|
|
152
|
+
(0, log_1.warning)(`Note: The page "${pageInTheOutline.nameOrTitle}" is in the outline, has content, and also points at other pages. It will be treated as a simple content page. This is no problem, unless you intended to have all your content pages in the database (kanban workflow) section.`);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
// a normal outline page that exists just to create the level, pointing at database pages that belong in this level
|
|
156
|
+
else if (pageInfo.childPageIdsAndOrder.length ||
|
|
157
|
+
pageInfo.linksPageIdsAndOrder.length) {
|
|
158
|
+
let layoutContext = incomingContext;
|
|
159
|
+
// don't make a level for "Outline" page at the root
|
|
160
|
+
if (!rootLevel && pageInTheOutline.nameOrTitle !== "Outline") {
|
|
161
|
+
layoutContext = layoutStrategy.newLevel(options.markdownOutputPath, pageInTheOutline.order, incomingContext, pageInTheOutline.nameOrTitle);
|
|
162
|
+
}
|
|
163
|
+
for (const childPageInfo of pageInfo.childPageIdsAndOrder) {
|
|
164
|
+
yield getPagesRecursively(options, layoutContext, childPageInfo.id, childPageInfo.order, false);
|
|
165
|
+
}
|
|
166
|
+
for (const linkPageInfo of pageInfo.linksPageIdsAndOrder) {
|
|
167
|
+
pages.push(yield fromPageId(layoutContext, linkPageInfo.id, linkPageInfo.order, false));
|
|
168
|
+
}
|
|
169
|
+
}
|
|
170
|
+
else {
|
|
171
|
+
console.info((0, log_1.warning)(`Warning: The page "${pageInTheOutline.nameOrTitle}" is in the outline but appears to not have content, links to other pages, or child pages. It will be skipped.`));
|
|
172
|
+
++counts.skipped_because_empty;
|
|
173
|
+
}
|
|
174
|
+
});
|
|
175
|
+
}
|
|
176
|
+
function writePage(page, finalMarkdown) {
|
|
177
|
+
const mdPath = layoutStrategy.getPathForPage(page, ".md");
|
|
178
|
+
(0, log_1.verbose)(`writing ${mdPath}`);
|
|
179
|
+
fs.writeFileSync(mdPath, finalMarkdown, {});
|
|
180
|
+
++counts.output_normally;
|
|
181
|
+
}
|
|
182
|
+
const notionLimiter = new limiter_1.RateLimiter({
|
|
183
|
+
tokensPerInterval: 3,
|
|
184
|
+
interval: "second",
|
|
185
|
+
});
|
|
186
|
+
let notionClient;
|
|
187
|
+
function getPageMetadata(id) {
|
|
188
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
189
|
+
yield rateLimit();
|
|
190
|
+
return yield notionClient.pages.retrieve({
|
|
191
|
+
page_id: id,
|
|
192
|
+
});
|
|
193
|
+
});
|
|
194
|
+
}
|
|
195
|
+
function rateLimit() {
|
|
196
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
197
|
+
if (notionLimiter.getTokensRemaining() < 1) {
|
|
198
|
+
(0, log_1.logDebug)("rateLimit", "*** delaying for rate limit");
|
|
199
|
+
}
|
|
200
|
+
yield notionLimiter.removeTokens(1);
|
|
201
|
+
});
|
|
202
|
+
}
|
|
203
|
+
function getBlockChildren(id) {
|
|
204
|
+
var _a, _b;
|
|
205
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
206
|
+
// we can only get so many responses per call, so we set this to
|
|
207
|
+
// the first response we get, then keep adding to its array of blocks
|
|
208
|
+
// with each subsequent response
|
|
209
|
+
let overallResult = undefined;
|
|
210
|
+
let start_cursor = undefined;
|
|
211
|
+
// Note: there is a now a collectPaginatedAPI() in the notion client, so
|
|
212
|
+
// we could switch to using that (I don't know if it does rate limiting?)
|
|
213
|
+
do {
|
|
214
|
+
yield rateLimit();
|
|
215
|
+
const response = yield notionClient.blocks.children.list({
|
|
216
|
+
start_cursor: start_cursor,
|
|
217
|
+
block_id: id,
|
|
218
|
+
});
|
|
219
|
+
if (!overallResult) {
|
|
220
|
+
overallResult = response;
|
|
221
|
+
}
|
|
222
|
+
else {
|
|
223
|
+
overallResult.results.push(...response.results);
|
|
224
|
+
}
|
|
225
|
+
start_cursor = response === null || response === void 0 ? void 0 : response.next_cursor;
|
|
226
|
+
} while (start_cursor != null);
|
|
227
|
+
if ((_a = overallResult === null || overallResult === void 0 ? void 0 : overallResult.results) === null || _a === void 0 ? void 0 : _a.some(b => !(0, client_1.isFullBlock)(b))) {
|
|
228
|
+
(0, log_1.error)(`The Notion API returned some blocks that were not full blocks. docu-notion does not handle this yet. Please report it.`);
|
|
229
|
+
(0, process_1.exit)(1);
|
|
230
|
+
}
|
|
231
|
+
return (_b = overallResult === null || overallResult === void 0 ? void 0 : overallResult.results) !== null && _b !== void 0 ? _b : [];
|
|
232
|
+
});
|
|
233
|
+
}
|
|
234
|
+
function initNotionClient(notionToken) {
|
|
235
|
+
notionClient = new client_1.Client({
|
|
236
|
+
auth: notionToken,
|
|
237
|
+
});
|
|
238
|
+
return notionClient;
|
|
239
|
+
}
|
|
240
|
+
exports.initNotionClient = initNotionClient;
|
|
241
|
+
function fromPageId(context, pageId, order, foundDirectlyInOutline) {
|
|
242
|
+
return __awaiter(this, void 0, void 0, function* () {
|
|
243
|
+
const metadata = yield getPageMetadata(pageId);
|
|
244
|
+
//logDebug("notion metadata", JSON.stringify(metadata));
|
|
245
|
+
return new NotionPage_1.NotionPage({
|
|
246
|
+
layoutContext: context,
|
|
247
|
+
pageId,
|
|
248
|
+
order,
|
|
249
|
+
metadata,
|
|
250
|
+
foundDirectlyInOutline,
|
|
251
|
+
});
|
|
252
|
+
});
|
|
253
|
+
}
|
package/dist/run.d.ts
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function run(): void;
|
package/dist/run.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.run = void 0;
|
|
4
|
+
const commander_1 = require("commander");
|
|
5
|
+
const log_1 = require("./log");
|
|
6
|
+
const pull_1 = require("./pull");
|
|
7
|
+
function run() {
|
|
8
|
+
const pkg = require("../package.json");
|
|
9
|
+
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
|
|
10
|
+
console.log(`docu-notion version ${pkg.version}`);
|
|
11
|
+
commander_1.program.name("docu-notion").description("");
|
|
12
|
+
commander_1.program.usage("-n <token> -r <root> [options]");
|
|
13
|
+
commander_1.program
|
|
14
|
+
.requiredOption("-n, --notion-token <string>", "notion api token, which looks like secret_3bc1b50XFYb15123RHF243x43450XFY33250XFYa343")
|
|
15
|
+
.requiredOption("-r, --root-page <string>", "The 31 character ID of the page which is the root of your docs page in notion. The code will look like 9120ec9960244ead80fa2ef4bc1bba25. This page must have a child page named 'Outline'")
|
|
16
|
+
.option("-m, --markdown-output-path <string>", "Root of the hierarchy for md files. WARNING: node-pull-mdx will delete files from this directory. Note also that if it finds localized images, it will create an i18n/ directory as a sibling.", "./docs")
|
|
17
|
+
.option("-t, --status-tag <string>", "Database pages without a Notion page property 'status' matching this will be ignored. Use '*' to ignore status altogether.", "Publish")
|
|
18
|
+
.option("--locales <codes>", "Comma-separated list of iso 639-2 codes, the same list as in docusaurus.config.js, minus the primary (i.e. 'en'). This is needed for image localization.", parseLocales, [])
|
|
19
|
+
.addOption(new commander_1.Option("-l, --log-level <level>", "Log level").choices([
|
|
20
|
+
"info",
|
|
21
|
+
"verbose",
|
|
22
|
+
"debug",
|
|
23
|
+
]))
|
|
24
|
+
.option("-i, --img-output-path <string>", "Path to directory where images will be stored. If this is not included, images will be placed in the same directory as the document that uses them, which then allows for localization of screenshots.")
|
|
25
|
+
.option("-p, --img-prefix-in-markdown <string>", "When referencing an image from markdown, prefix with this path instead of the full img-output-path. Should be used only in conjunction with --img-output-path.");
|
|
26
|
+
commander_1.program.showHelpAfterError();
|
|
27
|
+
commander_1.program.parse();
|
|
28
|
+
(0, log_1.setLogLevel)(commander_1.program.opts().logLevel);
|
|
29
|
+
console.log(JSON.stringify(commander_1.program.opts));
|
|
30
|
+
(0, pull_1.notionPull)(commander_1.program.opts()).then(() => console.log("docu-notion Finished."));
|
|
31
|
+
}
|
|
32
|
+
exports.run = run;
|
|
33
|
+
function parseLocales(value) {
|
|
34
|
+
return value.split(",").map(l => l.trim().toLowerCase());
|
|
35
|
+
}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { IDocuNotionContext } from "./plugins/pluginTypes";
|
|
2
|
+
import { NotionPage } from "./NotionPage";
|
|
3
|
+
import { IDocuNotionConfig } from "./config/configuration";
|
|
4
|
+
import { NotionBlock } from "./types";
|
|
5
|
+
export declare function getMarkdownForPage(config: IDocuNotionConfig, context: IDocuNotionContext, page: NotionPage): Promise<string>;
|
|
6
|
+
export declare function getMarkdownFromNotionBlocks(context: IDocuNotionContext, config: IDocuNotionConfig, blocks: Array<NotionBlock>): Promise<string>;
|