notion-to-md 3.1.7 → 4.0.0-alpha.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -302
- package/build/core/block-fetcher/index.d.ts +36 -0
- package/build/core/block-fetcher/index.d.ts.map +1 -0
- package/build/core/block-fetcher/index.js +241 -0
- package/build/core/block-fetcher/index.js.map +1 -0
- package/build/core/errors.d.ts +16 -0
- package/build/core/errors.d.ts.map +1 -0
- package/build/core/errors.js +31 -0
- package/build/core/errors.js.map +1 -0
- package/build/core/exporter/index.d.ts +12 -0
- package/build/core/exporter/index.d.ts.map +1 -0
- package/build/core/exporter/index.js +70 -0
- package/build/core/exporter/index.js.map +1 -0
- package/build/core/media-handler/index.d.ts +27 -0
- package/build/core/media-handler/index.d.ts.map +1 -0
- package/build/core/media-handler/index.js +135 -0
- package/build/core/media-handler/index.js.map +1 -0
- package/build/core/media-handler/strategies/download.d.ts +18 -0
- package/build/core/media-handler/strategies/download.d.ts.map +1 -0
- package/build/core/media-handler/strategies/download.js +258 -0
- package/build/core/media-handler/strategies/download.js.map +1 -0
- package/build/core/media-handler/strategies/upload.d.ts +11 -0
- package/build/core/media-handler/strategies/upload.d.ts.map +1 -0
- package/build/core/media-handler/strategies/upload.js +151 -0
- package/build/core/media-handler/strategies/upload.js.map +1 -0
- package/build/core/page-ref-handler/index.d.ts +30 -0
- package/build/core/page-ref-handler/index.d.ts.map +1 -0
- package/build/core/page-ref-handler/index.js +179 -0
- package/build/core/page-ref-handler/index.js.map +1 -0
- package/build/core/renderer/index.d.ts +113 -0
- package/build/core/renderer/index.d.ts.map +1 -0
- package/build/core/renderer/index.js +364 -0
- package/build/core/renderer/index.js.map +1 -0
- package/build/index.d.ts +62 -1
- package/build/index.d.ts.map +1 -1
- package/build/index.js +198 -15
- package/build/index.js.map +1 -1
- package/build/plugins/exporter/index.d.ts +40 -0
- package/build/plugins/exporter/index.d.ts.map +1 -0
- package/build/plugins/exporter/index.js +85 -0
- package/build/plugins/exporter/index.js.map +1 -0
- package/build/plugins/renderer/mdx/helpers.d.ts +7 -0
- package/build/plugins/renderer/mdx/helpers.d.ts.map +1 -0
- package/build/plugins/renderer/mdx/helpers.js +102 -0
- package/build/plugins/renderer/mdx/helpers.js.map +1 -0
- package/build/plugins/renderer/mdx/index.d.ts +16 -0
- package/build/plugins/renderer/mdx/index.d.ts.map +1 -0
- package/build/plugins/renderer/mdx/index.js +25 -0
- package/build/plugins/renderer/mdx/index.js.map +1 -0
- package/build/plugins/renderer/mdx/resolvers.d.ts +28 -0
- package/build/plugins/renderer/mdx/resolvers.d.ts.map +1 -0
- package/build/plugins/renderer/mdx/resolvers.js +99 -0
- package/build/plugins/renderer/mdx/resolvers.js.map +1 -0
- package/build/plugins/renderer/mdx/transformers/annotations.d.ts +3 -0
- package/build/plugins/renderer/mdx/transformers/annotations.d.ts.map +1 -0
- package/build/plugins/renderer/mdx/transformers/annotations.js +37 -0
- package/build/plugins/renderer/mdx/transformers/annotations.js.map +1 -0
- package/build/plugins/renderer/mdx/transformers/blocks.d.ts +3 -0
- package/build/plugins/renderer/mdx/transformers/blocks.d.ts.map +1 -0
- package/build/plugins/renderer/mdx/transformers/blocks.js +501 -0
- package/build/plugins/renderer/mdx/transformers/blocks.js.map +1 -0
- package/build/types/index.d.ts +5 -86
- package/build/types/index.d.ts.map +1 -1
- package/build/types/index.js +19 -0
- package/build/types/index.js.map +1 -1
- package/build/types/manifest-manager.d.ts +49 -0
- package/build/types/manifest-manager.d.ts.map +1 -0
- package/build/types/manifest-manager.js +18 -0
- package/build/types/manifest-manager.js.map +1 -0
- package/build/types/module.d.ts +24 -0
- package/build/types/module.d.ts.map +1 -0
- package/build/types/module.js +18 -0
- package/build/types/module.js.map +1 -0
- package/build/types/notion.d.ts +29 -0
- package/build/types/notion.d.ts.map +1 -0
- package/build/types/notion.js +3 -0
- package/build/types/notion.js.map +1 -0
- package/build/types/renderer.d.ts +42 -0
- package/build/types/renderer.d.ts.map +1 -0
- package/build/types/renderer.js +3 -0
- package/build/types/renderer.js.map +1 -0
- package/build/types/strategy.d.ts +43 -0
- package/build/types/strategy.d.ts.map +1 -0
- package/build/types/strategy.js +15 -0
- package/build/types/strategy.js.map +1 -0
- package/build/utils/manifest-manager/base.d.ts +25 -0
- package/build/utils/manifest-manager/base.d.ts.map +1 -0
- package/build/utils/manifest-manager/base.js +104 -0
- package/build/utils/manifest-manager/base.js.map +1 -0
- package/build/utils/manifest-manager/errors.d.ts +34 -0
- package/build/utils/manifest-manager/errors.d.ts.map +1 -0
- package/build/utils/manifest-manager/errors.js +77 -0
- package/build/utils/manifest-manager/errors.js.map +1 -0
- package/build/utils/manifest-manager/index.d.ts +3 -0
- package/build/utils/manifest-manager/index.d.ts.map +1 -0
- package/build/utils/manifest-manager/index.js +19 -0
- package/build/utils/manifest-manager/index.js.map +1 -0
- package/build/utils/manifest-manager/media.d.ts +56 -0
- package/build/utils/manifest-manager/media.d.ts.map +1 -0
- package/build/utils/manifest-manager/media.js +163 -0
- package/build/utils/manifest-manager/media.js.map +1 -0
- package/build/utils/manifest-manager/page.d.ts +63 -0
- package/build/utils/manifest-manager/page.d.ts.map +1 -0
- package/build/utils/manifest-manager/page.js +170 -0
- package/build/utils/manifest-manager/page.js.map +1 -0
- package/build/utils/notion/index.d.ts +4 -0
- package/build/utils/notion/index.d.ts.map +1 -0
- package/build/utils/notion/index.js +21 -0
- package/build/utils/notion/index.js.map +1 -0
- package/build/utils/page-ref-builder/index.d.ts +23 -0
- package/build/utils/page-ref-builder/index.d.ts.map +1 -0
- package/build/utils/page-ref-builder/index.js +133 -0
- package/build/utils/page-ref-builder/index.js.map +1 -0
- package/build/utils/url.d.ts +3 -0
- package/build/utils/url.d.ts.map +1 -0
- package/build/utils/url.js +8 -0
- package/build/utils/url.js.map +1 -0
- package/package.json +28 -9
- package/build/notion-to-md.d.ts +0 -47
- package/build/notion-to-md.d.ts.map +0 -1
- package/build/notion-to-md.js +0 -494
- package/build/notion-to-md.js.map +0 -1
- package/build/notion-to-md.spec.d.ts +0 -2
- package/build/notion-to-md.spec.d.ts.map +0 -1
- package/build/notion-to-md.spec.js +0 -66
- package/build/notion-to-md.spec.js.map +0 -1
- package/build/utils/md.d.ts +0 -23
- package/build/utils/md.d.ts.map +0 -1
- package/build/utils/md.js +0 -148
- package/build/utils/md.js.map +0 -1
- package/build/utils/md.spec.d.ts +0 -2
- package/build/utils/md.spec.d.ts.map +0 -1
- package/build/utils/md.spec.js +0 -105
- package/build/utils/md.spec.js.map +0 -1
- package/build/utils/notion.d.ts +0 -5
- package/build/utils/notion.d.ts.map +0 -1
- package/build/utils/notion.js +0 -36
- package/build/utils/notion.js.map +0 -1
package/README.md
CHANGED
|
@@ -1,322 +1,31 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
<!--
|
|
4
|
-
💡 For better readability and detailed instructions head over to the [wiki](https://github.com/souvikinator/notion-to-md/wiki).
|
|
5
|
-
-->
|
|
6
1
|
<h1 align="center">
|
|
7
2
|
<br>
|
|
8
|
-
<img src="https://
|
|
3
|
+
<img src="https://raw.githubusercontent.com/souvikinator/notion-to-md/refs/heads/v4.0.0-alpha.1/docs/notion-to-md/static/v4-banner.png" alt="notion-to-md banner" width="750" />
|
|
9
4
|
<br>
|
|
10
|
-
<b>Notion-to-MD
|
|
5
|
+
<b>Notion-to-MD v4 <sup>α</sup></b>
|
|
11
6
|
<br>
|
|
12
|
-
<sub><sup><b>
|
|
7
|
+
<sub><sup><b>Convert Notion to Markdown & Beyond</b></sup></sub>
|
|
13
8
|
<br>
|
|
14
9
|
</h1>
|
|
15
10
|
|
|
16
|
-
<div align="center">
|
|
17
|
-
<a href="https://github.com/souvikinator/notion-to-md/discussions/112">
|
|
18
|
-
<img src="https://i.ibb.co/Tmyf805/Group-167.png" />
|
|
19
|
-
</a>
|
|
20
|
-
</div>
|
|
21
|
-
|
|
22
|
-
<div align="center">
|
|
23
|
-
<a href="https://discord.gg/drWw5Ya535">
|
|
24
|
-
<img src="https://i.ibb.co/ZTk7QCW/Group-186.png" />
|
|
25
|
-
</a>
|
|
26
|
-
</div>
|
|
27
|
-
|
|
28
|
-
<div align="center">
|
|
29
|
-
<a href="https://www.reddit.com/r/notion_to_md/">
|
|
30
|
-
<img src="https://i.ibb.co/vBN5Cgz/Frame-4.png" />
|
|
31
|
-
</a>
|
|
32
|
-
</div>
|
|
33
|
-
|
|
34
11
|
<p align="center">
|
|
35
|
-
|
|
36
|
-
</p>
|
|
37
|
-
<p align="center">
|
|
38
|
-
Convert notion pages, blocks and list of blocks to markdown (supports nesting) using <a href="https://github.com/makenotion/notion-sdk-js">notion-sdk-js</a>
|
|
12
|
+
Powerful Notion conversion engine that lets you write once in Notion and seamlessly publish anywhere, in any format.
|
|
39
13
|
</p>
|
|
14
|
+
|
|
40
15
|
<p align="center">
|
|
41
16
|
<img src="https://img.shields.io/github/stars/souvikinator/notion-to-md?style=flat" alt="github stars">
|
|
42
17
|
<img src="https://img.shields.io/twitter/follow/souvikinator?color=blue&logo=x&style=flat" alt="twitter" />
|
|
43
18
|
<img src="https://komarev.com/ghpvc/?username=notion-to-md&abbreviated=true&label=REPO+VIEWS" alt="repo views" />
|
|
44
19
|
</p>
|
|
45
20
|
|
|
46
|
-
##
|
|
47
|
-
> If you've created a specific client, please open an issue to have it added here.
|
|
48
|
-
|
|
49
|
-
- [notion-to-md-py](https://github.com/SwordAndTea/notion-to-md-py) by [@SwordAndTea](https://github.com/SwordAndTea)
|
|
50
|
-
|
|
51
|
-
## Install
|
|
52
|
-
|
|
53
|
-
```Bash
|
|
54
|
-
npm install notion-to-md
|
|
55
|
-
```
|
|
56
|
-
|
|
57
|
-
## Usage
|
|
58
|
-
> ⚠️ **Note:** Before getting started, create [an integration and find the token](https://www.notion.so/my-integrations).
|
|
59
|
-
> Details on methods can be found in [API section](https://github.com/souvikinator/notion-to-md#api)
|
|
60
|
-
|
|
61
|
-
> ⚠️ **Note:** Starting from v2.7.0, `toMarkdownString` no longer automatically saves child pages.
|
|
62
|
-
> Now it provides an object containing the markdown content of child pages.
|
|
63
|
-
|
|
64
|
-
## converting markdown objects to markdown string
|
|
65
|
-
|
|
66
|
-
This is how the notion page looks for this example:
|
|
67
|
-
|
|
68
|
-
<img src="https://imgur.com/O6bKCmH.png" width="500" />
|
|
69
|
-
|
|
70
|
-
```javascript
|
|
71
|
-
const { Client } = require("@notionhq/client");
|
|
72
|
-
const { NotionToMarkdown } = require("notion-to-md");
|
|
73
|
-
const fs = require('fs');
|
|
74
|
-
// or
|
|
75
|
-
// import {NotionToMarkdown} from "notion-to-md";
|
|
76
|
-
|
|
77
|
-
const notion = new Client({
|
|
78
|
-
auth: "your integration token",
|
|
79
|
-
});
|
|
80
|
-
|
|
81
|
-
// passing notion client to the option
|
|
82
|
-
const n2m = new NotionToMarkdown({ notionClient: notion });
|
|
83
|
-
|
|
84
|
-
(async () => {
|
|
85
|
-
const mdblocks = await n2m.pageToMarkdown("target_page_id");
|
|
86
|
-
const mdString = n2m.toMarkdownString(mdblocks);
|
|
87
|
-
console.log(mdString.parent);
|
|
88
|
-
})();
|
|
89
|
-
```
|
|
90
|
-
<img src="https://imgur.com/XrUYrZ0.png" width="500" />
|
|
91
|
-
|
|
92
|
-
## Separate child page content
|
|
93
|
-
|
|
94
|
-
**parent page content:**
|
|
95
|
-
|
|
96
|
-
<img src="https://github.com/souvikinator/notion-to-md/assets/64456160/531ef45d-2dc7-47f4-bbb3-12d6fd44d299" width="500" />
|
|
97
|
-
|
|
98
|
-
**child page content:**
|
|
99
|
-
|
|
100
|
-
<img src="https://github.com/souvikinator/notion-to-md/assets/64456160/7dde090b-7333-46f8-b6df-e6c9a7b62fa9" width="500" />
|
|
101
|
-
|
|
102
|
-
`NotionToMarkdown` takes second argument, `config`
|
|
103
|
-
|
|
104
|
-
```javascript
|
|
105
|
-
const { Client } = require("@notionhq/client");
|
|
106
|
-
const { NotionToMarkdown } = require("notion-to-md");
|
|
107
|
-
const fs = require('fs');
|
|
108
|
-
// or
|
|
109
|
-
// import {NotionToMarkdown} from "notion-to-md";
|
|
110
|
-
|
|
111
|
-
const notion = new Client({
|
|
112
|
-
auth: "your integration token",
|
|
113
|
-
});
|
|
114
|
-
|
|
115
|
-
// passing notion client to the option
|
|
116
|
-
const n2m = new NotionToMarkdown({
|
|
117
|
-
notionClient: notion,
|
|
118
|
-
config:{
|
|
119
|
-
separateChildPage:true, // default: false
|
|
120
|
-
}
|
|
121
|
-
});
|
|
122
|
-
|
|
123
|
-
(async () => {
|
|
124
|
-
const mdblocks = await n2m.pageToMarkdown("target_page_id");
|
|
125
|
-
const mdString = n2m.toMarkdownString(mdblocks);
|
|
126
|
-
|
|
127
|
-
console.log(mdString);
|
|
128
|
-
})();
|
|
129
|
-
```
|
|
130
|
-
|
|
131
|
-
**Output:**
|
|
132
|
-
|
|
133
|
-
`toMarkdownString` returns an object with target page content corresponding to `parent` property and if any child page exists then it's included in the same object.
|
|
134
|
-
|
|
135
|
-
<img src="https://github.com/souvikinator/notion-to-md/assets/64456160/99bcc14e-46e6-4bed-912d-8b9300c214c1" width="500" />
|
|
136
|
-
|
|
137
|
-
User gets to save the content separately.
|
|
138
|
-
|
|
139
|
-
## Disable child page parsing
|
|
140
|
-
|
|
141
|
-
```javascript
|
|
142
|
-
...
|
|
143
|
-
|
|
144
|
-
const n2m = new NotionToMarkdown({
|
|
145
|
-
notionClient: notion,
|
|
146
|
-
config:{
|
|
147
|
-
parseChildPages:false, // default: parseChildPages
|
|
148
|
-
}
|
|
149
|
-
});
|
|
150
|
-
|
|
151
|
-
...
|
|
152
|
-
```
|
|
153
|
-
|
|
154
|
-
## converting page to markdown object
|
|
155
|
-
|
|
156
|
-
**Example notion page:**
|
|
157
|
-
|
|
158
|
-
<img src="https://imgur.com/9iqRpBl.png" width="500" />
|
|
159
|
-
|
|
160
|
-
```js
|
|
161
|
-
const { Client } = require("@notionhq/client");
|
|
162
|
-
const { NotionToMarkdown } = require("notion-to-md");
|
|
163
|
-
|
|
164
|
-
const notion = new Client({
|
|
165
|
-
auth: "your integration token",
|
|
166
|
-
});
|
|
167
|
-
|
|
168
|
-
// passing notion client to the option
|
|
169
|
-
const n2m = new NotionToMarkdown({ notionClient: notion });
|
|
170
|
-
|
|
171
|
-
(async () => {
|
|
172
|
-
// notice second argument, totalPage.
|
|
173
|
-
const x = await n2m.pageToMarkdown("target_page_id", 2);
|
|
174
|
-
console.log(x);
|
|
175
|
-
})();
|
|
176
|
-
```
|
|
177
|
-
|
|
178
|
-
**Output:**
|
|
179
|
-
|
|
180
|
-
```json
|
|
181
|
-
[
|
|
182
|
-
{
|
|
183
|
-
"parent": "# heading 1",
|
|
184
|
-
"children": []
|
|
185
|
-
},
|
|
186
|
-
{
|
|
187
|
-
"parent": "- bullet 1",
|
|
188
|
-
"children": [
|
|
189
|
-
{
|
|
190
|
-
"parent": "- bullet 1.1",
|
|
191
|
-
"children": []
|
|
192
|
-
},
|
|
193
|
-
{
|
|
194
|
-
"parent": "- bullet 1.2",
|
|
195
|
-
"children": []
|
|
196
|
-
}
|
|
197
|
-
]
|
|
198
|
-
},
|
|
199
|
-
{
|
|
200
|
-
"parent": "- bullet 2",
|
|
201
|
-
"children": []
|
|
202
|
-
},
|
|
203
|
-
{
|
|
204
|
-
"parent": "- [ ] check box 1",
|
|
205
|
-
"children": [
|
|
206
|
-
{
|
|
207
|
-
"parent": "- [x] check box 1.2",
|
|
208
|
-
"children": []
|
|
209
|
-
},
|
|
210
|
-
{
|
|
211
|
-
"parent": "- [ ] check box 1.3",
|
|
212
|
-
"children": []
|
|
213
|
-
}
|
|
214
|
-
]
|
|
215
|
-
},
|
|
216
|
-
{
|
|
217
|
-
"parent": "- [ ] checkbox 2",
|
|
218
|
-
"children": []
|
|
219
|
-
}
|
|
220
|
-
]
|
|
221
|
-
```
|
|
222
|
-
|
|
223
|
-
## converting list of blocks to markdown object
|
|
224
|
-
|
|
225
|
-
```js
|
|
226
|
-
const { Client } = require("@notionhq/client");
|
|
227
|
-
const { NotionToMarkdown } = require("notion-to-md");
|
|
228
|
-
|
|
229
|
-
const notion = new Client({
|
|
230
|
-
auth: "your integration token",
|
|
231
|
-
});
|
|
232
|
-
|
|
233
|
-
// passing notion client to the option
|
|
234
|
-
const n2m = new NotionToMarkdown({ notionClient: notion });
|
|
235
|
-
|
|
236
|
-
(async () => {
|
|
237
|
-
// get all blocks in the page
|
|
238
|
-
const { results } = await notion.blocks.children.list({
|
|
239
|
-
block_id,
|
|
240
|
-
});
|
|
241
|
-
|
|
242
|
-
//convert to markdown
|
|
243
|
-
const x = await n2m.blocksToMarkdown(results);
|
|
244
|
-
console.log(x);
|
|
245
|
-
})();
|
|
246
|
-
```
|
|
247
|
-
|
|
248
|
-
**Output**: same as before
|
|
249
|
-
|
|
250
|
-
## Converting a single block to markdown string
|
|
251
|
-
|
|
252
|
-
- only takes a single notion block and returns corresponding markdown string
|
|
253
|
-
- nesting is ignored
|
|
254
|
-
- depends on @notionhq/client
|
|
255
|
-
|
|
256
|
-
```js
|
|
257
|
-
const { NotionToMarkdown } = require("notion-to-md");
|
|
258
|
-
|
|
259
|
-
// passing notion client to the option
|
|
260
|
-
const n2m = new NotionToMarkdown({ notionClient: notion });
|
|
261
|
-
|
|
262
|
-
const result = n2m.blockToMarkdown(block);
|
|
263
|
-
console.log(result);
|
|
264
|
-
```
|
|
265
|
-
|
|
266
|
-
**result**:
|
|
267
|
-
|
|
268
|
-
```
|
|
269
|
-

|
|
270
|
-
```
|
|
271
|
-
|
|
272
|
-
## Custom Transformers
|
|
273
|
-
|
|
274
|
-
You can define your own custom transformer for a notion type, to parse and return your own string.
|
|
275
|
-
`setCustomTransformer(type, func)` will overload the parsing for the giving type.
|
|
276
|
-
|
|
277
|
-
```ts
|
|
278
|
-
const { NotionToMarkdown } = require("notion-to-md");
|
|
279
|
-
const n2m = new NotionToMarkdown({ notionClient: notion });
|
|
280
|
-
n2m.setCustomTransformer("embed", async (block) => {
|
|
281
|
-
const { embed } = block as any;
|
|
282
|
-
if (!embed?.url) return "";
|
|
283
|
-
return `<figure>
|
|
284
|
-
<iframe src="${embed?.url}"></iframe>
|
|
285
|
-
<figcaption>${await n2m.blockToMarkdown(embed?.caption)}</figcaption>
|
|
286
|
-
</figure>`;
|
|
287
|
-
});
|
|
288
|
-
const result = n2m.blockToMarkdown(block);
|
|
289
|
-
// Result will now parse the `embed` type with your custom function.
|
|
290
|
-
```
|
|
291
|
-
|
|
292
|
-
**Note** Be aware that `setCustomTransformer` will take only the last function for the given type. You can't set two different transforms for the same type.
|
|
293
|
-
|
|
294
|
-
You can also use the default parsing by returning `false` in your custom transformer.
|
|
295
|
-
|
|
296
|
-
```ts
|
|
297
|
-
// ...
|
|
298
|
-
n2m.setCustomTransformer("embed", async (block) => {
|
|
299
|
-
const { embed } = block as any;
|
|
300
|
-
if (embed?.url?.includes("myspecialurl.com")) {
|
|
301
|
-
return `...`; // some special rendering
|
|
302
|
-
}
|
|
303
|
-
return false; // use default behavior
|
|
304
|
-
});
|
|
305
|
-
const result = n2m.blockToMarkdown(block);
|
|
306
|
-
// Result will now only use custom parser if the embed url matches a specific url
|
|
307
|
-
```
|
|
308
|
-
|
|
309
|
-
## Contribution
|
|
21
|
+
## Vision
|
|
310
22
|
|
|
311
|
-
|
|
312
|
-
Please make sure to update tests as appropriate.
|
|
23
|
+
Notion-to-MD v4 represents a fundamental reimagining of what a Notion conversion library can be. Building on the solid foundation of previous versions, v4 introduces a modular, extensible architecture designed to meet the evolving needs of modern content workflows.
|
|
313
24
|
|
|
314
|
-
|
|
25
|
+
We aim to build more than a Markdown converter; our goal is to develop a comprehensive content transformation system that seamlessly adapts to diverse output formats, allows instant export anywhere, and integrates smoothly into any content workflow while preserving the structure and richness of Notion content.
|
|
315
26
|
|
|
316
|
-
|
|
317
|
-
<img src="https://contrib.rocks/image?repo=souvikinator/notion-to-md" />
|
|
318
|
-
</a>
|
|
27
|
+
## Documentation
|
|
319
28
|
|
|
320
|
-
|
|
29
|
+
For installation instructions, complete API references, and detailed examples, please visit our comprehensive documentation:
|
|
321
30
|
|
|
322
|
-
[
|
|
31
|
+
[https://notionconvert.com/notion-to-md/](https://notionconvert.com/notion-to-md/)
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import { Client } from '@notionhq/client';
|
|
2
|
+
import { ExtendedFetcherOutput, ProcessorChainNode, ChainData } from '../../types';
|
|
3
|
+
export interface BlockFetcherConfig {
|
|
4
|
+
fetchPageProperties?: boolean;
|
|
5
|
+
fetchComments?: boolean;
|
|
6
|
+
maxRequestsPerSecond?: number;
|
|
7
|
+
batchSize?: number;
|
|
8
|
+
trackMediaBlocks?: boolean;
|
|
9
|
+
trackPageRefBlocks?: boolean;
|
|
10
|
+
}
|
|
11
|
+
export declare class BlockFetcher implements ProcessorChainNode {
|
|
12
|
+
private client;
|
|
13
|
+
private config;
|
|
14
|
+
next?: ProcessorChainNode;
|
|
15
|
+
private queue;
|
|
16
|
+
private blocks;
|
|
17
|
+
private processedTasks;
|
|
18
|
+
private pageProperties?;
|
|
19
|
+
private rootComments;
|
|
20
|
+
private rootBlockId;
|
|
21
|
+
private mediaBlocks;
|
|
22
|
+
private pageRefBlocks;
|
|
23
|
+
private rateLimitWindow;
|
|
24
|
+
constructor(pageId: string, client: Client, config?: BlockFetcherConfig);
|
|
25
|
+
process(data: ChainData): Promise<ChainData>;
|
|
26
|
+
getBlocks(pageId: string): Promise<ExtendedFetcherOutput>;
|
|
27
|
+
private addTask;
|
|
28
|
+
private processTask;
|
|
29
|
+
private rateLimitRequest;
|
|
30
|
+
private fetchBlockChildren;
|
|
31
|
+
private fetchAllComments;
|
|
32
|
+
private fetchPageProperties;
|
|
33
|
+
private normalizeId;
|
|
34
|
+
private buildBlockTree;
|
|
35
|
+
}
|
|
36
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/block-fetcher/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAML,qBAAqB,EACrB,kBAAkB,EAClB,SAAS,EACV,MAAM,aAAa,CAAC;AAGrB,MAAM,WAAW,kBAAkB;IACjC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAQD,qBAAa,YAAa,YAAW,kBAAkB;IAmBnD,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IAnBhB,IAAI,CAAC,EAAE,kBAAkB,CAAC;IAE1B,OAAO,CAAC,KAAK,CAAmB;IAChC,OAAO,CAAC,MAAM,CAAsD;IACpE,OAAO,CAAC,cAAc,CAAqB;IAC3C,OAAO,CAAC,cAAc,CAAC,CAAuB;IAC9C,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,WAAW,CAAyC;IAC5D,OAAO,CAAC,aAAa,CAAyC;IAE9D,OAAO,CAAC,eAAe,CAGrB;gBAGA,MAAM,EAAE,MAAM,EACN,MAAM,EAAE,MAAM,EACd,MAAM,GAAE,kBAKf;IAOG,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IAW5C,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA6C/D,OAAO,CAAC,OAAO;YAOD,WAAW;YAuEX,gBAAgB;YA0BhB,kBAAkB;YA6BlB,gBAAgB;YAuBhB,mBAAmB;IAUjC,OAAO,CAAC,WAAW;IAInB,OAAO,CAAC,cAAc;CAmCvB"}
|
|
@@ -0,0 +1,241 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.BlockFetcher = void 0;
|
|
4
|
+
const notion_1 = require("../../utils/notion");
|
|
5
|
+
class BlockFetcher {
|
|
6
|
+
constructor(pageId, client, config = {
|
|
7
|
+
fetchPageProperties: false,
|
|
8
|
+
fetchComments: false,
|
|
9
|
+
maxRequestsPerSecond: 3,
|
|
10
|
+
batchSize: 3,
|
|
11
|
+
}) {
|
|
12
|
+
var _a, _b;
|
|
13
|
+
this.client = client;
|
|
14
|
+
this.config = config;
|
|
15
|
+
this.queue = [];
|
|
16
|
+
this.blocks = new Map();
|
|
17
|
+
this.processedTasks = new Set();
|
|
18
|
+
this.rootComments = [];
|
|
19
|
+
this.rootBlockId = '';
|
|
20
|
+
this.mediaBlocks = [];
|
|
21
|
+
this.pageRefBlocks = [];
|
|
22
|
+
this.rateLimitWindow = {
|
|
23
|
+
requests: 0,
|
|
24
|
+
startTime: Date.now(),
|
|
25
|
+
};
|
|
26
|
+
const moduleType = 'BlockFetcher';
|
|
27
|
+
this.config.maxRequestsPerSecond = (_a = config.maxRequestsPerSecond) !== null && _a !== void 0 ? _a : 3;
|
|
28
|
+
this.config.batchSize = (_b = config.batchSize) !== null && _b !== void 0 ? _b : 3;
|
|
29
|
+
}
|
|
30
|
+
async process(data) {
|
|
31
|
+
const blockTree = await this.getBlocks(data.pageId);
|
|
32
|
+
const updatedData = {
|
|
33
|
+
...data,
|
|
34
|
+
blockTree,
|
|
35
|
+
};
|
|
36
|
+
return this.next ? this.next.process(updatedData) : updatedData;
|
|
37
|
+
}
|
|
38
|
+
async getBlocks(pageId) {
|
|
39
|
+
this.rootBlockId = pageId;
|
|
40
|
+
this.queue = [];
|
|
41
|
+
this.blocks.clear();
|
|
42
|
+
this.processedTasks.clear();
|
|
43
|
+
this.pageProperties = undefined;
|
|
44
|
+
this.rootComments = [];
|
|
45
|
+
// Initialize queue with root level tasks
|
|
46
|
+
this.addTask({ type: 'fetch_blocks', id: pageId });
|
|
47
|
+
if (this.config.fetchPageProperties) {
|
|
48
|
+
this.addTask({ type: 'fetch_properties', id: pageId });
|
|
49
|
+
}
|
|
50
|
+
if (this.config.fetchComments) {
|
|
51
|
+
this.addTask({ type: 'fetch_comments', id: pageId });
|
|
52
|
+
}
|
|
53
|
+
// Process queue until empty
|
|
54
|
+
while (this.queue.length > 0) {
|
|
55
|
+
const batch = this.queue.splice(0, this.config.batchSize);
|
|
56
|
+
await Promise.all(batch.map((task) => this.processTask(task)));
|
|
57
|
+
}
|
|
58
|
+
const baseOutput = {
|
|
59
|
+
properties: this.pageProperties || {},
|
|
60
|
+
comments: this.rootComments,
|
|
61
|
+
blocks: this.buildBlockTree(pageId),
|
|
62
|
+
};
|
|
63
|
+
// Only include special arrays if tracking was enabled
|
|
64
|
+
if (this.config.trackMediaBlocks || this.config.trackPageRefBlocks) {
|
|
65
|
+
return {
|
|
66
|
+
...baseOutput,
|
|
67
|
+
...(this.config.trackMediaBlocks && { mediaBlocks: this.mediaBlocks }),
|
|
68
|
+
...(this.config.trackPageRefBlocks && {
|
|
69
|
+
pageRefBlocks: this.pageRefBlocks,
|
|
70
|
+
}),
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
return baseOutput;
|
|
74
|
+
}
|
|
75
|
+
addTask(task) {
|
|
76
|
+
const taskId = `${task.type}-${task.id}`;
|
|
77
|
+
if (!this.processedTasks.has(taskId)) {
|
|
78
|
+
this.queue.push(task);
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
async processTask(task) {
|
|
82
|
+
const taskId = `${task.type}-${task.id}`;
|
|
83
|
+
if (this.processedTasks.has(taskId))
|
|
84
|
+
return;
|
|
85
|
+
switch (task.type) {
|
|
86
|
+
case 'fetch_blocks': {
|
|
87
|
+
const blocks = await this.fetchBlockChildren(task.id);
|
|
88
|
+
for (const block of blocks) {
|
|
89
|
+
const storedBlock = {
|
|
90
|
+
...block,
|
|
91
|
+
children: [],
|
|
92
|
+
comments: [],
|
|
93
|
+
};
|
|
94
|
+
this.blocks.set(block.id, storedBlock);
|
|
95
|
+
// Track special blocks if enabled (storing references)
|
|
96
|
+
// these will be used by the next stage saving us from a
|
|
97
|
+
// lot of recursive calls to idenitfy blocks
|
|
98
|
+
if (this.config.trackMediaBlocks && (0, notion_1.isMediaBlock)(storedBlock)) {
|
|
99
|
+
this.mediaBlocks.push(storedBlock);
|
|
100
|
+
}
|
|
101
|
+
if (this.config.trackPageRefBlocks && (0, notion_1.isPageRefBlock)(storedBlock)) {
|
|
102
|
+
this.pageRefBlocks.push(storedBlock);
|
|
103
|
+
}
|
|
104
|
+
// If block has children, queue task to fetch them
|
|
105
|
+
if ('has_children' in block && block.has_children) {
|
|
106
|
+
this.addTask({
|
|
107
|
+
type: 'fetch_blocks',
|
|
108
|
+
id: block.id,
|
|
109
|
+
parentId: task.id,
|
|
110
|
+
});
|
|
111
|
+
}
|
|
112
|
+
// Queue comment fetching for each block if enabled
|
|
113
|
+
// extra api call for comment fetching, no way to know otherwise
|
|
114
|
+
if (this.config.fetchComments) {
|
|
115
|
+
this.addTask({
|
|
116
|
+
type: 'fetch_comments',
|
|
117
|
+
id: block.id,
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
break;
|
|
122
|
+
}
|
|
123
|
+
case 'fetch_comments': {
|
|
124
|
+
const comments = await this.fetchAllComments(task.id);
|
|
125
|
+
if (task.id === this.rootBlockId) {
|
|
126
|
+
// page level comments
|
|
127
|
+
this.rootComments = comments;
|
|
128
|
+
}
|
|
129
|
+
else {
|
|
130
|
+
const block = this.blocks.get(task.id);
|
|
131
|
+
if (block) {
|
|
132
|
+
block.comments = comments;
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
break;
|
|
136
|
+
}
|
|
137
|
+
case 'fetch_properties': {
|
|
138
|
+
const properties = await this.fetchPageProperties(task.id);
|
|
139
|
+
this.pageProperties = properties;
|
|
140
|
+
break;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
this.processedTasks.add(taskId);
|
|
144
|
+
}
|
|
145
|
+
async rateLimitRequest(request) {
|
|
146
|
+
const now = Date.now();
|
|
147
|
+
const windowSize = 1000;
|
|
148
|
+
// Reset window if it's expired
|
|
149
|
+
if (now - this.rateLimitWindow.startTime >= windowSize) {
|
|
150
|
+
this.rateLimitWindow = {
|
|
151
|
+
requests: 0,
|
|
152
|
+
startTime: now,
|
|
153
|
+
};
|
|
154
|
+
}
|
|
155
|
+
// Wait if we've hit the rate limit
|
|
156
|
+
if (this.rateLimitWindow.requests >= this.config.maxRequestsPerSecond) {
|
|
157
|
+
const waitTime = windowSize - (now - this.rateLimitWindow.startTime);
|
|
158
|
+
await new Promise((resolve) => setTimeout(resolve, waitTime));
|
|
159
|
+
this.rateLimitWindow = {
|
|
160
|
+
requests: 0,
|
|
161
|
+
startTime: Date.now(),
|
|
162
|
+
};
|
|
163
|
+
}
|
|
164
|
+
this.rateLimitWindow.requests++;
|
|
165
|
+
return request();
|
|
166
|
+
}
|
|
167
|
+
async fetchBlockChildren(blockId) {
|
|
168
|
+
var _a;
|
|
169
|
+
let allBlocks = [];
|
|
170
|
+
let hasMore = true;
|
|
171
|
+
let cursor;
|
|
172
|
+
while (hasMore) {
|
|
173
|
+
const response = await this.rateLimitRequest(() => this.client.blocks.children.list({
|
|
174
|
+
block_id: blockId,
|
|
175
|
+
start_cursor: cursor,
|
|
176
|
+
}));
|
|
177
|
+
// Filter out unsupported blocks
|
|
178
|
+
// should we leave it here or let user handle it in renderer?
|
|
179
|
+
const blocks = response.results.filter((block) => 'type' in block && block.type !== 'unsupported');
|
|
180
|
+
allBlocks = [...allBlocks, ...blocks];
|
|
181
|
+
hasMore = response.has_more;
|
|
182
|
+
cursor = (_a = response.next_cursor) !== null && _a !== void 0 ? _a : undefined;
|
|
183
|
+
}
|
|
184
|
+
return allBlocks;
|
|
185
|
+
}
|
|
186
|
+
async fetchAllComments(blockId) {
|
|
187
|
+
var _a;
|
|
188
|
+
let allComments = [];
|
|
189
|
+
let hasMore = true;
|
|
190
|
+
let cursor;
|
|
191
|
+
while (hasMore) {
|
|
192
|
+
const response = await this.rateLimitRequest(() => this.client.comments.list({
|
|
193
|
+
block_id: blockId,
|
|
194
|
+
start_cursor: cursor,
|
|
195
|
+
}));
|
|
196
|
+
allComments = [...allComments, ...response.results];
|
|
197
|
+
hasMore = response.has_more;
|
|
198
|
+
cursor = (_a = response.next_cursor) !== null && _a !== void 0 ? _a : undefined;
|
|
199
|
+
}
|
|
200
|
+
return allComments;
|
|
201
|
+
}
|
|
202
|
+
async fetchPageProperties(pageId) {
|
|
203
|
+
const response = await this.rateLimitRequest(() => this.client.pages.retrieve({ page_id: pageId }));
|
|
204
|
+
return 'properties' in response ? response.properties : {};
|
|
205
|
+
}
|
|
206
|
+
normalizeId(id) {
|
|
207
|
+
return id.replace(/-/g, '');
|
|
208
|
+
}
|
|
209
|
+
buildBlockTree(rootId) {
|
|
210
|
+
var _a, _b;
|
|
211
|
+
const childrenMap = new Map();
|
|
212
|
+
for (const [id, block] of this.blocks.entries()) {
|
|
213
|
+
const parentId =
|
|
214
|
+
// @ts-ignore
|
|
215
|
+
((_a = block.parent) === null || _a === void 0 ? void 0 : _a.type) === 'block_id'
|
|
216
|
+
? // @ts-ignore
|
|
217
|
+
this.normalizeId(block.parent.block_id)
|
|
218
|
+
: // @ts-ignore
|
|
219
|
+
((_b = block.parent) === null || _b === void 0 ? void 0 : _b.type) === 'page_id'
|
|
220
|
+
? // @ts-ignore
|
|
221
|
+
this.normalizeId(block.parent.page_id)
|
|
222
|
+
: undefined;
|
|
223
|
+
if (parentId) {
|
|
224
|
+
if (!childrenMap.has(parentId)) {
|
|
225
|
+
childrenMap.set(parentId, []);
|
|
226
|
+
}
|
|
227
|
+
childrenMap.get(parentId).push(block);
|
|
228
|
+
}
|
|
229
|
+
}
|
|
230
|
+
const buildChildren = (parentId) => {
|
|
231
|
+
const children = childrenMap.get(this.normalizeId(parentId)) || [];
|
|
232
|
+
return children.map((block) => {
|
|
233
|
+
block.children = buildChildren(block.id);
|
|
234
|
+
return block;
|
|
235
|
+
});
|
|
236
|
+
};
|
|
237
|
+
return buildChildren(rootId);
|
|
238
|
+
}
|
|
239
|
+
}
|
|
240
|
+
exports.BlockFetcher = BlockFetcher;
|
|
241
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/block-fetcher/index.ts"],"names":[],"mappings":";;;AAWA,+CAAkE;AAiBlE,MAAa,YAAY;IAiBvB,YACE,MAAc,EACN,MAAc,EACd,SAA6B;QACnC,mBAAmB,EAAE,KAAK;QAC1B,aAAa,EAAE,KAAK;QACpB,oBAAoB,EAAE,CAAC;QACvB,SAAS,EAAE,CAAC;KACb;;QANO,WAAM,GAAN,MAAM,CAAQ;QACd,WAAM,GAAN,MAAM,CAKb;QAtBK,UAAK,GAAgB,EAAE,CAAC;QACxB,WAAM,GAAG,IAAI,GAAG,EAA2C,CAAC;QAC5D,mBAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QAEnC,iBAAY,GAA2B,EAAE,CAAC;QAC1C,gBAAW,GAAW,EAAE,CAAC;QACzB,gBAAW,GAAsC,EAAE,CAAC;QACpD,kBAAa,GAAsC,EAAE,CAAC;QAEtD,oBAAe,GAAG;YACxB,QAAQ,EAAE,CAAC;YACX,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QAYA,MAAM,UAAU,GAAG,cAAc,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,oBAAoB,GAAG,MAAA,MAAM,CAAC,oBAAoB,mCAAI,CAAC,CAAC;QACpE,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,MAAA,MAAM,CAAC,SAAS,mCAAI,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAe;QAC3B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEpD,MAAM,WAAW,GAAc;YAC7B,GAAG,IAAI;YACP,SAAS;SACV,CAAC;QAEF,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAc;QAC5B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAChC,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QAEvB,yCAAyC;QACzC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAEnD,IAAI,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE;YACnC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;SACxD;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE;YAC7B,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;SACtD;QAED,4BAA4B;QAC5B,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC1D,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAChE;QAED,MAAM,UAAU,GAAkB;YAChC,UAAU,EAAE,IAAI,CAAC,cAAc,IAAI,EAAE;YACrC,QAAQ,EAAE,IAAI,CAAC,YAAY;YAC3B,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;SACpC,CAAC;QAEF,sDAAsD;QACtD,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE;YAClE,OAAO;gBACL,GAAG,UAAU;gBACb,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;gBACtE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI;oBACpC,aAAa,EAAE,IAAI,CAAC,aAAa;iBAClC,CAAC;aACH,CAAC;SACH;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,OAAO,CAAC,IAAe;QAC7B,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;YACpC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACvB;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAe;QACvC,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QAE5C,QAAQ,IAAI,CAAC,IAAI,EAAE;YACjB,KAAK,cAAc,CAAC,CAAC;gBACnB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEtD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;oBAC1B,MAAM,WAAW,GAAG;wBAClB,GAAG,KAAK;wBACR,QAAQ,EAAE,EAAE;wBACZ,QAAQ,EAAE,EAAE;qBACb,CAAC;oBACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;oBAEvC,uDAAuD;oBACvD,wDAAwD;oBACxD,4CAA4C;oBAC5C,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAA,qBAAY,EAAC,WAAW,CAAC,EAAE;wBAC7D,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;qBACpC;oBACD,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI,IAAA,uBAAc,EAAC,WAAW,CAAC,EAAE;wBACjE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;qBACtC;oBAED,kDAAkD;oBAClD,IAAI,cAAc,IAAI,KAAK,IAAI,KAAK,CAAC,YAAY,EAAE;wBACjD,IAAI,CAAC,OAAO,CAAC;4BACX,IAAI,EAAE,cAAc;4BACpB,EAAE,EAAE,KAAK,CAAC,EAAE;4BACZ,QAAQ,EAAE,IAAI,CAAC,EAAE;yBAClB,CAAC,CAAC;qBACJ;oBAED,mDAAmD;oBACnD,gEAAgE;oBAChE,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE;wBAC7B,IAAI,CAAC,OAAO,CAAC;4BACX,IAAI,EAAE,gBAAgB;4BACtB,EAAE,EAAE,KAAK,CAAC,EAAE;yBACb,CAAC,CAAC;qBACJ;iBACF;gBACD,MAAM;aACP;YAED,KAAK,gBAAgB,CAAC,CAAC;gBACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE;oBAChC,sBAAsB;oBACtB,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC;iBAC9B;qBAAM;oBACL,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACvC,IAAI,KAAK,EAAE;wBACT,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;qBAC3B;iBACF;gBACD,MAAM;aACP;YAED,KAAK,kBAAkB,CAAC,CAAC;gBACvB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC3D,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC;gBACjC,MAAM;aACP;SACF;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAI,OAAyB;QACzD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,UAAU,GAAG,IAAI,CAAC;QAExB,+BAA+B;QAC/B,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,IAAI,UAAU,EAAE;YACtD,IAAI,CAAC,eAAe,GAAG;gBACrB,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,GAAG;aACf,CAAC;SACH;QAED,mCAAmC;QACnC,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAqB,EAAE;YACtE,MAAM,QAAQ,GAAG,UAAU,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YACrE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;YAC9D,IAAI,CAAC,eAAe,GAAG;gBACrB,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC;SACH;QAED,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,EAAE,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,OAAe;;QAEf,IAAI,SAAS,GAAqC,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,IAAI,MAA0B,CAAC;QAE/B,OAAO,OAAO,EAAE;YACd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAChD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAC/B,QAAQ,EAAE,OAAO;gBACjB,YAAY,EAAE,MAAM;aACrB,CAAC,CACH,CAAC;YAEF,gCAAgC;YAChC,6DAA6D;YAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CACpC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,CACvB,CAAC;YAEtC,SAAS,GAAG,CAAC,GAAG,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC;YACtC,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC;YAC5B,MAAM,GAAG,MAAA,QAAQ,CAAC,WAAW,mCAAI,SAAS,CAAC;SAC5C;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,OAAe;;QAEf,IAAI,WAAW,GAA2B,EAAE,CAAC;QAC7C,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,IAAI,MAA0B,CAAC;QAE/B,OAAO,OAAO,EAAE;YACd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAChD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACxB,QAAQ,EAAE,OAAO;gBACjB,YAAY,EAAE,MAAM;aACrB,CAAC,CACH,CAAC;YAEF,WAAW,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;YACpD,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC;YAC5B,MAAM,GAAG,MAAA,QAAQ,CAAC,WAAW,mCAAI,SAAS,CAAC;SAC5C;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,MAAc;QAEd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAChD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAChD,CAAC;QAEF,OAAO,YAAY,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7D,CAAC;IAEO,WAAW,CAAC,EAAU;QAC5B,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC9B,CAAC;IAEO,cAAc,CAAC,MAAc;;QACnC,MAAM,WAAW,GAAG,IAAI,GAAG,EAA6C,CAAC;QAEzE,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE;YAC/C,MAAM,QAAQ;YACZ,aAAa;YACb,CAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,IAAI,MAAK,UAAU;gBAC/B,CAAC,CAAC,aAAa;oBACb,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC;gBACzC,CAAC,CAAC,aAAa;oBACb,CAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,IAAI,MAAK,SAAS;wBAChC,CAAC,CAAC,aAAa;4BACb,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;wBACxC,CAAC,CAAC,SAAS,CAAC;YAElB,IAAI,QAAQ,EAAE;gBACZ,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;oBAC9B,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBAC/B;gBACD,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACxC;SACF;QAED,MAAM,aAAa,GAAG,CACpB,QAAgB,EACkB,EAAE;YACpC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;YACnE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC5B,KAAK,CAAC,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACzC,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;QACL,CAAC,CAAC;QAEF,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;CACF;AArSD,oCAqSC","sourcesContent":["import { Client } from '@notionhq/client';\nimport {\n ListBlockChildrenResponseResult,\n PageObjectProperties,\n CommentResponseResults,\n FetcherOutput,\n ListBlockChildrenResponseResults,\n ExtendedFetcherOutput,\n ProcessorChainNode,\n ChainData,\n} from '../../types';\nimport { isMediaBlock, isPageRefBlock } from '../../utils/notion';\n\nexport interface BlockFetcherConfig {\n fetchPageProperties?: boolean;\n fetchComments?: boolean;\n maxRequestsPerSecond?: number;\n batchSize?: number;\n trackMediaBlocks?: boolean;\n trackPageRefBlocks?: boolean;\n}\n\ninterface QueueTask {\n type: 'fetch_properties' | 'fetch_comments' | 'fetch_blocks';\n id: string;\n parentId?: string;\n}\n\nexport class BlockFetcher implements ProcessorChainNode {\n next?: ProcessorChainNode;\n\n private queue: QueueTask[] = [];\n private blocks = new Map<string, ListBlockChildrenResponseResult>();\n private processedTasks = new Set<string>();\n private pageProperties?: PageObjectProperties;\n private rootComments: CommentResponseResults = [];\n private rootBlockId: string = '';\n private mediaBlocks: ListBlockChildrenResponseResult[] = [];\n private pageRefBlocks: ListBlockChildrenResponseResult[] = [];\n\n private rateLimitWindow = {\n requests: 0,\n startTime: Date.now(),\n };\n\n constructor(\n pageId: string,\n private client: Client,\n private config: BlockFetcherConfig = {\n fetchPageProperties: false,\n fetchComments: false,\n maxRequestsPerSecond: 3,\n batchSize: 3,\n },\n ) {\n const moduleType = 'BlockFetcher';\n this.config.maxRequestsPerSecond = config.maxRequestsPerSecond ?? 3;\n this.config.batchSize = config.batchSize ?? 3;\n }\n\n async process(data: ChainData): Promise<ChainData> {\n const blockTree = await this.getBlocks(data.pageId);\n\n const updatedData: ChainData = {\n ...data,\n blockTree,\n };\n\n return this.next ? this.next.process(updatedData) : updatedData;\n }\n\n async getBlocks(pageId: string): Promise<ExtendedFetcherOutput> {\n this.rootBlockId = pageId;\n this.queue = [];\n this.blocks.clear();\n this.processedTasks.clear();\n this.pageProperties = undefined;\n this.rootComments = [];\n\n // Initialize queue with root level tasks\n this.addTask({ type: 'fetch_blocks', id: pageId });\n\n if (this.config.fetchPageProperties) {\n this.addTask({ type: 'fetch_properties', id: pageId });\n }\n\n if (this.config.fetchComments) {\n this.addTask({ type: 'fetch_comments', id: pageId });\n }\n\n // Process queue until empty\n while (this.queue.length > 0) {\n const batch = this.queue.splice(0, this.config.batchSize);\n await Promise.all(batch.map((task) => this.processTask(task)));\n }\n\n const baseOutput: FetcherOutput = {\n properties: this.pageProperties || {},\n comments: this.rootComments,\n blocks: this.buildBlockTree(pageId),\n };\n\n // Only include special arrays if tracking was enabled\n if (this.config.trackMediaBlocks || this.config.trackPageRefBlocks) {\n return {\n ...baseOutput,\n ...(this.config.trackMediaBlocks && { mediaBlocks: this.mediaBlocks }),\n ...(this.config.trackPageRefBlocks && {\n pageRefBlocks: this.pageRefBlocks,\n }),\n };\n }\n\n return baseOutput;\n }\n\n private addTask(task: QueueTask): void {\n const taskId = `${task.type}-${task.id}`;\n if (!this.processedTasks.has(taskId)) {\n this.queue.push(task);\n }\n }\n\n private async processTask(task: QueueTask): Promise<void> {\n const taskId = `${task.type}-${task.id}`;\n if (this.processedTasks.has(taskId)) return;\n\n switch (task.type) {\n case 'fetch_blocks': {\n const blocks = await this.fetchBlockChildren(task.id);\n\n for (const block of blocks) {\n const storedBlock = {\n ...block,\n children: [],\n comments: [],\n };\n this.blocks.set(block.id, storedBlock);\n\n // Track special blocks if enabled (storing references)\n // these will be used by the next stage saving us from a\n // lot of recursive calls to idenitfy blocks\n if (this.config.trackMediaBlocks && isMediaBlock(storedBlock)) {\n this.mediaBlocks.push(storedBlock);\n }\n if (this.config.trackPageRefBlocks && isPageRefBlock(storedBlock)) {\n this.pageRefBlocks.push(storedBlock);\n }\n\n // If block has children, queue task to fetch them\n if ('has_children' in block && block.has_children) {\n this.addTask({\n type: 'fetch_blocks',\n id: block.id,\n parentId: task.id,\n });\n }\n\n // Queue comment fetching for each block if enabled\n // extra api call for comment fetching, no way to know otherwise\n if (this.config.fetchComments) {\n this.addTask({\n type: 'fetch_comments',\n id: block.id,\n });\n }\n }\n break;\n }\n\n case 'fetch_comments': {\n const comments = await this.fetchAllComments(task.id);\n if (task.id === this.rootBlockId) {\n // page level comments\n this.rootComments = comments;\n } else {\n const block = this.blocks.get(task.id);\n if (block) {\n block.comments = comments;\n }\n }\n break;\n }\n\n case 'fetch_properties': {\n const properties = await this.fetchPageProperties(task.id);\n this.pageProperties = properties;\n break;\n }\n }\n\n this.processedTasks.add(taskId);\n }\n\n private async rateLimitRequest<T>(request: () => Promise<T>): Promise<T> {\n const now = Date.now();\n const windowSize = 1000;\n\n // Reset window if it's expired\n if (now - this.rateLimitWindow.startTime >= windowSize) {\n this.rateLimitWindow = {\n requests: 0,\n startTime: now,\n };\n }\n\n // Wait if we've hit the rate limit\n if (this.rateLimitWindow.requests >= this.config.maxRequestsPerSecond!) {\n const waitTime = windowSize - (now - this.rateLimitWindow.startTime);\n await new Promise((resolve) => setTimeout(resolve, waitTime));\n this.rateLimitWindow = {\n requests: 0,\n startTime: Date.now(),\n };\n }\n\n this.rateLimitWindow.requests++;\n return request();\n }\n\n private async fetchBlockChildren(\n blockId: string,\n ): Promise<ListBlockChildrenResponseResults> {\n let allBlocks: ListBlockChildrenResponseResults = [];\n let hasMore = true;\n let cursor: string | undefined;\n\n while (hasMore) {\n const response = await this.rateLimitRequest(() =>\n this.client.blocks.children.list({\n block_id: blockId,\n start_cursor: cursor,\n }),\n );\n\n // Filter out unsupported blocks\n // should we leave it here or let user handle it in renderer?\n const blocks = response.results.filter(\n (block) => 'type' in block && block.type !== 'unsupported',\n ) as ListBlockChildrenResponseResults;\n\n allBlocks = [...allBlocks, ...blocks];\n hasMore = response.has_more;\n cursor = response.next_cursor ?? undefined;\n }\n\n return allBlocks;\n }\n\n private async fetchAllComments(\n blockId: string,\n ): Promise<CommentResponseResults> {\n let allComments: CommentResponseResults = [];\n let hasMore = true;\n let cursor: string | undefined;\n\n while (hasMore) {\n const response = await this.rateLimitRequest(() =>\n this.client.comments.list({\n block_id: blockId,\n start_cursor: cursor,\n }),\n );\n\n allComments = [...allComments, ...response.results];\n hasMore = response.has_more;\n cursor = response.next_cursor ?? undefined;\n }\n\n return allComments;\n }\n\n private async fetchPageProperties(\n pageId: string,\n ): Promise<PageObjectProperties> {\n const response = await this.rateLimitRequest(() =>\n this.client.pages.retrieve({ page_id: pageId }),\n );\n\n return 'properties' in response ? response.properties : {};\n }\n\n private normalizeId(id: string): string {\n return id.replace(/-/g, '');\n }\n\n private buildBlockTree(rootId: string): ListBlockChildrenResponseResults {\n const childrenMap = new Map<string, ListBlockChildrenResponseResult[]>();\n\n for (const [id, block] of this.blocks.entries()) {\n const parentId =\n // @ts-ignore\n block.parent?.type === 'block_id'\n ? // @ts-ignore\n this.normalizeId(block.parent.block_id)\n : // @ts-ignore\n block.parent?.type === 'page_id'\n ? // @ts-ignore\n this.normalizeId(block.parent.page_id)\n : undefined;\n\n if (parentId) {\n if (!childrenMap.has(parentId)) {\n childrenMap.set(parentId, []);\n }\n childrenMap.get(parentId)!.push(block);\n }\n }\n\n const buildChildren = (\n parentId: string,\n ): ListBlockChildrenResponseResults => {\n const children = childrenMap.get(this.normalizeId(parentId)) || [];\n return children.map((block) => {\n block.children = buildChildren(block.id);\n return block;\n });\n };\n\n return buildChildren(rootId);\n }\n}\n"]}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Specific error for when manifest management fails
|
|
3
|
+
*/
|
|
4
|
+
export declare class ManifestNotInitializedError extends Error {
|
|
5
|
+
constructor(moduleType: string);
|
|
6
|
+
}
|
|
7
|
+
export declare class MediaHandlerError extends Error {
|
|
8
|
+
constructor(message: string, cause?: Error);
|
|
9
|
+
}
|
|
10
|
+
/**
|
|
11
|
+
* Specific error class for page reference handling errors
|
|
12
|
+
*/
|
|
13
|
+
export declare class PageReferenceHandlerError extends Error {
|
|
14
|
+
constructor(message: string, cause?: Error);
|
|
15
|
+
}
|
|
16
|
+
//# sourceMappingURL=errors.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/core/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,2BAA4B,SAAQ,KAAK;gBACxC,UAAU,EAAE,MAAM;CAM/B;AAED,qBAAa,iBAAkB,SAAQ,KAAK;gBAC9B,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK;CAI3C;AAED;;GAEG;AACH,qBAAa,yBAA0B,SAAQ,KAAK;gBACtC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK;CAI3C"}
|