notion-to-md 3.1.7 → 4.0.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/README.md +11 -302
  2. package/build/core/block-fetcher/index.d.ts +36 -0
  3. package/build/core/block-fetcher/index.d.ts.map +1 -0
  4. package/build/core/block-fetcher/index.js +241 -0
  5. package/build/core/block-fetcher/index.js.map +1 -0
  6. package/build/core/errors.d.ts +16 -0
  7. package/build/core/errors.d.ts.map +1 -0
  8. package/build/core/errors.js +31 -0
  9. package/build/core/errors.js.map +1 -0
  10. package/build/core/exporter/index.d.ts +12 -0
  11. package/build/core/exporter/index.d.ts.map +1 -0
  12. package/build/core/exporter/index.js +70 -0
  13. package/build/core/exporter/index.js.map +1 -0
  14. package/build/core/media-handler/index.d.ts +27 -0
  15. package/build/core/media-handler/index.d.ts.map +1 -0
  16. package/build/core/media-handler/index.js +135 -0
  17. package/build/core/media-handler/index.js.map +1 -0
  18. package/build/core/media-handler/strategies/download.d.ts +18 -0
  19. package/build/core/media-handler/strategies/download.d.ts.map +1 -0
  20. package/build/core/media-handler/strategies/download.js +258 -0
  21. package/build/core/media-handler/strategies/download.js.map +1 -0
  22. package/build/core/media-handler/strategies/upload.d.ts +11 -0
  23. package/build/core/media-handler/strategies/upload.d.ts.map +1 -0
  24. package/build/core/media-handler/strategies/upload.js +151 -0
  25. package/build/core/media-handler/strategies/upload.js.map +1 -0
  26. package/build/core/page-ref-handler/index.d.ts +30 -0
  27. package/build/core/page-ref-handler/index.d.ts.map +1 -0
  28. package/build/core/page-ref-handler/index.js +179 -0
  29. package/build/core/page-ref-handler/index.js.map +1 -0
  30. package/build/core/renderer/index.d.ts +113 -0
  31. package/build/core/renderer/index.d.ts.map +1 -0
  32. package/build/core/renderer/index.js +364 -0
  33. package/build/core/renderer/index.js.map +1 -0
  34. package/build/index.d.ts +62 -1
  35. package/build/index.d.ts.map +1 -1
  36. package/build/index.js +198 -15
  37. package/build/index.js.map +1 -1
  38. package/build/plugins/exporter/index.d.ts +40 -0
  39. package/build/plugins/exporter/index.d.ts.map +1 -0
  40. package/build/plugins/exporter/index.js +85 -0
  41. package/build/plugins/exporter/index.js.map +1 -0
  42. package/build/plugins/renderer/mdx/helpers.d.ts +7 -0
  43. package/build/plugins/renderer/mdx/helpers.d.ts.map +1 -0
  44. package/build/plugins/renderer/mdx/helpers.js +102 -0
  45. package/build/plugins/renderer/mdx/helpers.js.map +1 -0
  46. package/build/plugins/renderer/mdx/index.d.ts +16 -0
  47. package/build/plugins/renderer/mdx/index.d.ts.map +1 -0
  48. package/build/plugins/renderer/mdx/index.js +25 -0
  49. package/build/plugins/renderer/mdx/index.js.map +1 -0
  50. package/build/plugins/renderer/mdx/resolvers.d.ts +28 -0
  51. package/build/plugins/renderer/mdx/resolvers.d.ts.map +1 -0
  52. package/build/plugins/renderer/mdx/resolvers.js +99 -0
  53. package/build/plugins/renderer/mdx/resolvers.js.map +1 -0
  54. package/build/plugins/renderer/mdx/transformers/annotations.d.ts +3 -0
  55. package/build/plugins/renderer/mdx/transformers/annotations.d.ts.map +1 -0
  56. package/build/plugins/renderer/mdx/transformers/annotations.js +37 -0
  57. package/build/plugins/renderer/mdx/transformers/annotations.js.map +1 -0
  58. package/build/plugins/renderer/mdx/transformers/blocks.d.ts +3 -0
  59. package/build/plugins/renderer/mdx/transformers/blocks.d.ts.map +1 -0
  60. package/build/plugins/renderer/mdx/transformers/blocks.js +501 -0
  61. package/build/plugins/renderer/mdx/transformers/blocks.js.map +1 -0
  62. package/build/types/index.d.ts +5 -86
  63. package/build/types/index.d.ts.map +1 -1
  64. package/build/types/index.js +19 -0
  65. package/build/types/index.js.map +1 -1
  66. package/build/types/manifest-manager.d.ts +49 -0
  67. package/build/types/manifest-manager.d.ts.map +1 -0
  68. package/build/types/manifest-manager.js +18 -0
  69. package/build/types/manifest-manager.js.map +1 -0
  70. package/build/types/module.d.ts +24 -0
  71. package/build/types/module.d.ts.map +1 -0
  72. package/build/types/module.js +18 -0
  73. package/build/types/module.js.map +1 -0
  74. package/build/types/notion.d.ts +29 -0
  75. package/build/types/notion.d.ts.map +1 -0
  76. package/build/types/notion.js +3 -0
  77. package/build/types/notion.js.map +1 -0
  78. package/build/types/renderer.d.ts +42 -0
  79. package/build/types/renderer.d.ts.map +1 -0
  80. package/build/types/renderer.js +3 -0
  81. package/build/types/renderer.js.map +1 -0
  82. package/build/types/strategy.d.ts +43 -0
  83. package/build/types/strategy.d.ts.map +1 -0
  84. package/build/types/strategy.js +15 -0
  85. package/build/types/strategy.js.map +1 -0
  86. package/build/utils/manifest-manager/base.d.ts +25 -0
  87. package/build/utils/manifest-manager/base.d.ts.map +1 -0
  88. package/build/utils/manifest-manager/base.js +104 -0
  89. package/build/utils/manifest-manager/base.js.map +1 -0
  90. package/build/utils/manifest-manager/errors.d.ts +34 -0
  91. package/build/utils/manifest-manager/errors.d.ts.map +1 -0
  92. package/build/utils/manifest-manager/errors.js +77 -0
  93. package/build/utils/manifest-manager/errors.js.map +1 -0
  94. package/build/utils/manifest-manager/index.d.ts +3 -0
  95. package/build/utils/manifest-manager/index.d.ts.map +1 -0
  96. package/build/utils/manifest-manager/index.js +19 -0
  97. package/build/utils/manifest-manager/index.js.map +1 -0
  98. package/build/utils/manifest-manager/media.d.ts +56 -0
  99. package/build/utils/manifest-manager/media.d.ts.map +1 -0
  100. package/build/utils/manifest-manager/media.js +163 -0
  101. package/build/utils/manifest-manager/media.js.map +1 -0
  102. package/build/utils/manifest-manager/page.d.ts +63 -0
  103. package/build/utils/manifest-manager/page.d.ts.map +1 -0
  104. package/build/utils/manifest-manager/page.js +170 -0
  105. package/build/utils/manifest-manager/page.js.map +1 -0
  106. package/build/utils/notion/index.d.ts +4 -0
  107. package/build/utils/notion/index.d.ts.map +1 -0
  108. package/build/utils/notion/index.js +21 -0
  109. package/build/utils/notion/index.js.map +1 -0
  110. package/build/utils/page-ref-builder/index.d.ts +23 -0
  111. package/build/utils/page-ref-builder/index.d.ts.map +1 -0
  112. package/build/utils/page-ref-builder/index.js +133 -0
  113. package/build/utils/page-ref-builder/index.js.map +1 -0
  114. package/build/utils/url.d.ts +3 -0
  115. package/build/utils/url.d.ts.map +1 -0
  116. package/build/utils/url.js +8 -0
  117. package/build/utils/url.js.map +1 -0
  118. package/package.json +28 -9
  119. package/build/notion-to-md.d.ts +0 -47
  120. package/build/notion-to-md.d.ts.map +0 -1
  121. package/build/notion-to-md.js +0 -494
  122. package/build/notion-to-md.js.map +0 -1
  123. package/build/notion-to-md.spec.d.ts +0 -2
  124. package/build/notion-to-md.spec.d.ts.map +0 -1
  125. package/build/notion-to-md.spec.js +0 -66
  126. package/build/notion-to-md.spec.js.map +0 -1
  127. package/build/utils/md.d.ts +0 -23
  128. package/build/utils/md.d.ts.map +0 -1
  129. package/build/utils/md.js +0 -148
  130. package/build/utils/md.js.map +0 -1
  131. package/build/utils/md.spec.d.ts +0 -2
  132. package/build/utils/md.spec.d.ts.map +0 -1
  133. package/build/utils/md.spec.js +0 -105
  134. package/build/utils/md.spec.js.map +0 -1
  135. package/build/utils/notion.d.ts +0 -5
  136. package/build/utils/notion.d.ts.map +0 -1
  137. package/build/utils/notion.js +0 -36
  138. package/build/utils/notion.js.map +0 -1
package/README.md CHANGED
@@ -1,322 +1,31 @@
1
-
2
-
3
- <!--
4
- 💡 For better readability and detailed instructions head over to the [wiki](https://github.com/souvikinator/notion-to-md/wiki).
5
- -->
6
1
  <h1 align="center">
7
2
  <br>
8
- <img src="https://i.ibb.co/Pzyf13S/Group-168.png" alt="notion-to-md banner" width="750" />
3
+ <img src="https://raw.githubusercontent.com/souvikinator/notion-to-md/refs/heads/v4.0.0-alpha.1/docs/notion-to-md/static/v4-banner.png" alt="notion-to-md banner" width="750" />
9
4
  <br>
10
- <b>Notion-to-MD</b>
5
+ <b>Notion-to-MD v4 <sup>α</sup></b>
11
6
  <br>
12
- <sub><sup><b>(Notion to Markdown)</b></sup></sub>
7
+ <sub><sup><b>Convert Notion to Markdown & Beyond</b></sup></sub>
13
8
  <br>
14
9
  </h1>
15
10
 
16
- <div align="center">
17
- <a href="https://github.com/souvikinator/notion-to-md/discussions/112">
18
- <img src="https://i.ibb.co/Tmyf805/Group-167.png" />
19
- </a>
20
- </div>
21
-
22
- <div align="center">
23
- <a href="https://discord.gg/drWw5Ya535">
24
- <img src="https://i.ibb.co/ZTk7QCW/Group-186.png" />
25
- </a>
26
- </div>
27
-
28
- <div align="center">
29
- <a href="https://www.reddit.com/r/notion_to_md/">
30
- <img src="https://i.ibb.co/vBN5Cgz/Frame-4.png" />
31
- </a>
32
- </div>
33
-
34
11
  <p align="center">
35
- Notion-to-MD is a Node.js package that allows you to convert Notion pages to Markdown format.
36
- </p>
37
- <p align="center">
38
- Convert notion pages, blocks and list of blocks to markdown (supports nesting) using <a href="https://github.com/makenotion/notion-sdk-js">notion-sdk-js</a>
12
+ Powerful Notion conversion engine that lets you write once in Notion and seamlessly publish anywhere, in any format.
39
13
  </p>
14
+
40
15
  <p align="center">
41
16
  <img src="https://img.shields.io/github/stars/souvikinator/notion-to-md?style=flat" alt="github stars">
42
17
  <img src="https://img.shields.io/twitter/follow/souvikinator?color=blue&logo=x&style=flat" alt="twitter" />
43
18
  <img src="https://komarev.com/ghpvc/?username=notion-to-md&abbreviated=true&label=REPO+VIEWS" alt="repo views" />
44
19
  </p>
45
20
 
46
- ## Looking for Support in Other Languages?
47
- > If you've created a specific client, please open an issue to have it added here.
48
-
49
- - [notion-to-md-py](https://github.com/SwordAndTea/notion-to-md-py) by [@SwordAndTea](https://github.com/SwordAndTea)
50
-
51
- ## Install
52
-
53
- ```Bash
54
- npm install notion-to-md
55
- ```
56
-
57
- ## Usage
58
- > ⚠️ **Note:** Before getting started, create [an integration and find the token](https://www.notion.so/my-integrations).
59
- > Details on methods can be found in [API section](https://github.com/souvikinator/notion-to-md#api)
60
-
61
- > ⚠️ **Note:** Starting from v2.7.0, `toMarkdownString` no longer automatically saves child pages.
62
- > Now it provides an object containing the markdown content of child pages.
63
-
64
- ## converting markdown objects to markdown string
65
-
66
- This is how the notion page looks for this example:
67
-
68
- <img src="https://imgur.com/O6bKCmH.png" width="500" />
69
-
70
- ```javascript
71
- const { Client } = require("@notionhq/client");
72
- const { NotionToMarkdown } = require("notion-to-md");
73
- const fs = require('fs');
74
- // or
75
- // import {NotionToMarkdown} from "notion-to-md";
76
-
77
- const notion = new Client({
78
- auth: "your integration token",
79
- });
80
-
81
- // passing notion client to the option
82
- const n2m = new NotionToMarkdown({ notionClient: notion });
83
-
84
- (async () => {
85
- const mdblocks = await n2m.pageToMarkdown("target_page_id");
86
- const mdString = n2m.toMarkdownString(mdblocks);
87
- console.log(mdString.parent);
88
- })();
89
- ```
90
- <img src="https://imgur.com/XrUYrZ0.png" width="500" />
91
-
92
- ## Separate child page content
93
-
94
- **parent page content:**
95
-
96
- <img src="https://github.com/souvikinator/notion-to-md/assets/64456160/531ef45d-2dc7-47f4-bbb3-12d6fd44d299" width="500" />
97
-
98
- **child page content:**
99
-
100
- <img src="https://github.com/souvikinator/notion-to-md/assets/64456160/7dde090b-7333-46f8-b6df-e6c9a7b62fa9" width="500" />
101
-
102
- `NotionToMarkdown` takes second argument, `config`
103
-
104
- ```javascript
105
- const { Client } = require("@notionhq/client");
106
- const { NotionToMarkdown } = require("notion-to-md");
107
- const fs = require('fs');
108
- // or
109
- // import {NotionToMarkdown} from "notion-to-md";
110
-
111
- const notion = new Client({
112
- auth: "your integration token",
113
- });
114
-
115
- // passing notion client to the option
116
- const n2m = new NotionToMarkdown({
117
- notionClient: notion,
118
- config:{
119
- separateChildPage:true, // default: false
120
- }
121
- });
122
-
123
- (async () => {
124
- const mdblocks = await n2m.pageToMarkdown("target_page_id");
125
- const mdString = n2m.toMarkdownString(mdblocks);
126
-
127
- console.log(mdString);
128
- })();
129
- ```
130
-
131
- **Output:**
132
-
133
- `toMarkdownString` returns an object with target page content corresponding to `parent` property and if any child page exists then it's included in the same object.
134
-
135
- <img src="https://github.com/souvikinator/notion-to-md/assets/64456160/99bcc14e-46e6-4bed-912d-8b9300c214c1" width="500" />
136
-
137
- User gets to save the content separately.
138
-
139
- ## Disable child page parsing
140
-
141
- ```javascript
142
- ...
143
-
144
- const n2m = new NotionToMarkdown({
145
- notionClient: notion,
146
- config:{
147
- parseChildPages:false, // default: parseChildPages
148
- }
149
- });
150
-
151
- ...
152
- ```
153
-
154
- ## converting page to markdown object
155
-
156
- **Example notion page:**
157
-
158
- <img src="https://imgur.com/9iqRpBl.png" width="500" />
159
-
160
- ```js
161
- const { Client } = require("@notionhq/client");
162
- const { NotionToMarkdown } = require("notion-to-md");
163
-
164
- const notion = new Client({
165
- auth: "your integration token",
166
- });
167
-
168
- // passing notion client to the option
169
- const n2m = new NotionToMarkdown({ notionClient: notion });
170
-
171
- (async () => {
172
- // notice second argument, totalPage.
173
- const x = await n2m.pageToMarkdown("target_page_id", 2);
174
- console.log(x);
175
- })();
176
- ```
177
-
178
- **Output:**
179
-
180
- ```json
181
- [
182
- {
183
- "parent": "# heading 1",
184
- "children": []
185
- },
186
- {
187
- "parent": "- bullet 1",
188
- "children": [
189
- {
190
- "parent": "- bullet 1.1",
191
- "children": []
192
- },
193
- {
194
- "parent": "- bullet 1.2",
195
- "children": []
196
- }
197
- ]
198
- },
199
- {
200
- "parent": "- bullet 2",
201
- "children": []
202
- },
203
- {
204
- "parent": "- [ ] check box 1",
205
- "children": [
206
- {
207
- "parent": "- [x] check box 1.2",
208
- "children": []
209
- },
210
- {
211
- "parent": "- [ ] check box 1.3",
212
- "children": []
213
- }
214
- ]
215
- },
216
- {
217
- "parent": "- [ ] checkbox 2",
218
- "children": []
219
- }
220
- ]
221
- ```
222
-
223
- ## converting list of blocks to markdown object
224
-
225
- ```js
226
- const { Client } = require("@notionhq/client");
227
- const { NotionToMarkdown } = require("notion-to-md");
228
-
229
- const notion = new Client({
230
- auth: "your integration token",
231
- });
232
-
233
- // passing notion client to the option
234
- const n2m = new NotionToMarkdown({ notionClient: notion });
235
-
236
- (async () => {
237
- // get all blocks in the page
238
- const { results } = await notion.blocks.children.list({
239
- block_id,
240
- });
241
-
242
- //convert to markdown
243
- const x = await n2m.blocksToMarkdown(results);
244
- console.log(x);
245
- })();
246
- ```
247
-
248
- **Output**: same as before
249
-
250
- ## Converting a single block to markdown string
251
-
252
- - only takes a single notion block and returns corresponding markdown string
253
- - nesting is ignored
254
- - depends on @notionhq/client
255
-
256
- ```js
257
- const { NotionToMarkdown } = require("notion-to-md");
258
-
259
- // passing notion client to the option
260
- const n2m = new NotionToMarkdown({ notionClient: notion });
261
-
262
- const result = n2m.blockToMarkdown(block);
263
- console.log(result);
264
- ```
265
-
266
- **result**:
267
-
268
- ```
269
- ![image](https://media.giphy.com/media/Ju7l5y9osyymQ/giphy.gif)
270
- ```
271
-
272
- ## Custom Transformers
273
-
274
- You can define your own custom transformer for a notion type, to parse and return your own string.
275
- `setCustomTransformer(type, func)` will overload the parsing for the giving type.
276
-
277
- ```ts
278
- const { NotionToMarkdown } = require("notion-to-md");
279
- const n2m = new NotionToMarkdown({ notionClient: notion });
280
- n2m.setCustomTransformer("embed", async (block) => {
281
- const { embed } = block as any;
282
- if (!embed?.url) return "";
283
- return `<figure>
284
- <iframe src="${embed?.url}"></iframe>
285
- <figcaption>${await n2m.blockToMarkdown(embed?.caption)}</figcaption>
286
- </figure>`;
287
- });
288
- const result = n2m.blockToMarkdown(block);
289
- // Result will now parse the `embed` type with your custom function.
290
- ```
291
-
292
- **Note** Be aware that `setCustomTransformer` will take only the last function for the given type. You can't set two different transforms for the same type.
293
-
294
- You can also use the default parsing by returning `false` in your custom transformer.
295
-
296
- ```ts
297
- // ...
298
- n2m.setCustomTransformer("embed", async (block) => {
299
- const { embed } = block as any;
300
- if (embed?.url?.includes("myspecialurl.com")) {
301
- return `...`; // some special rendering
302
- }
303
- return false; // use default behavior
304
- });
305
- const result = n2m.blockToMarkdown(block);
306
- // Result will now only use custom parser if the embed url matches a specific url
307
- ```
308
-
309
- ## Contribution
21
+ ## Vision
310
22
 
311
- Pull requests are welcome. For major changes, please open an issue first to discuss what you would like to change.
312
- Please make sure to update tests as appropriate.
23
+ Notion-to-MD v4 represents a fundamental reimagining of what a Notion conversion library can be. Building on the solid foundation of previous versions, v4 introduces a modular, extensible architecture designed to meet the evolving needs of modern content workflows.
313
24
 
314
- ## Contributers
25
+ We aim to build more than a Markdown converter; our goal is to develop a comprehensive content transformation system that seamlessly adapts to diverse output formats, allows instant export anywhere, and integrates smoothly into any content workflow while preserving the structure and richness of Notion content.
315
26
 
316
- <a href="https://github.com/souvikinator/notion-to-md/graphs/contributors">
317
- <img src="https://contrib.rocks/image?repo=souvikinator/notion-to-md" />
318
- </a>
27
+ ## Documentation
319
28
 
320
- ## License
29
+ For installation instructions, complete API references, and detailed examples, please visit our comprehensive documentation:
321
30
 
322
- [MIT](https://choosealicense.com/licenses/mit/)
31
+ [https://notionconvert.com/notion-to-md/](https://notionconvert.com/notion-to-md/)
@@ -0,0 +1,36 @@
1
+ import { Client } from '@notionhq/client';
2
+ import { ExtendedFetcherOutput, ProcessorChainNode, ChainData } from '../../types';
3
+ export interface BlockFetcherConfig {
4
+ fetchPageProperties?: boolean;
5
+ fetchComments?: boolean;
6
+ maxRequestsPerSecond?: number;
7
+ batchSize?: number;
8
+ trackMediaBlocks?: boolean;
9
+ trackPageRefBlocks?: boolean;
10
+ }
11
+ export declare class BlockFetcher implements ProcessorChainNode {
12
+ private client;
13
+ private config;
14
+ next?: ProcessorChainNode;
15
+ private queue;
16
+ private blocks;
17
+ private processedTasks;
18
+ private pageProperties?;
19
+ private rootComments;
20
+ private rootBlockId;
21
+ private mediaBlocks;
22
+ private pageRefBlocks;
23
+ private rateLimitWindow;
24
+ constructor(pageId: string, client: Client, config?: BlockFetcherConfig);
25
+ process(data: ChainData): Promise<ChainData>;
26
+ getBlocks(pageId: string): Promise<ExtendedFetcherOutput>;
27
+ private addTask;
28
+ private processTask;
29
+ private rateLimitRequest;
30
+ private fetchBlockChildren;
31
+ private fetchAllComments;
32
+ private fetchPageProperties;
33
+ private normalizeId;
34
+ private buildBlockTree;
35
+ }
36
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../src/core/block-fetcher/index.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,MAAM,EAAE,MAAM,kBAAkB,CAAC;AAC1C,OAAO,EAML,qBAAqB,EACrB,kBAAkB,EAClB,SAAS,EACV,MAAM,aAAa,CAAC;AAGrB,MAAM,WAAW,kBAAkB;IACjC,mBAAmB,CAAC,EAAE,OAAO,CAAC;IAC9B,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,gBAAgB,CAAC,EAAE,OAAO,CAAC;IAC3B,kBAAkB,CAAC,EAAE,OAAO,CAAC;CAC9B;AAQD,qBAAa,YAAa,YAAW,kBAAkB;IAmBnD,OAAO,CAAC,MAAM;IACd,OAAO,CAAC,MAAM;IAnBhB,IAAI,CAAC,EAAE,kBAAkB,CAAC;IAE1B,OAAO,CAAC,KAAK,CAAmB;IAChC,OAAO,CAAC,MAAM,CAAsD;IACpE,OAAO,CAAC,cAAc,CAAqB;IAC3C,OAAO,CAAC,cAAc,CAAC,CAAuB;IAC9C,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,WAAW,CAAc;IACjC,OAAO,CAAC,WAAW,CAAyC;IAC5D,OAAO,CAAC,aAAa,CAAyC;IAE9D,OAAO,CAAC,eAAe,CAGrB;gBAGA,MAAM,EAAE,MAAM,EACN,MAAM,EAAE,MAAM,EACd,MAAM,GAAE,kBAKf;IAOG,OAAO,CAAC,IAAI,EAAE,SAAS,GAAG,OAAO,CAAC,SAAS,CAAC;IAW5C,SAAS,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,qBAAqB,CAAC;IA6C/D,OAAO,CAAC,OAAO;YAOD,WAAW;YAuEX,gBAAgB;YA0BhB,kBAAkB;YA6BlB,gBAAgB;YAuBhB,mBAAmB;IAUjC,OAAO,CAAC,WAAW;IAInB,OAAO,CAAC,cAAc;CAmCvB"}
@@ -0,0 +1,241 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.BlockFetcher = void 0;
4
+ const notion_1 = require("../../utils/notion");
5
+ class BlockFetcher {
6
+ constructor(pageId, client, config = {
7
+ fetchPageProperties: false,
8
+ fetchComments: false,
9
+ maxRequestsPerSecond: 3,
10
+ batchSize: 3,
11
+ }) {
12
+ var _a, _b;
13
+ this.client = client;
14
+ this.config = config;
15
+ this.queue = [];
16
+ this.blocks = new Map();
17
+ this.processedTasks = new Set();
18
+ this.rootComments = [];
19
+ this.rootBlockId = '';
20
+ this.mediaBlocks = [];
21
+ this.pageRefBlocks = [];
22
+ this.rateLimitWindow = {
23
+ requests: 0,
24
+ startTime: Date.now(),
25
+ };
26
+ const moduleType = 'BlockFetcher';
27
+ this.config.maxRequestsPerSecond = (_a = config.maxRequestsPerSecond) !== null && _a !== void 0 ? _a : 3;
28
+ this.config.batchSize = (_b = config.batchSize) !== null && _b !== void 0 ? _b : 3;
29
+ }
30
+ async process(data) {
31
+ const blockTree = await this.getBlocks(data.pageId);
32
+ const updatedData = {
33
+ ...data,
34
+ blockTree,
35
+ };
36
+ return this.next ? this.next.process(updatedData) : updatedData;
37
+ }
38
+ async getBlocks(pageId) {
39
+ this.rootBlockId = pageId;
40
+ this.queue = [];
41
+ this.blocks.clear();
42
+ this.processedTasks.clear();
43
+ this.pageProperties = undefined;
44
+ this.rootComments = [];
45
+ // Initialize queue with root level tasks
46
+ this.addTask({ type: 'fetch_blocks', id: pageId });
47
+ if (this.config.fetchPageProperties) {
48
+ this.addTask({ type: 'fetch_properties', id: pageId });
49
+ }
50
+ if (this.config.fetchComments) {
51
+ this.addTask({ type: 'fetch_comments', id: pageId });
52
+ }
53
+ // Process queue until empty
54
+ while (this.queue.length > 0) {
55
+ const batch = this.queue.splice(0, this.config.batchSize);
56
+ await Promise.all(batch.map((task) => this.processTask(task)));
57
+ }
58
+ const baseOutput = {
59
+ properties: this.pageProperties || {},
60
+ comments: this.rootComments,
61
+ blocks: this.buildBlockTree(pageId),
62
+ };
63
+ // Only include special arrays if tracking was enabled
64
+ if (this.config.trackMediaBlocks || this.config.trackPageRefBlocks) {
65
+ return {
66
+ ...baseOutput,
67
+ ...(this.config.trackMediaBlocks && { mediaBlocks: this.mediaBlocks }),
68
+ ...(this.config.trackPageRefBlocks && {
69
+ pageRefBlocks: this.pageRefBlocks,
70
+ }),
71
+ };
72
+ }
73
+ return baseOutput;
74
+ }
75
+ addTask(task) {
76
+ const taskId = `${task.type}-${task.id}`;
77
+ if (!this.processedTasks.has(taskId)) {
78
+ this.queue.push(task);
79
+ }
80
+ }
81
+ async processTask(task) {
82
+ const taskId = `${task.type}-${task.id}`;
83
+ if (this.processedTasks.has(taskId))
84
+ return;
85
+ switch (task.type) {
86
+ case 'fetch_blocks': {
87
+ const blocks = await this.fetchBlockChildren(task.id);
88
+ for (const block of blocks) {
89
+ const storedBlock = {
90
+ ...block,
91
+ children: [],
92
+ comments: [],
93
+ };
94
+ this.blocks.set(block.id, storedBlock);
95
+ // Track special blocks if enabled (storing references)
96
+ // these will be used by the next stage saving us from a
97
+ // lot of recursive calls to idenitfy blocks
98
+ if (this.config.trackMediaBlocks && (0, notion_1.isMediaBlock)(storedBlock)) {
99
+ this.mediaBlocks.push(storedBlock);
100
+ }
101
+ if (this.config.trackPageRefBlocks && (0, notion_1.isPageRefBlock)(storedBlock)) {
102
+ this.pageRefBlocks.push(storedBlock);
103
+ }
104
+ // If block has children, queue task to fetch them
105
+ if ('has_children' in block && block.has_children) {
106
+ this.addTask({
107
+ type: 'fetch_blocks',
108
+ id: block.id,
109
+ parentId: task.id,
110
+ });
111
+ }
112
+ // Queue comment fetching for each block if enabled
113
+ // extra api call for comment fetching, no way to know otherwise
114
+ if (this.config.fetchComments) {
115
+ this.addTask({
116
+ type: 'fetch_comments',
117
+ id: block.id,
118
+ });
119
+ }
120
+ }
121
+ break;
122
+ }
123
+ case 'fetch_comments': {
124
+ const comments = await this.fetchAllComments(task.id);
125
+ if (task.id === this.rootBlockId) {
126
+ // page level comments
127
+ this.rootComments = comments;
128
+ }
129
+ else {
130
+ const block = this.blocks.get(task.id);
131
+ if (block) {
132
+ block.comments = comments;
133
+ }
134
+ }
135
+ break;
136
+ }
137
+ case 'fetch_properties': {
138
+ const properties = await this.fetchPageProperties(task.id);
139
+ this.pageProperties = properties;
140
+ break;
141
+ }
142
+ }
143
+ this.processedTasks.add(taskId);
144
+ }
145
+ async rateLimitRequest(request) {
146
+ const now = Date.now();
147
+ const windowSize = 1000;
148
+ // Reset window if it's expired
149
+ if (now - this.rateLimitWindow.startTime >= windowSize) {
150
+ this.rateLimitWindow = {
151
+ requests: 0,
152
+ startTime: now,
153
+ };
154
+ }
155
+ // Wait if we've hit the rate limit
156
+ if (this.rateLimitWindow.requests >= this.config.maxRequestsPerSecond) {
157
+ const waitTime = windowSize - (now - this.rateLimitWindow.startTime);
158
+ await new Promise((resolve) => setTimeout(resolve, waitTime));
159
+ this.rateLimitWindow = {
160
+ requests: 0,
161
+ startTime: Date.now(),
162
+ };
163
+ }
164
+ this.rateLimitWindow.requests++;
165
+ return request();
166
+ }
167
+ async fetchBlockChildren(blockId) {
168
+ var _a;
169
+ let allBlocks = [];
170
+ let hasMore = true;
171
+ let cursor;
172
+ while (hasMore) {
173
+ const response = await this.rateLimitRequest(() => this.client.blocks.children.list({
174
+ block_id: blockId,
175
+ start_cursor: cursor,
176
+ }));
177
+ // Filter out unsupported blocks
178
+ // should we leave it here or let user handle it in renderer?
179
+ const blocks = response.results.filter((block) => 'type' in block && block.type !== 'unsupported');
180
+ allBlocks = [...allBlocks, ...blocks];
181
+ hasMore = response.has_more;
182
+ cursor = (_a = response.next_cursor) !== null && _a !== void 0 ? _a : undefined;
183
+ }
184
+ return allBlocks;
185
+ }
186
+ async fetchAllComments(blockId) {
187
+ var _a;
188
+ let allComments = [];
189
+ let hasMore = true;
190
+ let cursor;
191
+ while (hasMore) {
192
+ const response = await this.rateLimitRequest(() => this.client.comments.list({
193
+ block_id: blockId,
194
+ start_cursor: cursor,
195
+ }));
196
+ allComments = [...allComments, ...response.results];
197
+ hasMore = response.has_more;
198
+ cursor = (_a = response.next_cursor) !== null && _a !== void 0 ? _a : undefined;
199
+ }
200
+ return allComments;
201
+ }
202
+ async fetchPageProperties(pageId) {
203
+ const response = await this.rateLimitRequest(() => this.client.pages.retrieve({ page_id: pageId }));
204
+ return 'properties' in response ? response.properties : {};
205
+ }
206
+ normalizeId(id) {
207
+ return id.replace(/-/g, '');
208
+ }
209
+ buildBlockTree(rootId) {
210
+ var _a, _b;
211
+ const childrenMap = new Map();
212
+ for (const [id, block] of this.blocks.entries()) {
213
+ const parentId =
214
+ // @ts-ignore
215
+ ((_a = block.parent) === null || _a === void 0 ? void 0 : _a.type) === 'block_id'
216
+ ? // @ts-ignore
217
+ this.normalizeId(block.parent.block_id)
218
+ : // @ts-ignore
219
+ ((_b = block.parent) === null || _b === void 0 ? void 0 : _b.type) === 'page_id'
220
+ ? // @ts-ignore
221
+ this.normalizeId(block.parent.page_id)
222
+ : undefined;
223
+ if (parentId) {
224
+ if (!childrenMap.has(parentId)) {
225
+ childrenMap.set(parentId, []);
226
+ }
227
+ childrenMap.get(parentId).push(block);
228
+ }
229
+ }
230
+ const buildChildren = (parentId) => {
231
+ const children = childrenMap.get(this.normalizeId(parentId)) || [];
232
+ return children.map((block) => {
233
+ block.children = buildChildren(block.id);
234
+ return block;
235
+ });
236
+ };
237
+ return buildChildren(rootId);
238
+ }
239
+ }
240
+ exports.BlockFetcher = BlockFetcher;
241
+ //# sourceMappingURL=index.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/core/block-fetcher/index.ts"],"names":[],"mappings":";;;AAWA,+CAAkE;AAiBlE,MAAa,YAAY;IAiBvB,YACE,MAAc,EACN,MAAc,EACd,SAA6B;QACnC,mBAAmB,EAAE,KAAK;QAC1B,aAAa,EAAE,KAAK;QACpB,oBAAoB,EAAE,CAAC;QACvB,SAAS,EAAE,CAAC;KACb;;QANO,WAAM,GAAN,MAAM,CAAQ;QACd,WAAM,GAAN,MAAM,CAKb;QAtBK,UAAK,GAAgB,EAAE,CAAC;QACxB,WAAM,GAAG,IAAI,GAAG,EAA2C,CAAC;QAC5D,mBAAc,GAAG,IAAI,GAAG,EAAU,CAAC;QAEnC,iBAAY,GAA2B,EAAE,CAAC;QAC1C,gBAAW,GAAW,EAAE,CAAC;QACzB,gBAAW,GAAsC,EAAE,CAAC;QACpD,kBAAa,GAAsC,EAAE,CAAC;QAEtD,oBAAe,GAAG;YACxB,QAAQ,EAAE,CAAC;YACX,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;SACtB,CAAC;QAYA,MAAM,UAAU,GAAG,cAAc,CAAC;QAClC,IAAI,CAAC,MAAM,CAAC,oBAAoB,GAAG,MAAA,MAAM,CAAC,oBAAoB,mCAAI,CAAC,CAAC;QACpE,IAAI,CAAC,MAAM,CAAC,SAAS,GAAG,MAAA,MAAM,CAAC,SAAS,mCAAI,CAAC,CAAC;IAChD,CAAC;IAED,KAAK,CAAC,OAAO,CAAC,IAAe;QAC3B,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAEpD,MAAM,WAAW,GAAc;YAC7B,GAAG,IAAI;YACP,SAAS;SACV,CAAC;QAEF,OAAO,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC;IAClE,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,MAAc;QAC5B,IAAI,CAAC,WAAW,GAAG,MAAM,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,EAAE,CAAC;QAChB,IAAI,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC;QACpB,IAAI,CAAC,cAAc,CAAC,KAAK,EAAE,CAAC;QAC5B,IAAI,CAAC,cAAc,GAAG,SAAS,CAAC;QAChC,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QAEvB,yCAAyC;QACzC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,cAAc,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;QAEnD,IAAI,IAAI,CAAC,MAAM,CAAC,mBAAmB,EAAE;YACnC,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;SACxD;QAED,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE;YAC7B,IAAI,CAAC,OAAO,CAAC,EAAE,IAAI,EAAE,gBAAgB,EAAE,EAAE,EAAE,MAAM,EAAE,CAAC,CAAC;SACtD;QAED,4BAA4B;QAC5B,OAAO,IAAI,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,EAAE;YAC5B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;YAC1D,MAAM,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;SAChE;QAED,MAAM,UAAU,GAAkB;YAChC,UAAU,EAAE,IAAI,CAAC,cAAc,IAAI,EAAE;YACrC,QAAQ,EAAE,IAAI,CAAC,YAAY;YAC3B,MAAM,EAAE,IAAI,CAAC,cAAc,CAAC,MAAM,CAAC;SACpC,CAAC;QAEF,sDAAsD;QACtD,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,EAAE;YAClE,OAAO;gBACL,GAAG,UAAU;gBACb,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,EAAE,WAAW,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC;gBACtE,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI;oBACpC,aAAa,EAAE,IAAI,CAAC,aAAa;iBAClC,CAAC;aACH,CAAC;SACH;QAED,OAAO,UAAU,CAAC;IACpB,CAAC;IAEO,OAAO,CAAC,IAAe;QAC7B,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,CAAC,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE;YACpC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;SACvB;IACH,CAAC;IAEO,KAAK,CAAC,WAAW,CAAC,IAAe;QACvC,MAAM,MAAM,GAAG,GAAG,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,EAAE,EAAE,CAAC;QACzC,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,OAAO;QAE5C,QAAQ,IAAI,CAAC,IAAI,EAAE;YACjB,KAAK,cAAc,CAAC,CAAC;gBACnB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,kBAAkB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAEtD,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE;oBAC1B,MAAM,WAAW,GAAG;wBAClB,GAAG,KAAK;wBACR,QAAQ,EAAE,EAAE;wBACZ,QAAQ,EAAE,EAAE;qBACb,CAAC;oBACF,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;oBAEvC,uDAAuD;oBACvD,wDAAwD;oBACxD,4CAA4C;oBAC5C,IAAI,IAAI,CAAC,MAAM,CAAC,gBAAgB,IAAI,IAAA,qBAAY,EAAC,WAAW,CAAC,EAAE;wBAC7D,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;qBACpC;oBACD,IAAI,IAAI,CAAC,MAAM,CAAC,kBAAkB,IAAI,IAAA,uBAAc,EAAC,WAAW,CAAC,EAAE;wBACjE,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC;qBACtC;oBAED,kDAAkD;oBAClD,IAAI,cAAc,IAAI,KAAK,IAAI,KAAK,CAAC,YAAY,EAAE;wBACjD,IAAI,CAAC,OAAO,CAAC;4BACX,IAAI,EAAE,cAAc;4BACpB,EAAE,EAAE,KAAK,CAAC,EAAE;4BACZ,QAAQ,EAAE,IAAI,CAAC,EAAE;yBAClB,CAAC,CAAC;qBACJ;oBAED,mDAAmD;oBACnD,gEAAgE;oBAChE,IAAI,IAAI,CAAC,MAAM,CAAC,aAAa,EAAE;wBAC7B,IAAI,CAAC,OAAO,CAAC;4BACX,IAAI,EAAE,gBAAgB;4BACtB,EAAE,EAAE,KAAK,CAAC,EAAE;yBACb,CAAC,CAAC;qBACJ;iBACF;gBACD,MAAM;aACP;YAED,KAAK,gBAAgB,CAAC,CAAC;gBACrB,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBACtD,IAAI,IAAI,CAAC,EAAE,KAAK,IAAI,CAAC,WAAW,EAAE;oBAChC,sBAAsB;oBACtB,IAAI,CAAC,YAAY,GAAG,QAAQ,CAAC;iBAC9B;qBAAM;oBACL,MAAM,KAAK,GAAG,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;oBACvC,IAAI,KAAK,EAAE;wBACT,KAAK,CAAC,QAAQ,GAAG,QAAQ,CAAC;qBAC3B;iBACF;gBACD,MAAM;aACP;YAED,KAAK,kBAAkB,CAAC,CAAC;gBACvB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;gBAC3D,IAAI,CAAC,cAAc,GAAG,UAAU,CAAC;gBACjC,MAAM;aACP;SACF;QAED,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IAClC,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAI,OAAyB;QACzD,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,UAAU,GAAG,IAAI,CAAC;QAExB,+BAA+B;QAC/B,IAAI,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,IAAI,UAAU,EAAE;YACtD,IAAI,CAAC,eAAe,GAAG;gBACrB,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,GAAG;aACf,CAAC;SACH;QAED,mCAAmC;QACnC,IAAI,IAAI,CAAC,eAAe,CAAC,QAAQ,IAAI,IAAI,CAAC,MAAM,CAAC,oBAAqB,EAAE;YACtE,MAAM,QAAQ,GAAG,UAAU,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,eAAe,CAAC,SAAS,CAAC,CAAC;YACrE,MAAM,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC,CAAC;YAC9D,IAAI,CAAC,eAAe,GAAG;gBACrB,QAAQ,EAAE,CAAC;gBACX,SAAS,EAAE,IAAI,CAAC,GAAG,EAAE;aACtB,CAAC;SACH;QAED,IAAI,CAAC,eAAe,CAAC,QAAQ,EAAE,CAAC;QAChC,OAAO,OAAO,EAAE,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAC9B,OAAe;;QAEf,IAAI,SAAS,GAAqC,EAAE,CAAC;QACrD,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,IAAI,MAA0B,CAAC;QAE/B,OAAO,OAAO,EAAE;YACd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAChD,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;gBAC/B,QAAQ,EAAE,OAAO;gBACjB,YAAY,EAAE,MAAM;aACrB,CAAC,CACH,CAAC;YAEF,gCAAgC;YAChC,6DAA6D;YAC7D,MAAM,MAAM,GAAG,QAAQ,CAAC,OAAO,CAAC,MAAM,CACpC,CAAC,KAAK,EAAE,EAAE,CAAC,MAAM,IAAI,KAAK,IAAI,KAAK,CAAC,IAAI,KAAK,aAAa,CACvB,CAAC;YAEtC,SAAS,GAAG,CAAC,GAAG,SAAS,EAAE,GAAG,MAAM,CAAC,CAAC;YACtC,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC;YAC5B,MAAM,GAAG,MAAA,QAAQ,CAAC,WAAW,mCAAI,SAAS,CAAC;SAC5C;QAED,OAAO,SAAS,CAAC;IACnB,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAC5B,OAAe;;QAEf,IAAI,WAAW,GAA2B,EAAE,CAAC;QAC7C,IAAI,OAAO,GAAG,IAAI,CAAC;QACnB,IAAI,MAA0B,CAAC;QAE/B,OAAO,OAAO,EAAE;YACd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAChD,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;gBACxB,QAAQ,EAAE,OAAO;gBACjB,YAAY,EAAE,MAAM;aACrB,CAAC,CACH,CAAC;YAEF,WAAW,GAAG,CAAC,GAAG,WAAW,EAAE,GAAG,QAAQ,CAAC,OAAO,CAAC,CAAC;YACpD,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC;YAC5B,MAAM,GAAG,MAAA,QAAQ,CAAC,WAAW,mCAAI,SAAS,CAAC;SAC5C;QAED,OAAO,WAAW,CAAC;IACrB,CAAC;IAEO,KAAK,CAAC,mBAAmB,CAC/B,MAAc;QAEd,MAAM,QAAQ,GAAG,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,EAAE,CAChD,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAChD,CAAC;QAEF,OAAO,YAAY,IAAI,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAAC,EAAE,CAAC;IAC7D,CAAC;IAEO,WAAW,CAAC,EAAU;QAC5B,OAAO,EAAE,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IAC9B,CAAC;IAEO,cAAc,CAAC,MAAc;;QACnC,MAAM,WAAW,GAAG,IAAI,GAAG,EAA6C,CAAC;QAEzE,KAAK,MAAM,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,EAAE;YAC/C,MAAM,QAAQ;YACZ,aAAa;YACb,CAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,IAAI,MAAK,UAAU;gBAC/B,CAAC,CAAC,aAAa;oBACb,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,QAAQ,CAAC;gBACzC,CAAC,CAAC,aAAa;oBACb,CAAA,MAAA,KAAK,CAAC,MAAM,0CAAE,IAAI,MAAK,SAAS;wBAChC,CAAC,CAAC,aAAa;4BACb,IAAI,CAAC,WAAW,CAAC,KAAK,CAAC,MAAM,CAAC,OAAO,CAAC;wBACxC,CAAC,CAAC,SAAS,CAAC;YAElB,IAAI,QAAQ,EAAE;gBACZ,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE;oBAC9B,WAAW,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC;iBAC/B;gBACD,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aACxC;SACF;QAED,MAAM,aAAa,GAAG,CACpB,QAAgB,EACkB,EAAE;YACpC,MAAM,QAAQ,GAAG,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;YACnE,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;gBAC5B,KAAK,CAAC,QAAQ,GAAG,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;gBACzC,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;QACL,CAAC,CAAC;QAEF,OAAO,aAAa,CAAC,MAAM,CAAC,CAAC;IAC/B,CAAC;CACF;AArSD,oCAqSC","sourcesContent":["import { Client } from '@notionhq/client';\nimport {\n ListBlockChildrenResponseResult,\n PageObjectProperties,\n CommentResponseResults,\n FetcherOutput,\n ListBlockChildrenResponseResults,\n ExtendedFetcherOutput,\n ProcessorChainNode,\n ChainData,\n} from '../../types';\nimport { isMediaBlock, isPageRefBlock } from '../../utils/notion';\n\nexport interface BlockFetcherConfig {\n fetchPageProperties?: boolean;\n fetchComments?: boolean;\n maxRequestsPerSecond?: number;\n batchSize?: number;\n trackMediaBlocks?: boolean;\n trackPageRefBlocks?: boolean;\n}\n\ninterface QueueTask {\n type: 'fetch_properties' | 'fetch_comments' | 'fetch_blocks';\n id: string;\n parentId?: string;\n}\n\nexport class BlockFetcher implements ProcessorChainNode {\n next?: ProcessorChainNode;\n\n private queue: QueueTask[] = [];\n private blocks = new Map<string, ListBlockChildrenResponseResult>();\n private processedTasks = new Set<string>();\n private pageProperties?: PageObjectProperties;\n private rootComments: CommentResponseResults = [];\n private rootBlockId: string = '';\n private mediaBlocks: ListBlockChildrenResponseResult[] = [];\n private pageRefBlocks: ListBlockChildrenResponseResult[] = [];\n\n private rateLimitWindow = {\n requests: 0,\n startTime: Date.now(),\n };\n\n constructor(\n pageId: string,\n private client: Client,\n private config: BlockFetcherConfig = {\n fetchPageProperties: false,\n fetchComments: false,\n maxRequestsPerSecond: 3,\n batchSize: 3,\n },\n ) {\n const moduleType = 'BlockFetcher';\n this.config.maxRequestsPerSecond = config.maxRequestsPerSecond ?? 3;\n this.config.batchSize = config.batchSize ?? 3;\n }\n\n async process(data: ChainData): Promise<ChainData> {\n const blockTree = await this.getBlocks(data.pageId);\n\n const updatedData: ChainData = {\n ...data,\n blockTree,\n };\n\n return this.next ? this.next.process(updatedData) : updatedData;\n }\n\n async getBlocks(pageId: string): Promise<ExtendedFetcherOutput> {\n this.rootBlockId = pageId;\n this.queue = [];\n this.blocks.clear();\n this.processedTasks.clear();\n this.pageProperties = undefined;\n this.rootComments = [];\n\n // Initialize queue with root level tasks\n this.addTask({ type: 'fetch_blocks', id: pageId });\n\n if (this.config.fetchPageProperties) {\n this.addTask({ type: 'fetch_properties', id: pageId });\n }\n\n if (this.config.fetchComments) {\n this.addTask({ type: 'fetch_comments', id: pageId });\n }\n\n // Process queue until empty\n while (this.queue.length > 0) {\n const batch = this.queue.splice(0, this.config.batchSize);\n await Promise.all(batch.map((task) => this.processTask(task)));\n }\n\n const baseOutput: FetcherOutput = {\n properties: this.pageProperties || {},\n comments: this.rootComments,\n blocks: this.buildBlockTree(pageId),\n };\n\n // Only include special arrays if tracking was enabled\n if (this.config.trackMediaBlocks || this.config.trackPageRefBlocks) {\n return {\n ...baseOutput,\n ...(this.config.trackMediaBlocks && { mediaBlocks: this.mediaBlocks }),\n ...(this.config.trackPageRefBlocks && {\n pageRefBlocks: this.pageRefBlocks,\n }),\n };\n }\n\n return baseOutput;\n }\n\n private addTask(task: QueueTask): void {\n const taskId = `${task.type}-${task.id}`;\n if (!this.processedTasks.has(taskId)) {\n this.queue.push(task);\n }\n }\n\n private async processTask(task: QueueTask): Promise<void> {\n const taskId = `${task.type}-${task.id}`;\n if (this.processedTasks.has(taskId)) return;\n\n switch (task.type) {\n case 'fetch_blocks': {\n const blocks = await this.fetchBlockChildren(task.id);\n\n for (const block of blocks) {\n const storedBlock = {\n ...block,\n children: [],\n comments: [],\n };\n this.blocks.set(block.id, storedBlock);\n\n // Track special blocks if enabled (storing references)\n // these will be used by the next stage saving us from a\n // lot of recursive calls to idenitfy blocks\n if (this.config.trackMediaBlocks && isMediaBlock(storedBlock)) {\n this.mediaBlocks.push(storedBlock);\n }\n if (this.config.trackPageRefBlocks && isPageRefBlock(storedBlock)) {\n this.pageRefBlocks.push(storedBlock);\n }\n\n // If block has children, queue task to fetch them\n if ('has_children' in block && block.has_children) {\n this.addTask({\n type: 'fetch_blocks',\n id: block.id,\n parentId: task.id,\n });\n }\n\n // Queue comment fetching for each block if enabled\n // extra api call for comment fetching, no way to know otherwise\n if (this.config.fetchComments) {\n this.addTask({\n type: 'fetch_comments',\n id: block.id,\n });\n }\n }\n break;\n }\n\n case 'fetch_comments': {\n const comments = await this.fetchAllComments(task.id);\n if (task.id === this.rootBlockId) {\n // page level comments\n this.rootComments = comments;\n } else {\n const block = this.blocks.get(task.id);\n if (block) {\n block.comments = comments;\n }\n }\n break;\n }\n\n case 'fetch_properties': {\n const properties = await this.fetchPageProperties(task.id);\n this.pageProperties = properties;\n break;\n }\n }\n\n this.processedTasks.add(taskId);\n }\n\n private async rateLimitRequest<T>(request: () => Promise<T>): Promise<T> {\n const now = Date.now();\n const windowSize = 1000;\n\n // Reset window if it's expired\n if (now - this.rateLimitWindow.startTime >= windowSize) {\n this.rateLimitWindow = {\n requests: 0,\n startTime: now,\n };\n }\n\n // Wait if we've hit the rate limit\n if (this.rateLimitWindow.requests >= this.config.maxRequestsPerSecond!) {\n const waitTime = windowSize - (now - this.rateLimitWindow.startTime);\n await new Promise((resolve) => setTimeout(resolve, waitTime));\n this.rateLimitWindow = {\n requests: 0,\n startTime: Date.now(),\n };\n }\n\n this.rateLimitWindow.requests++;\n return request();\n }\n\n private async fetchBlockChildren(\n blockId: string,\n ): Promise<ListBlockChildrenResponseResults> {\n let allBlocks: ListBlockChildrenResponseResults = [];\n let hasMore = true;\n let cursor: string | undefined;\n\n while (hasMore) {\n const response = await this.rateLimitRequest(() =>\n this.client.blocks.children.list({\n block_id: blockId,\n start_cursor: cursor,\n }),\n );\n\n // Filter out unsupported blocks\n // should we leave it here or let user handle it in renderer?\n const blocks = response.results.filter(\n (block) => 'type' in block && block.type !== 'unsupported',\n ) as ListBlockChildrenResponseResults;\n\n allBlocks = [...allBlocks, ...blocks];\n hasMore = response.has_more;\n cursor = response.next_cursor ?? undefined;\n }\n\n return allBlocks;\n }\n\n private async fetchAllComments(\n blockId: string,\n ): Promise<CommentResponseResults> {\n let allComments: CommentResponseResults = [];\n let hasMore = true;\n let cursor: string | undefined;\n\n while (hasMore) {\n const response = await this.rateLimitRequest(() =>\n this.client.comments.list({\n block_id: blockId,\n start_cursor: cursor,\n }),\n );\n\n allComments = [...allComments, ...response.results];\n hasMore = response.has_more;\n cursor = response.next_cursor ?? undefined;\n }\n\n return allComments;\n }\n\n private async fetchPageProperties(\n pageId: string,\n ): Promise<PageObjectProperties> {\n const response = await this.rateLimitRequest(() =>\n this.client.pages.retrieve({ page_id: pageId }),\n );\n\n return 'properties' in response ? response.properties : {};\n }\n\n private normalizeId(id: string): string {\n return id.replace(/-/g, '');\n }\n\n private buildBlockTree(rootId: string): ListBlockChildrenResponseResults {\n const childrenMap = new Map<string, ListBlockChildrenResponseResult[]>();\n\n for (const [id, block] of this.blocks.entries()) {\n const parentId =\n // @ts-ignore\n block.parent?.type === 'block_id'\n ? // @ts-ignore\n this.normalizeId(block.parent.block_id)\n : // @ts-ignore\n block.parent?.type === 'page_id'\n ? // @ts-ignore\n this.normalizeId(block.parent.page_id)\n : undefined;\n\n if (parentId) {\n if (!childrenMap.has(parentId)) {\n childrenMap.set(parentId, []);\n }\n childrenMap.get(parentId)!.push(block);\n }\n }\n\n const buildChildren = (\n parentId: string,\n ): ListBlockChildrenResponseResults => {\n const children = childrenMap.get(this.normalizeId(parentId)) || [];\n return children.map((block) => {\n block.children = buildChildren(block.id);\n return block;\n });\n };\n\n return buildChildren(rootId);\n }\n}\n"]}
@@ -0,0 +1,16 @@
1
+ /**
2
+ * Specific error for when manifest management fails
3
+ */
4
+ export declare class ManifestNotInitializedError extends Error {
5
+ constructor(moduleType: string);
6
+ }
7
+ export declare class MediaHandlerError extends Error {
8
+ constructor(message: string, cause?: Error);
9
+ }
10
+ /**
11
+ * Specific error class for page reference handling errors
12
+ */
13
+ export declare class PageReferenceHandlerError extends Error {
14
+ constructor(message: string, cause?: Error);
15
+ }
16
+ //# sourceMappingURL=errors.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"errors.d.ts","sourceRoot":"","sources":["../../src/core/errors.ts"],"names":[],"mappings":"AAAA;;GAEG;AACH,qBAAa,2BAA4B,SAAQ,KAAK;gBACxC,UAAU,EAAE,MAAM;CAM/B;AAED,qBAAa,iBAAkB,SAAQ,KAAK;gBAC9B,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK;CAI3C;AAED;;GAEG;AACH,qBAAa,yBAA0B,SAAQ,KAAK;gBACtC,OAAO,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,KAAK;CAI3C"}