gatsby-source-notion-churnotion 1.1.28 → 1.1.29
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/getPages.js +229 -214
- package/dist/api/service/index.d.ts +1 -0
- package/dist/api/service/index.js +17 -0
- package/dist/api/service/notionService.d.ts +37 -0
- package/dist/api/service/notionService.js +105 -0
- package/dist/util/blocks/blockProcessor.d.ts +35 -0
- package/dist/util/blocks/blockProcessor.js +44 -0
- package/dist/util/blocks/blockProcessorRegistry.d.ts +10 -0
- package/dist/util/blocks/blockProcessorRegistry.js +40 -0
- package/dist/util/blocks/imageBlockProcessor.d.ts +6 -0
- package/dist/util/blocks/imageBlockProcessor.js +113 -0
- package/dist/util/blocks/index.d.ts +6 -0
- package/dist/util/blocks/index.js +22 -0
- package/dist/util/blocks/mediaBlockProcessor.d.ts +6 -0
- package/dist/util/blocks/mediaBlockProcessor.js +48 -0
- package/dist/util/blocks/structureBlockProcessor.d.ts +6 -0
- package/dist/util/blocks/structureBlockProcessor.js +81 -0
- package/dist/util/blocks/textBlockProcessor.d.ts +6 -0
- package/dist/util/blocks/textBlockProcessor.js +23 -0
- package/dist/util/processor.js +35 -149
- package/dist/util/timeLimit.d.ts +8 -0
- package/dist/util/timeLimit.js +25 -1
- package/package.json +2 -1
package/dist/api/getPages.js
CHANGED
@@ -6,237 +6,115 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.getPages = void 0;
|
7
7
|
const crypto_1 = __importDefault(require("crypto"));
|
8
8
|
const constants_1 = require("../constants");
|
9
|
-
const fetchData_1 = require("../util/fetchData");
|
10
9
|
const processor_1 = require("../util/processor");
|
11
10
|
const slugify_1 = require("../util/slugify");
|
12
|
-
const bookCategoryMap_1 = __importDefault(require("../util/bookCategoryMap"));
|
13
11
|
const formatDate_1 = require("../util/formatDate");
|
12
|
+
const service_1 = require("./service");
|
13
|
+
const timeLimit_1 = require("../util/timeLimit");
|
14
|
+
// 최대 동시 요청 수 설정
|
15
|
+
const MAX_CONCURRENT_REQUESTS = 5;
|
14
16
|
const getPages = async ({ databaseId, reporter, getCache, actions, createNode, createNodeId, createParentChildLink, getNode, cache, }) => {
|
17
|
+
// Notion Service 초기화
|
18
|
+
const notionService = new service_1.NotionService({
|
19
|
+
reporter,
|
20
|
+
parallelLimit: MAX_CONCURRENT_REQUESTS,
|
21
|
+
enableCaching: true,
|
22
|
+
});
|
23
|
+
// 태그 매핑을 위한 객체
|
24
|
+
const tagMap = {};
|
15
25
|
/**
|
16
26
|
* 데이터베이스 내에 페이지들을 읽어서 재귀적으로 추가하는 서브 메서드드
|
17
27
|
* @param databaseId 데이터베이스 아이디
|
18
28
|
* @param parentCategoryId 부모 데이터베이스 아이디
|
19
29
|
*/
|
20
|
-
const processDatabase = async (databaseId, parentCategoryId = null, categoryPath = [],
|
30
|
+
const processDatabase = async (databaseId, parentCategoryId = null, categoryPath = [], parentCategoryUrl = ``) => {
|
21
31
|
let hasMore = true;
|
22
32
|
try {
|
33
|
+
// 동시에 처리될 페이지 목록
|
34
|
+
const pagesToProcess = [];
|
23
35
|
while (hasMore) {
|
24
|
-
|
25
|
-
const
|
26
|
-
|
36
|
+
// 데이터베이스 쿼리
|
37
|
+
const result = await notionService.queryDatabase(databaseId);
|
38
|
+
hasMore = false; // Notion API가 페이지네이션을 완전히 지원하지 않으므로 일단 한 번만 처리
|
27
39
|
if (result?.results?.length) {
|
28
40
|
reporter.info(`[SUCCESS] total pages > ${result.results.length}`);
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
const
|
36
|
-
|
37
|
-
|
38
|
-
if (!title) {
|
39
|
-
reporter.warn(`[WARNING] Category without a title detected: ${categoryJsonData.id}`);
|
40
|
-
}
|
41
|
-
const nodeId = createNodeId(`${categoryJsonData.id}-category`);
|
42
|
-
const categoryUrl = `${parentCategoryUrl}/${slug}`;
|
43
|
-
const categoryNode = {
|
44
|
-
id: nodeId,
|
45
|
-
category_name: title,
|
46
|
-
parent: parentCategoryId,
|
47
|
-
slug: slug,
|
48
|
-
children: [],
|
49
|
-
internal: {
|
50
|
-
type: constants_1.NODE_TYPE.Category,
|
51
|
-
contentDigest: crypto_1.default
|
52
|
-
.createHash(`md5`)
|
53
|
-
.update(JSON.stringify(categoryJsonData))
|
54
|
-
.digest(`hex`),
|
55
|
-
},
|
56
|
-
url: `${constants_1.COMMON_URI}/${constants_1.CATEGORY_URI}${categoryUrl}`,
|
57
|
-
books: [],
|
41
|
+
// 페이지 ID 목록 수집
|
42
|
+
const pageIds = result.results.map((page) => page.id);
|
43
|
+
// 페이지 블록들을 병렬로 가져오기
|
44
|
+
const pagesBlocks = await notionService.getMultiplePagesBlocks(pageIds);
|
45
|
+
// 페이지 데이터와 블록 결합
|
46
|
+
for (const page of result.results) {
|
47
|
+
const pageData = {
|
48
|
+
page,
|
49
|
+
blocks: pagesBlocks[page.id] || [],
|
58
50
|
};
|
59
|
-
|
60
|
-
const bookRelations = page.properties?.book?.relation || null;
|
61
|
-
if (bookRelations) {
|
62
|
-
bookRelations.forEach((relation) => {
|
63
|
-
const bookId = relation.id;
|
64
|
-
const bookNodeId = createNodeId(`${bookId}-book`);
|
65
|
-
const bookNode = getNode(bookNodeId);
|
66
|
-
if (bookNode) {
|
67
|
-
createParentChildLink({
|
68
|
-
parent: categoryNode,
|
69
|
-
child: bookNode,
|
70
|
-
});
|
71
|
-
const updatedBookNode = {
|
72
|
-
...bookNode,
|
73
|
-
book_category: categoryNode.id,
|
74
|
-
internal: {
|
75
|
-
type: bookNode.internal.type,
|
76
|
-
contentDigest: crypto_1.default
|
77
|
-
.createHash(`md5`)
|
78
|
-
.update(JSON.stringify(bookNode))
|
79
|
-
.digest(`hex`),
|
80
|
-
},
|
81
|
-
};
|
82
|
-
createNode(updatedBookNode);
|
83
|
-
reporter.info(`[SUCCESS] Linked Category-Book: ${categoryNode.category_name} -> child: ${bookNode.book_name}`);
|
84
|
-
}
|
85
|
-
});
|
86
|
-
}
|
87
|
-
if (parentCategoryId && categoryNode) {
|
88
|
-
const parentNode = getNode(parentCategoryId); // Gatsby에서 노드를 검색
|
89
|
-
if (parentNode) {
|
90
|
-
createParentChildLink({
|
91
|
-
parent: parentNode,
|
92
|
-
child: categoryNode,
|
93
|
-
});
|
94
|
-
reporter.info(`[SUCCESS] Linked parent: ${parentNode.category_name} -> child: ${categoryNode.category_name}`);
|
95
|
-
}
|
96
|
-
else {
|
97
|
-
reporter.warn(`[WARNING] Parent node not found for ID: ${parentCategoryId}`);
|
98
|
-
}
|
99
|
-
}
|
100
|
-
const newCategoryPath = [...categoryPath, categoryNode];
|
101
|
-
await processDatabase(categoryJsonData.id, nodeId, newCategoryPath, tagMap, categoryUrl);
|
102
|
-
}
|
103
|
-
else {
|
104
|
-
// 페이지인 경우
|
105
|
-
const title = page.properties?.[`이름`]?.title?.[0]?.plain_text || `Unnamed`;
|
106
|
-
const slug = (0, slugify_1.slugify)(page.properties?.[`slug`]?.rich_text?.[0]?.plain_text ||
|
107
|
-
crypto_1.default
|
108
|
-
.createHash(`md5`)
|
109
|
-
.update(JSON.stringify(title))
|
110
|
-
.digest(`hex`));
|
111
|
-
if (!page.properties?.[`이름`]?.title?.[0]?.plain_text) {
|
112
|
-
reporter.warn(`[WARNING] Category without a title detected: ${page.id}`);
|
113
|
-
}
|
114
|
-
const nodeId = createNodeId(`${page.id}-page`);
|
115
|
-
// Tag 노드 만들기
|
116
|
-
const tagIds = [];
|
117
|
-
if (page.properties.tags && page.properties.tags.multi_select) {
|
118
|
-
page.properties.tags.multi_select.forEach((tagData) => {
|
119
|
-
if (tagMap[tagData.name]) {
|
120
|
-
// 이미 존재하는 태그라면 tagMap에서 가져오기
|
121
|
-
const existingTagId = tagMap[tagData.name];
|
122
|
-
tagIds.push(existingTagId); // 기존 태그 ID 추가
|
123
|
-
reporter.info(`[INFO] Reusing existing tag: ${tagData.name}`);
|
124
|
-
}
|
125
|
-
else {
|
126
|
-
// 새로운 태그 생성
|
127
|
-
const tagNodeId = createNodeId(`${tagData.id}-tag`);
|
128
|
-
tagMap[tagData.name] = tagNodeId; // tagMap에 저장
|
129
|
-
tagIds.push(tagNodeId); // 새로운 태그 ID 추가
|
130
|
-
const slug = (0, slugify_1.slugify)(tagData.name) || `no-tag-${tagNodeId}`;
|
131
|
-
// 태그 노드 생성
|
132
|
-
const tagNode = {
|
133
|
-
id: tagNodeId,
|
134
|
-
tag_name: tagData.name,
|
135
|
-
slug: slug,
|
136
|
-
color: tagData.color,
|
137
|
-
children: [],
|
138
|
-
internal: {
|
139
|
-
type: constants_1.NODE_TYPE.Tag,
|
140
|
-
contentDigest: crypto_1.default
|
141
|
-
.createHash(`md5`)
|
142
|
-
.update(JSON.stringify(tagData))
|
143
|
-
.digest(`hex`),
|
144
|
-
},
|
145
|
-
url: `${constants_1.COMMON_URI}/${constants_1.TAG_URI}/${slug}`,
|
146
|
-
churnotions: [],
|
147
|
-
parent: null,
|
148
|
-
};
|
149
|
-
createNode(tagNode);
|
150
|
-
reporter.info(`[SUCCESS] Created new tag: ${tagData.name}`);
|
151
|
-
}
|
152
|
-
});
|
153
|
-
}
|
154
|
-
const bookId = page.properties?.book?.relation?.[0]?.id || null;
|
155
|
-
const bookNodeId = createNodeId(`${bookId}-book`);
|
156
|
-
const bookNode = getNode(bookNodeId);
|
157
|
-
const [imageNode, tableOfContents, updatedBlocks, rawText] = await (0, processor_1.processor)(pageData.results, actions, getCache, createNodeId, reporter, cache);
|
158
|
-
const postNode = {
|
159
|
-
id: nodeId,
|
160
|
-
category: parentCategoryId,
|
161
|
-
book: bookNode?.id,
|
162
|
-
book_index: page.properties?.bookIndex?.number || 0,
|
163
|
-
title: title,
|
164
|
-
content: updatedBlocks,
|
165
|
-
create_date: (0, formatDate_1.useFormatDate)(page.created_time),
|
166
|
-
update_date: (0, formatDate_1.useFormatDate)(page.last_edited_time),
|
167
|
-
version: page.properties?.version?.number || null,
|
168
|
-
description: page.properties?.description?.rich_text?.[0]?.plain_text ||
|
169
|
-
rawText.substring(0, 400),
|
170
|
-
slug: slug,
|
171
|
-
category_list: categoryPath,
|
172
|
-
children: [],
|
173
|
-
tableOfContents,
|
174
|
-
internal: {
|
175
|
-
type: constants_1.NODE_TYPE.Post,
|
176
|
-
contentDigest: crypto_1.default
|
177
|
-
.createHash(`md5`)
|
178
|
-
.update(JSON.stringify(nodeId))
|
179
|
-
.digest(`hex`),
|
180
|
-
},
|
181
|
-
tags: tagIds,
|
182
|
-
parent: null,
|
183
|
-
url: `${constants_1.COMMON_URI}/${constants_1.POST_URI}${parentCategoryUrl}/${slug}`,
|
184
|
-
thumbnail: imageNode,
|
185
|
-
rawText,
|
186
|
-
};
|
187
|
-
await createNode(postNode);
|
188
|
-
if (bookNode) {
|
189
|
-
createParentChildLink({
|
190
|
-
parent: bookNode,
|
191
|
-
child: postNode,
|
192
|
-
});
|
193
|
-
reporter.info(`[SUCCESS] Linked book: ${bookNode.book_name} -> page: ${postNode.title}`);
|
194
|
-
}
|
195
|
-
// tag와 post 부모-자식 관계 설정
|
196
|
-
tagIds.forEach((tagId) => {
|
197
|
-
const tagNode = getNode(tagId);
|
198
|
-
if (tagNode) {
|
199
|
-
createParentChildLink({
|
200
|
-
parent: tagNode,
|
201
|
-
child: postNode,
|
202
|
-
});
|
203
|
-
reporter.info(`[SUCCESS] Linked tag: ${tagNode.tag_name} -> page: ${postNode.title}`);
|
204
|
-
}
|
205
|
-
else {
|
206
|
-
reporter.warn(`[WARNING] Tag node not found for ID: ${tagId}`);
|
207
|
-
}
|
208
|
-
});
|
209
|
-
// category와 post 부모-자식 관계 설정
|
210
|
-
if (parentCategoryId && postNode) {
|
211
|
-
const parentNode = getNode(parentCategoryId); // Gatsby에서 노드를 검색
|
212
|
-
if (parentNode) {
|
213
|
-
createParentChildLink({
|
214
|
-
parent: parentNode,
|
215
|
-
child: postNode,
|
216
|
-
});
|
217
|
-
reporter.info(`[SUCCESS] Linked parent: ${parentNode.category_name} -> child: ${postNode.title}`);
|
218
|
-
}
|
219
|
-
else {
|
220
|
-
reporter.warn(`[WARNING] Parent node not found for ID: ${parentCategoryId}`);
|
221
|
-
}
|
222
|
-
}
|
51
|
+
pagesToProcess.push(pageData);
|
223
52
|
}
|
224
53
|
}
|
225
|
-
hasMore = result.has_more;
|
226
54
|
}
|
55
|
+
// 모든 페이지 병렬 처리
|
56
|
+
await Promise.all(pagesToProcess.map(async ({ page, blocks }) => {
|
57
|
+
try {
|
58
|
+
// Timeout 설정으로 너무 오래 걸리는 페이지는 건너뛰기
|
59
|
+
await (0, timeLimit_1.timeLimit)(processPageData(page, blocks, parentCategoryId, categoryPath, tagMap, parentCategoryUrl), 30000, // 30초 제한
|
60
|
+
`Processing page ${page.id} timed out after 30 seconds`);
|
61
|
+
}
|
62
|
+
catch (error) {
|
63
|
+
reporter.warn(`[WARNING] Error processing page ${page.id}: ${error}`);
|
64
|
+
}
|
65
|
+
}));
|
227
66
|
}
|
228
67
|
catch (error) {
|
229
|
-
reporter.error(`[ERROR]
|
230
|
-
hasMore = false;
|
68
|
+
reporter.error(`[ERROR] Processing database ${databaseId} failed: ${error}`);
|
231
69
|
}
|
232
70
|
};
|
233
|
-
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
239
|
-
|
71
|
+
/**
|
72
|
+
* 페이지 데이터 처리 메서드
|
73
|
+
*/
|
74
|
+
const processPageData = async (page, pageBlocks, parentCategoryId, categoryPath, tagMap, parentCategoryUrl) => {
|
75
|
+
// 첫 번째 블록이 child_database인지 확인
|
76
|
+
if (pageBlocks?.[0]?.type === `child_database`) {
|
77
|
+
await processCategory(page, pageBlocks, parentCategoryId, categoryPath, tagMap, parentCategoryUrl);
|
78
|
+
}
|
79
|
+
else {
|
80
|
+
await processPost(page, pageBlocks, parentCategoryId, categoryPath, tagMap, parentCategoryUrl);
|
81
|
+
}
|
82
|
+
};
|
83
|
+
/**
|
84
|
+
* 카테고리 처리 메서드
|
85
|
+
*/
|
86
|
+
const processCategory = async (page, pageBlocks, parentCategoryId, categoryPath, tagMap, parentCategoryUrl) => {
|
87
|
+
const categoryJsonData = pageBlocks[0];
|
88
|
+
const title = categoryJsonData.child_database?.title || `Unnamed Category`;
|
89
|
+
const slug = (0, slugify_1.slugify)(title) || `no-title-${categoryJsonData.id}`;
|
90
|
+
if (!title) {
|
91
|
+
reporter.warn(`[WARNING] Category without a title detected: ${categoryJsonData.id}`);
|
92
|
+
}
|
93
|
+
const nodeId = createNodeId(`${categoryJsonData.id}-category`);
|
94
|
+
const categoryUrl = `${parentCategoryUrl}/${slug}`;
|
95
|
+
const categoryNode = {
|
96
|
+
id: nodeId,
|
97
|
+
category_name: title,
|
98
|
+
parent: parentCategoryId,
|
99
|
+
slug: slug,
|
100
|
+
children: [],
|
101
|
+
internal: {
|
102
|
+
type: constants_1.NODE_TYPE.Category,
|
103
|
+
contentDigest: crypto_1.default
|
104
|
+
.createHash(`md5`)
|
105
|
+
.update(JSON.stringify(categoryJsonData))
|
106
|
+
.digest(`hex`),
|
107
|
+
},
|
108
|
+
url: `${constants_1.COMMON_URI}/${constants_1.CATEGORY_URI}${categoryUrl}`,
|
109
|
+
books: [],
|
110
|
+
};
|
111
|
+
await createNode(categoryNode);
|
112
|
+
const bookRelations = page.properties?.book?.relation || null;
|
113
|
+
if (bookRelations) {
|
114
|
+
bookRelations.forEach((relation) => {
|
115
|
+
const bookId = relation.id;
|
116
|
+
const bookNodeId = createNodeId(`${bookId}-book`);
|
117
|
+
const bookNode = getNode(bookNodeId);
|
240
118
|
if (bookNode) {
|
241
119
|
createParentChildLink({
|
242
120
|
parent: categoryNode,
|
@@ -254,16 +132,153 @@ const getPages = async ({ databaseId, reporter, getCache, actions, createNode, c
|
|
254
132
|
},
|
255
133
|
};
|
256
134
|
createNode(updatedBookNode);
|
257
|
-
reporter.info(`[SUCCESS] Linked Book
|
135
|
+
reporter.info(`[SUCCESS] Linked Category-Book: ${categoryNode.category_name} -> child: ${bookNode.book_name}`);
|
136
|
+
}
|
137
|
+
});
|
138
|
+
}
|
139
|
+
if (parentCategoryId && categoryNode) {
|
140
|
+
const parentNode = getNode(parentCategoryId);
|
141
|
+
if (parentNode) {
|
142
|
+
createParentChildLink({
|
143
|
+
parent: parentNode,
|
144
|
+
child: categoryNode,
|
145
|
+
});
|
146
|
+
reporter.info(`[SUCCESS] Linked parent: ${parentNode.category_name} -> child: ${categoryNode.category_name}`);
|
147
|
+
}
|
148
|
+
else {
|
149
|
+
reporter.warn(`[WARNING] Parent node not found for ID: ${parentCategoryId}`);
|
150
|
+
}
|
151
|
+
}
|
152
|
+
const newCategoryPath = [...categoryPath, categoryNode];
|
153
|
+
await processDatabase(categoryJsonData.id, nodeId, newCategoryPath, categoryUrl);
|
154
|
+
};
|
155
|
+
/**
|
156
|
+
* 포스트 처리 메서드
|
157
|
+
*/
|
158
|
+
const processPost = async (page, pageBlocks, parentCategoryId, categoryPath, tagMap, parentCategoryUrl) => {
|
159
|
+
const title = page.properties?.[`이름`]?.title?.[0]?.plain_text || `Unnamed`;
|
160
|
+
const slug = (0, slugify_1.slugify)(page.properties?.[`slug`]?.rich_text?.[0]?.plain_text ||
|
161
|
+
crypto_1.default.createHash(`md5`).update(JSON.stringify(title)).digest(`hex`));
|
162
|
+
if (!page.properties?.[`이름`]?.title?.[0]?.plain_text) {
|
163
|
+
reporter.warn(`[WARNING] Post without a title detected: ${page.id}`);
|
164
|
+
}
|
165
|
+
const nodeId = createNodeId(`${page.id}-page`);
|
166
|
+
// Tag 노드 만들기
|
167
|
+
const tagIds = [];
|
168
|
+
if (page.properties.tags && page.properties.tags.multi_select) {
|
169
|
+
page.properties.tags.multi_select.forEach((tagData) => {
|
170
|
+
if (tagMap[tagData.name]) {
|
171
|
+
// 이미 존재하는 태그라면 tagMap에서 가져오기
|
172
|
+
const existingTagId = tagMap[tagData.name];
|
173
|
+
tagIds.push(existingTagId); // 기존 태그 ID 추가
|
174
|
+
reporter.info(`[INFO] Reusing existing tag: ${tagData.name}`);
|
258
175
|
}
|
259
176
|
else {
|
260
|
-
|
177
|
+
// 새로운 태그 생성
|
178
|
+
const tagNodeId = createNodeId(`${tagData.id}-tag`);
|
179
|
+
tagMap[tagData.name] = tagNodeId; // tagMap에 저장
|
180
|
+
tagIds.push(tagNodeId); // 새로운 태그 ID 추가
|
181
|
+
const slug = (0, slugify_1.slugify)(tagData.name) || `no-tag-${tagNodeId}`;
|
182
|
+
// 태그 노드 생성
|
183
|
+
const tagNode = {
|
184
|
+
id: tagNodeId,
|
185
|
+
tag_name: tagData.name,
|
186
|
+
slug: slug,
|
187
|
+
color: tagData.color,
|
188
|
+
children: [],
|
189
|
+
internal: {
|
190
|
+
type: constants_1.NODE_TYPE.Tag,
|
191
|
+
contentDigest: crypto_1.default
|
192
|
+
.createHash(`md5`)
|
193
|
+
.update(JSON.stringify(tagData))
|
194
|
+
.digest(`hex`),
|
195
|
+
},
|
196
|
+
url: `${constants_1.COMMON_URI}/${constants_1.TAG_URI}/${slug}`,
|
197
|
+
churnotions: [],
|
198
|
+
parent: null,
|
199
|
+
};
|
200
|
+
createNode(tagNode);
|
201
|
+
reporter.info(`[SUCCESS] Created new tag: ${tagData.name}`);
|
261
202
|
}
|
203
|
+
});
|
204
|
+
}
|
205
|
+
const bookId = page.properties?.book?.relation?.[0]?.id || null;
|
206
|
+
const bookNodeId = createNodeId(`${bookId}-book`);
|
207
|
+
const bookNode = getNode(bookNodeId);
|
208
|
+
const [imageNode, tableOfContents, updatedBlocks, rawText] = await (0, processor_1.processor)(pageBlocks, actions, getCache, createNodeId, reporter, cache);
|
209
|
+
const postNode = {
|
210
|
+
id: nodeId,
|
211
|
+
category: parentCategoryId,
|
212
|
+
book: bookNode?.id,
|
213
|
+
book_index: page.properties?.bookIndex?.number || 0,
|
214
|
+
title: title,
|
215
|
+
content: updatedBlocks,
|
216
|
+
create_date: (0, formatDate_1.useFormatDate)(page.created_time),
|
217
|
+
update_date: (0, formatDate_1.useFormatDate)(page.last_edited_time),
|
218
|
+
version: page.properties?.version?.number || null,
|
219
|
+
description: page.properties?.description?.rich_text?.[0]?.plain_text ||
|
220
|
+
rawText.substring(0, 400),
|
221
|
+
slug: slug,
|
222
|
+
category_list: categoryPath,
|
223
|
+
children: [],
|
224
|
+
tags: tagIds,
|
225
|
+
tableOfContents,
|
226
|
+
internal: {
|
227
|
+
type: constants_1.NODE_TYPE.Post,
|
228
|
+
contentDigest: crypto_1.default
|
229
|
+
.createHash(`md5`)
|
230
|
+
.update(JSON.stringify(nodeId))
|
231
|
+
.digest(`hex`),
|
232
|
+
},
|
233
|
+
url: `${constants_1.COMMON_URI}/${constants_1.POST_URI}/${slug}`,
|
234
|
+
thumbnail: imageNode,
|
235
|
+
parent: parentCategoryId,
|
236
|
+
};
|
237
|
+
await createNode(postNode);
|
238
|
+
if (parentCategoryId) {
|
239
|
+
const parentNode = getNode(parentCategoryId);
|
240
|
+
if (parentNode) {
|
241
|
+
createParentChildLink({
|
242
|
+
parent: parentNode,
|
243
|
+
child: postNode,
|
244
|
+
});
|
245
|
+
reporter.info(`[SUCCESS] Linked parent: ${parentNode.category_name} -> child: ${postNode.title}`);
|
262
246
|
}
|
263
247
|
}
|
264
|
-
|
265
|
-
|
248
|
+
// 태그와 포스트 연결
|
249
|
+
tagIds.forEach((tagId) => {
|
250
|
+
const tagNode = getNode(tagId);
|
251
|
+
if (tagNode) {
|
252
|
+
const updatedTagNode = {
|
253
|
+
...tagNode,
|
254
|
+
churnotions: [...(tagNode.churnotions || []), nodeId],
|
255
|
+
internal: {
|
256
|
+
type: tagNode.internal.type,
|
257
|
+
contentDigest: crypto_1.default
|
258
|
+
.createHash(`md5`)
|
259
|
+
.update(JSON.stringify({
|
260
|
+
...tagNode,
|
261
|
+
churnotions: [...(tagNode.churnotions || []), nodeId],
|
262
|
+
}))
|
263
|
+
.digest(`hex`),
|
264
|
+
},
|
265
|
+
};
|
266
|
+
createNode(updatedTagNode);
|
267
|
+
reporter.info(`[SUCCESS] Added post to tag: ${tagNode.tag_name} -> ${postNode.title}`);
|
268
|
+
}
|
269
|
+
});
|
270
|
+
// 책과 포스트 연결
|
271
|
+
if (bookNode && postNode) {
|
272
|
+
createParentChildLink({
|
273
|
+
parent: bookNode,
|
274
|
+
child: postNode,
|
275
|
+
});
|
276
|
+
reporter.info(`[SUCCESS] Linked Book-Post: ${bookNode.book_name} -> child: ${postNode.title}`);
|
266
277
|
}
|
267
|
-
}
|
278
|
+
};
|
279
|
+
// 초기 데이터베이스 처리 시작
|
280
|
+
await processDatabase(databaseId);
|
281
|
+
// 캐시 정리
|
282
|
+
notionService.clearCache();
|
268
283
|
};
|
269
284
|
exports.getPages = getPages;
|
@@ -0,0 +1 @@
|
|
1
|
+
export * from "./notionService";
|
@@ -0,0 +1,17 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
3
|
+
if (k2 === undefined) k2 = k;
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
7
|
+
}
|
8
|
+
Object.defineProperty(o, k2, desc);
|
9
|
+
}) : (function(o, m, k, k2) {
|
10
|
+
if (k2 === undefined) k2 = k;
|
11
|
+
o[k2] = m[k];
|
12
|
+
}));
|
13
|
+
var __exportStar = (this && this.__exportStar) || function(m, exports) {
|
14
|
+
for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
|
15
|
+
};
|
16
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
17
|
+
__exportStar(require("./notionService"), exports);
|
@@ -0,0 +1,37 @@
|
|
1
|
+
import { Reporter } from "gatsby";
|
2
|
+
import { BaseContentBlock } from "notion-types";
|
3
|
+
export interface NotionServiceOptions {
|
4
|
+
reporter: Reporter;
|
5
|
+
parallelLimit?: number;
|
6
|
+
enableCaching?: boolean;
|
7
|
+
}
|
8
|
+
export declare class NotionService {
|
9
|
+
private reporter;
|
10
|
+
private parallelLimit;
|
11
|
+
private enableCaching;
|
12
|
+
private cache;
|
13
|
+
private limiter;
|
14
|
+
constructor(options: NotionServiceOptions);
|
15
|
+
/**
|
16
|
+
* 데이터베이스 쿼리
|
17
|
+
*/
|
18
|
+
queryDatabase(databaseId: string, body?: any): Promise<any>;
|
19
|
+
/**
|
20
|
+
* 페이지의 자식 블록 가져오기
|
21
|
+
*/
|
22
|
+
getPageBlocks(pageId: string): Promise<BaseContentBlock[]>;
|
23
|
+
/**
|
24
|
+
* 여러 페이지의 블록 병렬 처리
|
25
|
+
*/
|
26
|
+
getMultiplePagesBlocks(pageIds: string[]): Promise<{
|
27
|
+
[id: string]: BaseContentBlock[];
|
28
|
+
}>;
|
29
|
+
/**
|
30
|
+
* 캐시 초기화
|
31
|
+
*/
|
32
|
+
clearCache(): void;
|
33
|
+
/**
|
34
|
+
* 병렬 처리 제한 설정
|
35
|
+
*/
|
36
|
+
setParallelLimit(limit: number): void;
|
37
|
+
}
|
@@ -0,0 +1,105 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
4
|
+
};
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
6
|
+
exports.NotionService = void 0;
|
7
|
+
const fetchData_1 = require("../../util/fetchData");
|
8
|
+
const p_limit_1 = __importDefault(require("p-limit"));
|
9
|
+
class NotionService {
|
10
|
+
reporter;
|
11
|
+
parallelLimit;
|
12
|
+
enableCaching;
|
13
|
+
cache;
|
14
|
+
limiter;
|
15
|
+
constructor(options) {
|
16
|
+
this.reporter = options.reporter;
|
17
|
+
this.parallelLimit = options.parallelLimit || 5; // 기본 동시 요청 수
|
18
|
+
this.enableCaching = options.enableCaching !== false; // 기본값은 캐싱 활성화
|
19
|
+
this.cache = new Map();
|
20
|
+
this.limiter = (0, p_limit_1.default)(this.parallelLimit);
|
21
|
+
}
|
22
|
+
/**
|
23
|
+
* 데이터베이스 쿼리
|
24
|
+
*/
|
25
|
+
async queryDatabase(databaseId, body = {}) {
|
26
|
+
const cacheKey = `database-${databaseId}-${JSON.stringify(body)}`;
|
27
|
+
if (this.enableCaching && this.cache.has(cacheKey)) {
|
28
|
+
this.reporter.info(`[CACHE] Using cached database query for ${databaseId}`);
|
29
|
+
return this.cache.get(cacheKey);
|
30
|
+
}
|
31
|
+
const databaseUrl = `databases/${databaseId}/query`;
|
32
|
+
try {
|
33
|
+
const result = await (0, fetchData_1.fetchPostWithRetry)(databaseUrl, body);
|
34
|
+
if (this.enableCaching) {
|
35
|
+
this.cache.set(cacheKey, result);
|
36
|
+
}
|
37
|
+
this.reporter.info(`[SUCCESS] Database query ${databaseId} - results: ${result?.results?.length || 0}`);
|
38
|
+
return result;
|
39
|
+
}
|
40
|
+
catch (error) {
|
41
|
+
this.reporter.error(`[ERROR] Failed to query database ${databaseId}: ${error}`);
|
42
|
+
throw error;
|
43
|
+
}
|
44
|
+
}
|
45
|
+
/**
|
46
|
+
* 페이지의 자식 블록 가져오기
|
47
|
+
*/
|
48
|
+
async getPageBlocks(pageId) {
|
49
|
+
const cacheKey = `page-blocks-${pageId}`;
|
50
|
+
if (this.enableCaching && this.cache.has(cacheKey)) {
|
51
|
+
this.reporter.info(`[CACHE] Using cached page blocks for ${pageId}`);
|
52
|
+
return this.cache.get(cacheKey);
|
53
|
+
}
|
54
|
+
const pageUrl = `blocks/${pageId}/children?page_size=100`;
|
55
|
+
try {
|
56
|
+
const data = await (0, fetchData_1.fetchGetWithRetry)(pageUrl);
|
57
|
+
const blocks = data.results;
|
58
|
+
if (this.enableCaching) {
|
59
|
+
this.cache.set(cacheKey, blocks);
|
60
|
+
}
|
61
|
+
return blocks;
|
62
|
+
}
|
63
|
+
catch (error) {
|
64
|
+
this.reporter.error(`[ERROR] Failed to get page blocks ${pageId}: ${error}`);
|
65
|
+
throw error;
|
66
|
+
}
|
67
|
+
}
|
68
|
+
/**
|
69
|
+
* 여러 페이지의 블록 병렬 처리
|
70
|
+
*/
|
71
|
+
async getMultiplePagesBlocks(pageIds) {
|
72
|
+
this.reporter.info(`[NOTION] Fetching blocks for ${pageIds.length} pages in parallel (limit: ${this.parallelLimit})`);
|
73
|
+
const tasks = pageIds.map((pageId) => this.limiter(async () => {
|
74
|
+
try {
|
75
|
+
const blocks = await this.getPageBlocks(pageId);
|
76
|
+
return { pageId, blocks };
|
77
|
+
}
|
78
|
+
catch (error) {
|
79
|
+
this.reporter.warn(`[WARNING] Failed to fetch blocks for page ${pageId}: ${error}`);
|
80
|
+
return { pageId, blocks: [] };
|
81
|
+
}
|
82
|
+
}));
|
83
|
+
const results = await Promise.all(tasks);
|
84
|
+
return results.reduce((acc, { pageId, blocks }) => {
|
85
|
+
acc[pageId] = blocks;
|
86
|
+
return acc;
|
87
|
+
}, {});
|
88
|
+
}
|
89
|
+
/**
|
90
|
+
* 캐시 초기화
|
91
|
+
*/
|
92
|
+
clearCache() {
|
93
|
+
this.reporter.info(`[NOTION] Clearing service cache - entries: ${this.cache.size}`);
|
94
|
+
this.cache.clear();
|
95
|
+
}
|
96
|
+
/**
|
97
|
+
* 병렬 처리 제한 설정
|
98
|
+
*/
|
99
|
+
setParallelLimit(limit) {
|
100
|
+
this.parallelLimit = limit;
|
101
|
+
this.limiter = (0, p_limit_1.default)(limit);
|
102
|
+
this.reporter.info(`[NOTION] Updated parallel request limit to ${limit}`);
|
103
|
+
}
|
104
|
+
}
|
105
|
+
exports.NotionService = NotionService;
|