gatsby-source-notion-churnotion 1.1.40 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/getBooks.js +8 -3
- package/dist/api/getPages.js +20 -4
- package/dist/api/service/notionService.d.ts +12 -0
- package/dist/api/service/notionService.js +98 -3
- package/dist/rust-bindings.d.ts +46 -0
- package/dist/rust-bindings.js +134 -0
- package/package.json +4 -2
- package/dist/util/fetchData.d.ts +0 -20
- package/dist/util/fetchData.js +0 -112
package/dist/api/getBooks.js
CHANGED
@@ -6,17 +6,22 @@ Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
6
|
exports.getBooks = void 0;
|
7
7
|
const crypto_1 = __importDefault(require("crypto"));
|
8
8
|
const constants_1 = require("../constants");
|
9
|
-
const fetchData_1 = require("../util/fetchData");
|
10
9
|
const gatsby_source_filesystem_1 = require("gatsby-source-filesystem");
|
11
10
|
const bookCategoryMap_1 = __importDefault(require("../util/bookCategoryMap"));
|
12
11
|
const formatDate_1 = require("../util/formatDate");
|
12
|
+
const service_1 = require("./service");
|
13
13
|
const getBooks = async ({ bookDatabaseId, reporter, getCache, createNode, createNodeId, getNode, cache, }) => {
|
14
|
-
|
14
|
+
// Initialize the TypeScript Notion Service for database queries
|
15
|
+
const notionService = new service_1.NotionService({
|
16
|
+
reporter,
|
17
|
+
parallelLimit: 5,
|
18
|
+
enableCaching: true,
|
19
|
+
});
|
15
20
|
const cacheKey = `booksDatabase-${bookDatabaseId}`;
|
16
21
|
let result = await cache.get(cacheKey);
|
17
22
|
if (!result) {
|
18
23
|
const body = {};
|
19
|
-
result = await
|
24
|
+
result = await notionService.queryDatabase(bookDatabaseId, body);
|
20
25
|
await cache.set(cacheKey, result);
|
21
26
|
}
|
22
27
|
if (result?.results?.length) {
|
package/dist/api/getPages.js
CHANGED
@@ -9,17 +9,32 @@ const constants_1 = require("../constants");
|
|
9
9
|
const processor_1 = require("../util/processor");
|
10
10
|
const slugify_1 = require("../util/slugify");
|
11
11
|
const formatDate_1 = require("../util/formatDate");
|
12
|
+
const rust_bindings_1 = require("../rust-bindings");
|
12
13
|
const service_1 = require("./service");
|
13
14
|
const timeLimit_1 = require("../util/timeLimit");
|
14
15
|
// 최대 동시 요청 수 설정
|
15
|
-
const MAX_CONCURRENT_REQUESTS =
|
16
|
+
const MAX_CONCURRENT_REQUESTS = 10;
|
16
17
|
const getPages = async ({ databaseId, reporter, getCache, actions, createNode, createNodeId, createParentChildLink, getNode, cache, }) => {
|
17
|
-
// Notion
|
18
|
+
// Get the Notion API key from environment variables
|
19
|
+
const notionApiKey = process.env.NOTION_API_KEY;
|
20
|
+
if (!notionApiKey) {
|
21
|
+
reporter.error("[ERROR] NOTION_API_KEY environment variable is not set!");
|
22
|
+
throw new Error("NOTION_API_KEY environment variable is required");
|
23
|
+
}
|
24
|
+
// Initialize the TypeScript Notion Service for methods not implemented in Rust
|
18
25
|
const notionService = new service_1.NotionService({
|
19
26
|
reporter,
|
20
27
|
parallelLimit: MAX_CONCURRENT_REQUESTS,
|
21
28
|
enableCaching: true,
|
22
29
|
});
|
30
|
+
// Initialize Rust-based Notion Service
|
31
|
+
const rustNotionService = new rust_bindings_1.RustNotionService({
|
32
|
+
reporter,
|
33
|
+
notionApiKey,
|
34
|
+
parallelLimit: MAX_CONCURRENT_REQUESTS,
|
35
|
+
enableCaching: true,
|
36
|
+
});
|
37
|
+
reporter.info(`[INIT] Initialized Rust-based Notion service with parallel limit: ${MAX_CONCURRENT_REQUESTS}`);
|
23
38
|
// 태그 매핑을 위한 객체
|
24
39
|
const tagMap = {};
|
25
40
|
/**
|
@@ -33,7 +48,7 @@ const getPages = async ({ databaseId, reporter, getCache, actions, createNode, c
|
|
33
48
|
// 동시에 처리될 페이지 목록
|
34
49
|
const pagesToProcess = [];
|
35
50
|
while (hasMore) {
|
36
|
-
// 데이터베이스 쿼리
|
51
|
+
// 데이터베이스 쿼리 (still use the TypeScript implementation for this)
|
37
52
|
const result = await notionService.queryDatabase(databaseId);
|
38
53
|
hasMore = false; // Notion API가 페이지네이션을 완전히 지원하지 않으므로 일단 한 번만 처리
|
39
54
|
if (result?.results?.length) {
|
@@ -44,7 +59,8 @@ const getPages = async ({ databaseId, reporter, getCache, actions, createNode, c
|
|
44
59
|
for (let i = 0; i < pageIds.length; i += 20) {
|
45
60
|
const batch = pageIds.slice(i, i + 20);
|
46
61
|
reporter.info(`[BATCH] Processing pages ${i + 1} to ${i + batch.length} of ${pageIds.length}`);
|
47
|
-
|
62
|
+
// Use the Rust implementation for parallel fetching
|
63
|
+
const batchBlocks = await rustNotionService.getMultiplePagesBlocks(batch);
|
48
64
|
// 페이지 데이터와 블록 결합
|
49
65
|
for (const pageId of batch) {
|
50
66
|
const page = result.results.find((p) => p.id === pageId);
|
@@ -15,6 +15,18 @@ export declare class NotionService {
|
|
15
15
|
private cache;
|
16
16
|
private limiter;
|
17
17
|
constructor(options: NotionServiceOptions);
|
18
|
+
/**
|
19
|
+
* GET 요청을 수행하는 함수 (재시도 로직 포함)
|
20
|
+
*/
|
21
|
+
private fetchGetWithRetry;
|
22
|
+
/**
|
23
|
+
* POST 요청을 수행하는 함수 (재시도 로직 포함)
|
24
|
+
*/
|
25
|
+
private fetchPostWithRetry;
|
26
|
+
/**
|
27
|
+
* 모든 블록 데이터를 가져오는 함수 (next_cursor 지원)
|
28
|
+
*/
|
29
|
+
private fetchAllBlocks;
|
18
30
|
/**
|
19
31
|
* 데이터베이스 쿼리
|
20
32
|
*/
|
@@ -1,7 +1,9 @@
|
|
1
1
|
"use strict";
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
3
3
|
exports.NotionService = void 0;
|
4
|
-
const
|
4
|
+
const connector_1 = require("../connector");
|
5
|
+
const constants_1 = require("../../constants");
|
6
|
+
const timeLimit_1 = require("../../util/timeLimit");
|
5
7
|
/**
|
6
8
|
* 간단한 병렬 제한 큐
|
7
9
|
*/
|
@@ -60,6 +62,99 @@ class NotionService {
|
|
60
62
|
this.cache = new Map();
|
61
63
|
this.limiter = new ParallelLimiter(this.parallelLimit);
|
62
64
|
}
|
65
|
+
/**
|
66
|
+
* GET 요청을 수행하는 함수 (재시도 로직 포함)
|
67
|
+
*/
|
68
|
+
async fetchGetWithRetry(url, tryCount = 0, maxRetries = 5) {
|
69
|
+
try {
|
70
|
+
const response = await connector_1.instance.get(url);
|
71
|
+
let results = response.data.results;
|
72
|
+
// Notion API 응답에 has_more가 있으면 추가 블록을 가져와 병합
|
73
|
+
if (response.data.has_more) {
|
74
|
+
const pageId = url.match(/blocks\/([^\/]*)\/children/)?.[1]; // 정확한 pageId 추출
|
75
|
+
if (pageId) {
|
76
|
+
const additionalResults = await this.fetchAllBlocks(pageId, response.data.next_cursor);
|
77
|
+
results = [...results, ...additionalResults]; // 기존 results에 추가 데이터 병합
|
78
|
+
}
|
79
|
+
else {
|
80
|
+
this.reporter.warn(`[WARNING] Failed to extract pageId from URL: ${url}`);
|
81
|
+
}
|
82
|
+
}
|
83
|
+
return { ...response.data, results }; // 병합된 전체 데이터 반환
|
84
|
+
}
|
85
|
+
catch (error) {
|
86
|
+
const status = error.response?.status;
|
87
|
+
const message = error.response?.data?.message || `Unknown error`;
|
88
|
+
// Rate limit 오류 처리 (Exponential Backoff)
|
89
|
+
if (status === constants_1.NOTION_LIMIT_ERROR && tryCount < maxRetries) {
|
90
|
+
const delay = Math.pow(2, tryCount) * 1000;
|
91
|
+
this.reporter.info(`[${constants_1.PLUGIN_NAME}] Rate limit hit. Retrying in ${delay / 1000}s...`);
|
92
|
+
await (0, timeLimit_1.sleep)(delay);
|
93
|
+
return this.fetchGetWithRetry(url, tryCount + 1, maxRetries);
|
94
|
+
}
|
95
|
+
// 기타 예외 처리 (Exponential Backoff)
|
96
|
+
if (tryCount < maxRetries) {
|
97
|
+
const delay = Math.pow(2, tryCount) * 1000;
|
98
|
+
this.reporter.info(`[${constants_1.PLUGIN_NAME}] Unexpected error. Retrying in ${delay / 1000}s...`);
|
99
|
+
await (0, timeLimit_1.sleep)(delay);
|
100
|
+
return this.fetchGetWithRetry(url, tryCount + 1, maxRetries);
|
101
|
+
}
|
102
|
+
// 최대 재시도 초과 시 오류 출력 후 throw
|
103
|
+
this.reporter.error(`[${constants_1.PLUGIN_NAME}] Failed after ${tryCount} retries:`, message);
|
104
|
+
throw error;
|
105
|
+
}
|
106
|
+
}
|
107
|
+
/**
|
108
|
+
* POST 요청을 수행하는 함수 (재시도 로직 포함)
|
109
|
+
*/
|
110
|
+
async fetchPostWithRetry(url, body, tryCount = 0, maxRetries = 5) {
|
111
|
+
try {
|
112
|
+
const response = await connector_1.instance.post(url, body);
|
113
|
+
return response.data;
|
114
|
+
}
|
115
|
+
catch (error) {
|
116
|
+
const status = error.response?.status;
|
117
|
+
const message = error.response?.data?.message || `Unknown error`;
|
118
|
+
// Rate limit 오류 처리 (Exponential Backoff)
|
119
|
+
if (status === constants_1.NOTION_LIMIT_ERROR && tryCount < maxRetries) {
|
120
|
+
const delay = Math.pow(2, tryCount) * 1000;
|
121
|
+
this.reporter.info(`[${constants_1.PLUGIN_NAME}] Rate limit hit. Retrying in ${delay / 1000}s...`);
|
122
|
+
await (0, timeLimit_1.sleep)(delay);
|
123
|
+
return this.fetchPostWithRetry(url, body, tryCount + 1, maxRetries);
|
124
|
+
}
|
125
|
+
// 기타 예외 처리 (Exponential Backoff)
|
126
|
+
if (tryCount < maxRetries) {
|
127
|
+
const delay = Math.pow(2, tryCount) * 1000;
|
128
|
+
this.reporter.info(`[${constants_1.PLUGIN_NAME}] Unexpected error. Retrying in ${delay / 1000}s...`);
|
129
|
+
await (0, timeLimit_1.sleep)(delay);
|
130
|
+
return this.fetchPostWithRetry(url, body, tryCount + 1, maxRetries);
|
131
|
+
}
|
132
|
+
// 최대 재시도 초과 시 오류 출력 후 throw
|
133
|
+
this.reporter.error(`[${constants_1.PLUGIN_NAME}] Failed after ${tryCount} retries:`, message);
|
134
|
+
throw error;
|
135
|
+
}
|
136
|
+
}
|
137
|
+
/**
|
138
|
+
* 모든 블록 데이터를 가져오는 함수 (next_cursor 지원)
|
139
|
+
*/
|
140
|
+
async fetchAllBlocks(pageId, startCursor = null) {
|
141
|
+
let hasMore = true;
|
142
|
+
let nextCursor = startCursor;
|
143
|
+
let allResults = [];
|
144
|
+
while (hasMore) {
|
145
|
+
const pageUrl = `blocks/${pageId}/children?page_size=100${nextCursor ? `&start_cursor=${nextCursor}` : ""}`;
|
146
|
+
const result = await this.fetchGetWithRetry(pageUrl);
|
147
|
+
if (result?.results?.length) {
|
148
|
+
allResults = [...allResults, ...result.results];
|
149
|
+
}
|
150
|
+
hasMore = result.has_more || false;
|
151
|
+
nextCursor = result.next_cursor || null;
|
152
|
+
if (!nextCursor) {
|
153
|
+
hasMore = false;
|
154
|
+
}
|
155
|
+
}
|
156
|
+
return allResults;
|
157
|
+
}
|
63
158
|
/**
|
64
159
|
* 데이터베이스 쿼리
|
65
160
|
*/
|
@@ -71,7 +166,7 @@ class NotionService {
|
|
71
166
|
}
|
72
167
|
const databaseUrl = `databases/${databaseId}/query`;
|
73
168
|
try {
|
74
|
-
const result = await
|
169
|
+
const result = await this.fetchPostWithRetry(databaseUrl, body);
|
75
170
|
if (this.enableCaching) {
|
76
171
|
this.cache.set(cacheKey, result);
|
77
172
|
}
|
@@ -94,7 +189,7 @@ class NotionService {
|
|
94
189
|
}
|
95
190
|
const pageUrl = `blocks/${pageId}/children?page_size=100`;
|
96
191
|
try {
|
97
|
-
const data = await
|
192
|
+
const data = await this.fetchGetWithRetry(pageUrl);
|
98
193
|
const blocks = data.results;
|
99
194
|
// 재귀적으로 has_children이 true인 블록의 자식들을 가져옴
|
100
195
|
const blocksWithChildren = await this.fetchChildBlocks(blocks);
|
@@ -0,0 +1,46 @@
|
|
1
|
+
/**
|
2
|
+
* This file provides TypeScript bindings to the Rust implementation
|
3
|
+
* of parallel Notion API processing.
|
4
|
+
*/
|
5
|
+
import { Reporter } from "gatsby";
|
6
|
+
/**
|
7
|
+
* Notion Block type from the Notion API
|
8
|
+
*/
|
9
|
+
export type NotionBlock = any;
|
10
|
+
/**
|
11
|
+
* Options for the RustNotionService
|
12
|
+
*/
|
13
|
+
export interface RustNotionServiceOptions {
|
14
|
+
reporter: Reporter;
|
15
|
+
notionApiKey: string;
|
16
|
+
parallelLimit?: number;
|
17
|
+
enableCaching?: boolean;
|
18
|
+
}
|
19
|
+
/**
|
20
|
+
* A wrapper class that provides access to the Rust implementation
|
21
|
+
* of parallel processing for Notion API, with a fallback to the
|
22
|
+
* TypeScript implementation if the Rust module is not available.
|
23
|
+
*/
|
24
|
+
export declare class RustNotionService {
|
25
|
+
private reporter;
|
26
|
+
private notionApiKey;
|
27
|
+
private parallelLimit;
|
28
|
+
private enableCaching;
|
29
|
+
private cache;
|
30
|
+
private rustInstance;
|
31
|
+
constructor(options: RustNotionServiceOptions);
|
32
|
+
/**
|
33
|
+
* Get blocks for a single page, with recursive fetching of child blocks
|
34
|
+
*/
|
35
|
+
getPageBlocks(pageId: string): Promise<NotionBlock[]>;
|
36
|
+
/**
|
37
|
+
* Get blocks for multiple pages in parallel, using Rust implementation if available
|
38
|
+
*/
|
39
|
+
getMultiplePagesBlocks(pageIds: string[]): Promise<{
|
40
|
+
[id: string]: NotionBlock[];
|
41
|
+
}>;
|
42
|
+
/**
|
43
|
+
* Set the parallel processing limit
|
44
|
+
*/
|
45
|
+
setParallelLimit(limit: number): void;
|
46
|
+
}
|
@@ -0,0 +1,134 @@
|
|
1
|
+
"use strict";
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
3
|
+
if (k2 === undefined) k2 = k;
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
7
|
+
}
|
8
|
+
Object.defineProperty(o, k2, desc);
|
9
|
+
}) : (function(o, m, k, k2) {
|
10
|
+
if (k2 === undefined) k2 = k;
|
11
|
+
o[k2] = m[k];
|
12
|
+
}));
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
15
|
+
}) : function(o, v) {
|
16
|
+
o["default"] = v;
|
17
|
+
});
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
19
|
+
var ownKeys = function(o) {
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
21
|
+
var ar = [];
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
23
|
+
return ar;
|
24
|
+
};
|
25
|
+
return ownKeys(o);
|
26
|
+
};
|
27
|
+
return function (mod) {
|
28
|
+
if (mod && mod.__esModule) return mod;
|
29
|
+
var result = {};
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
31
|
+
__setModuleDefault(result, mod);
|
32
|
+
return result;
|
33
|
+
};
|
34
|
+
})();
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
36
|
+
exports.RustNotionService = void 0;
|
37
|
+
// Use dynamic import with try-catch to handle module loading
|
38
|
+
let notionParallel = null;
|
39
|
+
try {
|
40
|
+
// When the compiled Rust library exists, load it
|
41
|
+
notionParallel = require("../rust/notion-parallel/index.node");
|
42
|
+
}
|
43
|
+
catch (error) {
|
44
|
+
console.warn("Rust bindings for notion-parallel not found. Using TypeScript fallback implementation.");
|
45
|
+
}
|
46
|
+
/**
|
47
|
+
* A wrapper class that provides access to the Rust implementation
|
48
|
+
* of parallel processing for Notion API, with a fallback to the
|
49
|
+
* TypeScript implementation if the Rust module is not available.
|
50
|
+
*/
|
51
|
+
class RustNotionService {
|
52
|
+
reporter;
|
53
|
+
notionApiKey;
|
54
|
+
parallelLimit;
|
55
|
+
enableCaching;
|
56
|
+
cache;
|
57
|
+
rustInstance = null;
|
58
|
+
constructor(options) {
|
59
|
+
this.reporter = options.reporter;
|
60
|
+
this.notionApiKey = options.notionApiKey;
|
61
|
+
this.parallelLimit = options.parallelLimit || 5;
|
62
|
+
this.enableCaching = options.enableCaching !== false;
|
63
|
+
this.cache = new Map();
|
64
|
+
// Initialize Rust instance if available
|
65
|
+
if (notionParallel) {
|
66
|
+
try {
|
67
|
+
this.rustInstance = new notionParallel.NotionParallel(this.notionApiKey, this.parallelLimit);
|
68
|
+
this.reporter.info(`[RUST] Initialized Rust-based parallel processing with limit: ${this.parallelLimit}`);
|
69
|
+
}
|
70
|
+
catch (error) {
|
71
|
+
this.reporter.warn(`[RUST] Failed to initialize Rust implementation: ${error}`);
|
72
|
+
this.rustInstance = null;
|
73
|
+
}
|
74
|
+
}
|
75
|
+
}
|
76
|
+
/**
|
77
|
+
* Get blocks for a single page, with recursive fetching of child blocks
|
78
|
+
*/
|
79
|
+
async getPageBlocks(pageId) {
|
80
|
+
const cacheKey = `page-blocks-${pageId}`;
|
81
|
+
if (this.enableCaching && this.cache.has(cacheKey)) {
|
82
|
+
this.reporter.info(`[CACHE] Using cached page blocks for ${pageId}`);
|
83
|
+
return this.cache.get(cacheKey);
|
84
|
+
}
|
85
|
+
// Currently we don't have a direct Rust counterpart for a single page
|
86
|
+
// So we use the multiple pages method with a single ID
|
87
|
+
const results = await this.getMultiplePagesBlocks([pageId]);
|
88
|
+
const blocks = results[pageId] || [];
|
89
|
+
if (this.enableCaching) {
|
90
|
+
this.cache.set(cacheKey, blocks);
|
91
|
+
}
|
92
|
+
return blocks;
|
93
|
+
}
|
94
|
+
/**
|
95
|
+
* Get blocks for multiple pages in parallel, using Rust implementation if available
|
96
|
+
*/
|
97
|
+
async getMultiplePagesBlocks(pageIds) {
|
98
|
+
this.reporter.info(`[NOTION] Fetching blocks for ${pageIds.length} pages in parallel (limit: ${this.parallelLimit})`);
|
99
|
+
// If Rust implementation is available and initialized, use it
|
100
|
+
if (this.rustInstance) {
|
101
|
+
try {
|
102
|
+
this.reporter.info(`[RUST] Using Rust implementation for parallel processing`);
|
103
|
+
const results = await this.rustInstance.getMultiplePagesBlocks(pageIds);
|
104
|
+
return results;
|
105
|
+
}
|
106
|
+
catch (error) {
|
107
|
+
this.reporter.warn(`[RUST] Error using Rust implementation, falling back to TypeScript: ${error}`);
|
108
|
+
// Fall back to TypeScript implementation
|
109
|
+
}
|
110
|
+
}
|
111
|
+
// Fallback to TypeScript implementation
|
112
|
+
// Import the TypeScript implementation dynamically
|
113
|
+
const { NotionService } = await Promise.resolve().then(() => __importStar(require("./api/service/notionService")));
|
114
|
+
// Create a NotionService instance with the same configuration
|
115
|
+
const tsService = new NotionService({
|
116
|
+
reporter: this.reporter,
|
117
|
+
parallelLimit: this.parallelLimit,
|
118
|
+
enableCaching: this.enableCaching,
|
119
|
+
});
|
120
|
+
return tsService.getMultiplePagesBlocks(pageIds);
|
121
|
+
}
|
122
|
+
/**
|
123
|
+
* Set the parallel processing limit
|
124
|
+
*/
|
125
|
+
setParallelLimit(limit) {
|
126
|
+
this.reporter.info(`[NOTION] Updated parallel request limit to ${limit}`);
|
127
|
+
this.parallelLimit = limit;
|
128
|
+
// Update the Rust instance if available
|
129
|
+
if (this.rustInstance) {
|
130
|
+
this.rustInstance.setParallelLimit(limit);
|
131
|
+
}
|
132
|
+
}
|
133
|
+
}
|
134
|
+
exports.RustNotionService = RustNotionService;
|
package/package.json
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
{
|
2
2
|
"name": "gatsby-source-notion-churnotion",
|
3
3
|
"description": "Gatsby plugin that can connect with One Notion Database RECURSIVELY using official API",
|
4
|
-
"version": "1.
|
4
|
+
"version": "1.2.0",
|
5
5
|
"skipLibCheck": true,
|
6
6
|
"license": "0BSD",
|
7
7
|
"main": "./dist/gatsby-node.js",
|
@@ -11,7 +11,8 @@
|
|
11
11
|
],
|
12
12
|
"scripts": {
|
13
13
|
"clean": "del-cli dist",
|
14
|
-
"build": "tsc",
|
14
|
+
"build": "npm run build:rust && tsc",
|
15
|
+
"build:rust": "cd rust/notion-parallel && (cargo build --release || cargo build --release --target x86_64-pc-windows-gnu || echo 'Rust build failed, using TypeScript fallback') && cd ../..",
|
15
16
|
"develop": "tsc --watch",
|
16
17
|
"test": "jest",
|
17
18
|
"prepare": "npm run clean && npm run build"
|
@@ -54,6 +55,7 @@
|
|
54
55
|
"typescript": "^5.7.2"
|
55
56
|
},
|
56
57
|
"devDependencies": {
|
58
|
+
"@netlify/plugin-cachedir": "^1.0.0",
|
57
59
|
"@types/node-fetch": "^2.6.12",
|
58
60
|
"gatsby": "^5.14.0"
|
59
61
|
},
|
package/dist/util/fetchData.d.ts
DELETED
@@ -1,20 +0,0 @@
|
|
1
|
-
/**
|
2
|
-
* Notion API에서 모든 블록 데이터를 가져오는 함수 (next_cursor 지원)
|
3
|
-
* @param {string} pageId - Notion 페이지 ID
|
4
|
-
* @param {string | null} startCursor - 처음 호출 시 넘길 next_cursor (기본값 null)
|
5
|
-
* @returns {Promise<any[]>} 모든 블록 데이터 리스트
|
6
|
-
*/
|
7
|
-
export declare const fetchAllBlocks: (pageId: string, startCursor?: string | null) => Promise<any[]>;
|
8
|
-
/**
|
9
|
-
* Notion API에서 GET 요청을 수행하는 함수 (재시도 로직 포함)
|
10
|
-
*/
|
11
|
-
export declare const fetchGetWithRetry: (url: string, options?: any, tryCount?: number, maxRetries?: number) => Promise<any>;
|
12
|
-
/**
|
13
|
-
* Notion API에서 POST 요청을 수행하는 함수 (재시도 로직 포함)
|
14
|
-
* @param {string} url - API endpoint URL
|
15
|
-
* @param {any} body - 요청 데이터
|
16
|
-
* @param {number} tryCount - 현재 재시도 횟수
|
17
|
-
* @param {number} maxRetries - 최대 재시도 횟수
|
18
|
-
* @returns {Promise<any>} API 응답 데이터
|
19
|
-
*/
|
20
|
-
export declare const fetchPostWithRetry: (url: string, body: any, tryCount?: number, maxRetries?: number) => Promise<any>;
|
package/dist/util/fetchData.js
DELETED
@@ -1,112 +0,0 @@
|
|
1
|
-
"use strict";
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
3
|
-
exports.fetchPostWithRetry = exports.fetchGetWithRetry = exports.fetchAllBlocks = void 0;
|
4
|
-
const connector_1 = require("../api/connector");
|
5
|
-
const constants_1 = require("../constants");
|
6
|
-
const timeLimit_1 = require("./timeLimit");
|
7
|
-
/**
|
8
|
-
* Notion API에서 모든 블록 데이터를 가져오는 함수 (next_cursor 지원)
|
9
|
-
* @param {string} pageId - Notion 페이지 ID
|
10
|
-
* @param {string | null} startCursor - 처음 호출 시 넘길 next_cursor (기본값 null)
|
11
|
-
* @returns {Promise<any[]>} 모든 블록 데이터 리스트
|
12
|
-
*/
|
13
|
-
const fetchAllBlocks = async (pageId, startCursor = null) => {
|
14
|
-
let hasMore = true;
|
15
|
-
let nextCursor = startCursor; // 초기 next_cursor를 설정
|
16
|
-
let allResults = [];
|
17
|
-
while (hasMore) {
|
18
|
-
const pageUrl = `blocks/${pageId}/children?page_size=100${nextCursor ? `&start_cursor=${nextCursor}` : ""}`;
|
19
|
-
const result = await (0, exports.fetchGetWithRetry)(pageUrl);
|
20
|
-
if (result?.results?.length) {
|
21
|
-
allResults = [...allResults, ...result.results]; // 기존 데이터와 병합
|
22
|
-
}
|
23
|
-
hasMore = result.has_more || false; // 다음 데이터가 있는지 확인
|
24
|
-
nextCursor = result.next_cursor || null; // 다음 페이지 커서 업데이트
|
25
|
-
// 만약 nextCursor가 계속 null이라면 루프 탈출
|
26
|
-
if (!nextCursor) {
|
27
|
-
hasMore = false;
|
28
|
-
}
|
29
|
-
}
|
30
|
-
return allResults;
|
31
|
-
};
|
32
|
-
exports.fetchAllBlocks = fetchAllBlocks;
|
33
|
-
/**
|
34
|
-
* Notion API에서 GET 요청을 수행하는 함수 (재시도 로직 포함)
|
35
|
-
*/
|
36
|
-
const fetchGetWithRetry = async (url, options = {}, tryCount = 0, maxRetries = 5) => {
|
37
|
-
try {
|
38
|
-
const response = await connector_1.instance.get(url);
|
39
|
-
let results = response.data.results;
|
40
|
-
// Notion API 응답에 has_more가 있으면 추가 블록을 가져와 병합
|
41
|
-
if (response.data.has_more) {
|
42
|
-
const pageId = url.match(/blocks\/([^\/]*)\/children/)?.[1]; // 정확한 pageId 추출
|
43
|
-
if (pageId) {
|
44
|
-
const additionalResults = await (0, exports.fetchAllBlocks)(pageId, response.data.next_cursor); // next_cursor도 함께 전달
|
45
|
-
results = [...results, ...additionalResults]; // 기존 results에 추가 데이터 병합
|
46
|
-
}
|
47
|
-
else {
|
48
|
-
console.warn(`[WARNING] Failed to extract pageId from URL: ${url}`);
|
49
|
-
}
|
50
|
-
}
|
51
|
-
return { ...response.data, results }; // 병합된 전체 데이터 반환
|
52
|
-
}
|
53
|
-
catch (error) {
|
54
|
-
const status = error.response?.status;
|
55
|
-
const message = error.response?.data?.message || `Unknown error`;
|
56
|
-
// Rate limit 오류 처리 (Exponential Backoff)
|
57
|
-
if (status === constants_1.NOTION_LIMIT_ERROR && tryCount < maxRetries) {
|
58
|
-
const delay = Math.pow(2, tryCount) * 1000;
|
59
|
-
console.log(`[${constants_1.PLUGIN_NAME}] Rate limit hit. Retrying in ${delay / 1000}s...`);
|
60
|
-
await (0, timeLimit_1.sleep)(delay);
|
61
|
-
return (0, exports.fetchGetWithRetry)(url, options, tryCount + 1, maxRetries);
|
62
|
-
}
|
63
|
-
// 기타 예외 처리 (Exponential Backoff)
|
64
|
-
if (tryCount < maxRetries) {
|
65
|
-
const delay = Math.pow(2, tryCount) * 1000;
|
66
|
-
console.log(`[${constants_1.PLUGIN_NAME}] Unexpected error. Retrying in ${delay / 1000}s...`);
|
67
|
-
await (0, timeLimit_1.sleep)(delay);
|
68
|
-
return (0, exports.fetchGetWithRetry)(url, options, tryCount + 1, maxRetries);
|
69
|
-
}
|
70
|
-
// 최대 재시도 초과 시 오류 출력 후 throw
|
71
|
-
console.error(`[${constants_1.PLUGIN_NAME}] Failed after ${tryCount} retries:`, message);
|
72
|
-
throw error;
|
73
|
-
}
|
74
|
-
};
|
75
|
-
exports.fetchGetWithRetry = fetchGetWithRetry;
|
76
|
-
/**
|
77
|
-
* Notion API에서 POST 요청을 수행하는 함수 (재시도 로직 포함)
|
78
|
-
* @param {string} url - API endpoint URL
|
79
|
-
* @param {any} body - 요청 데이터
|
80
|
-
* @param {number} tryCount - 현재 재시도 횟수
|
81
|
-
* @param {number} maxRetries - 최대 재시도 횟수
|
82
|
-
* @returns {Promise<any>} API 응답 데이터
|
83
|
-
*/
|
84
|
-
const fetchPostWithRetry = async (url, body, tryCount = 0, maxRetries = 5) => {
|
85
|
-
try {
|
86
|
-
const response = await connector_1.instance.post(url, body);
|
87
|
-
// 요청 성공 시 데이터 반환
|
88
|
-
return response.data;
|
89
|
-
}
|
90
|
-
catch (error) {
|
91
|
-
const status = error.response?.status;
|
92
|
-
const message = error.response?.data?.message || `Unknown error`;
|
93
|
-
// Rate limit 오류 처리 (Exponential Backoff)
|
94
|
-
if (status === constants_1.NOTION_LIMIT_ERROR && tryCount < maxRetries) {
|
95
|
-
const delay = Math.pow(2, tryCount) * 1000;
|
96
|
-
console.log(`[${constants_1.PLUGIN_NAME}] Rate limit hit. Retrying in ${delay / 1000}s...`);
|
97
|
-
await (0, timeLimit_1.sleep)(delay);
|
98
|
-
return (0, exports.fetchPostWithRetry)(url, body, tryCount + 1, maxRetries);
|
99
|
-
}
|
100
|
-
// 기타 예외 처리 (Exponential Backoff)
|
101
|
-
if (tryCount < maxRetries) {
|
102
|
-
const delay = Math.pow(2, tryCount) * 1000;
|
103
|
-
console.log(`[${constants_1.PLUGIN_NAME}] Unexpected error. Retrying in ${delay / 1000}s...`);
|
104
|
-
await (0, timeLimit_1.sleep)(delay);
|
105
|
-
return (0, exports.fetchPostWithRetry)(url, body, tryCount + 1, maxRetries);
|
106
|
-
}
|
107
|
-
// 최대 재시도 초과 시 오류 출력 후 throw
|
108
|
-
console.error(`[${constants_1.PLUGIN_NAME}] Failed after ${tryCount} retries:`, message);
|
109
|
-
throw error;
|
110
|
-
}
|
111
|
-
};
|
112
|
-
exports.fetchPostWithRetry = fetchPostWithRetry;
|