@moxn/kb-migrate 0.4.22 → 0.4.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.js +12 -2
- package/dist/index.js +18 -4
- package/dist/sources/notion-blocks.d.ts +4 -0
- package/dist/sources/notion-blocks.js +25 -21
- package/dist/sources/notion.d.ts +8 -1
- package/dist/sources/notion.js +127 -11
- package/package.json +1 -1
package/dist/client.js
CHANGED
|
@@ -299,7 +299,12 @@ export class MoxnClient {
|
|
|
299
299
|
error.branchId = body.branchId;
|
|
300
300
|
throw error;
|
|
301
301
|
}
|
|
302
|
-
|
|
302
|
+
const message = body.details
|
|
303
|
+
? `API error ${response.status}: ${body.details}`
|
|
304
|
+
: body.error
|
|
305
|
+
? `API error ${response.status}: ${body.error}`
|
|
306
|
+
: `API error: ${response.status}`;
|
|
307
|
+
throw new Error(message);
|
|
303
308
|
}
|
|
304
309
|
return response.json();
|
|
305
310
|
}
|
|
@@ -314,7 +319,12 @@ export class MoxnClient {
|
|
|
314
319
|
});
|
|
315
320
|
if (!response.ok) {
|
|
316
321
|
const body = await response.json().catch(() => ({}));
|
|
317
|
-
|
|
322
|
+
const message = body.details
|
|
323
|
+
? `API error ${response.status}: ${body.details}`
|
|
324
|
+
: body.error
|
|
325
|
+
? `API error ${response.status}: ${body.error}`
|
|
326
|
+
: `API error: ${response.status}`;
|
|
327
|
+
throw new Error(message);
|
|
318
328
|
}
|
|
319
329
|
return response.json();
|
|
320
330
|
}
|
package/dist/index.js
CHANGED
|
@@ -27,21 +27,28 @@ async function runMigration(source, options) {
|
|
|
27
27
|
const results = [];
|
|
28
28
|
// Validate source
|
|
29
29
|
await source.validate();
|
|
30
|
-
// Get document count for progress
|
|
30
|
+
// Get document count for progress (this is an upper bound — some pages
|
|
31
|
+
// may be skipped during extraction if they are empty or have errors)
|
|
31
32
|
const totalCount = await source.getDocumentCount();
|
|
32
|
-
if (totalCount !== undefined) {
|
|
33
|
-
console.log(`Found ${totalCount} documents to migrate`);
|
|
34
|
-
}
|
|
35
33
|
// Create client
|
|
36
34
|
const client = new MoxnClient(options);
|
|
37
35
|
// Process documents
|
|
38
36
|
let processed = 0;
|
|
37
|
+
let consecutiveFailures = 0;
|
|
38
|
+
const MAX_CONSECUTIVE_FAILURES = 10;
|
|
39
39
|
for await (const doc of source.extract()) {
|
|
40
40
|
processed++;
|
|
41
41
|
const progress = totalCount ? ` (${processed}/${totalCount})` : '';
|
|
42
42
|
console.log(`Processing: ${doc.sourcePath}${progress}`);
|
|
43
43
|
const result = await client.migrateDocument(doc, options.basePath, options.onConflict, options.dryRun);
|
|
44
44
|
results.push(result);
|
|
45
|
+
// Track consecutive failures for circuit breaker
|
|
46
|
+
if (result.status === 'failed') {
|
|
47
|
+
consecutiveFailures++;
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
consecutiveFailures = 0;
|
|
51
|
+
}
|
|
45
52
|
// Log result
|
|
46
53
|
const statusIcon = {
|
|
47
54
|
created: '\u2713',
|
|
@@ -53,6 +60,13 @@ async function runMigration(source, options) {
|
|
|
53
60
|
if (result.error) {
|
|
54
61
|
console.log(` Error: ${result.error}`);
|
|
55
62
|
}
|
|
63
|
+
// Circuit breaker: abort after too many consecutive failures
|
|
64
|
+
if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
|
|
65
|
+
console.error(`\nAborting: ${MAX_CONSECUTIVE_FAILURES} consecutive failures. ` +
|
|
66
|
+
'Last error: ' + (result.error || 'unknown') + '\n' +
|
|
67
|
+
'Fix the underlying issue and retry. Remaining documents will be skipped.');
|
|
68
|
+
break;
|
|
69
|
+
}
|
|
56
70
|
}
|
|
57
71
|
// Build summary
|
|
58
72
|
const summary = {
|
|
@@ -8,6 +8,8 @@ import type { SectionInput } from '../types.js';
|
|
|
8
8
|
import type { NotionBlock, NotionRichText, NotionApiClient, NotionPage } from './notion-api.js';
|
|
9
9
|
/** Map of page IDs to their KB paths (for cross-link resolution). */
|
|
10
10
|
export type PagePathMap = Map<string, string>;
|
|
11
|
+
/** Tracks unsupported Notion block types encountered during conversion. */
|
|
12
|
+
export type UnsupportedBlockTracker = Map<string, number>;
|
|
11
13
|
/**
|
|
12
14
|
* Convert a page's blocks into Moxn sections.
|
|
13
15
|
*
|
|
@@ -19,6 +21,8 @@ export declare function blocksToSections(blocks: NotionBlock[], client: NotionAp
|
|
|
19
21
|
visitedSyncedBlocks?: Set<string>;
|
|
20
22
|
/** Map of Notion database IDs to KB database IDs (for child_database blocks). */
|
|
21
23
|
databaseIdMap?: Map<string, string>;
|
|
24
|
+
/** Accumulates counts of unsupported block types (caller-owned). */
|
|
25
|
+
unsupportedBlocks?: UnsupportedBlockTracker;
|
|
22
26
|
}): Promise<SectionInput[]>;
|
|
23
27
|
/** Convert rich text array to markdown string. */
|
|
24
28
|
export declare function richTextToMarkdown(richText: NotionRichText[]): string;
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
export async function blocksToSections(blocks, client, pagePathMap, options) {
|
|
17
17
|
const visitedSyncedBlocks = options?.visitedSyncedBlocks ?? new Set();
|
|
18
18
|
const databaseIdMap = options?.databaseIdMap;
|
|
19
|
+
const unsupportedBlocks = options?.unsupportedBlocks;
|
|
19
20
|
const sections = [];
|
|
20
21
|
let currentSectionName = 'Introduction';
|
|
21
22
|
let currentBlocks = [];
|
|
@@ -35,7 +36,7 @@ export async function blocksToSections(blocks, client, pagePathMap, options) {
|
|
|
35
36
|
continue;
|
|
36
37
|
}
|
|
37
38
|
// Convert block to content blocks
|
|
38
|
-
const converted = await convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap);
|
|
39
|
+
const converted = await convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks);
|
|
39
40
|
currentBlocks.push(...converted);
|
|
40
41
|
}
|
|
41
42
|
// Flush last section
|
|
@@ -51,7 +52,7 @@ export async function blocksToSections(blocks, client, pagePathMap, options) {
|
|
|
51
52
|
// ============================================
|
|
52
53
|
// Block conversion
|
|
53
54
|
// ============================================
|
|
54
|
-
async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
55
|
+
async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
|
|
55
56
|
const results = [];
|
|
56
57
|
switch (block.type) {
|
|
57
58
|
case 'paragraph':
|
|
@@ -81,10 +82,10 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
|
|
|
81
82
|
results.push(...convertToDo(block));
|
|
82
83
|
break;
|
|
83
84
|
case 'quote':
|
|
84
|
-
results.push(...(await convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
85
|
+
results.push(...(await convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
|
|
85
86
|
break;
|
|
86
87
|
case 'callout':
|
|
87
|
-
results.push(...(await convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
88
|
+
results.push(...(await convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
|
|
88
89
|
break;
|
|
89
90
|
case 'divider':
|
|
90
91
|
results.push(textBlock('---'));
|
|
@@ -93,7 +94,7 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
|
|
|
93
94
|
results.push(...(await convertTable(block, client)));
|
|
94
95
|
break;
|
|
95
96
|
case 'toggle':
|
|
96
|
-
results.push(...(await convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
97
|
+
results.push(...(await convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
|
|
97
98
|
break;
|
|
98
99
|
case 'bookmark':
|
|
99
100
|
results.push(...convertBookmark(block));
|
|
@@ -122,10 +123,10 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
|
|
|
122
123
|
results.push(...convertChildDatabase(block, databaseIdMap));
|
|
123
124
|
break;
|
|
124
125
|
case 'synced_block':
|
|
125
|
-
results.push(...(await convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
126
|
+
results.push(...(await convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
|
|
126
127
|
break;
|
|
127
128
|
case 'column_list':
|
|
128
|
-
results.push(...(await convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
129
|
+
results.push(...(await convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
|
|
129
130
|
break;
|
|
130
131
|
case 'column':
|
|
131
132
|
// Columns are handled by column_list
|
|
@@ -146,7 +147,10 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
|
|
|
146
147
|
results.push(...convertLinkPreview(block));
|
|
147
148
|
break;
|
|
148
149
|
default:
|
|
149
|
-
// Unknown block type —
|
|
150
|
+
// Unknown block type — track and skip
|
|
151
|
+
if (unsupportedBlocks) {
|
|
152
|
+
unsupportedBlocks.set(block.type, (unsupportedBlocks.get(block.type) ?? 0) + 1);
|
|
153
|
+
}
|
|
150
154
|
console.warn(` Skipping unsupported Notion block type: ${block.type}`);
|
|
151
155
|
break;
|
|
152
156
|
}
|
|
@@ -165,7 +169,7 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
|
|
|
165
169
|
? ' '
|
|
166
170
|
: undefined;
|
|
167
171
|
const children = await client.getBlockChildren(block.id);
|
|
168
|
-
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap);
|
|
172
|
+
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap, unsupportedBlocks);
|
|
169
173
|
results.push(...childBlocks);
|
|
170
174
|
}
|
|
171
175
|
return results;
|
|
@@ -211,7 +215,7 @@ function convertToDo(block) {
|
|
|
211
215
|
const text = richTextToMarkdown(td.to_do.rich_text);
|
|
212
216
|
return [textBlock(`${checkbox} ${text}`)];
|
|
213
217
|
}
|
|
214
|
-
async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
218
|
+
async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
|
|
215
219
|
const q = block;
|
|
216
220
|
const text = richTextToMarkdown(q.quote.rich_text);
|
|
217
221
|
if (!text && !block.has_children)
|
|
@@ -225,12 +229,12 @@ async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks, dat
|
|
|
225
229
|
const results = quoted ? [textBlock(quoted)] : [];
|
|
226
230
|
if (block.has_children) {
|
|
227
231
|
const children = await client.getBlockChildren(block.id);
|
|
228
|
-
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
|
|
232
|
+
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap, unsupportedBlocks);
|
|
229
233
|
results.push(...childBlocks);
|
|
230
234
|
}
|
|
231
235
|
return results;
|
|
232
236
|
}
|
|
233
|
-
async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
237
|
+
async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
|
|
234
238
|
const c = block;
|
|
235
239
|
const text = richTextToMarkdown(c.callout.rich_text);
|
|
236
240
|
const emoji = c.callout.icon?.emoji ?? '';
|
|
@@ -242,7 +246,7 @@ async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks, d
|
|
|
242
246
|
const results = [textBlock(quoted)];
|
|
243
247
|
if (block.has_children) {
|
|
244
248
|
const children = await client.getBlockChildren(block.id);
|
|
245
|
-
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
|
|
249
|
+
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap, unsupportedBlocks);
|
|
246
250
|
results.push(...childBlocks);
|
|
247
251
|
}
|
|
248
252
|
return results;
|
|
@@ -271,13 +275,13 @@ async function convertTable(block, client) {
|
|
|
271
275
|
}
|
|
272
276
|
return [textBlock(lines.join('\n'))];
|
|
273
277
|
}
|
|
274
|
-
async function convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
278
|
+
async function convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
|
|
275
279
|
const t = block;
|
|
276
280
|
const header = richTextToMarkdown(t.toggle.rich_text);
|
|
277
281
|
const results = [textBlock(`**${header}**`)];
|
|
278
282
|
if (block.has_children) {
|
|
279
283
|
const children = await client.getBlockChildren(block.id);
|
|
280
|
-
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
|
|
284
|
+
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap, unsupportedBlocks);
|
|
281
285
|
results.push(...childBlocks);
|
|
282
286
|
}
|
|
283
287
|
return results;
|
|
@@ -402,7 +406,7 @@ function convertLinkToPage(block, pagePathMap) {
|
|
|
402
406
|
// Target not in import set
|
|
403
407
|
return [textBlock(`*(Link to Notion page: ${targetId})*`)];
|
|
404
408
|
}
|
|
405
|
-
async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
409
|
+
async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
|
|
406
410
|
const sb = block;
|
|
407
411
|
// Get the source block ID (either this block or the original)
|
|
408
412
|
const sourceId = sb.synced_block.synced_from?.block_id ?? block.id;
|
|
@@ -413,19 +417,19 @@ async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlock
|
|
|
413
417
|
visitedSyncedBlocks.add(sourceId);
|
|
414
418
|
try {
|
|
415
419
|
const children = await client.getBlockChildren(sourceId);
|
|
416
|
-
return await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap);
|
|
420
|
+
return await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap, unsupportedBlocks);
|
|
417
421
|
}
|
|
418
422
|
finally {
|
|
419
423
|
visitedSyncedBlocks.delete(sourceId);
|
|
420
424
|
}
|
|
421
425
|
}
|
|
422
|
-
async function convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
426
|
+
async function convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
|
|
423
427
|
const children = await client.getBlockChildren(block.id);
|
|
424
428
|
const results = [];
|
|
425
429
|
for (const column of children) {
|
|
426
430
|
if (column.type === 'column' && column.has_children) {
|
|
427
431
|
const columnChildren = await client.getBlockChildren(column.id);
|
|
428
|
-
const converted = await convertAndMergeChildren(columnChildren, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap);
|
|
432
|
+
const converted = await convertAndMergeChildren(columnChildren, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap, unsupportedBlocks);
|
|
429
433
|
results.push(...converted);
|
|
430
434
|
}
|
|
431
435
|
}
|
|
@@ -574,10 +578,10 @@ function mergeConsecutiveListBlocks(blocks) {
|
|
|
574
578
|
/**
|
|
575
579
|
* Convert children blocks, merge consecutive list items, and optionally indent.
|
|
576
580
|
*/
|
|
577
|
-
async function convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap) {
|
|
581
|
+
async function convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap, unsupportedBlocks) {
|
|
578
582
|
const blocks = [];
|
|
579
583
|
for (const child of children) {
|
|
580
|
-
const converted = await convertBlock(child, client, pagePathMap, visitedSyncedBlocks, databaseIdMap);
|
|
584
|
+
const converted = await convertBlock(child, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks);
|
|
581
585
|
blocks.push(...converted);
|
|
582
586
|
}
|
|
583
587
|
const merged = mergeConsecutiveListBlocks(blocks);
|
package/dist/sources/notion.d.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
import type { ExtractedDocument } from '../types.js';
|
|
11
11
|
import { MigrationSource, type SourceConfig } from './base.js';
|
|
12
|
-
import type { NotionPage } from './notion-api.js';
|
|
12
|
+
import type { NotionPage, NotionDatabase } from './notion-api.js';
|
|
13
13
|
import { type PagePathMap } from './notion-blocks.js';
|
|
14
14
|
import { type ParsedDatabaseSchema } from './notion-databases.js';
|
|
15
15
|
export interface NotionSourceConfig extends SourceConfig {
|
|
@@ -26,6 +26,7 @@ export interface PageTreeNode {
|
|
|
26
26
|
children: PageTreeNode[];
|
|
27
27
|
kbPath: string;
|
|
28
28
|
isDatabaseEntry: boolean;
|
|
29
|
+
isDatabaseNode?: boolean;
|
|
29
30
|
parentDatabaseId?: string;
|
|
30
31
|
}
|
|
31
32
|
/** Info yielded to the migration runner for database import (post-document pass). */
|
|
@@ -51,6 +52,11 @@ export declare class NotionSource extends MigrationSource<NotionSourceConfig> {
|
|
|
51
52
|
private databaseEntryPageIds;
|
|
52
53
|
private _documentCount;
|
|
53
54
|
private _validated;
|
|
55
|
+
private _skippedEmpty;
|
|
56
|
+
private _skippedNoContent;
|
|
57
|
+
private _extractionErrors;
|
|
58
|
+
private _syntheticDatabaseNodes;
|
|
59
|
+
private _unsupportedBlocks;
|
|
54
60
|
/** Map of Notion database ID → KB database ID, set before extraction. */
|
|
55
61
|
private _databaseIdMap;
|
|
56
62
|
constructor(config: NotionSourceConfig);
|
|
@@ -83,6 +89,7 @@ export declare class NotionSource extends MigrationSource<NotionSourceConfig> {
|
|
|
83
89
|
export interface BuildPageTreeOptions {
|
|
84
90
|
rootPageId?: string;
|
|
85
91
|
maxDepth?: number;
|
|
92
|
+
databases?: NotionDatabase[];
|
|
86
93
|
}
|
|
87
94
|
/** Result of building the page tree. */
|
|
88
95
|
export interface PageTreeResult {
|
package/dist/sources/notion.js
CHANGED
|
@@ -28,6 +28,12 @@ export class NotionSource extends MigrationSource {
|
|
|
28
28
|
databaseEntryPageIds = new Set();
|
|
29
29
|
_documentCount = 0;
|
|
30
30
|
_validated = false;
|
|
31
|
+
// Tracking for extraction-time skips (populated during extract())
|
|
32
|
+
_skippedEmpty = [];
|
|
33
|
+
_skippedNoContent = [];
|
|
34
|
+
_extractionErrors = [];
|
|
35
|
+
_syntheticDatabaseNodes = 0;
|
|
36
|
+
_unsupportedBlocks = new Map();
|
|
31
37
|
/** Map of Notion database ID → KB database ID, set before extraction. */
|
|
32
38
|
_databaseIdMap = new Map();
|
|
33
39
|
constructor(config) {
|
|
@@ -84,18 +90,23 @@ export class NotionSource extends MigrationSource {
|
|
|
84
90
|
console.log('Discovering databases...');
|
|
85
91
|
const allNotionDatabases = await this.client.searchDatabases();
|
|
86
92
|
console.log(` Found ${allNotionDatabases.length} databases.`);
|
|
87
|
-
// 4. Build page tree
|
|
88
|
-
this.buildPageTreeInternal(allNotionPages);
|
|
93
|
+
// 4. Build page tree (pass databases so they become tree nodes for nesting entries)
|
|
94
|
+
this.buildPageTreeInternal(allNotionPages, allNotionDatabases);
|
|
89
95
|
// 5. Process databases — identify entries and build schemas
|
|
90
96
|
await this.processDatabases(allNotionDatabases);
|
|
91
|
-
// 6. Count and validate
|
|
92
|
-
this.
|
|
97
|
+
// 6. Count and validate (exclude synthetic database container nodes)
|
|
98
|
+
this._syntheticDatabaseNodes = this.allPages.filter((n) => n.isDatabaseNode).length;
|
|
99
|
+
const pagesInTree = this.allPages.filter((n) => !n.isDatabaseNode).length;
|
|
100
|
+
const databaseEntriesInTree = this.allPages.filter((n) => n.isDatabaseEntry && !n.isDatabaseNode).length;
|
|
101
|
+
this._documentCount = pagesInTree;
|
|
93
102
|
// Add database-only entries (pages that are in databases but not in page tree)
|
|
103
|
+
let databaseEntriesOutsideTree = 0;
|
|
94
104
|
for (const dbInfo of this.databases) {
|
|
95
105
|
for (const entry of dbInfo.entries) {
|
|
96
106
|
const nid = normalizeId(entry.id);
|
|
97
107
|
if (!this.pagePathMap.has(nid)) {
|
|
98
108
|
this._documentCount++;
|
|
109
|
+
databaseEntriesOutsideTree++;
|
|
99
110
|
}
|
|
100
111
|
}
|
|
101
112
|
}
|
|
@@ -103,7 +114,12 @@ export class NotionSource extends MigrationSource {
|
|
|
103
114
|
throw new Error(`Workspace has ${this._documentCount} documents, exceeding the ${MAX_DOCUMENT_COUNT} limit. ` +
|
|
104
115
|
'Use --root-page-id to import a subtree.');
|
|
105
116
|
}
|
|
106
|
-
|
|
117
|
+
// Log clear breakdown of what was discovered
|
|
118
|
+
const standalonePages = pagesInTree - databaseEntriesInTree;
|
|
119
|
+
const totalDbEntries = databaseEntriesInTree + databaseEntriesOutsideTree;
|
|
120
|
+
console.log(` ${this._documentCount} documents to import: ` +
|
|
121
|
+
`${standalonePages} pages + ${totalDbEntries} database entries ` +
|
|
122
|
+
`(across ${this.databases.length} databases)`);
|
|
107
123
|
// 7. Initialize media downloader
|
|
108
124
|
await this.mediaDownloader.init();
|
|
109
125
|
this._validated = true;
|
|
@@ -135,15 +151,26 @@ export class NotionSource extends MigrationSource {
|
|
|
135
151
|
// Pass 2: Extraction
|
|
136
152
|
// ============================================
|
|
137
153
|
async *extract() {
|
|
154
|
+
// Reset extraction tracking
|
|
155
|
+
this._skippedEmpty = [];
|
|
156
|
+
this._skippedNoContent = [];
|
|
157
|
+
this._extractionErrors = [];
|
|
158
|
+
this._unsupportedBlocks = new Map();
|
|
159
|
+
let yielded = 0;
|
|
138
160
|
// Walk pages depth-first
|
|
139
161
|
for (const node of this.allPages) {
|
|
140
162
|
// Skip database entries that will be created during database import
|
|
141
163
|
// unless they also appear in the page tree (child_page)
|
|
142
164
|
if (node.isDatabaseEntry)
|
|
143
165
|
continue;
|
|
166
|
+
// Skip synthetic database container nodes — they're for nesting, not documents
|
|
167
|
+
if (node.isDatabaseNode)
|
|
168
|
+
continue;
|
|
144
169
|
const doc = await this.extractPage(node);
|
|
145
|
-
if (doc)
|
|
170
|
+
if (doc) {
|
|
171
|
+
yielded++;
|
|
146
172
|
yield doc;
|
|
173
|
+
}
|
|
147
174
|
}
|
|
148
175
|
// Extract database-only entries (pages not in page tree)
|
|
149
176
|
for (const dbInfo of this.databases) {
|
|
@@ -159,10 +186,43 @@ export class NotionSource extends MigrationSource {
|
|
|
159
186
|
}
|
|
160
187
|
}
|
|
161
188
|
const doc = await this.extractDatabaseEntry(entry, dbInfo);
|
|
162
|
-
if (doc)
|
|
189
|
+
if (doc) {
|
|
190
|
+
yielded++;
|
|
163
191
|
yield doc;
|
|
192
|
+
}
|
|
164
193
|
}
|
|
165
194
|
}
|
|
195
|
+
// Log extraction summary so user can reconcile counts
|
|
196
|
+
const totalSkipped = this._skippedEmpty.length +
|
|
197
|
+
this._skippedNoContent.length +
|
|
198
|
+
this._extractionErrors.length;
|
|
199
|
+
console.log(`\nExtraction complete: ${yielded} of ${this._documentCount} documents extracted`);
|
|
200
|
+
if (totalSkipped > 0) {
|
|
201
|
+
console.log(` ${totalSkipped} pages skipped:`);
|
|
202
|
+
if (this._skippedEmpty.length > 0) {
|
|
203
|
+
console.log(` Empty pages (no blocks): ${this._skippedEmpty.map((t) => `"${t}"`).join(', ')}`);
|
|
204
|
+
}
|
|
205
|
+
if (this._skippedNoContent.length > 0) {
|
|
206
|
+
console.log(` No extractable content: ${this._skippedNoContent.map((t) => `"${t}"`).join(', ')}`);
|
|
207
|
+
}
|
|
208
|
+
if (this._extractionErrors.length > 0) {
|
|
209
|
+
console.log(` Extraction errors: ${this._extractionErrors.map((t) => `"${t}"`).join(', ')}`);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
// Report unsupported block types
|
|
213
|
+
if (this._unsupportedBlocks.size > 0) {
|
|
214
|
+
const sorted = [...this._unsupportedBlocks.entries()].sort((a, b) => b[1] - a[1]);
|
|
215
|
+
const totalUnsupported = sorted.reduce((sum, [, count]) => sum + count, 0);
|
|
216
|
+
console.log(` ${totalUnsupported} unsupported block(s) across ${sorted.length} type(s):`);
|
|
217
|
+
for (const [type, count] of sorted) {
|
|
218
|
+
console.log(` ${type}: ${count}`);
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
// Warn if counts don't reconcile (helps catch counting bugs)
|
|
222
|
+
const expectedYield = this._documentCount - totalSkipped;
|
|
223
|
+
if (yielded !== expectedYield) {
|
|
224
|
+
console.log(` ⚠ Count mismatch: expected ${expectedYield} (${this._documentCount} - ${totalSkipped} skipped) but extracted ${yielded}`);
|
|
225
|
+
}
|
|
166
226
|
}
|
|
167
227
|
/** Clean up temp files after migration completes. */
|
|
168
228
|
async cleanup() {
|
|
@@ -171,10 +231,11 @@ export class NotionSource extends MigrationSource {
|
|
|
171
231
|
// ============================================
|
|
172
232
|
// Tree building (delegates to standalone function)
|
|
173
233
|
// ============================================
|
|
174
|
-
buildPageTreeInternal(pages) {
|
|
234
|
+
buildPageTreeInternal(pages, databases) {
|
|
175
235
|
const result = buildPageTree(pages, {
|
|
176
236
|
rootPageId: this.config.rootPageId,
|
|
177
237
|
maxDepth: this.config.maxDepth,
|
|
238
|
+
databases,
|
|
178
239
|
});
|
|
179
240
|
this.pageTree = result.tree;
|
|
180
241
|
this.allPages = result.allPages;
|
|
@@ -220,11 +281,12 @@ export class NotionSource extends MigrationSource {
|
|
|
220
281
|
try {
|
|
221
282
|
const blocks = await this.client.getBlockChildren(node.page.id);
|
|
222
283
|
if (blocks.length === 0) {
|
|
223
|
-
|
|
284
|
+
this._skippedEmpty.push(node.title);
|
|
224
285
|
return null;
|
|
225
286
|
}
|
|
226
287
|
let sections = await blocksToSections(blocks, this.client, this.pagePathMap, {
|
|
227
288
|
databaseIdMap: this._databaseIdMap.size > 0 ? this._databaseIdMap : undefined,
|
|
289
|
+
unsupportedBlocks: this._unsupportedBlocks,
|
|
228
290
|
});
|
|
229
291
|
// Download Notion-hosted media files
|
|
230
292
|
sections = await this.downloadSectionMedia(sections);
|
|
@@ -232,7 +294,7 @@ export class NotionSource extends MigrationSource {
|
|
|
232
294
|
const { sections: resolvedSections, references } = resolveNotionReferences(sections, { notionIdToKbPath: this.pagePathMap });
|
|
233
295
|
sections = resolvedSections;
|
|
234
296
|
if (sections.length === 0) {
|
|
235
|
-
|
|
297
|
+
this._skippedNoContent.push(node.title);
|
|
236
298
|
return null;
|
|
237
299
|
}
|
|
238
300
|
return {
|
|
@@ -248,6 +310,7 @@ export class NotionSource extends MigrationSource {
|
|
|
248
310
|
};
|
|
249
311
|
}
|
|
250
312
|
catch (error) {
|
|
313
|
+
this._extractionErrors.push(node.title);
|
|
251
314
|
console.error(` Error extracting page "${node.title}": ${error instanceof Error ? error.message : error}`);
|
|
252
315
|
return null;
|
|
253
316
|
}
|
|
@@ -269,6 +332,7 @@ export class NotionSource extends MigrationSource {
|
|
|
269
332
|
if (blocks.length > 0) {
|
|
270
333
|
const contentSections = await blocksToSections(blocks, this.client, this.pagePathMap, {
|
|
271
334
|
databaseIdMap: this._databaseIdMap.size > 0 ? this._databaseIdMap : undefined,
|
|
335
|
+
unsupportedBlocks: this._unsupportedBlocks,
|
|
272
336
|
});
|
|
273
337
|
sections.push(...contentSections);
|
|
274
338
|
}
|
|
@@ -299,7 +363,9 @@ export class NotionSource extends MigrationSource {
|
|
|
299
363
|
};
|
|
300
364
|
}
|
|
301
365
|
catch (error) {
|
|
302
|
-
|
|
366
|
+
const entryTitle = getPageTitle(entry) || `DB entry ${entry.id}`;
|
|
367
|
+
this._extractionErrors.push(entryTitle);
|
|
368
|
+
console.error(` Error extracting database entry "${entryTitle}": ${error instanceof Error ? error.message : error}`);
|
|
303
369
|
return null;
|
|
304
370
|
}
|
|
305
371
|
}
|
|
@@ -332,6 +398,7 @@ export function buildPageTree(pages, options) {
|
|
|
332
398
|
// Build lookup maps
|
|
333
399
|
const pageById = new Map();
|
|
334
400
|
const childrenByParent = new Map();
|
|
401
|
+
const syntheticDatabaseIds = new Set();
|
|
335
402
|
for (const page of pages) {
|
|
336
403
|
const nid = normalizeId(page.id);
|
|
337
404
|
pageById.set(nid, page);
|
|
@@ -344,6 +411,39 @@ export function buildPageTree(pages, options) {
|
|
|
344
411
|
childrenByParent.get(npid).push(page);
|
|
345
412
|
}
|
|
346
413
|
}
|
|
414
|
+
// Inject synthetic page nodes for databases so entries nest under them
|
|
415
|
+
if (options?.databases?.length) {
|
|
416
|
+
for (const db of options.databases) {
|
|
417
|
+
const dbNid = normalizeId(db.id);
|
|
418
|
+
// Skip if a real page already has this ID
|
|
419
|
+
if (pageById.has(dbNid))
|
|
420
|
+
continue;
|
|
421
|
+
// Create synthetic NotionPage wrapper
|
|
422
|
+
const syntheticPage = {
|
|
423
|
+
id: db.id,
|
|
424
|
+
object: 'page',
|
|
425
|
+
parent: db.parent,
|
|
426
|
+
created_time: db.created_time,
|
|
427
|
+
last_edited_time: db.last_edited_time,
|
|
428
|
+
created_by: { id: '', object: 'user' },
|
|
429
|
+
last_edited_by: { id: '', object: 'user' },
|
|
430
|
+
url: '',
|
|
431
|
+
properties: {
|
|
432
|
+
title: { id: 'title', type: 'title', title: db.title },
|
|
433
|
+
},
|
|
434
|
+
};
|
|
435
|
+
pageById.set(dbNid, syntheticPage);
|
|
436
|
+
// Add to childrenByParent
|
|
437
|
+
const parentId = getParentPageId(syntheticPage);
|
|
438
|
+
if (parentId) {
|
|
439
|
+
const npid = normalizeId(parentId);
|
|
440
|
+
if (!childrenByParent.has(npid))
|
|
441
|
+
childrenByParent.set(npid, []);
|
|
442
|
+
childrenByParent.get(npid).push(syntheticPage);
|
|
443
|
+
}
|
|
444
|
+
syntheticDatabaseIds.add(dbNid);
|
|
445
|
+
}
|
|
446
|
+
}
|
|
347
447
|
// Identify database entries
|
|
348
448
|
for (const page of pages) {
|
|
349
449
|
if (page.parent.type === 'database_id' && page.parent.database_id) {
|
|
@@ -377,6 +477,19 @@ export function buildPageTree(pages, options) {
|
|
|
377
477
|
}
|
|
378
478
|
}
|
|
379
479
|
}
|
|
480
|
+
// Also add synthetic database nodes whose parent is workspace or not in the tree
|
|
481
|
+
for (const dbNid of syntheticDatabaseIds) {
|
|
482
|
+
const synth = pageById.get(dbNid);
|
|
483
|
+
if (synth.parent.type === 'workspace') {
|
|
484
|
+
rootPages.push(synth);
|
|
485
|
+
}
|
|
486
|
+
else {
|
|
487
|
+
const parentId = getParentPageId(synth);
|
|
488
|
+
if (parentId && !pageById.has(normalizeId(parentId))) {
|
|
489
|
+
rootPages.push(synth);
|
|
490
|
+
}
|
|
491
|
+
}
|
|
492
|
+
}
|
|
380
493
|
}
|
|
381
494
|
// Build tree recursively
|
|
382
495
|
const buildNode = (page, parentPath, depth, siblingSlugCounts) => {
|
|
@@ -400,6 +513,7 @@ export function buildPageTree(pages, options) {
|
|
|
400
513
|
slug,
|
|
401
514
|
kbPath,
|
|
402
515
|
isDatabaseEntry,
|
|
516
|
+
isDatabaseNode: syntheticDatabaseIds.has(nid) || undefined,
|
|
403
517
|
parentDatabaseId: page.parent.type === 'database_id' ? page.parent.database_id : undefined,
|
|
404
518
|
children: [],
|
|
405
519
|
};
|
|
@@ -438,6 +552,8 @@ export function buildPageTree(pages, options) {
|
|
|
438
552
|
function getParentPageId(page) {
|
|
439
553
|
if (page.parent.type === 'page_id')
|
|
440
554
|
return page.parent.page_id ?? null;
|
|
555
|
+
if (page.parent.type === 'database_id')
|
|
556
|
+
return page.parent.database_id ?? null;
|
|
441
557
|
if (page.parent.type === 'block_id')
|
|
442
558
|
return page.parent.block_id ?? null;
|
|
443
559
|
return null;
|
package/package.json
CHANGED