@moxn/kb-migrate 0.4.23 → 0.4.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/client.d.ts +1 -1
- package/dist/client.js +12 -2
- package/dist/index.js +38 -4
- package/dist/sources/notion-blocks.d.ts +4 -0
- package/dist/sources/notion-blocks.js +25 -21
- package/dist/sources/notion-databases.d.ts +3 -3
- package/dist/sources/notion-databases.js +26 -3
- package/dist/sources/notion-references.d.ts +1 -0
- package/dist/sources/notion-references.js +9 -6
- package/dist/sources/notion.d.ts +7 -0
- package/dist/sources/notion.js +82 -13
- package/package.json +1 -1
package/dist/client.d.ts
CHANGED
|
@@ -53,7 +53,7 @@ export declare class MoxnClient {
|
|
|
53
53
|
*/
|
|
54
54
|
addDatabaseColumn(databaseId: string, input: {
|
|
55
55
|
name: string;
|
|
56
|
-
type: 'select' | 'multi_select' | 'status' | 'checkbox' | 'number' | 'date' | 'text' | 'url' | 'email';
|
|
56
|
+
type: 'select' | 'multi_select' | 'status' | 'checkbox' | 'number' | 'date' | 'text' | 'url' | 'email' | 'page_ref';
|
|
57
57
|
optionTagIds?: string[];
|
|
58
58
|
newOptionParentPath?: string;
|
|
59
59
|
config?: Record<string, unknown>;
|
package/dist/client.js
CHANGED
|
@@ -299,7 +299,12 @@ export class MoxnClient {
|
|
|
299
299
|
error.branchId = body.branchId;
|
|
300
300
|
throw error;
|
|
301
301
|
}
|
|
302
|
-
|
|
302
|
+
const message = body.details
|
|
303
|
+
? `API error ${response.status}: ${body.details}`
|
|
304
|
+
: body.error
|
|
305
|
+
? `API error ${response.status}: ${body.error}`
|
|
306
|
+
: `API error: ${response.status}`;
|
|
307
|
+
throw new Error(message);
|
|
303
308
|
}
|
|
304
309
|
return response.json();
|
|
305
310
|
}
|
|
@@ -314,7 +319,12 @@ export class MoxnClient {
|
|
|
314
319
|
});
|
|
315
320
|
if (!response.ok) {
|
|
316
321
|
const body = await response.json().catch(() => ({}));
|
|
317
|
-
|
|
322
|
+
const message = body.details
|
|
323
|
+
? `API error ${response.status}: ${body.details}`
|
|
324
|
+
: body.error
|
|
325
|
+
? `API error ${response.status}: ${body.error}`
|
|
326
|
+
: `API error: ${response.status}`;
|
|
327
|
+
throw new Error(message);
|
|
318
328
|
}
|
|
319
329
|
return response.json();
|
|
320
330
|
}
|
package/dist/index.js
CHANGED
|
@@ -27,21 +27,28 @@ async function runMigration(source, options) {
|
|
|
27
27
|
const results = [];
|
|
28
28
|
// Validate source
|
|
29
29
|
await source.validate();
|
|
30
|
-
// Get document count for progress
|
|
30
|
+
// Get document count for progress (this is an upper bound — some pages
|
|
31
|
+
// may be skipped during extraction if they are empty or have errors)
|
|
31
32
|
const totalCount = await source.getDocumentCount();
|
|
32
|
-
if (totalCount !== undefined) {
|
|
33
|
-
console.log(`Found ${totalCount} documents to migrate`);
|
|
34
|
-
}
|
|
35
33
|
// Create client
|
|
36
34
|
const client = new MoxnClient(options);
|
|
37
35
|
// Process documents
|
|
38
36
|
let processed = 0;
|
|
37
|
+
let consecutiveFailures = 0;
|
|
38
|
+
const MAX_CONSECUTIVE_FAILURES = 10;
|
|
39
39
|
for await (const doc of source.extract()) {
|
|
40
40
|
processed++;
|
|
41
41
|
const progress = totalCount ? ` (${processed}/${totalCount})` : '';
|
|
42
42
|
console.log(`Processing: ${doc.sourcePath}${progress}`);
|
|
43
43
|
const result = await client.migrateDocument(doc, options.basePath, options.onConflict, options.dryRun);
|
|
44
44
|
results.push(result);
|
|
45
|
+
// Track consecutive failures for circuit breaker
|
|
46
|
+
if (result.status === 'failed') {
|
|
47
|
+
consecutiveFailures++;
|
|
48
|
+
}
|
|
49
|
+
else {
|
|
50
|
+
consecutiveFailures = 0;
|
|
51
|
+
}
|
|
45
52
|
// Log result
|
|
46
53
|
const statusIcon = {
|
|
47
54
|
created: '\u2713',
|
|
@@ -53,6 +60,15 @@ async function runMigration(source, options) {
|
|
|
53
60
|
if (result.error) {
|
|
54
61
|
console.log(` Error: ${result.error}`);
|
|
55
62
|
}
|
|
63
|
+
// Circuit breaker: abort after too many consecutive failures
|
|
64
|
+
if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
|
|
65
|
+
console.error(`\nAborting: ${MAX_CONSECUTIVE_FAILURES} consecutive failures. ` +
|
|
66
|
+
'Last error: ' +
|
|
67
|
+
(result.error || 'unknown') +
|
|
68
|
+
'\n' +
|
|
69
|
+
'Fix the underlying issue and retry. Remaining documents will be skipped.');
|
|
70
|
+
break;
|
|
71
|
+
}
|
|
56
72
|
}
|
|
57
73
|
// Build summary
|
|
58
74
|
const summary = {
|
|
@@ -674,6 +690,8 @@ async function importNotionDatabase(client, dbImport, log, options, preCreatedDb
|
|
|
674
690
|
// 3. Link entries and assign tags
|
|
675
691
|
// Build a map of KB path → { documentId, branchId } from migration results
|
|
676
692
|
const docByPath = new Map();
|
|
693
|
+
// Build a map of Notion page ID → KB document ID for page_ref resolution
|
|
694
|
+
const notionIdToDocId = new Map();
|
|
677
695
|
for (const result of log.results) {
|
|
678
696
|
if (result.documentId && result.branchId) {
|
|
679
697
|
docByPath.set(result.documentPath, {
|
|
@@ -681,6 +699,9 @@ async function importNotionDatabase(client, dbImport, log, options, preCreatedDb
|
|
|
681
699
|
branchId: result.branchId,
|
|
682
700
|
});
|
|
683
701
|
}
|
|
702
|
+
if (result.sourcePageId && result.documentId) {
|
|
703
|
+
notionIdToDocId.set(result.sourcePageId, result.documentId);
|
|
704
|
+
}
|
|
684
705
|
}
|
|
685
706
|
let linkedCount = 0;
|
|
686
707
|
for (const entry of entries) {
|
|
@@ -735,6 +756,19 @@ async function importNotionDatabase(client, dbImport, log, options, preCreatedDb
|
|
|
735
756
|
case 'email':
|
|
736
757
|
value = scalarVal.textValue ?? null;
|
|
737
758
|
break;
|
|
759
|
+
case 'page_ref': {
|
|
760
|
+
// Resolve notion:XXX markers to KB document IDs
|
|
761
|
+
const refs = JSON.parse(scalarVal.textValue ?? '[]');
|
|
762
|
+
const resolvedRefs = refs.map((ref) => {
|
|
763
|
+
if (ref.startsWith('notion:')) {
|
|
764
|
+
const notionId = ref.substring(7);
|
|
765
|
+
return notionIdToDocId.get(notionId) ?? ref;
|
|
766
|
+
}
|
|
767
|
+
return ref;
|
|
768
|
+
});
|
|
769
|
+
value = JSON.stringify(resolvedRefs);
|
|
770
|
+
break;
|
|
771
|
+
}
|
|
738
772
|
}
|
|
739
773
|
if (value != null) {
|
|
740
774
|
try {
|
|
@@ -8,6 +8,8 @@ import type { SectionInput } from '../types.js';
|
|
|
8
8
|
import type { NotionBlock, NotionRichText, NotionApiClient, NotionPage } from './notion-api.js';
|
|
9
9
|
/** Map of page IDs to their KB paths (for cross-link resolution). */
|
|
10
10
|
export type PagePathMap = Map<string, string>;
|
|
11
|
+
/** Tracks unsupported Notion block types encountered during conversion. */
|
|
12
|
+
export type UnsupportedBlockTracker = Map<string, number>;
|
|
11
13
|
/**
|
|
12
14
|
* Convert a page's blocks into Moxn sections.
|
|
13
15
|
*
|
|
@@ -19,6 +21,8 @@ export declare function blocksToSections(blocks: NotionBlock[], client: NotionAp
|
|
|
19
21
|
visitedSyncedBlocks?: Set<string>;
|
|
20
22
|
/** Map of Notion database IDs to KB database IDs (for child_database blocks). */
|
|
21
23
|
databaseIdMap?: Map<string, string>;
|
|
24
|
+
/** Accumulates counts of unsupported block types (caller-owned). */
|
|
25
|
+
unsupportedBlocks?: UnsupportedBlockTracker;
|
|
22
26
|
}): Promise<SectionInput[]>;
|
|
23
27
|
/** Convert rich text array to markdown string. */
|
|
24
28
|
export declare function richTextToMarkdown(richText: NotionRichText[]): string;
|
|
@@ -16,6 +16,7 @@
|
|
|
16
16
|
export async function blocksToSections(blocks, client, pagePathMap, options) {
|
|
17
17
|
const visitedSyncedBlocks = options?.visitedSyncedBlocks ?? new Set();
|
|
18
18
|
const databaseIdMap = options?.databaseIdMap;
|
|
19
|
+
const unsupportedBlocks = options?.unsupportedBlocks;
|
|
19
20
|
const sections = [];
|
|
20
21
|
let currentSectionName = 'Introduction';
|
|
21
22
|
let currentBlocks = [];
|
|
@@ -35,7 +36,7 @@ export async function blocksToSections(blocks, client, pagePathMap, options) {
|
|
|
35
36
|
continue;
|
|
36
37
|
}
|
|
37
38
|
// Convert block to content blocks
|
|
38
|
-
const converted = await convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap);
|
|
39
|
+
const converted = await convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks);
|
|
39
40
|
currentBlocks.push(...converted);
|
|
40
41
|
}
|
|
41
42
|
// Flush last section
|
|
@@ -51,7 +52,7 @@ export async function blocksToSections(blocks, client, pagePathMap, options) {
|
|
|
51
52
|
// ============================================
|
|
52
53
|
// Block conversion
|
|
53
54
|
// ============================================
|
|
54
|
-
async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
55
|
+
async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
|
|
55
56
|
const results = [];
|
|
56
57
|
switch (block.type) {
|
|
57
58
|
case 'paragraph':
|
|
@@ -81,10 +82,10 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
|
|
|
81
82
|
results.push(...convertToDo(block));
|
|
82
83
|
break;
|
|
83
84
|
case 'quote':
|
|
84
|
-
results.push(...(await convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
85
|
+
results.push(...(await convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
|
|
85
86
|
break;
|
|
86
87
|
case 'callout':
|
|
87
|
-
results.push(...(await convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
88
|
+
results.push(...(await convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
|
|
88
89
|
break;
|
|
89
90
|
case 'divider':
|
|
90
91
|
results.push(textBlock('---'));
|
|
@@ -93,7 +94,7 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
|
|
|
93
94
|
results.push(...(await convertTable(block, client)));
|
|
94
95
|
break;
|
|
95
96
|
case 'toggle':
|
|
96
|
-
results.push(...(await convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
97
|
+
results.push(...(await convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
|
|
97
98
|
break;
|
|
98
99
|
case 'bookmark':
|
|
99
100
|
results.push(...convertBookmark(block));
|
|
@@ -122,10 +123,10 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
|
|
|
122
123
|
results.push(...convertChildDatabase(block, databaseIdMap));
|
|
123
124
|
break;
|
|
124
125
|
case 'synced_block':
|
|
125
|
-
results.push(...(await convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
126
|
+
results.push(...(await convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
|
|
126
127
|
break;
|
|
127
128
|
case 'column_list':
|
|
128
|
-
results.push(...(await convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
|
|
129
|
+
results.push(...(await convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
|
|
129
130
|
break;
|
|
130
131
|
case 'column':
|
|
131
132
|
// Columns are handled by column_list
|
|
@@ -146,7 +147,10 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
|
|
|
146
147
|
results.push(...convertLinkPreview(block));
|
|
147
148
|
break;
|
|
148
149
|
default:
|
|
149
|
-
// Unknown block type —
|
|
150
|
+
// Unknown block type — track and skip
|
|
151
|
+
if (unsupportedBlocks) {
|
|
152
|
+
unsupportedBlocks.set(block.type, (unsupportedBlocks.get(block.type) ?? 0) + 1);
|
|
153
|
+
}
|
|
150
154
|
console.warn(` Skipping unsupported Notion block type: ${block.type}`);
|
|
151
155
|
break;
|
|
152
156
|
}
|
|
@@ -165,7 +169,7 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
|
|
|
165
169
|
? ' '
|
|
166
170
|
: undefined;
|
|
167
171
|
const children = await client.getBlockChildren(block.id);
|
|
168
|
-
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap);
|
|
172
|
+
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap, unsupportedBlocks);
|
|
169
173
|
results.push(...childBlocks);
|
|
170
174
|
}
|
|
171
175
|
return results;
|
|
@@ -211,7 +215,7 @@ function convertToDo(block) {
|
|
|
211
215
|
const text = richTextToMarkdown(td.to_do.rich_text);
|
|
212
216
|
return [textBlock(`${checkbox} ${text}`)];
|
|
213
217
|
}
|
|
214
|
-
async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
218
|
+
async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
|
|
215
219
|
const q = block;
|
|
216
220
|
const text = richTextToMarkdown(q.quote.rich_text);
|
|
217
221
|
if (!text && !block.has_children)
|
|
@@ -225,12 +229,12 @@ async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks, dat
|
|
|
225
229
|
const results = quoted ? [textBlock(quoted)] : [];
|
|
226
230
|
if (block.has_children) {
|
|
227
231
|
const children = await client.getBlockChildren(block.id);
|
|
228
|
-
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
|
|
232
|
+
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap, unsupportedBlocks);
|
|
229
233
|
results.push(...childBlocks);
|
|
230
234
|
}
|
|
231
235
|
return results;
|
|
232
236
|
}
|
|
233
|
-
async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
237
|
+
async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
|
|
234
238
|
const c = block;
|
|
235
239
|
const text = richTextToMarkdown(c.callout.rich_text);
|
|
236
240
|
const emoji = c.callout.icon?.emoji ?? '';
|
|
@@ -242,7 +246,7 @@ async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks, d
|
|
|
242
246
|
const results = [textBlock(quoted)];
|
|
243
247
|
if (block.has_children) {
|
|
244
248
|
const children = await client.getBlockChildren(block.id);
|
|
245
|
-
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
|
|
249
|
+
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap, unsupportedBlocks);
|
|
246
250
|
results.push(...childBlocks);
|
|
247
251
|
}
|
|
248
252
|
return results;
|
|
@@ -271,13 +275,13 @@ async function convertTable(block, client) {
|
|
|
271
275
|
}
|
|
272
276
|
return [textBlock(lines.join('\n'))];
|
|
273
277
|
}
|
|
274
|
-
async function convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
278
|
+
async function convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
|
|
275
279
|
const t = block;
|
|
276
280
|
const header = richTextToMarkdown(t.toggle.rich_text);
|
|
277
281
|
const results = [textBlock(`**${header}**`)];
|
|
278
282
|
if (block.has_children) {
|
|
279
283
|
const children = await client.getBlockChildren(block.id);
|
|
280
|
-
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
|
|
284
|
+
const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap, unsupportedBlocks);
|
|
281
285
|
results.push(...childBlocks);
|
|
282
286
|
}
|
|
283
287
|
return results;
|
|
@@ -402,7 +406,7 @@ function convertLinkToPage(block, pagePathMap) {
|
|
|
402
406
|
// Target not in import set
|
|
403
407
|
return [textBlock(`*(Link to Notion page: ${targetId})*`)];
|
|
404
408
|
}
|
|
405
|
-
async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
409
|
+
async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
|
|
406
410
|
const sb = block;
|
|
407
411
|
// Get the source block ID (either this block or the original)
|
|
408
412
|
const sourceId = sb.synced_block.synced_from?.block_id ?? block.id;
|
|
@@ -413,19 +417,19 @@ async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlock
|
|
|
413
417
|
visitedSyncedBlocks.add(sourceId);
|
|
414
418
|
try {
|
|
415
419
|
const children = await client.getBlockChildren(sourceId);
|
|
416
|
-
return await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap);
|
|
420
|
+
return await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap, unsupportedBlocks);
|
|
417
421
|
}
|
|
418
422
|
finally {
|
|
419
423
|
visitedSyncedBlocks.delete(sourceId);
|
|
420
424
|
}
|
|
421
425
|
}
|
|
422
|
-
async function convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
|
|
426
|
+
async function convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
|
|
423
427
|
const children = await client.getBlockChildren(block.id);
|
|
424
428
|
const results = [];
|
|
425
429
|
for (const column of children) {
|
|
426
430
|
if (column.type === 'column' && column.has_children) {
|
|
427
431
|
const columnChildren = await client.getBlockChildren(column.id);
|
|
428
|
-
const converted = await convertAndMergeChildren(columnChildren, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap);
|
|
432
|
+
const converted = await convertAndMergeChildren(columnChildren, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap, unsupportedBlocks);
|
|
429
433
|
results.push(...converted);
|
|
430
434
|
}
|
|
431
435
|
}
|
|
@@ -574,10 +578,10 @@ function mergeConsecutiveListBlocks(blocks) {
|
|
|
574
578
|
/**
|
|
575
579
|
* Convert children blocks, merge consecutive list items, and optionally indent.
|
|
576
580
|
*/
|
|
577
|
-
async function convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap) {
|
|
581
|
+
async function convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap, unsupportedBlocks) {
|
|
578
582
|
const blocks = [];
|
|
579
583
|
for (const child of children) {
|
|
580
|
-
const converted = await convertBlock(child, client, pagePathMap, visitedSyncedBlocks, databaseIdMap);
|
|
584
|
+
const converted = await convertBlock(child, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks);
|
|
581
585
|
blocks.push(...converted);
|
|
582
586
|
}
|
|
583
587
|
const merged = mergeConsecutiveListBlocks(blocks);
|
|
@@ -30,8 +30,8 @@ export interface MappedColumn {
|
|
|
30
30
|
export interface ScalarColumn {
|
|
31
31
|
notionPropertyId: string;
|
|
32
32
|
notionPropertyName: string;
|
|
33
|
-
notionType: 'number' | 'date' | 'url' | 'email' | 'rich_text' | 'checkbox';
|
|
34
|
-
moxnType: 'number' | 'date' | 'url' | 'email' | 'text';
|
|
33
|
+
notionType: 'number' | 'date' | 'url' | 'email' | 'rich_text' | 'checkbox' | 'relation';
|
|
34
|
+
moxnType: 'number' | 'date' | 'url' | 'email' | 'text' | 'page_ref';
|
|
35
35
|
}
|
|
36
36
|
/** Parsed Notion column that will be rendered as text in a Properties section. */
|
|
37
37
|
export interface UnmappedColumn {
|
|
@@ -50,7 +50,7 @@ export interface ParsedDatabaseSchema {
|
|
|
50
50
|
/** Scalar value for a single database entry property. */
|
|
51
51
|
export interface ScalarValue {
|
|
52
52
|
columnName: string;
|
|
53
|
-
moxnType: 'number' | 'date' | 'url' | 'email' | 'text';
|
|
53
|
+
moxnType: 'number' | 'date' | 'url' | 'email' | 'text' | 'page_ref';
|
|
54
54
|
textValue?: string | null;
|
|
55
55
|
numberValue?: number | null;
|
|
56
56
|
dateValue?: string | null;
|
|
@@ -123,6 +123,14 @@ export function parseDatabaseSchema(db) {
|
|
|
123
123
|
moxnType: 'text',
|
|
124
124
|
});
|
|
125
125
|
}
|
|
126
|
+
else if (prop.type === 'relation') {
|
|
127
|
+
scalarColumns.push({
|
|
128
|
+
notionPropertyId: prop.id,
|
|
129
|
+
notionPropertyName: propName,
|
|
130
|
+
notionType: 'relation',
|
|
131
|
+
moxnType: 'page_ref',
|
|
132
|
+
});
|
|
133
|
+
}
|
|
126
134
|
else {
|
|
127
135
|
// Warn on property types that have no meaningful static value
|
|
128
136
|
if (prop.type === 'button' || prop.type === 'ai_text') {
|
|
@@ -175,9 +183,24 @@ export function parseEntryValues(page, schema) {
|
|
|
175
183
|
// Check if this is a scalar column
|
|
176
184
|
const scalar = scalarByName.get(propName);
|
|
177
185
|
if (scalar) {
|
|
178
|
-
|
|
179
|
-
if (
|
|
180
|
-
|
|
186
|
+
// page_ref columns extract relation page IDs (handled differently from standard scalars)
|
|
187
|
+
if (scalar.moxnType === 'page_ref') {
|
|
188
|
+
const refs = (propValue.relation ?? [])
|
|
189
|
+
.map((r) => normalizeId(r.id))
|
|
190
|
+
.filter(Boolean);
|
|
191
|
+
if (refs.length > 0) {
|
|
192
|
+
scalarValues.set(propName, {
|
|
193
|
+
columnName: propName,
|
|
194
|
+
moxnType: 'page_ref',
|
|
195
|
+
textValue: JSON.stringify(refs.map((id) => `notion:${id}`)),
|
|
196
|
+
});
|
|
197
|
+
}
|
|
198
|
+
}
|
|
199
|
+
else {
|
|
200
|
+
const extracted = extractScalarValue(propValue, scalar);
|
|
201
|
+
if (extracted) {
|
|
202
|
+
scalarValues.set(propName, extracted);
|
|
203
|
+
}
|
|
181
204
|
}
|
|
182
205
|
continue;
|
|
183
206
|
}
|
|
@@ -125,17 +125,19 @@ export function resolveNotionReferences(sections, mapping) {
|
|
|
125
125
|
text = text.replace(RELATION_REF_RE, (_match, rawId) => {
|
|
126
126
|
const nid = normalizeId(rawId);
|
|
127
127
|
const kbPath = mapping.notionIdToKbPath.get(nid);
|
|
128
|
+
const title = mapping.notionIdToTitle?.get(nid);
|
|
128
129
|
if (kbPath) {
|
|
130
|
+
const displayText = title || kbPath;
|
|
129
131
|
references.push({
|
|
130
132
|
sectionIndex,
|
|
131
133
|
targetNotionId: nid,
|
|
132
134
|
targetKbPath: kbPath,
|
|
133
|
-
displayText
|
|
135
|
+
displayText,
|
|
134
136
|
});
|
|
135
|
-
return `[${
|
|
137
|
+
return `[${displayText}](${kbPath})`;
|
|
136
138
|
}
|
|
137
|
-
// Unresolved — keep the ID
|
|
138
|
-
return nid;
|
|
139
|
+
// Unresolved — show title if available, otherwise keep the ID
|
|
140
|
+
return title || nid;
|
|
139
141
|
});
|
|
140
142
|
return { blockType: 'text', text };
|
|
141
143
|
});
|
|
@@ -151,9 +153,10 @@ export function resolveRelationIds(value, mapping) {
|
|
|
151
153
|
return value.replace(RELATION_REF_RE, (_match, rawId) => {
|
|
152
154
|
const nid = normalizeId(rawId);
|
|
153
155
|
const kbPath = mapping.notionIdToKbPath.get(nid);
|
|
156
|
+
const title = mapping.notionIdToTitle?.get(nid);
|
|
154
157
|
if (kbPath) {
|
|
155
|
-
return `[${kbPath}](${kbPath})`;
|
|
158
|
+
return `[${title || kbPath}](${kbPath})`;
|
|
156
159
|
}
|
|
157
|
-
return nid;
|
|
160
|
+
return title || nid;
|
|
158
161
|
});
|
|
159
162
|
}
|
package/dist/sources/notion.d.ts
CHANGED
|
@@ -48,10 +48,16 @@ export declare class NotionSource extends MigrationSource<NotionSourceConfig> {
|
|
|
48
48
|
private pageTree;
|
|
49
49
|
private allPages;
|
|
50
50
|
private pagePathMap;
|
|
51
|
+
private pageTitleMap;
|
|
51
52
|
private databases;
|
|
52
53
|
private databaseEntryPageIds;
|
|
53
54
|
private _documentCount;
|
|
54
55
|
private _validated;
|
|
56
|
+
private _skippedEmpty;
|
|
57
|
+
private _skippedNoContent;
|
|
58
|
+
private _extractionErrors;
|
|
59
|
+
private _syntheticDatabaseNodes;
|
|
60
|
+
private _unsupportedBlocks;
|
|
55
61
|
/** Map of Notion database ID → KB database ID, set before extraction. */
|
|
56
62
|
private _databaseIdMap;
|
|
57
63
|
constructor(config: NotionSourceConfig);
|
|
@@ -91,6 +97,7 @@ export interface PageTreeResult {
|
|
|
91
97
|
tree: PageTreeNode[];
|
|
92
98
|
allPages: PageTreeNode[];
|
|
93
99
|
pagePathMap: PagePathMap;
|
|
100
|
+
pageTitleMap: Map<string, string>;
|
|
94
101
|
databaseEntryPageIds: Set<string>;
|
|
95
102
|
}
|
|
96
103
|
/**
|
package/dist/sources/notion.js
CHANGED
|
@@ -24,10 +24,17 @@ export class NotionSource extends MigrationSource {
|
|
|
24
24
|
pageTree = [];
|
|
25
25
|
allPages = []; // flat list, depth-first order
|
|
26
26
|
pagePathMap = new Map();
|
|
27
|
+
pageTitleMap = new Map();
|
|
27
28
|
databases = [];
|
|
28
29
|
databaseEntryPageIds = new Set();
|
|
29
30
|
_documentCount = 0;
|
|
30
31
|
_validated = false;
|
|
32
|
+
// Tracking for extraction-time skips (populated during extract())
|
|
33
|
+
_skippedEmpty = [];
|
|
34
|
+
_skippedNoContent = [];
|
|
35
|
+
_extractionErrors = [];
|
|
36
|
+
_syntheticDatabaseNodes = 0;
|
|
37
|
+
_unsupportedBlocks = new Map();
|
|
31
38
|
/** Map of Notion database ID → KB database ID, set before extraction. */
|
|
32
39
|
_databaseIdMap = new Map();
|
|
33
40
|
constructor(config) {
|
|
@@ -89,13 +96,18 @@ export class NotionSource extends MigrationSource {
|
|
|
89
96
|
// 5. Process databases — identify entries and build schemas
|
|
90
97
|
await this.processDatabases(allNotionDatabases);
|
|
91
98
|
// 6. Count and validate (exclude synthetic database container nodes)
|
|
92
|
-
this.
|
|
99
|
+
this._syntheticDatabaseNodes = this.allPages.filter((n) => n.isDatabaseNode).length;
|
|
100
|
+
const pagesInTree = this.allPages.filter((n) => !n.isDatabaseNode).length;
|
|
101
|
+
const databaseEntriesInTree = this.allPages.filter((n) => n.isDatabaseEntry && !n.isDatabaseNode).length;
|
|
102
|
+
this._documentCount = pagesInTree;
|
|
93
103
|
// Add database-only entries (pages that are in databases but not in page tree)
|
|
104
|
+
let databaseEntriesOutsideTree = 0;
|
|
94
105
|
for (const dbInfo of this.databases) {
|
|
95
106
|
for (const entry of dbInfo.entries) {
|
|
96
107
|
const nid = normalizeId(entry.id);
|
|
97
108
|
if (!this.pagePathMap.has(nid)) {
|
|
98
109
|
this._documentCount++;
|
|
110
|
+
databaseEntriesOutsideTree++;
|
|
99
111
|
}
|
|
100
112
|
}
|
|
101
113
|
}
|
|
@@ -103,7 +115,12 @@ export class NotionSource extends MigrationSource {
|
|
|
103
115
|
throw new Error(`Workspace has ${this._documentCount} documents, exceeding the ${MAX_DOCUMENT_COUNT} limit. ` +
|
|
104
116
|
'Use --root-page-id to import a subtree.');
|
|
105
117
|
}
|
|
106
|
-
|
|
118
|
+
// Log clear breakdown of what was discovered
|
|
119
|
+
const standalonePages = pagesInTree - databaseEntriesInTree;
|
|
120
|
+
const totalDbEntries = databaseEntriesInTree + databaseEntriesOutsideTree;
|
|
121
|
+
console.log(` ${this._documentCount} documents to import: ` +
|
|
122
|
+
`${standalonePages} pages + ${totalDbEntries} database entries ` +
|
|
123
|
+
`(across ${this.databases.length} databases)`);
|
|
107
124
|
// 7. Initialize media downloader
|
|
108
125
|
await this.mediaDownloader.init();
|
|
109
126
|
this._validated = true;
|
|
@@ -135,6 +152,12 @@ export class NotionSource extends MigrationSource {
|
|
|
135
152
|
// Pass 2: Extraction
|
|
136
153
|
// ============================================
|
|
137
154
|
async *extract() {
|
|
155
|
+
// Reset extraction tracking
|
|
156
|
+
this._skippedEmpty = [];
|
|
157
|
+
this._skippedNoContent = [];
|
|
158
|
+
this._extractionErrors = [];
|
|
159
|
+
this._unsupportedBlocks = new Map();
|
|
160
|
+
let yielded = 0;
|
|
138
161
|
// Walk pages depth-first
|
|
139
162
|
for (const node of this.allPages) {
|
|
140
163
|
// Skip database entries that will be created during database import
|
|
@@ -145,8 +168,10 @@ export class NotionSource extends MigrationSource {
|
|
|
145
168
|
if (node.isDatabaseNode)
|
|
146
169
|
continue;
|
|
147
170
|
const doc = await this.extractPage(node);
|
|
148
|
-
if (doc)
|
|
171
|
+
if (doc) {
|
|
172
|
+
yielded++;
|
|
149
173
|
yield doc;
|
|
174
|
+
}
|
|
150
175
|
}
|
|
151
176
|
// Extract database-only entries (pages not in page tree)
|
|
152
177
|
for (const dbInfo of this.databases) {
|
|
@@ -162,10 +187,43 @@ export class NotionSource extends MigrationSource {
|
|
|
162
187
|
}
|
|
163
188
|
}
|
|
164
189
|
const doc = await this.extractDatabaseEntry(entry, dbInfo);
|
|
165
|
-
if (doc)
|
|
190
|
+
if (doc) {
|
|
191
|
+
yielded++;
|
|
166
192
|
yield doc;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
// Log extraction summary so user can reconcile counts
|
|
197
|
+
const totalSkipped = this._skippedEmpty.length +
|
|
198
|
+
this._skippedNoContent.length +
|
|
199
|
+
this._extractionErrors.length;
|
|
200
|
+
console.log(`\nExtraction complete: ${yielded} of ${this._documentCount} documents extracted`);
|
|
201
|
+
if (totalSkipped > 0) {
|
|
202
|
+
console.log(` ${totalSkipped} pages skipped:`);
|
|
203
|
+
if (this._skippedEmpty.length > 0) {
|
|
204
|
+
console.log(` Empty pages (no blocks): ${this._skippedEmpty.map((t) => `"${t}"`).join(', ')}`);
|
|
205
|
+
}
|
|
206
|
+
if (this._skippedNoContent.length > 0) {
|
|
207
|
+
console.log(` No extractable content: ${this._skippedNoContent.map((t) => `"${t}"`).join(', ')}`);
|
|
208
|
+
}
|
|
209
|
+
if (this._extractionErrors.length > 0) {
|
|
210
|
+
console.log(` Extraction errors: ${this._extractionErrors.map((t) => `"${t}"`).join(', ')}`);
|
|
167
211
|
}
|
|
168
212
|
}
|
|
213
|
+
// Report unsupported block types
|
|
214
|
+
if (this._unsupportedBlocks.size > 0) {
|
|
215
|
+
const sorted = [...this._unsupportedBlocks.entries()].sort((a, b) => b[1] - a[1]);
|
|
216
|
+
const totalUnsupported = sorted.reduce((sum, [, count]) => sum + count, 0);
|
|
217
|
+
console.log(` ${totalUnsupported} unsupported block(s) across ${sorted.length} type(s):`);
|
|
218
|
+
for (const [type, count] of sorted) {
|
|
219
|
+
console.log(` ${type}: ${count}`);
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
// Warn if counts don't reconcile (helps catch counting bugs)
|
|
223
|
+
const expectedYield = this._documentCount - totalSkipped;
|
|
224
|
+
if (yielded !== expectedYield) {
|
|
225
|
+
console.log(` ⚠ Count mismatch: expected ${expectedYield} (${this._documentCount} - ${totalSkipped} skipped) but extracted ${yielded}`);
|
|
226
|
+
}
|
|
169
227
|
}
|
|
170
228
|
/** Clean up temp files after migration completes. */
|
|
171
229
|
async cleanup() {
|
|
@@ -183,6 +241,7 @@ export class NotionSource extends MigrationSource {
|
|
|
183
241
|
this.pageTree = result.tree;
|
|
184
242
|
this.allPages = result.allPages;
|
|
185
243
|
this.pagePathMap = result.pagePathMap;
|
|
244
|
+
this.pageTitleMap = result.pageTitleMap;
|
|
186
245
|
this.databaseEntryPageIds = result.databaseEntryPageIds;
|
|
187
246
|
}
|
|
188
247
|
// ============================================
|
|
@@ -224,19 +283,20 @@ export class NotionSource extends MigrationSource {
|
|
|
224
283
|
try {
|
|
225
284
|
const blocks = await this.client.getBlockChildren(node.page.id);
|
|
226
285
|
if (blocks.length === 0) {
|
|
227
|
-
|
|
286
|
+
this._skippedEmpty.push(node.title);
|
|
228
287
|
return null;
|
|
229
288
|
}
|
|
230
289
|
let sections = await blocksToSections(blocks, this.client, this.pagePathMap, {
|
|
231
290
|
databaseIdMap: this._databaseIdMap.size > 0 ? this._databaseIdMap : undefined,
|
|
291
|
+
unsupportedBlocks: this._unsupportedBlocks,
|
|
232
292
|
});
|
|
233
293
|
// Download Notion-hosted media files
|
|
234
294
|
sections = await this.downloadSectionMedia(sections);
|
|
235
295
|
// Resolve cross-references
|
|
236
|
-
const { sections: resolvedSections, references } = resolveNotionReferences(sections, { notionIdToKbPath: this.pagePathMap });
|
|
296
|
+
const { sections: resolvedSections, references } = resolveNotionReferences(sections, { notionIdToKbPath: this.pagePathMap, notionIdToTitle: this.pageTitleMap });
|
|
237
297
|
sections = resolvedSections;
|
|
238
298
|
if (sections.length === 0) {
|
|
239
|
-
|
|
299
|
+
this._skippedNoContent.push(node.title);
|
|
240
300
|
return null;
|
|
241
301
|
}
|
|
242
302
|
return {
|
|
@@ -252,6 +312,7 @@ export class NotionSource extends MigrationSource {
|
|
|
252
312
|
};
|
|
253
313
|
}
|
|
254
314
|
catch (error) {
|
|
315
|
+
this._extractionErrors.push(node.title);
|
|
255
316
|
console.error(` Error extracting page "${node.title}": ${error instanceof Error ? error.message : error}`);
|
|
256
317
|
return null;
|
|
257
318
|
}
|
|
@@ -273,13 +334,14 @@ export class NotionSource extends MigrationSource {
|
|
|
273
334
|
if (blocks.length > 0) {
|
|
274
335
|
const contentSections = await blocksToSections(blocks, this.client, this.pagePathMap, {
|
|
275
336
|
databaseIdMap: this._databaseIdMap.size > 0 ? this._databaseIdMap : undefined,
|
|
337
|
+
unsupportedBlocks: this._unsupportedBlocks,
|
|
276
338
|
});
|
|
277
339
|
sections.push(...contentSections);
|
|
278
340
|
}
|
|
279
341
|
// Download media
|
|
280
342
|
let processedSections = await this.downloadSectionMedia(sections);
|
|
281
343
|
// Resolve cross-references
|
|
282
|
-
const { sections: resolvedSections, references } = resolveNotionReferences(processedSections, { notionIdToKbPath: this.pagePathMap });
|
|
344
|
+
const { sections: resolvedSections, references } = resolveNotionReferences(processedSections, { notionIdToKbPath: this.pagePathMap, notionIdToTitle: this.pageTitleMap });
|
|
283
345
|
processedSections = resolvedSections;
|
|
284
346
|
const nid = normalizeId(entry.id);
|
|
285
347
|
const kbPath = this.pagePathMap.get(nid) ?? slug;
|
|
@@ -303,7 +365,9 @@ export class NotionSource extends MigrationSource {
|
|
|
303
365
|
};
|
|
304
366
|
}
|
|
305
367
|
catch (error) {
|
|
306
|
-
|
|
368
|
+
const entryTitle = getPageTitle(entry) || `DB entry ${entry.id}`;
|
|
369
|
+
this._extractionErrors.push(entryTitle);
|
|
370
|
+
console.error(` Error extracting database entry "${entryTitle}": ${error instanceof Error ? error.message : error}`);
|
|
307
371
|
return null;
|
|
308
372
|
}
|
|
309
373
|
}
|
|
@@ -330,6 +394,7 @@ export class NotionSource extends MigrationSource {
|
|
|
330
394
|
*/
|
|
331
395
|
export function buildPageTree(pages, options) {
|
|
332
396
|
const pagePathMap = new Map();
|
|
397
|
+
const pageTitleMap = new Map();
|
|
333
398
|
const databaseEntryPageIds = new Set();
|
|
334
399
|
const tree = [];
|
|
335
400
|
const allPages = [];
|
|
@@ -431,7 +496,11 @@ export function buildPageTree(pages, options) {
|
|
|
431
496
|
}
|
|
432
497
|
// Build tree recursively
|
|
433
498
|
const buildNode = (page, parentPath, depth, siblingSlugCounts) => {
|
|
434
|
-
|
|
499
|
+
const nid = normalizeId(page.id);
|
|
500
|
+
const isDatabaseSubtree = syntheticDatabaseIds.has(nid) || databaseEntryPageIds.has(nid);
|
|
501
|
+
if (options?.maxDepth !== undefined &&
|
|
502
|
+
depth > options.maxDepth &&
|
|
503
|
+
!isDatabaseSubtree) {
|
|
435
504
|
return null;
|
|
436
505
|
}
|
|
437
506
|
const title = getPageTitle(page);
|
|
@@ -443,7 +512,6 @@ export function buildPageTree(pages, options) {
|
|
|
443
512
|
slug = `${slug}-${existing + 1}`;
|
|
444
513
|
}
|
|
445
514
|
const kbPath = parentPath ? `${parentPath}/${slug}` : slug;
|
|
446
|
-
const nid = normalizeId(page.id);
|
|
447
515
|
const isDatabaseEntry = databaseEntryPageIds.has(nid);
|
|
448
516
|
const node = {
|
|
449
517
|
page,
|
|
@@ -455,8 +523,9 @@ export function buildPageTree(pages, options) {
|
|
|
455
523
|
parentDatabaseId: page.parent.type === 'database_id' ? page.parent.database_id : undefined,
|
|
456
524
|
children: [],
|
|
457
525
|
};
|
|
458
|
-
// Register in path map
|
|
526
|
+
// Register in path map and title map
|
|
459
527
|
pagePathMap.set(nid, kbPath);
|
|
528
|
+
pageTitleMap.set(nid, title);
|
|
460
529
|
// Process children
|
|
461
530
|
const childPages = childrenByParent.get(nid) ?? [];
|
|
462
531
|
const childSlugCounts = new Map();
|
|
@@ -484,7 +553,7 @@ export function buildPageTree(pages, options) {
|
|
|
484
553
|
}
|
|
485
554
|
};
|
|
486
555
|
flatten(tree);
|
|
487
|
-
return { tree, allPages, pagePathMap, databaseEntryPageIds };
|
|
556
|
+
return { tree, allPages, pagePathMap, pageTitleMap, databaseEntryPageIds };
|
|
488
557
|
}
|
|
489
558
|
/** Extract the parent page ID from a Notion page's parent field. */
|
|
490
559
|
function getParentPageId(page) {
|
package/package.json
CHANGED