@moxn/kb-migrate 0.4.23 → 0.4.25

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/client.d.ts CHANGED
@@ -53,7 +53,7 @@ export declare class MoxnClient {
53
53
  */
54
54
  addDatabaseColumn(databaseId: string, input: {
55
55
  name: string;
56
- type: 'select' | 'multi_select' | 'status' | 'checkbox' | 'number' | 'date' | 'text' | 'url' | 'email';
56
+ type: 'select' | 'multi_select' | 'status' | 'checkbox' | 'number' | 'date' | 'text' | 'url' | 'email' | 'page_ref';
57
57
  optionTagIds?: string[];
58
58
  newOptionParentPath?: string;
59
59
  config?: Record<string, unknown>;
package/dist/client.js CHANGED
@@ -299,7 +299,12 @@ export class MoxnClient {
299
299
  error.branchId = body.branchId;
300
300
  throw error;
301
301
  }
302
- throw new Error(body.error || `API error: ${response.status}`);
302
+ const message = body.details
303
+ ? `API error ${response.status}: ${body.details}`
304
+ : body.error
305
+ ? `API error ${response.status}: ${body.error}`
306
+ : `API error: ${response.status}`;
307
+ throw new Error(message);
303
308
  }
304
309
  return response.json();
305
310
  }
@@ -314,7 +319,12 @@ export class MoxnClient {
314
319
  });
315
320
  if (!response.ok) {
316
321
  const body = await response.json().catch(() => ({}));
317
- throw new Error(body.error || `API error: ${response.status}`);
322
+ const message = body.details
323
+ ? `API error ${response.status}: ${body.details}`
324
+ : body.error
325
+ ? `API error ${response.status}: ${body.error}`
326
+ : `API error: ${response.status}`;
327
+ throw new Error(message);
318
328
  }
319
329
  return response.json();
320
330
  }
package/dist/index.js CHANGED
@@ -27,21 +27,28 @@ async function runMigration(source, options) {
27
27
  const results = [];
28
28
  // Validate source
29
29
  await source.validate();
30
- // Get document count for progress
30
+ // Get document count for progress (this is an upper bound — some pages
31
+ // may be skipped during extraction if they are empty or have errors)
31
32
  const totalCount = await source.getDocumentCount();
32
- if (totalCount !== undefined) {
33
- console.log(`Found ${totalCount} documents to migrate`);
34
- }
35
33
  // Create client
36
34
  const client = new MoxnClient(options);
37
35
  // Process documents
38
36
  let processed = 0;
37
+ let consecutiveFailures = 0;
38
+ const MAX_CONSECUTIVE_FAILURES = 10;
39
39
  for await (const doc of source.extract()) {
40
40
  processed++;
41
41
  const progress = totalCount ? ` (${processed}/${totalCount})` : '';
42
42
  console.log(`Processing: ${doc.sourcePath}${progress}`);
43
43
  const result = await client.migrateDocument(doc, options.basePath, options.onConflict, options.dryRun);
44
44
  results.push(result);
45
+ // Track consecutive failures for circuit breaker
46
+ if (result.status === 'failed') {
47
+ consecutiveFailures++;
48
+ }
49
+ else {
50
+ consecutiveFailures = 0;
51
+ }
45
52
  // Log result
46
53
  const statusIcon = {
47
54
  created: '\u2713',
@@ -53,6 +60,15 @@ async function runMigration(source, options) {
53
60
  if (result.error) {
54
61
  console.log(` Error: ${result.error}`);
55
62
  }
63
+ // Circuit breaker: abort after too many consecutive failures
64
+ if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
65
+ console.error(`\nAborting: ${MAX_CONSECUTIVE_FAILURES} consecutive failures. ` +
66
+ 'Last error: ' +
67
+ (result.error || 'unknown') +
68
+ '\n' +
69
+ 'Fix the underlying issue and retry. Remaining documents will be skipped.');
70
+ break;
71
+ }
56
72
  }
57
73
  // Build summary
58
74
  const summary = {
@@ -674,6 +690,8 @@ async function importNotionDatabase(client, dbImport, log, options, preCreatedDb
674
690
  // 3. Link entries and assign tags
675
691
  // Build a map of KB path → { documentId, branchId } from migration results
676
692
  const docByPath = new Map();
693
+ // Build a map of Notion page ID → KB document ID for page_ref resolution
694
+ const notionIdToDocId = new Map();
677
695
  for (const result of log.results) {
678
696
  if (result.documentId && result.branchId) {
679
697
  docByPath.set(result.documentPath, {
@@ -681,6 +699,9 @@ async function importNotionDatabase(client, dbImport, log, options, preCreatedDb
681
699
  branchId: result.branchId,
682
700
  });
683
701
  }
702
+ if (result.sourcePageId && result.documentId) {
703
+ notionIdToDocId.set(result.sourcePageId, result.documentId);
704
+ }
684
705
  }
685
706
  let linkedCount = 0;
686
707
  for (const entry of entries) {
@@ -735,6 +756,19 @@ async function importNotionDatabase(client, dbImport, log, options, preCreatedDb
735
756
  case 'email':
736
757
  value = scalarVal.textValue ?? null;
737
758
  break;
759
+ case 'page_ref': {
760
+ // Resolve notion:XXX markers to KB document IDs
761
+ const refs = JSON.parse(scalarVal.textValue ?? '[]');
762
+ const resolvedRefs = refs.map((ref) => {
763
+ if (ref.startsWith('notion:')) {
764
+ const notionId = ref.substring(7);
765
+ return notionIdToDocId.get(notionId) ?? ref;
766
+ }
767
+ return ref;
768
+ });
769
+ value = JSON.stringify(resolvedRefs);
770
+ break;
771
+ }
738
772
  }
739
773
  if (value != null) {
740
774
  try {
@@ -8,6 +8,8 @@ import type { SectionInput } from '../types.js';
8
8
  import type { NotionBlock, NotionRichText, NotionApiClient, NotionPage } from './notion-api.js';
9
9
  /** Map of page IDs to their KB paths (for cross-link resolution). */
10
10
  export type PagePathMap = Map<string, string>;
11
+ /** Tracks unsupported Notion block types encountered during conversion. */
12
+ export type UnsupportedBlockTracker = Map<string, number>;
11
13
  /**
12
14
  * Convert a page's blocks into Moxn sections.
13
15
  *
@@ -19,6 +21,8 @@ export declare function blocksToSections(blocks: NotionBlock[], client: NotionAp
19
21
  visitedSyncedBlocks?: Set<string>;
20
22
  /** Map of Notion database IDs to KB database IDs (for child_database blocks). */
21
23
  databaseIdMap?: Map<string, string>;
24
+ /** Accumulates counts of unsupported block types (caller-owned). */
25
+ unsupportedBlocks?: UnsupportedBlockTracker;
22
26
  }): Promise<SectionInput[]>;
23
27
  /** Convert rich text array to markdown string. */
24
28
  export declare function richTextToMarkdown(richText: NotionRichText[]): string;
@@ -16,6 +16,7 @@
16
16
  export async function blocksToSections(blocks, client, pagePathMap, options) {
17
17
  const visitedSyncedBlocks = options?.visitedSyncedBlocks ?? new Set();
18
18
  const databaseIdMap = options?.databaseIdMap;
19
+ const unsupportedBlocks = options?.unsupportedBlocks;
19
20
  const sections = [];
20
21
  let currentSectionName = 'Introduction';
21
22
  let currentBlocks = [];
@@ -35,7 +36,7 @@ export async function blocksToSections(blocks, client, pagePathMap, options) {
35
36
  continue;
36
37
  }
37
38
  // Convert block to content blocks
38
- const converted = await convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap);
39
+ const converted = await convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks);
39
40
  currentBlocks.push(...converted);
40
41
  }
41
42
  // Flush last section
@@ -51,7 +52,7 @@ export async function blocksToSections(blocks, client, pagePathMap, options) {
51
52
  // ============================================
52
53
  // Block conversion
53
54
  // ============================================
54
- async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
55
+ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
55
56
  const results = [];
56
57
  switch (block.type) {
57
58
  case 'paragraph':
@@ -81,10 +82,10 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
81
82
  results.push(...convertToDo(block));
82
83
  break;
83
84
  case 'quote':
84
- results.push(...(await convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
85
+ results.push(...(await convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
85
86
  break;
86
87
  case 'callout':
87
- results.push(...(await convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
88
+ results.push(...(await convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
88
89
  break;
89
90
  case 'divider':
90
91
  results.push(textBlock('---'));
@@ -93,7 +94,7 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
93
94
  results.push(...(await convertTable(block, client)));
94
95
  break;
95
96
  case 'toggle':
96
- results.push(...(await convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
97
+ results.push(...(await convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
97
98
  break;
98
99
  case 'bookmark':
99
100
  results.push(...convertBookmark(block));
@@ -122,10 +123,10 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
122
123
  results.push(...convertChildDatabase(block, databaseIdMap));
123
124
  break;
124
125
  case 'synced_block':
125
- results.push(...(await convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
126
+ results.push(...(await convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
126
127
  break;
127
128
  case 'column_list':
128
- results.push(...(await convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
129
+ results.push(...(await convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
129
130
  break;
130
131
  case 'column':
131
132
  // Columns are handled by column_list
@@ -146,7 +147,10 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
146
147
  results.push(...convertLinkPreview(block));
147
148
  break;
148
149
  default:
149
- // Unknown block type — log and skip
150
+ // Unknown block type — track and skip
151
+ if (unsupportedBlocks) {
152
+ unsupportedBlocks.set(block.type, (unsupportedBlocks.get(block.type) ?? 0) + 1);
153
+ }
150
154
  console.warn(` Skipping unsupported Notion block type: ${block.type}`);
151
155
  break;
152
156
  }
@@ -165,7 +169,7 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
165
169
  ? ' '
166
170
  : undefined;
167
171
  const children = await client.getBlockChildren(block.id);
168
- const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap);
172
+ const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap, unsupportedBlocks);
169
173
  results.push(...childBlocks);
170
174
  }
171
175
  return results;
@@ -211,7 +215,7 @@ function convertToDo(block) {
211
215
  const text = richTextToMarkdown(td.to_do.rich_text);
212
216
  return [textBlock(`${checkbox} ${text}`)];
213
217
  }
214
- async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
218
+ async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
215
219
  const q = block;
216
220
  const text = richTextToMarkdown(q.quote.rich_text);
217
221
  if (!text && !block.has_children)
@@ -225,12 +229,12 @@ async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks, dat
225
229
  const results = quoted ? [textBlock(quoted)] : [];
226
230
  if (block.has_children) {
227
231
  const children = await client.getBlockChildren(block.id);
228
- const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
232
+ const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap, unsupportedBlocks);
229
233
  results.push(...childBlocks);
230
234
  }
231
235
  return results;
232
236
  }
233
- async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
237
+ async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
234
238
  const c = block;
235
239
  const text = richTextToMarkdown(c.callout.rich_text);
236
240
  const emoji = c.callout.icon?.emoji ?? '';
@@ -242,7 +246,7 @@ async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks, d
242
246
  const results = [textBlock(quoted)];
243
247
  if (block.has_children) {
244
248
  const children = await client.getBlockChildren(block.id);
245
- const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
249
+ const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap, unsupportedBlocks);
246
250
  results.push(...childBlocks);
247
251
  }
248
252
  return results;
@@ -271,13 +275,13 @@ async function convertTable(block, client) {
271
275
  }
272
276
  return [textBlock(lines.join('\n'))];
273
277
  }
274
- async function convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
278
+ async function convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
275
279
  const t = block;
276
280
  const header = richTextToMarkdown(t.toggle.rich_text);
277
281
  const results = [textBlock(`**${header}**`)];
278
282
  if (block.has_children) {
279
283
  const children = await client.getBlockChildren(block.id);
280
- const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
284
+ const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap, unsupportedBlocks);
281
285
  results.push(...childBlocks);
282
286
  }
283
287
  return results;
@@ -402,7 +406,7 @@ function convertLinkToPage(block, pagePathMap) {
402
406
  // Target not in import set
403
407
  return [textBlock(`*(Link to Notion page: ${targetId})*`)];
404
408
  }
405
- async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
409
+ async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
406
410
  const sb = block;
407
411
  // Get the source block ID (either this block or the original)
408
412
  const sourceId = sb.synced_block.synced_from?.block_id ?? block.id;
@@ -413,19 +417,19 @@ async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlock
413
417
  visitedSyncedBlocks.add(sourceId);
414
418
  try {
415
419
  const children = await client.getBlockChildren(sourceId);
416
- return await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap);
420
+ return await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap, unsupportedBlocks);
417
421
  }
418
422
  finally {
419
423
  visitedSyncedBlocks.delete(sourceId);
420
424
  }
421
425
  }
422
- async function convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
426
+ async function convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
423
427
  const children = await client.getBlockChildren(block.id);
424
428
  const results = [];
425
429
  for (const column of children) {
426
430
  if (column.type === 'column' && column.has_children) {
427
431
  const columnChildren = await client.getBlockChildren(column.id);
428
- const converted = await convertAndMergeChildren(columnChildren, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap);
432
+ const converted = await convertAndMergeChildren(columnChildren, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap, unsupportedBlocks);
429
433
  results.push(...converted);
430
434
  }
431
435
  }
@@ -574,10 +578,10 @@ function mergeConsecutiveListBlocks(blocks) {
574
578
  /**
575
579
  * Convert children blocks, merge consecutive list items, and optionally indent.
576
580
  */
577
- async function convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap) {
581
+ async function convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap, unsupportedBlocks) {
578
582
  const blocks = [];
579
583
  for (const child of children) {
580
- const converted = await convertBlock(child, client, pagePathMap, visitedSyncedBlocks, databaseIdMap);
584
+ const converted = await convertBlock(child, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks);
581
585
  blocks.push(...converted);
582
586
  }
583
587
  const merged = mergeConsecutiveListBlocks(blocks);
@@ -30,8 +30,8 @@ export interface MappedColumn {
30
30
  export interface ScalarColumn {
31
31
  notionPropertyId: string;
32
32
  notionPropertyName: string;
33
- notionType: 'number' | 'date' | 'url' | 'email' | 'rich_text' | 'checkbox';
34
- moxnType: 'number' | 'date' | 'url' | 'email' | 'text';
33
+ notionType: 'number' | 'date' | 'url' | 'email' | 'rich_text' | 'checkbox' | 'relation';
34
+ moxnType: 'number' | 'date' | 'url' | 'email' | 'text' | 'page_ref';
35
35
  }
36
36
  /** Parsed Notion column that will be rendered as text in a Properties section. */
37
37
  export interface UnmappedColumn {
@@ -50,7 +50,7 @@ export interface ParsedDatabaseSchema {
50
50
  /** Scalar value for a single database entry property. */
51
51
  export interface ScalarValue {
52
52
  columnName: string;
53
- moxnType: 'number' | 'date' | 'url' | 'email' | 'text';
53
+ moxnType: 'number' | 'date' | 'url' | 'email' | 'text' | 'page_ref';
54
54
  textValue?: string | null;
55
55
  numberValue?: number | null;
56
56
  dateValue?: string | null;
@@ -123,6 +123,14 @@ export function parseDatabaseSchema(db) {
123
123
  moxnType: 'text',
124
124
  });
125
125
  }
126
+ else if (prop.type === 'relation') {
127
+ scalarColumns.push({
128
+ notionPropertyId: prop.id,
129
+ notionPropertyName: propName,
130
+ notionType: 'relation',
131
+ moxnType: 'page_ref',
132
+ });
133
+ }
126
134
  else {
127
135
  // Warn on property types that have no meaningful static value
128
136
  if (prop.type === 'button' || prop.type === 'ai_text') {
@@ -175,9 +183,24 @@ export function parseEntryValues(page, schema) {
175
183
  // Check if this is a scalar column
176
184
  const scalar = scalarByName.get(propName);
177
185
  if (scalar) {
178
- const extracted = extractScalarValue(propValue, scalar);
179
- if (extracted) {
180
- scalarValues.set(propName, extracted);
186
+ // page_ref columns extract relation page IDs (handled differently from standard scalars)
187
+ if (scalar.moxnType === 'page_ref') {
188
+ const refs = (propValue.relation ?? [])
189
+ .map((r) => normalizeId(r.id))
190
+ .filter(Boolean);
191
+ if (refs.length > 0) {
192
+ scalarValues.set(propName, {
193
+ columnName: propName,
194
+ moxnType: 'page_ref',
195
+ textValue: JSON.stringify(refs.map((id) => `notion:${id}`)),
196
+ });
197
+ }
198
+ }
199
+ else {
200
+ const extracted = extractScalarValue(propValue, scalar);
201
+ if (extracted) {
202
+ scalarValues.set(propName, extracted);
203
+ }
181
204
  }
182
205
  continue;
183
206
  }
@@ -10,6 +10,7 @@
10
10
  import type { SectionInput } from '../types.js';
11
11
  export interface NotionIdMapping {
12
12
  notionIdToKbPath: Map<string, string>;
13
+ notionIdToTitle?: Map<string, string>;
13
14
  }
14
15
  export interface ExtractedReference {
15
16
  sectionIndex: number;
@@ -125,17 +125,19 @@ export function resolveNotionReferences(sections, mapping) {
125
125
  text = text.replace(RELATION_REF_RE, (_match, rawId) => {
126
126
  const nid = normalizeId(rawId);
127
127
  const kbPath = mapping.notionIdToKbPath.get(nid);
128
+ const title = mapping.notionIdToTitle?.get(nid);
128
129
  if (kbPath) {
130
+ const displayText = title || kbPath;
129
131
  references.push({
130
132
  sectionIndex,
131
133
  targetNotionId: nid,
132
134
  targetKbPath: kbPath,
133
- displayText: kbPath,
135
+ displayText,
134
136
  });
135
- return `[${kbPath}](${kbPath})`;
137
+ return `[${displayText}](${kbPath})`;
136
138
  }
137
- // Unresolved — keep the ID
138
- return nid;
139
+ // Unresolved — show title if available, otherwise keep the ID
140
+ return title || nid;
139
141
  });
140
142
  return { blockType: 'text', text };
141
143
  });
@@ -151,9 +153,10 @@ export function resolveRelationIds(value, mapping) {
151
153
  return value.replace(RELATION_REF_RE, (_match, rawId) => {
152
154
  const nid = normalizeId(rawId);
153
155
  const kbPath = mapping.notionIdToKbPath.get(nid);
156
+ const title = mapping.notionIdToTitle?.get(nid);
154
157
  if (kbPath) {
155
- return `[${kbPath}](${kbPath})`;
158
+ return `[${title || kbPath}](${kbPath})`;
156
159
  }
157
- return nid;
160
+ return title || nid;
158
161
  });
159
162
  }
@@ -48,10 +48,16 @@ export declare class NotionSource extends MigrationSource<NotionSourceConfig> {
48
48
  private pageTree;
49
49
  private allPages;
50
50
  private pagePathMap;
51
+ private pageTitleMap;
51
52
  private databases;
52
53
  private databaseEntryPageIds;
53
54
  private _documentCount;
54
55
  private _validated;
56
+ private _skippedEmpty;
57
+ private _skippedNoContent;
58
+ private _extractionErrors;
59
+ private _syntheticDatabaseNodes;
60
+ private _unsupportedBlocks;
55
61
  /** Map of Notion database ID → KB database ID, set before extraction. */
56
62
  private _databaseIdMap;
57
63
  constructor(config: NotionSourceConfig);
@@ -91,6 +97,7 @@ export interface PageTreeResult {
91
97
  tree: PageTreeNode[];
92
98
  allPages: PageTreeNode[];
93
99
  pagePathMap: PagePathMap;
100
+ pageTitleMap: Map<string, string>;
94
101
  databaseEntryPageIds: Set<string>;
95
102
  }
96
103
  /**
@@ -24,10 +24,17 @@ export class NotionSource extends MigrationSource {
24
24
  pageTree = [];
25
25
  allPages = []; // flat list, depth-first order
26
26
  pagePathMap = new Map();
27
+ pageTitleMap = new Map();
27
28
  databases = [];
28
29
  databaseEntryPageIds = new Set();
29
30
  _documentCount = 0;
30
31
  _validated = false;
32
+ // Tracking for extraction-time skips (populated during extract())
33
+ _skippedEmpty = [];
34
+ _skippedNoContent = [];
35
+ _extractionErrors = [];
36
+ _syntheticDatabaseNodes = 0;
37
+ _unsupportedBlocks = new Map();
31
38
  /** Map of Notion database ID → KB database ID, set before extraction. */
32
39
  _databaseIdMap = new Map();
33
40
  constructor(config) {
@@ -89,13 +96,18 @@ export class NotionSource extends MigrationSource {
89
96
  // 5. Process databases — identify entries and build schemas
90
97
  await this.processDatabases(allNotionDatabases);
91
98
  // 6. Count and validate (exclude synthetic database container nodes)
92
- this._documentCount = this.allPages.filter((n) => !n.isDatabaseNode).length;
99
+ this._syntheticDatabaseNodes = this.allPages.filter((n) => n.isDatabaseNode).length;
100
+ const pagesInTree = this.allPages.filter((n) => !n.isDatabaseNode).length;
101
+ const databaseEntriesInTree = this.allPages.filter((n) => n.isDatabaseEntry && !n.isDatabaseNode).length;
102
+ this._documentCount = pagesInTree;
93
103
  // Add database-only entries (pages that are in databases but not in page tree)
104
+ let databaseEntriesOutsideTree = 0;
94
105
  for (const dbInfo of this.databases) {
95
106
  for (const entry of dbInfo.entries) {
96
107
  const nid = normalizeId(entry.id);
97
108
  if (!this.pagePathMap.has(nid)) {
98
109
  this._documentCount++;
110
+ databaseEntriesOutsideTree++;
99
111
  }
100
112
  }
101
113
  }
@@ -103,7 +115,12 @@ export class NotionSource extends MigrationSource {
103
115
  throw new Error(`Workspace has ${this._documentCount} documents, exceeding the ${MAX_DOCUMENT_COUNT} limit. ` +
104
116
  'Use --root-page-id to import a subtree.');
105
117
  }
106
- console.log(` ${this.allPages.length} pages + ${this.databases.length} databases ready for import.`);
118
+ // Log clear breakdown of what was discovered
119
+ const standalonePages = pagesInTree - databaseEntriesInTree;
120
+ const totalDbEntries = databaseEntriesInTree + databaseEntriesOutsideTree;
121
+ console.log(` ${this._documentCount} documents to import: ` +
122
+ `${standalonePages} pages + ${totalDbEntries} database entries ` +
123
+ `(across ${this.databases.length} databases)`);
107
124
  // 7. Initialize media downloader
108
125
  await this.mediaDownloader.init();
109
126
  this._validated = true;
@@ -135,6 +152,12 @@ export class NotionSource extends MigrationSource {
135
152
  // Pass 2: Extraction
136
153
  // ============================================
137
154
  async *extract() {
155
+ // Reset extraction tracking
156
+ this._skippedEmpty = [];
157
+ this._skippedNoContent = [];
158
+ this._extractionErrors = [];
159
+ this._unsupportedBlocks = new Map();
160
+ let yielded = 0;
138
161
  // Walk pages depth-first
139
162
  for (const node of this.allPages) {
140
163
  // Skip database entries that will be created during database import
@@ -145,8 +168,10 @@ export class NotionSource extends MigrationSource {
145
168
  if (node.isDatabaseNode)
146
169
  continue;
147
170
  const doc = await this.extractPage(node);
148
- if (doc)
171
+ if (doc) {
172
+ yielded++;
149
173
  yield doc;
174
+ }
150
175
  }
151
176
  // Extract database-only entries (pages not in page tree)
152
177
  for (const dbInfo of this.databases) {
@@ -162,10 +187,43 @@ export class NotionSource extends MigrationSource {
162
187
  }
163
188
  }
164
189
  const doc = await this.extractDatabaseEntry(entry, dbInfo);
165
- if (doc)
190
+ if (doc) {
191
+ yielded++;
166
192
  yield doc;
193
+ }
194
+ }
195
+ }
196
+ // Log extraction summary so user can reconcile counts
197
+ const totalSkipped = this._skippedEmpty.length +
198
+ this._skippedNoContent.length +
199
+ this._extractionErrors.length;
200
+ console.log(`\nExtraction complete: ${yielded} of ${this._documentCount} documents extracted`);
201
+ if (totalSkipped > 0) {
202
+ console.log(` ${totalSkipped} pages skipped:`);
203
+ if (this._skippedEmpty.length > 0) {
204
+ console.log(` Empty pages (no blocks): ${this._skippedEmpty.map((t) => `"${t}"`).join(', ')}`);
205
+ }
206
+ if (this._skippedNoContent.length > 0) {
207
+ console.log(` No extractable content: ${this._skippedNoContent.map((t) => `"${t}"`).join(', ')}`);
208
+ }
209
+ if (this._extractionErrors.length > 0) {
210
+ console.log(` Extraction errors: ${this._extractionErrors.map((t) => `"${t}"`).join(', ')}`);
167
211
  }
168
212
  }
213
+ // Report unsupported block types
214
+ if (this._unsupportedBlocks.size > 0) {
215
+ const sorted = [...this._unsupportedBlocks.entries()].sort((a, b) => b[1] - a[1]);
216
+ const totalUnsupported = sorted.reduce((sum, [, count]) => sum + count, 0);
217
+ console.log(` ${totalUnsupported} unsupported block(s) across ${sorted.length} type(s):`);
218
+ for (const [type, count] of sorted) {
219
+ console.log(` ${type}: ${count}`);
220
+ }
221
+ }
222
+ // Warn if counts don't reconcile (helps catch counting bugs)
223
+ const expectedYield = this._documentCount - totalSkipped;
224
+ if (yielded !== expectedYield) {
225
+ console.log(` ⚠ Count mismatch: expected ${expectedYield} (${this._documentCount} - ${totalSkipped} skipped) but extracted ${yielded}`);
226
+ }
169
227
  }
170
228
  /** Clean up temp files after migration completes. */
171
229
  async cleanup() {
@@ -183,6 +241,7 @@ export class NotionSource extends MigrationSource {
183
241
  this.pageTree = result.tree;
184
242
  this.allPages = result.allPages;
185
243
  this.pagePathMap = result.pagePathMap;
244
+ this.pageTitleMap = result.pageTitleMap;
186
245
  this.databaseEntryPageIds = result.databaseEntryPageIds;
187
246
  }
188
247
  // ============================================
@@ -224,19 +283,20 @@ export class NotionSource extends MigrationSource {
224
283
  try {
225
284
  const blocks = await this.client.getBlockChildren(node.page.id);
226
285
  if (blocks.length === 0) {
227
- console.log(` Skipping empty page: ${node.title}`);
286
+ this._skippedEmpty.push(node.title);
228
287
  return null;
229
288
  }
230
289
  let sections = await blocksToSections(blocks, this.client, this.pagePathMap, {
231
290
  databaseIdMap: this._databaseIdMap.size > 0 ? this._databaseIdMap : undefined,
291
+ unsupportedBlocks: this._unsupportedBlocks,
232
292
  });
233
293
  // Download Notion-hosted media files
234
294
  sections = await this.downloadSectionMedia(sections);
235
295
  // Resolve cross-references
236
- const { sections: resolvedSections, references } = resolveNotionReferences(sections, { notionIdToKbPath: this.pagePathMap });
296
+ const { sections: resolvedSections, references } = resolveNotionReferences(sections, { notionIdToKbPath: this.pagePathMap, notionIdToTitle: this.pageTitleMap });
237
297
  sections = resolvedSections;
238
298
  if (sections.length === 0) {
239
- console.log(` Skipping page with no content: ${node.title}`);
299
+ this._skippedNoContent.push(node.title);
240
300
  return null;
241
301
  }
242
302
  return {
@@ -252,6 +312,7 @@ export class NotionSource extends MigrationSource {
252
312
  };
253
313
  }
254
314
  catch (error) {
315
+ this._extractionErrors.push(node.title);
255
316
  console.error(` Error extracting page "${node.title}": ${error instanceof Error ? error.message : error}`);
256
317
  return null;
257
318
  }
@@ -273,13 +334,14 @@ export class NotionSource extends MigrationSource {
273
334
  if (blocks.length > 0) {
274
335
  const contentSections = await blocksToSections(blocks, this.client, this.pagePathMap, {
275
336
  databaseIdMap: this._databaseIdMap.size > 0 ? this._databaseIdMap : undefined,
337
+ unsupportedBlocks: this._unsupportedBlocks,
276
338
  });
277
339
  sections.push(...contentSections);
278
340
  }
279
341
  // Download media
280
342
  let processedSections = await this.downloadSectionMedia(sections);
281
343
  // Resolve cross-references
282
- const { sections: resolvedSections, references } = resolveNotionReferences(processedSections, { notionIdToKbPath: this.pagePathMap });
344
+ const { sections: resolvedSections, references } = resolveNotionReferences(processedSections, { notionIdToKbPath: this.pagePathMap, notionIdToTitle: this.pageTitleMap });
283
345
  processedSections = resolvedSections;
284
346
  const nid = normalizeId(entry.id);
285
347
  const kbPath = this.pagePathMap.get(nid) ?? slug;
@@ -303,7 +365,9 @@ export class NotionSource extends MigrationSource {
303
365
  };
304
366
  }
305
367
  catch (error) {
306
- console.error(` Error extracting database entry: ${error instanceof Error ? error.message : error}`);
368
+ const entryTitle = getPageTitle(entry) || `DB entry ${entry.id}`;
369
+ this._extractionErrors.push(entryTitle);
370
+ console.error(` Error extracting database entry "${entryTitle}": ${error instanceof Error ? error.message : error}`);
307
371
  return null;
308
372
  }
309
373
  }
@@ -330,6 +394,7 @@ export class NotionSource extends MigrationSource {
330
394
  */
331
395
  export function buildPageTree(pages, options) {
332
396
  const pagePathMap = new Map();
397
+ const pageTitleMap = new Map();
333
398
  const databaseEntryPageIds = new Set();
334
399
  const tree = [];
335
400
  const allPages = [];
@@ -431,7 +496,11 @@ export function buildPageTree(pages, options) {
431
496
  }
432
497
  // Build tree recursively
433
498
  const buildNode = (page, parentPath, depth, siblingSlugCounts) => {
434
- if (options?.maxDepth !== undefined && depth > options.maxDepth) {
499
+ const nid = normalizeId(page.id);
500
+ const isDatabaseSubtree = syntheticDatabaseIds.has(nid) || databaseEntryPageIds.has(nid);
501
+ if (options?.maxDepth !== undefined &&
502
+ depth > options.maxDepth &&
503
+ !isDatabaseSubtree) {
435
504
  return null;
436
505
  }
437
506
  const title = getPageTitle(page);
@@ -443,7 +512,6 @@ export function buildPageTree(pages, options) {
443
512
  slug = `${slug}-${existing + 1}`;
444
513
  }
445
514
  const kbPath = parentPath ? `${parentPath}/${slug}` : slug;
446
- const nid = normalizeId(page.id);
447
515
  const isDatabaseEntry = databaseEntryPageIds.has(nid);
448
516
  const node = {
449
517
  page,
@@ -455,8 +523,9 @@ export function buildPageTree(pages, options) {
455
523
  parentDatabaseId: page.parent.type === 'database_id' ? page.parent.database_id : undefined,
456
524
  children: [],
457
525
  };
458
- // Register in path map
526
+ // Register in path map and title map
459
527
  pagePathMap.set(nid, kbPath);
528
+ pageTitleMap.set(nid, title);
460
529
  // Process children
461
530
  const childPages = childrenByParent.get(nid) ?? [];
462
531
  const childSlugCounts = new Map();
@@ -484,7 +553,7 @@ export function buildPageTree(pages, options) {
484
553
  }
485
554
  };
486
555
  flatten(tree);
487
- return { tree, allPages, pagePathMap, databaseEntryPageIds };
556
+ return { tree, allPages, pagePathMap, pageTitleMap, databaseEntryPageIds };
488
557
  }
489
558
  /** Extract the parent page ID from a Notion page's parent field. */
490
559
  function getParentPageId(page) {
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@moxn/kb-migrate",
3
- "version": "0.4.23",
3
+ "version": "0.4.25",
4
4
  "description": "Migration tool for importing documents into Moxn Knowledge Base from local files, Notion, Google Docs, and more",
5
5
  "type": "module",
6
6
  "main": "dist/index.js",