@moxn/kb-migrate 0.4.23 → 0.4.24

package/dist/client.js

@@ -299,7 +299,12 @@ export class MoxnClient {
                 error.branchId = body.branchId;
                 throw error;
             }
-            throw new Error(body.error || `API error: ${response.status}`);
+            const message = body.details
+                ? `API error ${response.status}: ${body.details}`
+                : body.error
+                    ? `API error ${response.status}: ${body.error}`
+                    : `API error: ${response.status}`;
+            throw new Error(message);
         }
         return response.json();
     }
@@ -314,7 +319,12 @@ export class MoxnClient {
         });
         if (!response.ok) {
             const body = await response.json().catch(() => ({}));
-            throw new Error(body.error || `API error: ${response.status}`);
+            const message = body.details
+                ? `API error ${response.status}: ${body.details}`
+                : body.error
+                    ? `API error ${response.status}: ${body.error}`
+                    : `API error: ${response.status}`;
+            throw new Error(message);
         }
         return response.json();
     }

package/dist/index.js

@@ -27,21 +27,28 @@ async function runMigration(source, options) {
     const results = [];
     // Validate source
     await source.validate();
-    // Get document count for progress
+    // Get document count for progress (this is an upper bound — some pages
+    // may be skipped during extraction if they are empty or have errors)
     const totalCount = await source.getDocumentCount();
-    if (totalCount !== undefined) {
-        console.log(`Found ${totalCount} documents to migrate`);
-    }
     // Create client
     const client = new MoxnClient(options);
     // Process documents
     let processed = 0;
+    let consecutiveFailures = 0;
+    const MAX_CONSECUTIVE_FAILURES = 10;
     for await (const doc of source.extract()) {
         processed++;
         const progress = totalCount ? ` (${processed}/${totalCount})` : '';
         console.log(`Processing: ${doc.sourcePath}${progress}`);
         const result = await client.migrateDocument(doc, options.basePath, options.onConflict, options.dryRun);
         results.push(result);
+        // Track consecutive failures for circuit breaker
+        if (result.status === 'failed') {
+            consecutiveFailures++;
+        }
+        else {
+            consecutiveFailures = 0;
+        }
         // Log result
         const statusIcon = {
             created: '\u2713',
@@ -53,6 +60,13 @@ async function runMigration(source, options) {
         if (result.error) {
             console.log(`    Error: ${result.error}`);
         }
+        // Circuit breaker: abort after too many consecutive failures
+        if (consecutiveFailures >= MAX_CONSECUTIVE_FAILURES) {
+            console.error(`\nAborting: ${MAX_CONSECUTIVE_FAILURES} consecutive failures. ` +
+                'Last error: ' + (result.error || 'unknown') + '\n' +
+                'Fix the underlying issue and retry. Remaining documents will be skipped.');
+            break;
+        }
     }
     // Build summary
     const summary = {

package/dist/sources/notion-blocks.d.ts

@@ -8,6 +8,8 @@ import type { SectionInput } from '../types.js';
 import type { NotionBlock, NotionRichText, NotionApiClient, NotionPage } from './notion-api.js';
 /** Map of page IDs to their KB paths (for cross-link resolution). */
 export type PagePathMap = Map<string, string>;
+/** Tracks unsupported Notion block types encountered during conversion. */
+export type UnsupportedBlockTracker = Map<string, number>;
 /**
  * Convert a page's blocks into Moxn sections.
  *
@@ -19,6 +21,8 @@ export declare function blocksToSections(blocks: NotionBlock[], client: NotionAp
     visitedSyncedBlocks?: Set<string>;
     /** Map of Notion database IDs to KB database IDs (for child_database blocks). */
     databaseIdMap?: Map<string, string>;
+    /** Accumulates counts of unsupported block types (caller-owned). */
+    unsupportedBlocks?: UnsupportedBlockTracker;
 }): Promise<SectionInput[]>;
 /** Convert rich text array to markdown string. */
 export declare function richTextToMarkdown(richText: NotionRichText[]): string;

package/dist/sources/notion-blocks.js

@@ -16,6 +16,7 @@
 export async function blocksToSections(blocks, client, pagePathMap, options) {
     const visitedSyncedBlocks = options?.visitedSyncedBlocks ?? new Set();
     const databaseIdMap = options?.databaseIdMap;
+    const unsupportedBlocks = options?.unsupportedBlocks;
     const sections = [];
     let currentSectionName = 'Introduction';
     let currentBlocks = [];
@@ -35,7 +36,7 @@ export async function blocksToSections(blocks, client, pagePathMap, options) {
             continue;
         }
         // Convert block to content blocks
-        const converted = await convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap);
+        const converted = await convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks);
         currentBlocks.push(...converted);
     }
     // Flush last section
@@ -51,7 +52,7 @@ export async function blocksToSections(blocks, client, pagePathMap, options) {
 // ============================================
 // Block conversion
 // ============================================
-async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
+async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
     const results = [];
     switch (block.type) {
         case 'paragraph':
@@ -81,10 +82,10 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
             results.push(...convertToDo(block));
             break;
         case 'quote':
-            results.push(...(await convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
+            results.push(...(await convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
             break;
         case 'callout':
-            results.push(...(await convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
+            results.push(...(await convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
             break;
         case 'divider':
             results.push(textBlock('---'));
@@ -93,7 +94,7 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
             results.push(...(await convertTable(block, client)));
             break;
         case 'toggle':
-            results.push(...(await convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
+            results.push(...(await convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
             break;
         case 'bookmark':
             results.push(...convertBookmark(block));
@@ -122,10 +123,10 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
             results.push(...convertChildDatabase(block, databaseIdMap));
             break;
         case 'synced_block':
-            results.push(...(await convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
+            results.push(...(await convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
             break;
         case 'column_list':
-            results.push(...(await convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap)));
+            results.push(...(await convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks)));
             break;
         case 'column':
             // Columns are handled by column_list
@@ -146,7 +147,10 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
             results.push(...convertLinkPreview(block));
             break;
         default:
-            // Unknown block type — log and skip
+            // Unknown block type — track and skip
+            if (unsupportedBlocks) {
+                unsupportedBlocks.set(block.type, (unsupportedBlocks.get(block.type) ?? 0) + 1);
+            }
             console.warn(`  Skipping unsupported Notion block type: ${block.type}`);
             break;
     }
@@ -165,7 +169,7 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
             ? '  '
             : undefined;
         const children = await client.getBlockChildren(block.id);
-        const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap);
+        const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap, unsupportedBlocks);
         results.push(...childBlocks);
     }
     return results;
@@ -211,7 +215,7 @@ function convertToDo(block) {
     const text = richTextToMarkdown(td.to_do.rich_text);
     return [textBlock(`${checkbox} ${text}`)];
 }
-async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
+async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
     const q = block;
     const text = richTextToMarkdown(q.quote.rich_text);
     if (!text && !block.has_children)
@@ -225,12 +229,12 @@ async function convertQuote(block, client, pagePathMap, visitedSyncedBlocks, dat
     const results = quoted ? [textBlock(quoted)] : [];
     if (block.has_children) {
         const children = await client.getBlockChildren(block.id);
-        const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
+        const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap, unsupportedBlocks);
         results.push(...childBlocks);
     }
     return results;
 }
-async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
+async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
     const c = block;
     const text = richTextToMarkdown(c.callout.rich_text);
     const emoji = c.callout.icon?.emoji ?? '';
@@ -242,7 +246,7 @@ async function convertCallout(block, client, pagePathMap, visitedSyncedBlocks, d
     const results = [textBlock(quoted)];
     if (block.has_children) {
         const children = await client.getBlockChildren(block.id);
-        const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
+        const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap, unsupportedBlocks);
         results.push(...childBlocks);
     }
     return results;
@@ -271,13 +275,13 @@ async function convertTable(block, client) {
     }
     return [textBlock(lines.join('\n'))];
 }
-async function convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
+async function convertToggle(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
     const t = block;
     const header = richTextToMarkdown(t.toggle.rich_text);
     const results = [textBlock(`**${header}**`)];
     if (block.has_children) {
         const children = await client.getBlockChildren(block.id);
-        const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap);
+        const childBlocks = await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, '> ', databaseIdMap, unsupportedBlocks);
         results.push(...childBlocks);
     }
     return results;
@@ -402,7 +406,7 @@ function convertLinkToPage(block, pagePathMap) {
     // Target not in import set
     return [textBlock(`*(Link to Notion page: ${targetId})*`)];
 }
-async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
+async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
     const sb = block;
     // Get the source block ID (either this block or the original)
     const sourceId = sb.synced_block.synced_from?.block_id ?? block.id;
@@ -413,19 +417,19 @@ async function convertSyncedBlock(block, client, pagePathMap, visitedSyncedBlock
     visitedSyncedBlocks.add(sourceId);
     try {
         const children = await client.getBlockChildren(sourceId);
-        return await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap);
+        return await convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap, unsupportedBlocks);
     }
     finally {
         visitedSyncedBlocks.delete(sourceId);
     }
 }
-async function convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap) {
+async function convertColumnList(block, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks) {
     const children = await client.getBlockChildren(block.id);
     const results = [];
     for (const column of children) {
         if (column.type === 'column' && column.has_children) {
             const columnChildren = await client.getBlockChildren(column.id);
-            const converted = await convertAndMergeChildren(columnChildren, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap);
+            const converted = await convertAndMergeChildren(columnChildren, client, pagePathMap, visitedSyncedBlocks, undefined, databaseIdMap, unsupportedBlocks);
             results.push(...converted);
         }
     }
@@ -574,10 +578,10 @@ function mergeConsecutiveListBlocks(blocks) {
 /**
  * Convert children blocks, merge consecutive list items, and optionally indent.
  */
-async function convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap) {
+async function convertAndMergeChildren(children, client, pagePathMap, visitedSyncedBlocks, indent, databaseIdMap, unsupportedBlocks) {
     const blocks = [];
     for (const child of children) {
-        const converted = await convertBlock(child, client, pagePathMap, visitedSyncedBlocks, databaseIdMap);
+        const converted = await convertBlock(child, client, pagePathMap, visitedSyncedBlocks, databaseIdMap, unsupportedBlocks);
         blocks.push(...converted);
     }
     const merged = mergeConsecutiveListBlocks(blocks);

package/dist/sources/notion.d.ts

@@ -52,6 +52,11 @@ export declare class NotionSource extends MigrationSource<NotionSourceConfig> {
     private databaseEntryPageIds;
     private _documentCount;
     private _validated;
+    private _skippedEmpty;
+    private _skippedNoContent;
+    private _extractionErrors;
+    private _syntheticDatabaseNodes;
+    private _unsupportedBlocks;
     /** Map of Notion database ID → KB database ID, set before extraction. */
     private _databaseIdMap;
     constructor(config: NotionSourceConfig);

package/dist/sources/notion.js

@@ -28,6 +28,12 @@ export class NotionSource extends MigrationSource {
     databaseEntryPageIds = new Set();
     _documentCount = 0;
     _validated = false;
+    // Tracking for extraction-time skips (populated during extract())
+    _skippedEmpty = [];
+    _skippedNoContent = [];
+    _extractionErrors = [];
+    _syntheticDatabaseNodes = 0;
+    _unsupportedBlocks = new Map();
     /** Map of Notion database ID → KB database ID, set before extraction. */
     _databaseIdMap = new Map();
     constructor(config) {
@@ -89,13 +95,18 @@ export class NotionSource extends MigrationSource {
         // 5. Process databases — identify entries and build schemas
         await this.processDatabases(allNotionDatabases);
         // 6. Count and validate (exclude synthetic database container nodes)
-        this._documentCount = this.allPages.filter((n) => !n.isDatabaseNode).length;
+        this._syntheticDatabaseNodes = this.allPages.filter((n) => n.isDatabaseNode).length;
+        const pagesInTree = this.allPages.filter((n) => !n.isDatabaseNode).length;
+        const databaseEntriesInTree = this.allPages.filter((n) => n.isDatabaseEntry && !n.isDatabaseNode).length;
+        this._documentCount = pagesInTree;
         // Add database-only entries (pages that are in databases but not in page tree)
+        let databaseEntriesOutsideTree = 0;
         for (const dbInfo of this.databases) {
             for (const entry of dbInfo.entries) {
                 const nid = normalizeId(entry.id);
                 if (!this.pagePathMap.has(nid)) {
                     this._documentCount++;
+                    databaseEntriesOutsideTree++;
                 }
             }
         }
@@ -103,7 +114,12 @@ export class NotionSource extends MigrationSource {
             throw new Error(`Workspace has ${this._documentCount} documents, exceeding the ${MAX_DOCUMENT_COUNT} limit. ` +
                 'Use --root-page-id to import a subtree.');
         }
-        console.log(`  ${this.allPages.length} pages + ${this.databases.length} databases ready for import.`);
+        // Log clear breakdown of what was discovered
+        const standalonePages = pagesInTree - databaseEntriesInTree;
+        const totalDbEntries = databaseEntriesInTree + databaseEntriesOutsideTree;
+        console.log(`  ${this._documentCount} documents to import: ` +
+            `${standalonePages} pages + ${totalDbEntries} database entries ` +
+            `(across ${this.databases.length} databases)`);
         // 7. Initialize media downloader
         await this.mediaDownloader.init();
         this._validated = true;
@@ -135,6 +151,12 @@ export class NotionSource extends MigrationSource {
     // Pass 2: Extraction
     // ============================================
     async *extract() {
+        // Reset extraction tracking
+        this._skippedEmpty = [];
+        this._skippedNoContent = [];
+        this._extractionErrors = [];
+        this._unsupportedBlocks = new Map();
+        let yielded = 0;
         // Walk pages depth-first
         for (const node of this.allPages) {
             // Skip database entries that will be created during database import
@@ -145,8 +167,10 @@ export class NotionSource extends MigrationSource {
             if (node.isDatabaseNode)
                 continue;
             const doc = await this.extractPage(node);
-            if (doc)
+            if (doc) {
+                yielded++;
                 yield doc;
+            }
         }
         // Extract database-only entries (pages not in page tree)
         for (const dbInfo of this.databases) {
@@ -162,9 +186,42 @@ export class NotionSource extends MigrationSource {
                     }
                 }
                 const doc = await this.extractDatabaseEntry(entry, dbInfo);
-                if (doc)
+                if (doc) {
+                    yielded++;
                     yield doc;
+                }
+            }
+        }
+        // Log extraction summary so user can reconcile counts
+        const totalSkipped = this._skippedEmpty.length +
+            this._skippedNoContent.length +
+            this._extractionErrors.length;
+        console.log(`\nExtraction complete: ${yielded} of ${this._documentCount} documents extracted`);
+        if (totalSkipped > 0) {
+            console.log(`  ${totalSkipped} pages skipped:`);
+            if (this._skippedEmpty.length > 0) {
+                console.log(`    Empty pages (no blocks): ${this._skippedEmpty.map((t) => `"${t}"`).join(', ')}`);
+            }
+            if (this._skippedNoContent.length > 0) {
+                console.log(`    No extractable content: ${this._skippedNoContent.map((t) => `"${t}"`).join(', ')}`);
             }
+            if (this._extractionErrors.length > 0) {
+                console.log(`    Extraction errors: ${this._extractionErrors.map((t) => `"${t}"`).join(', ')}`);
+            }
+        }
+        // Report unsupported block types
+        if (this._unsupportedBlocks.size > 0) {
+            const sorted = [...this._unsupportedBlocks.entries()].sort((a, b) => b[1] - a[1]);
+            const totalUnsupported = sorted.reduce((sum, [, count]) => sum + count, 0);
+            console.log(`  ${totalUnsupported} unsupported block(s) across ${sorted.length} type(s):`);
+            for (const [type, count] of sorted) {
+                console.log(`    ${type}: ${count}`);
+            }
+        }
+        // Warn if counts don't reconcile (helps catch counting bugs)
+        const expectedYield = this._documentCount - totalSkipped;
+        if (yielded !== expectedYield) {
+            console.log(`  ⚠ Count mismatch: expected ${expectedYield} (${this._documentCount} - ${totalSkipped} skipped) but extracted ${yielded}`);
         }
     }
     /** Clean up temp files after migration completes. */
@@ -224,11 +281,12 @@ export class NotionSource extends MigrationSource {
         try {
             const blocks = await this.client.getBlockChildren(node.page.id);
             if (blocks.length === 0) {
-                console.log(`  Skipping empty page: ${node.title}`);
+                this._skippedEmpty.push(node.title);
                 return null;
             }
             let sections = await blocksToSections(blocks, this.client, this.pagePathMap, {
                 databaseIdMap: this._databaseIdMap.size > 0 ? this._databaseIdMap : undefined,
+                unsupportedBlocks: this._unsupportedBlocks,
             });
             // Download Notion-hosted media files
             sections = await this.downloadSectionMedia(sections);
@@ -236,7 +294,7 @@ export class NotionSource extends MigrationSource {
             const { sections: resolvedSections, references } = resolveNotionReferences(sections, { notionIdToKbPath: this.pagePathMap });
             sections = resolvedSections;
             if (sections.length === 0) {
-                console.log(`  Skipping page with no content: ${node.title}`);
+                this._skippedNoContent.push(node.title);
                 return null;
             }
             return {
@@ -252,6 +310,7 @@ export class NotionSource extends MigrationSource {
             };
         }
         catch (error) {
+            this._extractionErrors.push(node.title);
             console.error(`  Error extracting page "${node.title}": ${error instanceof Error ? error.message : error}`);
             return null;
         }
@@ -273,6 +332,7 @@ export class NotionSource extends MigrationSource {
             if (blocks.length > 0) {
                 const contentSections = await blocksToSections(blocks, this.client, this.pagePathMap, {
                     databaseIdMap: this._databaseIdMap.size > 0 ? this._databaseIdMap : undefined,
+                    unsupportedBlocks: this._unsupportedBlocks,
                 });
                 sections.push(...contentSections);
             }
@@ -303,7 +363,9 @@ export class NotionSource extends MigrationSource {
             };
         }
         catch (error) {
-            console.error(`  Error extracting database entry: ${error instanceof Error ? error.message : error}`);
+            const entryTitle = getPageTitle(entry) || `DB entry ${entry.id}`;
+            this._extractionErrors.push(entryTitle);
+            console.error(`  Error extracting database entry "${entryTitle}": ${error instanceof Error ? error.message : error}`);
             return null;
         }
     }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moxn/kb-migrate",
-  "version": "0.4.23",
+  "version": "0.4.24",
   "description": "Migration tool for importing documents into Moxn Knowledge Base from local files, Notion, Google Docs, and more",
   "type": "module",
   "main": "dist/index.js",