@moxn/kb-migrate 0.4.6 → 0.4.8

package/dist/sources/notion.d.ts

@@ -15,6 +15,8 @@ export interface NotionSourceConfig extends SourceConfig {
     token: string;
     rootPageId?: string;
     maxDepth?: number;
+    /** Date filter for source pages */
+    dateFilter?: import('../date-filter.js').DateFilter;
 }
 /** Info yielded to the migration runner for database import (post-document pass). */
 export interface NotionDatabaseImportInfo {
@@ -38,7 +40,16 @@ export declare class NotionSource extends MigrationSource<NotionSourceConfig> {
     private databases;
     private databaseEntryPageIds;
     private _documentCount;
+    private _validated;
+    /** Map of Notion database ID → KB database ID, set before extraction. */
+    private _databaseIdMap;
     constructor(config: NotionSourceConfig);
+    /**
+     * Set the database ID map (Notion DB ID → KB DB ID).
+     * Must be called after validate() and before extract() so that
+     * child_database blocks can be converted to database_embed blocks.
+     */
+    setDatabaseIdMap(map: Map<string, string>): void;
     get sourceType(): string;
     get sourceLocation(): string;
     validate(): Promise<void>;

package/dist/sources/notion.js

@@ -27,11 +27,22 @@ export class NotionSource extends MigrationSource {
     databases = [];
     databaseEntryPageIds = new Set();
     _documentCount = 0;
+    _validated = false;
+    /** Map of Notion database ID → KB database ID, set before extraction. */
+    _databaseIdMap = new Map();
     constructor(config) {
         super(config);
         this.client = new NotionApiClient(config.token);
         this.mediaDownloader = new NotionMediaDownloader();
     }
+    /**
+     * Set the database ID map (Notion DB ID → KB DB ID).
+     * Must be called after validate() and before extract() so that
+     * child_database blocks can be converted to database_embed blocks.
+     */
+    setDatabaseIdMap(map) {
+        this._databaseIdMap = map;
+    }
     get sourceType() {
         return 'notion';
     }
@@ -44,14 +55,31 @@ export class NotionSource extends MigrationSource {
     // Pass 1: Discovery (validate)
     // ============================================
     async validate() {
+        // Idempotent — safe to call multiple times (e.g. once explicitly for
+        // pre-creating databases, then again inside runMigration).
+        if (this._validated)
+            return;
         // 1. Test token
         console.log('Validating Notion API token...');
         await this.client.validateToken();
         console.log('  Token valid.');
         // 2. Discover all pages
         console.log('Discovering pages...');
-        const allNotionPages = await this.client.searchPages();
+        let allNotionPages = await this.client.searchPages();
         console.log(`  Found ${allNotionPages.length} pages in workspace.`);
+        // 2b. Apply date filter if configured
+        if (this.config.dateFilter) {
+            const { matchesDateFilter } = await import('../date-filter.js');
+            const beforeCount = allNotionPages.length;
+            allNotionPages = allNotionPages.filter((page) => matchesDateFilter(this.config.dateFilter, {
+                createdAt: page.created_time,
+                modifiedAt: page.last_edited_time,
+            }));
+            const skipped = beforeCount - allNotionPages.length;
+            if (skipped > 0) {
+                console.log(`  Filtered ${skipped} pages by date criteria`);
+            }
+        }
         // 3. Discover all databases
         console.log('Discovering databases...');
         const allNotionDatabases = await this.client.searchDatabases();
@@ -78,6 +106,7 @@ export class NotionSource extends MigrationSource {
         console.log(`  ${this.allPages.length} pages + ${this.databases.length} databases ready for import.`);
         // 7. Initialize media downloader
         await this.mediaDownloader.init();
+        this._validated = true;
     }
     async getDocumentCount() {
         return this._documentCount;
@@ -300,7 +329,9 @@ export class NotionSource extends MigrationSource {
                 console.log(`  Skipping empty page: ${node.title}`);
                 return null;
             }
-            let sections = await blocksToSections(blocks, this.client, this.pagePathMap);
+            let sections = await blocksToSections(blocks, this.client, this.pagePathMap, {
+                databaseIdMap: this._databaseIdMap.size > 0 ? this._databaseIdMap : undefined,
+            });
             // Download Notion-hosted media files
             sections = await this.downloadSectionMedia(sections);
             // Resolve cross-references
@@ -316,6 +347,10 @@ export class NotionSource extends MigrationSource {
                 sections,
                 sourcePath: `notion://${node.page.id}`,
                 references: references.length > 0 ? references : undefined,
+                metadata: {
+                    notionPageId: normalizeId(node.page.id),
+                    notionTitle: node.title,
+                },
             };
         }
         catch (error) {
@@ -338,7 +373,9 @@ export class NotionSource extends MigrationSource {
             // Get page blocks
             const blocks = await this.client.getBlockChildren(entry.id);
             if (blocks.length > 0) {
-                const contentSections = await blocksToSections(blocks, this.client, this.pagePathMap);
+                const contentSections = await blocksToSections(blocks, this.client, this.pagePathMap, {
+                    databaseIdMap: this._databaseIdMap.size > 0 ? this._databaseIdMap : undefined,
+                });
                 sections.push(...contentSections);
             }
             // Download media
@@ -361,6 +398,10 @@ export class NotionSource extends MigrationSource {
                     ],
                 sourcePath: `notion://${entry.id}`,
                 references: references.length > 0 ? references : undefined,
+                metadata: {
+                    notionPageId: normalizeId(entry.id),
+                    notionTitle: title,
+                },
             };
         }
         catch (error) {

package/dist/targets/base.d.ts

@@ -0,0 +1,77 @@
+/**
+ * Base interface for export targets
+ *
+ * Each target (Notion, Google Docs, etc.) implements this interface
+ * to provide a consistent way to export KB documents.
+ */
+import type { DocumentDetail, DocumentListItem } from '../types.js';
+/**
+ * Result of exporting a single document to an external target
+ */
+export interface ExportTargetResult {
+    documentId: string;
+    documentPath: string;
+    status: 'created' | 'updated' | 'skipped' | 'failed';
+    externalId?: string;
+    error?: string;
+    duration: number;
+}
+/**
+ * Summary log for an export-to-target operation
+ */
+export interface ExportTargetLog {
+    timestamp: string;
+    sourceApi: string;
+    target: {
+        type: string;
+        location: string;
+    };
+    basePath: string;
+    options: {
+        dryRun: boolean;
+        conflictStrategy: 'skip' | 'update';
+    };
+    results: ExportTargetResult[];
+    summary: {
+        total: number;
+        created: number;
+        updated: number;
+        skipped: number;
+        failed: number;
+        duration: number;
+    };
+}
+/**
+ * Configuration for export targets
+ */
+export interface ExportTargetConfig {
+    [key: string]: unknown;
+}
+/**
+ * Abstract base class for export targets
+ */
+export declare abstract class ExportTarget<TConfig extends ExportTargetConfig = ExportTargetConfig> {
+    protected config: TConfig;
+    constructor(config: TConfig);
+    /**
+     * Human-readable name of the target type
+     */
+    abstract get targetType(): string;
+    /**
+     * Human-readable location identifier for logging
+     */
+    abstract get targetLocation(): string;
+    /**
+     * Validate the target configuration and connectivity
+     * @throws Error if configuration is invalid or target is not accessible
+     */
+    abstract validate(): Promise<void>;
+    /**
+     * Export a single document to the target
+     */
+    abstract exportDocument(doc: DocumentDetail, listItem: DocumentListItem, dryRun: boolean): Promise<ExportTargetResult>;
+    /**
+     * Clean up resources after export completes
+     */
+    cleanup(): Promise<void>;
+}

package/dist/targets/base.js

@@ -0,0 +1,21 @@
+/**
+ * Base interface for export targets
+ *
+ * Each target (Notion, Google Docs, etc.) implements this interface
+ * to provide a consistent way to export KB documents.
+ */
+/**
+ * Abstract base class for export targets
+ */
+export class ExportTarget {
+    config;
+    constructor(config) {
+        this.config = config;
+    }
+    /**
+     * Clean up resources after export completes
+     */
+    async cleanup() {
+        // Default no-op, override if needed
+    }
+}

package/dist/targets/index.d.ts

@@ -0,0 +1,6 @@
+/**
+ * Target registry and factory
+ */
+export { ExportTarget, type ExportTargetConfig, type ExportTargetResult, type ExportTargetLog } from './base.js';
+export { NotionExportTarget, type NotionExportConfig } from './notion.js';
+export { exportDatabase, type DatabaseExportResult, type DatabaseExportContext } from './notion-database-export.js';

package/dist/targets/index.js

@@ -0,0 +1,6 @@
+/**
+ * Target registry and factory
+ */
+export { ExportTarget } from './base.js';
+export { NotionExportTarget } from './notion.js';
+export { exportDatabase } from './notion-database-export.js';

package/dist/targets/notion-database-export.d.ts

@@ -0,0 +1,36 @@
+/**
+ * Notion Database Export
+ *
+ * Exports KB databases to Notion databases.
+ * Handles:
+ * - Creating new Notion databases from KB database schemas
+ * - Updating existing Notion databases (clear + recreate entries)
+ * - Column mapping (select/multi_select backed by tags -> Notion select/multi_select)
+ * - Color mapping (hex -> Notion color names)
+ * - Entry creation with correct property values
+ */
+import { Client } from '@notionhq/client';
+import type { MoxnClient } from '../client.js';
+export interface DatabaseExportResult {
+    kbDatabaseId: string;
+    kbDatabaseName: string;
+    notionDatabaseId: string;
+    status: 'created' | 'updated' | 'skipped' | 'failed';
+    entriesCreated: number;
+    error?: string;
+}
+export interface DatabaseExportContext {
+    notionClient: Client;
+    moxnClient: MoxnClient;
+    conflictStrategy: 'skip' | 'update';
+    dryRun: boolean;
+}
+/**
+ * Export a single KB database to Notion.
+ *
+ * @param ctx Export context with Notion client, Moxn client, and options
+ * @param kbDatabaseId KB database ID to export
+ * @param parentNotionPageId Notion page ID to create the database under
+ * @returns Export result
+ */
+export declare function exportDatabase(ctx: DatabaseExportContext, kbDatabaseId: string, parentNotionPageId: string): Promise<DatabaseExportResult>;

package/dist/targets/notion-database-export.js

@@ -0,0 +1,255 @@
+/**
+ * Notion Database Export
+ *
+ * Exports KB databases to Notion databases.
+ * Handles:
+ * - Creating new Notion databases from KB database schemas
+ * - Updating existing Notion databases (clear + recreate entries)
+ * - Column mapping (select/multi_select backed by tags -> Notion select/multi_select)
+ * - Color mapping (hex -> Notion color names)
+ * - Entry creation with correct property values
+ */
+const RATE_LIMIT_MS = 350;
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+// Reverse mapping: hex color -> Notion color name
+const HEX_TO_NOTION_COLOR = {
+    '#6B7280': 'gray',
+    '#92400E': 'brown',
+    '#F97316': 'orange',
+    '#EAB308': 'yellow',
+    '#22C55E': 'green',
+    '#3B82F6': 'blue',
+    '#8B5CF6': 'purple',
+    '#EC4899': 'pink',
+    '#EF4444': 'red',
+};
+function hexToNotionColor(hex) {
+    if (!hex)
+        return 'default';
+    return HEX_TO_NOTION_COLOR[hex.toUpperCase()] ?? HEX_TO_NOTION_COLOR[hex] ?? 'default';
+}
+/**
+ * Export a single KB database to Notion.
+ *
+ * @param ctx Export context with Notion client, Moxn client, and options
+ * @param kbDatabaseId KB database ID to export
+ * @param parentNotionPageId Notion page ID to create the database under
+ * @returns Export result
+ */
+export async function exportDatabase(ctx, kbDatabaseId, parentNotionPageId) {
+    try {
+        // 1. Fetch resolved database from KB API
+        const resolved = await ctx.moxnClient.resolveDatabase(kbDatabaseId);
+        // 2. Check for existing Notion database mapping
+        let existingMapping = null;
+        try {
+            const mappingResult = await ctx.moxnClient.getNotionDatabaseMapping(kbDatabaseId);
+            existingMapping = mappingResult.mapping;
+        }
+        catch {
+            // No mapping found -- that's fine
+        }
+        // 3. Handle conflict strategy
+        if (existingMapping) {
+            if (ctx.conflictStrategy === 'skip') {
+                return {
+                    kbDatabaseId,
+                    kbDatabaseName: resolved.databaseName,
+                    notionDatabaseId: existingMapping.notionDatabaseId,
+                    status: 'skipped',
+                    entriesCreated: 0,
+                };
+            }
+            if (ctx.dryRun) {
+                return {
+                    kbDatabaseId,
+                    kbDatabaseName: resolved.databaseName,
+                    notionDatabaseId: existingMapping.notionDatabaseId,
+                    status: 'updated',
+                    entriesCreated: resolved.documents.length,
+                };
+            }
+            // Update existing: archive all pages, then recreate entries
+            const entriesCreated = await updateExistingNotionDatabase(ctx, existingMapping.notionDatabaseId, resolved);
+            return {
+                kbDatabaseId,
+                kbDatabaseName: resolved.databaseName,
+                notionDatabaseId: existingMapping.notionDatabaseId,
+                status: 'updated',
+                entriesCreated,
+            };
+        }
+        // 4. No existing mapping -- create new Notion database
+        if (ctx.dryRun) {
+            return {
+                kbDatabaseId,
+                kbDatabaseName: resolved.databaseName,
+                notionDatabaseId: 'dry-run',
+                status: 'created',
+                entriesCreated: resolved.documents.length,
+            };
+        }
+        const { notionDatabaseId, dataSourceId } = await createNotionDatabase(ctx, parentNotionPageId, resolved);
+        // 5. Create entries
+        const entriesCreated = await createDatabaseEntries(ctx, dataSourceId, resolved);
+        // 6. Upsert mapping
+        await ctx.moxnClient.createNotionDatabaseMapping({
+            kbDatabaseId,
+            notionDatabaseId: notionDatabaseId.replace(/-/g, ''),
+            notionDatabaseTitle: resolved.databaseName,
+        });
+        return {
+            kbDatabaseId,
+            kbDatabaseName: resolved.databaseName,
+            notionDatabaseId: notionDatabaseId.replace(/-/g, ''),
+            status: 'created',
+            entriesCreated,
+        };
+    }
+    catch (error) {
+        return {
+            kbDatabaseId,
+            kbDatabaseName: 'Unknown',
+            notionDatabaseId: '',
+            status: 'failed',
+            entriesCreated: 0,
+            error: error instanceof Error ? error.message : String(error),
+        };
+    }
+}
+/**
+ * Create a new Notion database with the correct schema.
+ */
+async function createNotionDatabase(ctx, parentPageId, resolved) {
+    // Build properties from KB columns
+    const properties = {};
+    for (const col of resolved.columns) {
+        const options = col.options.map((opt) => ({
+            name: opt.tagName,
+            color: hexToNotionColor(opt.tagColor),
+        }));
+        if (col.type === 'select') {
+            properties[col.name] = {
+                type: 'select',
+                select: { options },
+            };
+        }
+        else if (col.type === 'multi_select') {
+            properties[col.name] = {
+                type: 'multi_select',
+                multi_select: { options },
+            };
+        }
+    }
+    await sleep(RATE_LIMIT_MS);
+    // Use client.request() for v5 API compatibility
+    // Properties go under initial_data_source.properties (not top-level)
+    const response = (await ctx.notionClient.request({
+        method: 'post',
+        path: 'databases',
+        body: {
+            parent: { type: 'page_id', page_id: parentPageId },
+            title: [{ type: 'text', text: { content: resolved.databaseName } }],
+            initial_data_source: {
+                type: 'external',
+                external: {},
+                properties,
+            },
+        },
+    }));
+    const notionDatabaseId = response.id;
+    const dataSourceId = response.data_sources?.[0]?.id ?? notionDatabaseId;
+    return { notionDatabaseId, dataSourceId };
+}
+/**
+ * Update an existing Notion database by archiving all pages and recreating entries.
+ */
+async function updateExistingNotionDatabase(ctx, notionDatabaseId, resolved) {
+    // Archive existing pages (Notion v5: query database, archive each page)
+    await sleep(RATE_LIMIT_MS);
+    let hasMore = true;
+    let startCursor;
+    while (hasMore) {
+        const queryResult = await ctx.notionClient.databases.query({
+            database_id: notionDatabaseId,
+            start_cursor: startCursor,
+            page_size: 100,
+        });
+        for (const page of queryResult.results) {
+            await sleep(RATE_LIMIT_MS);
+            try {
+                await ctx.notionClient.pages.update({
+                    page_id: page.id,
+                    archived: true,
+                });
+            }
+            catch {
+                // Continue if individual archive fails
+            }
+        }
+        hasMore = queryResult.has_more;
+        startCursor = queryResult.next_cursor ?? undefined;
+    }
+    // Now get the data_source_id for entry creation
+    // Query the database to get its data_sources
+    await sleep(RATE_LIMIT_MS);
+    const dbInfo = (await ctx.notionClient.request({
+        method: 'get',
+        path: `databases/${notionDatabaseId}`,
+    }));
+    const dataSourceId = dbInfo.data_sources?.[0]?.id ?? notionDatabaseId;
+    // Create new entries
+    return createDatabaseEntries(ctx, dataSourceId, resolved);
+}
+/**
+ * Create database entries as Notion pages.
+ */
+async function createDatabaseEntries(ctx, dataSourceId, resolved) {
+    let created = 0;
+    for (const doc of resolved.documents) {
+        try {
+            // Build property values for this entry
+            const properties = {
+                // Title property (always "Name" for databases)
+                Name: {
+                    type: 'title',
+                    title: [{ type: 'text', text: { content: doc.documentName ?? 'Untitled' } }],
+                },
+            };
+            // Add column values
+            for (const [colName, propValue] of Object.entries(doc.properties)) {
+                if (propValue.values.length === 0)
+                    continue;
+                if (propValue.columnType === 'select') {
+                    properties[colName] = {
+                        type: 'select',
+                        select: { name: propValue.values[0].tagName },
+                    };
+                }
+                else if (propValue.columnType === 'multi_select') {
+                    properties[colName] = {
+                        type: 'multi_select',
+                        multi_select: propValue.values.map((v) => ({ name: v.tagName })),
+                    };
+                }
+            }
+            await sleep(RATE_LIMIT_MS);
+            // Create entry as page with data_source_id parent
+            await ctx.notionClient.request({
+                method: 'post',
+                path: 'pages',
+                body: {
+                    parent: { type: 'data_source_id', data_source_id: dataSourceId },
+                    properties,
+                },
+            });
+            created++;
+        }
+        catch (error) {
+            console.error(`  Warning: Failed to create entry "${doc.documentName}": ${error instanceof Error ? error.message : error}`);
+        }
+    }
+    return created;
+}

package/dist/targets/notion.d.ts

@@ -0,0 +1,93 @@
+/**
+ * NotionExportTarget — exports KB documents to Notion pages.
+ *
+ * Converts KB section content (markdown) to Notion blocks using @tryfabric/martian,
+ * then creates or updates Notion pages via the Notion API.
+ *
+ * Supports a two-pass export algorithm:
+ * - Pass 1: Create all pages (without cross-references)
+ * - Pass 2: Append reference blocks using the KB doc ID -> Notion page ID mapping
+ *
+ * Conflict detection uses the MoxnClient's notion-mapping API endpoints
+ * (when available) to find existing Notion pages for a KB document.
+ */
+import { ExportTarget, type ExportTargetConfig, type ExportTargetResult } from './base.js';
+import type { DocumentDetail, DocumentListItem } from '../types.js';
+export interface NotionExportConfig extends ExportTargetConfig {
+    /** Notion integration token */
+    notionToken: string;
+    /** Parent page ID under which to create pages */
+    parentPageId: string;
+    /** How to handle pages that already exist in Notion */
+    conflictStrategy: 'skip' | 'update';
+    /** Moxn API URL (for notion-mapping lookups) */
+    apiUrl: string;
+    /** Moxn API key */
+    apiKey: string;
+}
+/** A cross-reference extracted from section content */
+export interface ExtractedKBReference {
+    /** The KB document ID of the source document */
+    sourceDocumentId: string;
+    /** Target KB path found in markdown links */
+    targetKbPath: string;
+    /** Display text for the reference */
+    displayText: string;
+}
+export declare class NotionExportTarget extends ExportTarget<NotionExportConfig> {
+    private client;
+    /** Cache: kbDocumentId -> notionPageId (from mapping API) */
+    private mappingCache;
+    /** Cache: notionPageId -> true (pages we've verified exist) */
+    private verifiedPages;
+    /** Map from KB path prefix -> Notion page ID (for nested page creation) */
+    private pathToNotionId;
+    /** KB path -> KB document ID mapping (built during two-pass export) */
+    private kbPathToDocId;
+    /** KB doc ID -> Notion page ID mapping (built during pass 1) */
+    private docIdToNotionPageId;
+    constructor(config: NotionExportConfig);
+    get targetType(): string;
+    get targetLocation(): string;
+    validate(): Promise<void>;
+    /**
+     * Register a KB document in the path -> ID index.
+     * Called before pass 1 so references can be resolved in pass 2.
+     */
+    registerDocument(doc: DocumentDetail): void;
+    /**
+     * After pass 1, record a successful export mapping.
+     * Called by the runner when a doc is created/updated successfully.
+     */
+    registerExportedPage(kbDocumentId: string, notionPageId: string): void;
+    /**
+     * Pass 2: Resolve references for a document and append link blocks to its Notion page.
+     *
+     * Finds KB path links in the document content, resolves them to Notion page IDs
+     * (from the current export batch or pre-existing mappings), and appends
+     * mention/link blocks to the Notion page.
+     *
+     * Returns the number of references resolved.
+     */
+    resolveAndAppendReferences(doc: DocumentDetail, notionPageId: string): Promise<{
+        resolved: number;
+        unresolved: number;
+    }>;
+    /**
+     * Resolve a KB path to a Notion page ID.
+     *
+     * Checks three sources in order:
+     * 1. Current export batch (docIdToNotionPageId from pass 1)
+     * 2. Local mapping cache
+     * 3. Remote notion-mapping API
+     */
+    private resolveKBPathToNotionPage;
+    exportDocument(doc: DocumentDetail, listItem: DocumentListItem, dryRun: boolean): Promise<ExportTargetResult>;
+    cleanup(): Promise<void>;
+    private createNotionPage;
+    private updateNotionPage;
+    private clearPageContent;
+    private appendRemainingBlocks;
+    private findExistingNotionPage;
+    private recordMapping;
+}