@moxn/kb-migrate 0.4.6 → 0.4.8

package/dist/targets/notion.js

@@ -0,0 +1,486 @@
+/**
+ * NotionExportTarget — exports KB documents to Notion pages.
+ *
+ * Converts KB section content (markdown) to Notion blocks using @tryfabric/martian,
+ * then creates or updates Notion pages via the Notion API.
+ *
+ * Supports a two-pass export algorithm:
+ * - Pass 1: Create all pages (without cross-references)
+ * - Pass 2: Append reference blocks using the KB doc ID -> Notion page ID mapping
+ *
+ * Conflict detection uses the MoxnClient's notion-mapping API endpoints
+ * (when available) to find existing Notion pages for a KB document.
+ */
+import { Client } from '@notionhq/client';
+import { markdownToBlocks } from '@tryfabric/martian';
+import { ExportTarget, } from './base.js';
+// ============================================
+// Helpers
+// ============================================
+const RATE_LIMIT_MS = 350;
+function sleep(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
+/** Strip <moxn:comment> tags from text, preserving the inner text. */
+function stripCommentTags(text) {
+    return text.replace(/<moxn:comment[^>]*>([\s\S]*?)<\/moxn:comment>/g, '$1');
+}
+/**
+ * Extract KB path references from markdown text.
+ * Matches markdown links where the URL starts with / (KB internal paths).
+ * Returns the extracted references and the text with references removed.
+ */
+function extractKBPathReferences(text) {
+    const references = [];
+    // Match [display text](/kb/path) - links starting with /
+    const KB_PATH_RE = /\[([^\]]+)\]\((\/[^)]+)\)/g;
+    const cleanedText = text.replace(KB_PATH_RE, (match, displayText, path) => {
+        // Only treat as KB reference if it looks like a KB document path
+        // (starts with / and doesn't look like an external URL fragment)
+        if (path.startsWith('/') && !path.startsWith('//')) {
+            references.push({ targetKbPath: path, displayText });
+            // Replace with plain text (the link text) since we'll add references separately
+            return displayText;
+        }
+        return match;
+    });
+    return { cleanedText, references };
+}
+/**
+ * Convert a KB document's sections into a single markdown string.
+ * Section names become H2 headings (mirrors the import convention).
+ *
+ * If extractReferences is true, also extracts KB path references from
+ * the content and returns them separately (for two-pass export).
+ */
+function sectionsToMarkdown(sections, options) {
+    const parts = [];
+    const allReferences = [];
+    const databaseIds = [];
+    const extractRefs = options?.extractReferences ?? false;
+    for (const section of sections) {
+        parts.push(`## ${section.name}\n`);
+        for (const block of section.content) {
+            if (block.blockType === 'text' && block.text) {
+                let text = stripCommentTags(block.text);
+                if (extractRefs) {
+                    const { cleanedText, references } = extractKBPathReferences(text);
+                    text = cleanedText;
+                    allReferences.push(...references);
+                }
+                parts.push(text);
+                parts.push('');
+            }
+            else if (block.blockType === 'image' && block.url) {
+                parts.push(`![${block.alt || ''}](${block.url})`);
+                parts.push('');
+            }
+            else if (block.blockType === 'document' && block.url) {
+                parts.push(`[${block.filename || 'document'}](${block.url})`);
+                parts.push('');
+            }
+            else if (block.blockType === 'csv' && block.url) {
+                parts.push(`[${block.filename || 'data.csv'}](${block.url})`);
+                parts.push('');
+            }
+            else if (block.blockType === 'database_embed' && block.databaseId) {
+                // Collect database ID for Pass 1.5 export
+                databaseIds.push(block.databaseId);
+                // Add a placeholder in the markdown
+                parts.push(`> **[Database]** *(exported as inline database)*`);
+                parts.push('');
+            }
+        }
+    }
+    return { markdown: parts.join('\n').trim(), references: allReferences, databaseIds };
+}
+// Max 100 blocks per API call
+const MAX_BLOCKS_PER_APPEND = 100;
+// ============================================
+// Target
+// ============================================
+export class NotionExportTarget extends ExportTarget {
+    client;
+    /** Cache: kbDocumentId -> notionPageId (from mapping API) */
+    mappingCache = new Map();
+    /** Cache: notionPageId -> true (pages we've verified exist) */
+    verifiedPages = new Set();
+    /** Map from KB path prefix -> Notion page ID (for nested page creation) */
+    pathToNotionId = new Map();
+    /** KB path -> KB document ID mapping (built during two-pass export) */
+    kbPathToDocId = new Map();
+    /** KB doc ID -> Notion page ID mapping (built during pass 1) */
+    docIdToNotionPageId = new Map();
+    constructor(config) {
+        super(config);
+        this.client = new Client({ auth: config.notionToken });
+    }
+    get targetType() {
+        return 'notion';
+    }
+    get targetLocation() {
+        return `Notion (parent: ${this.config.parentPageId})`;
+    }
+    // ============================================
+    // Validation
+    // ============================================
+    async validate() {
+        console.log('Validating Notion API token...');
+        try {
+            await this.client.users.me({});
+            console.log('  Token valid.');
+        }
+        catch (error) {
+            throw new Error(`Notion token invalid: ${error instanceof Error ? error.message : error}`);
+        }
+        console.log('Verifying parent page...');
+        try {
+            await sleep(RATE_LIMIT_MS);
+            await this.client.pages.retrieve({ page_id: this.config.parentPageId });
+            console.log('  Parent page accessible.');
+        }
+        catch (error) {
+            throw new Error(`Parent page not accessible: ${error instanceof Error ? error.message : error}`);
+        }
+    }
+    // ============================================
+    // Two-Pass Export Support
+    // ============================================
+    /**
+     * Register a KB document in the path -> ID index.
+     * Called before pass 1 so references can be resolved in pass 2.
+     */
+    registerDocument(doc) {
+        this.kbPathToDocId.set(doc.path, doc.id);
+    }
+    /**
+     * After pass 1, record a successful export mapping.
+     * Called by the runner when a doc is created/updated successfully.
+     */
+    registerExportedPage(kbDocumentId, notionPageId) {
+        this.docIdToNotionPageId.set(kbDocumentId, notionPageId);
+    }
+    /**
+     * Pass 2: Resolve references for a document and append link blocks to its Notion page.
+     *
+     * Finds KB path links in the document content, resolves them to Notion page IDs
+     * (from the current export batch or pre-existing mappings), and appends
+     * mention/link blocks to the Notion page.
+     *
+     * Returns the number of references resolved.
+     */
+    async resolveAndAppendReferences(doc, notionPageId) {
+        // Extract references from section content
+        const { references } = sectionsToMarkdown(doc.sections, { extractReferences: true });
+        if (references.length === 0) {
+            return { resolved: 0, unresolved: 0 };
+        }
+        // Deduplicate references by target path
+        const seen = new Set();
+        const uniqueRefs = references.filter((ref) => {
+            if (seen.has(ref.targetKbPath))
+                return false;
+            seen.add(ref.targetKbPath);
+            return true;
+        });
+        const referenceBlocks = [];
+        let resolved = 0;
+        let unresolved = 0;
+        for (const ref of uniqueRefs) {
+            const targetNotionPageId = await this.resolveKBPathToNotionPage(ref.targetKbPath);
+            if (targetNotionPageId) {
+                // Create a mention block linking to the Notion page
+                referenceBlocks.push({
+                    object: 'block',
+                    type: 'paragraph',
+                    paragraph: {
+                        rich_text: [
+                            { type: 'text', text: { content: 'See also: ' } },
+                            {
+                                type: 'mention',
+                                mention: {
+                                    type: 'page',
+                                    page: { id: targetNotionPageId },
+                                },
+                            },
+                        ],
+                    },
+                });
+                resolved++;
+            }
+            else {
+                // Graceful degradation: plain text reference
+                referenceBlocks.push({
+                    object: 'block',
+                    type: 'paragraph',
+                    paragraph: {
+                        rich_text: [
+                            {
+                                type: 'text',
+                                text: { content: `See also: ${ref.displayText} (${ref.targetKbPath})` },
+                                annotations: { italic: true, color: 'gray' },
+                            },
+                        ],
+                    },
+                });
+                unresolved++;
+            }
+        }
+        if (referenceBlocks.length > 0) {
+            // Add a divider before references section
+            const blocksToAppend = [
+                { object: 'block', type: 'divider', divider: {} },
+                {
+                    object: 'block',
+                    type: 'heading_3',
+                    heading_3: {
+                        rich_text: [{ type: 'text', text: { content: 'References' } }],
+                    },
+                },
+                ...referenceBlocks,
+            ];
+            await this.appendRemainingBlocks(notionPageId, blocksToAppend);
+        }
+        return { resolved, unresolved };
+    }
+    /**
+     * Resolve a KB path to a Notion page ID.
+     *
+     * Checks three sources in order:
+     * 1. Current export batch (docIdToNotionPageId from pass 1)
+     * 2. Local mapping cache
+     * 3. Remote notion-mapping API
+     */
+    async resolveKBPathToNotionPage(kbPath) {
+        // 1. Try to find the KB doc ID from path
+        const kbDocId = this.kbPathToDocId.get(kbPath);
+        if (kbDocId) {
+            // 2. Check if it was exported in this batch
+            const batchNotionId = this.docIdToNotionPageId.get(kbDocId);
+            if (batchNotionId)
+                return batchNotionId;
+            // 3. Check mapping cache / API
+            const mappedNotionId = await this.findExistingNotionPage(kbDocId);
+            if (mappedNotionId)
+                return mappedNotionId;
+        }
+        return null;
+    }
+    // ============================================
+    // Export (Pass 1)
+    // ============================================
+    async exportDocument(doc, listItem, dryRun) {
+        const startTime = Date.now();
+        try {
+            const existingNotionPageId = await this.findExistingNotionPage(doc.id);
+            if (existingNotionPageId) {
+                if (this.config.conflictStrategy === 'skip') {
+                    this.registerExportedPage(doc.id, existingNotionPageId);
+                    return {
+                        documentId: doc.id,
+                        documentPath: doc.path,
+                        status: 'skipped',
+                        externalId: existingNotionPageId,
+                        duration: Date.now() - startTime,
+                    };
+                }
+                if (dryRun) {
+                    return {
+                        documentId: doc.id,
+                        documentPath: doc.path,
+                        status: 'updated',
+                        externalId: existingNotionPageId,
+                        duration: Date.now() - startTime,
+                    };
+                }
+                await this.updateNotionPage(existingNotionPageId, doc);
+                await this.recordMapping(doc.id, existingNotionPageId);
+                this.registerExportedPage(doc.id, existingNotionPageId);
+                return {
+                    documentId: doc.id,
+                    documentPath: doc.path,
+                    status: 'updated',
+                    externalId: existingNotionPageId,
+                    duration: Date.now() - startTime,
+                };
+            }
+            if (dryRun) {
+                return {
+                    documentId: doc.id,
+                    documentPath: doc.path,
+                    status: 'created',
+                    duration: Date.now() - startTime,
+                };
+            }
+            const notionPageId = await this.createNotionPage(doc);
+            await this.recordMapping(doc.id, notionPageId);
+            this.registerExportedPage(doc.id, notionPageId);
+            return {
+                documentId: doc.id,
+                documentPath: doc.path,
+                status: 'created',
+                externalId: notionPageId,
+                duration: Date.now() - startTime,
+            };
+        }
+        catch (error) {
+            console.error(`Export failed for ${doc.path}: ${error instanceof Error ? error.message : error}`);
+            return {
+                documentId: doc.id,
+                documentPath: doc.path,
+                status: 'failed',
+                error: error instanceof Error ? error.message : String(error),
+                duration: Date.now() - startTime,
+            };
+        }
+    }
+    async cleanup() {
+        // No-op
+    }
+    // ============================================
+    // Notion page creation / update
+    // ============================================
+    async createNotionPage(doc) {
+        const { markdown } = sectionsToMarkdown(doc.sections, { extractReferences: true });
+        const blocks = markdownToBlocks(markdown);
+        // First batch: up to 100 blocks as children of the new page
+        const firstBatch = blocks.slice(0, MAX_BLOCKS_PER_APPEND);
+        const remainingBlocks = blocks.slice(MAX_BLOCKS_PER_APPEND);
+        await sleep(RATE_LIMIT_MS);
+        const response = await this.client.pages.create({
+            parent: { page_id: this.config.parentPageId },
+            properties: {
+                title: {
+                    type: 'title',
+                    title: [{ type: 'text', text: { content: doc.name } }],
+                },
+            },
+            children: firstBatch,
+        });
+        const pageId = response.id;
+        // Append remaining blocks in batches
+        await this.appendRemainingBlocks(pageId, remainingBlocks);
+        return pageId;
+    }
+    async updateNotionPage(notionPageId, doc) {
+        await sleep(RATE_LIMIT_MS);
+        await this.client.pages.update({
+            page_id: notionPageId,
+            properties: {
+                title: {
+                    type: 'title',
+                    title: [{ type: 'text', text: { content: doc.name } }],
+                },
+            },
+        });
+        await this.clearPageContent(notionPageId);
+        const { markdown } = sectionsToMarkdown(doc.sections, { extractReferences: true });
+        const blocks = markdownToBlocks(markdown);
+        await this.appendRemainingBlocks(notionPageId, blocks);
+    }
+    async clearPageContent(pageId) {
+        await sleep(RATE_LIMIT_MS);
+        const children = await this.client.blocks.children.list({
+            block_id: pageId,
+            page_size: 100,
+        });
+        for (const block of children.results) {
+            try {
+                await sleep(RATE_LIMIT_MS);
+                const deletePromise = this.client.blocks.delete({ block_id: block.id });
+                const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Block delete timeout')), 10000));
+                await Promise.race([deletePromise, timeoutPromise]);
+            }
+            catch {
+                // Continue with other blocks even if one fails
+            }
+        }
+        // Handle pagination
+        if (children.has_more && children.next_cursor) {
+            let cursor = children.next_cursor;
+            while (cursor) {
+                try {
+                    await sleep(RATE_LIMIT_MS);
+                    const more = await this.client.blocks.children.list({
+                        block_id: pageId,
+                        start_cursor: cursor,
+                        page_size: 100,
+                    });
+                    for (const block of more.results) {
+                        try {
+                            await sleep(RATE_LIMIT_MS);
+                            const deletePromise = this.client.blocks.delete({ block_id: block.id });
+                            const timeoutPromise = new Promise((_, reject) => setTimeout(() => reject(new Error('Block delete timeout')), 10000));
+                            await Promise.race([deletePromise, timeoutPromise]);
+                        }
+                        catch {
+                            // Continue on failure
+                        }
+                    }
+                    cursor = more.has_more ? more.next_cursor : null;
+                }
+                catch {
+                    break;
+                }
+            }
+        }
+    }
+    async appendRemainingBlocks(pageId, blocks) {
+        for (let i = 0; i < blocks.length; i += MAX_BLOCKS_PER_APPEND) {
+            const batch = blocks.slice(i, i + MAX_BLOCKS_PER_APPEND);
+            await sleep(RATE_LIMIT_MS);
+            await this.client.blocks.children.append({
+                block_id: pageId,
+                children: batch,
+            });
+        }
+    }
+    // ============================================
+    // Mapping lookups
+    // ============================================
+    async findExistingNotionPage(kbDocumentId) {
+        // Check cache first
+        const cached = this.mappingCache.get(kbDocumentId);
+        if (cached)
+            return cached;
+        // Try notion-mapping API
+        try {
+            const response = await fetch(`${this.config.apiUrl}/api/v1/kb/notion-mappings/by-document/${kbDocumentId}`, {
+                headers: { 'x-api-key': this.config.apiKey },
+            });
+            if (response.ok) {
+                const data = (await response.json());
+                if (data.mapping?.notionPageId) {
+                    this.mappingCache.set(kbDocumentId, data.mapping.notionPageId);
+                    return data.mapping.notionPageId;
+                }
+            }
+        }
+        catch {
+            // Mapping API might not be deployed yet - that's OK, treat as no mapping
+        }
+        return null;
+    }
+    async recordMapping(kbDocumentId, notionPageId) {
+        this.mappingCache.set(kbDocumentId, notionPageId);
+        // Try to save mapping via API (best-effort)
+        try {
+            await fetch(`${this.config.apiUrl}/api/v1/kb/notion-mappings`, {
+                method: 'POST',
+                headers: {
+                    'Content-Type': 'application/json',
+                    'x-api-key': this.config.apiKey,
+                },
+                body: JSON.stringify({
+                    kbDocumentId,
+                    notionPageId,
+                    importSource: 'export',
+                    syncDirection: 'kb_to_notion',
+                }),
+            });
+        }
+        catch {
+            // Best-effort — mapping API might not be available
+        }
+    }
+}

package/dist/types.d.ts

@@ -8,7 +8,7 @@
  * to kb-migrate (local filesystem paths) — MoxnClient converts them to
  * `type: 'storage'` before sending to the KB API.
  */
-export type ContentBlock = TextBlock | ImageRemoteBlock | ImageFileBlock | DocumentRemoteBlock | DocumentFileBlock | CsvRemoteBlock | CsvFileBlock;
+export type ContentBlock = TextBlock | ImageRemoteBlock | ImageFileBlock | DocumentRemoteBlock | DocumentFileBlock | CsvRemoteBlock | CsvFileBlock | DatabaseEmbedBlock;
 export interface TextBlock {
     blockType: 'text';
     text: string;
@@ -65,6 +65,10 @@ export interface CsvFileBlock {
     headers?: string[];
     rowCount?: number;
 }
+export interface DatabaseEmbedBlock {
+    blockType: 'database_embed';
+    databaseId: string;
+}
 /**
  * A section to be created in a document
  */
@@ -98,6 +102,12 @@ export interface ExtractedDocument {
     sourcePath: string;
     /** Cross-references discovered during resolution */
     references?: ExtractedReference[];
+    /** Source-specific metadata (e.g., Notion page ID for bidirectional sync) */
+    metadata?: {
+        notionPageId?: string;
+        notionTitle?: string;
+        [key: string]: unknown;
+    };
 }
 /**
  * Status of a single document migration
@@ -162,6 +172,8 @@ export interface MigrationOptions {
     aiAccess?: 'edit' | 'read' | 'none';
     /** Convenience flag: 'team' = read, 'private' = none */
     visibility?: 'team' | 'private';
+    /** Date filter for source documents */
+    dateFilter?: import('./date-filter.js').DateFilter;
 }
 /**
  * Error response from API when document already exists
@@ -175,13 +187,14 @@ export interface ConflictError {
  * A content block as returned by the KB API
  */
 export interface ExportContentBlock {
-    blockType: 'text' | 'image' | 'document' | 'csv';
+    blockType: 'text' | 'image' | 'document' | 'csv' | 'database_embed';
     text?: string;
     url?: string;
     mimeType?: string;
     alt?: string;
     filename?: string;
     storageKey?: string;
+    databaseId?: string;
 }
 /**
  * A section within a document (API response)
@@ -201,6 +214,7 @@ export interface DocumentListItem {
     name: string;
     description: string | null;
     createdAt: string;
+    updatedAt: string | null;
 }
 /**
  * Full document detail with sections
@@ -271,4 +285,6 @@ export interface ExportOptions {
     pdfDir: string;
     csvDir: string;
     dryRun: boolean;
+    /** Date filter for exported documents */
+    dateFilter?: import('./date-filter.js').DateFilter;
 }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moxn/kb-migrate",
-  "version": "0.4.6",
+  "version": "0.4.8",
   "description": "Migration tool for importing documents into Moxn Knowledge Base from local files, Notion, Google Docs, and more",
   "type": "module",
   "main": "dist/index.js",
@@ -49,6 +49,26 @@
     "./client": {
       "types": "./dist/client.d.ts",
       "default": "./dist/client.js"
+    },
+    "./date-filter": {
+      "types": "./dist/date-filter.d.ts",
+      "default": "./dist/date-filter.js"
+    },
+    "./targets/base": {
+      "types": "./dist/targets/base.d.ts",
+      "default": "./dist/targets/base.js"
+    },
+    "./targets/notion": {
+      "types": "./dist/targets/notion.d.ts",
+      "default": "./dist/targets/notion.js"
+    },
+    "./targets": {
+      "types": "./dist/targets/index.d.ts",
+      "default": "./dist/targets/index.js"
+    },
+    "./export-notion": {
+      "types": "./dist/export-notion.d.ts",
+      "default": "./dist/export-notion.js"
     }
   },
   "bin": {
@@ -62,6 +82,8 @@
     "prepublishOnly": "npm run build"
   },
   "dependencies": {
+    "@notionhq/client": "^2.3.0",
+    "@tryfabric/martian": "^1.2.4",
     "commander": "^12.0.0",
     "glob": "^10.0.0",
     "remark-parse": "^11.0.0",