@moxn/kb-migrate 0.2.2 → 0.4.0

package/dist/client.d.ts

@@ -28,7 +28,56 @@ export declare class MoxnClient {
     private buildPath;
     private processSections;
     private processContentBlocks;
+    /**
+     * Upload a file to Moxn storage and return the storage key.
+     */
+    uploadFile(data: Buffer, mimeType: string, filename?: string): Promise<{
+        key: string;
+    }>;
+    private getUploadUrl;
     private createDocument;
     private updateDocument;
+    /**
+     * Create a KB database.
+     */
+    createDatabase(input: {
+        name: string;
+        description?: string;
+    }): Promise<{
+        id: string;
+        name: string;
+    }>;
+    /**
+     * Add a column to a KB database.
+     */
+    addDatabaseColumn(databaseId: string, input: {
+        name: string;
+        type: 'select' | 'multi_select';
+        optionTagIds?: string[];
+        newOptionParentPath?: string;
+    }): Promise<{
+        id: string;
+        name: string;
+    }>;
+    /**
+     * Add a document to a KB database.
+     */
+    addDocumentToDatabase(databaseId: string, documentId: string): Promise<void>;
+    /**
+     * Create a tag (with automatic ancestor creation).
+     */
+    createTag(input: {
+        path: string;
+        description?: string;
+        color?: string;
+    }): Promise<{
+        id: string;
+        path: string;
+        name: string;
+    }>;
+    /**
+     * Assign a tag to a document.
+     */
+    assignTag(documentId: string, tagId: string, branchId: string): Promise<void>;
     private isConflictError;
 }

package/dist/client.js

@@ -179,20 +179,82 @@ export class MoxnClient {
     }
     async processContentBlocks(blocks) {
         return Promise.all(blocks.map(async (block) => {
+            // Handle image files - upload to storage
             if (block.blockType === 'image' && block.type === 'file' && block.path) {
-                // Convert local file to base64
                 const data = await fs.readFile(block.path);
+                const filename = block.path.split('/').pop();
+                const { key } = await this.uploadFile(data, block.mediaType, filename);
                 return {
                     blockType: block.blockType,
-                    type: 'base64',
-                    base64: data.toString('base64'),
+                    type: 'storage',
+                    key,
                     mediaType: block.mediaType,
                     alt: block.alt,
                 };
             }
+            // Handle PDF/document files - upload to storage
+            if (block.blockType === 'document' && block.type === 'file' && block.path) {
+                const data = await fs.readFile(block.path);
+                const filename = block.filename || block.path.split('/').pop();
+                const { key } = await this.uploadFile(data, block.mediaType || 'application/pdf', filename);
+                return {
+                    blockType: block.blockType,
+                    type: 'storage',
+                    key,
+                    mediaType: block.mediaType || 'application/pdf',
+                    filename: block.filename,
+                };
+            }
+            // Handle CSV files - upload to storage
+            if (block.blockType === 'csv' && block.type === 'file' && block.path) {
+                const data = await fs.readFile(block.path);
+                const { key } = await this.uploadFile(data, 'text/csv', block.filename);
+                return {
+                    blockType: block.blockType,
+                    type: 'storage',
+                    key,
+                    mediaType: 'text/csv',
+                    filename: block.filename,
+                    headers: block.headers,
+                    rowCount: block.rowCount,
+                };
+            }
             return block;
         }));
     }
+    /**
+     * Upload a file to Moxn storage and return the storage key.
+     */
+    async uploadFile(data, mimeType, filename) {
+        // 1. Get presigned upload URL
+        const { key, uploadUrl } = await this.getUploadUrl(mimeType, filename);
+        // 2. PUT file to presigned URL
+        const response = await fetch(uploadUrl, {
+            method: 'PUT',
+            headers: { 'Content-Type': mimeType },
+            body: new Uint8Array(data),
+        });
+        if (!response.ok) {
+            throw new Error(`File upload failed: ${response.status}`);
+        }
+        return { key };
+    }
+    async getUploadUrl(type, filename) {
+        const response = await fetch(`${this.apiUrl}/api/v1/kb/upload`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'x-api-key': this.apiKey,
+            },
+            body: JSON.stringify({ type, filename }),
+        });
+        if (!response.ok) {
+            const body = await response.text();
+            throw new Error(`Upload URL request failed: ${response.status} ${body}`);
+        }
+        const data = await response.json();
+        return { key: data.key, uploadUrl: data.uploadUrl };
+    }
     async createDocument(request) {
         const response = await fetch(`${this.apiUrl}/api/v1/kb/documents`, {
             method: 'POST',
@@ -229,6 +291,108 @@ export class MoxnClient {
         }
         return response.json();
     }
+    // ──────────────────────────────────────────────
+    // Database & tag methods (for Notion import)
+    // ──────────────────────────────────────────────
+    /**
+     * Create a KB database.
+     */
+    async createDatabase(input) {
+        const response = await fetch(`${this.apiUrl}/api/v1/kb/databases`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'x-api-key': this.apiKey,
+            },
+            body: JSON.stringify(input),
+        });
+        if (!response.ok) {
+            const body = await response.json().catch(() => ({}));
+            throw new Error(body.error || `Failed to create database: ${response.status}`);
+        }
+        return response.json();
+    }
+    /**
+     * Add a column to a KB database.
+     */
+    async addDatabaseColumn(databaseId, input) {
+        const response = await fetch(`${this.apiUrl}/api/v1/kb/databases/${databaseId}/columns`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'x-api-key': this.apiKey,
+            },
+            body: JSON.stringify(input),
+        });
+        if (!response.ok) {
+            const body = await response.json().catch(() => ({}));
+            throw new Error(body.error || `Failed to add column: ${response.status}`);
+        }
+        return response.json();
+    }
+    /**
+     * Add a document to a KB database.
+     */
+    async addDocumentToDatabase(databaseId, documentId) {
+        const response = await fetch(`${this.apiUrl}/api/v1/kb/databases/${databaseId}/documents`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'x-api-key': this.apiKey,
+            },
+            body: JSON.stringify({ documentId }),
+        });
+        if (!response.ok) {
+            const body = await response.json().catch(() => ({}));
+            // Ignore 409 — document already in database
+            if (response.status === 409)
+                return;
+            throw new Error(body.error || `Failed to add document to database: ${response.status}`);
+        }
+    }
+    /**
+     * Create a tag (with automatic ancestor creation).
+     */
+    async createTag(input) {
+        const response = await fetch(`${this.apiUrl}/api/v1/kb/tags`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'x-api-key': this.apiKey,
+            },
+            body: JSON.stringify(input),
+        });
+        if (!response.ok) {
+            const body = await response.json().catch(() => ({}));
+            // If tag already exists (409), try to find it
+            if (response.status === 409 && body.tagId) {
+                return {
+                    id: body.tagId,
+                    path: input.path,
+                    name: input.path.split('/').pop() || '',
+                };
+            }
+            throw new Error(body.error || `Failed to create tag: ${response.status}`);
+        }
+        return response.json();
+    }
+    /**
+     * Assign a tag to a document.
+     */
+    async assignTag(documentId, tagId, branchId) {
+        const response = await fetch(`${this.apiUrl}/api/v1/kb/documents/${documentId}/tags`, {
+            method: 'POST',
+            headers: {
+                'Content-Type': 'application/json',
+                'x-api-key': this.apiKey,
+            },
+            body: JSON.stringify({ tagId, branchId }),
+        });
+        if (!response.ok) {
+            const body = await response.json().catch(() => ({}));
+            throw new Error(body.error || `Failed to assign tag: ${response.status}`);
+        }
+    }
     isConflictError(error) {
         return (error instanceof Error &&
             'documentId' in error &&

package/dist/index.js

@@ -14,6 +14,8 @@
  */
 import { Command } from 'commander';
 import { LocalSource } from './sources/local.js';
+import { NotionSource } from './sources/notion.js';
+import { notionColorToHex } from './sources/notion-api.js';
 import { MoxnClient } from './client.js';
 import { runExport } from './export.js';
 const DEFAULT_API_URL = 'https://moxn.dev';
@@ -216,10 +218,211 @@ program
         process.exit(1);
     }
 });
-// Future: Add notion and google-docs commands here
-// program
-//   .command('notion')
-//   .description('Migrate documents from Notion')
-//   .option('--token <token>', 'Notion integration token')
-//   .action(...)
+program
+    .command('notion')
+    .description('Migrate documents from a Notion workspace')
+    .option('--token <token>', 'Notion integration token (or set NOTION_TOKEN env var)')
+    .option('--api-key <key>', 'Moxn API key (or set MOXN_API_KEY env var)')
+    .option('--api-url <url>', 'Moxn API base URL', DEFAULT_API_URL)
+    .option('--base-path <path>', 'Base path for imported documents', '/imported-from-notion')
+    .option('--root-page-id <id>', 'Import subtree starting from this Notion page ID')
+    .option('--max-depth <n>', 'Maximum nesting depth')
+    .option('--on-conflict <action>', 'Action on conflict: skip or update', 'skip')
+    .option('--default-permission <perm>', 'Default permission: edit, read, or none')
+    .option('--ai-access <perm>', 'AI access permission: edit, read, or none')
+    .option('--visibility <vis>', 'Convenience flag: team (read) or private (none)')
+    .option('--dry-run', 'Preview without making changes', false)
+    .option('--json', 'Output results as JSON', false)
+    .action(async (opts) => {
+    const token = opts.token || process.env.NOTION_TOKEN;
+    if (!token) {
+        console.error('Error: Notion token required. Use --token or set NOTION_TOKEN env var.');
+        process.exit(1);
+    }
+    const apiKey = opts.apiKey || process.env.MOXN_API_KEY;
+    if (!apiKey) {
+        console.error('Error: Moxn API key required. Use --api-key or set MOXN_API_KEY env var.');
+        process.exit(1);
+    }
+    const onConflict = opts.onConflict;
+    if (!['skip', 'update'].includes(onConflict)) {
+        console.error('Error: --on-conflict must be "skip" or "update"');
+        process.exit(1);
+    }
+    // Resolve visibility → defaultPermission (visibility is syntactic sugar)
+    let defaultPermission = opts.defaultPermission;
+    if (!defaultPermission && opts.visibility) {
+        defaultPermission = opts.visibility === 'private' ? 'none' : 'read';
+    }
+    const source = new NotionSource({
+        token,
+        rootPageId: opts.rootPageId,
+        maxDepth: opts.maxDepth ? parseInt(opts.maxDepth, 10) : undefined,
+    });
+    const migrationOptions = {
+        apiUrl: opts.apiUrl,
+        apiKey,
+        basePath: opts.basePath,
+        onConflict,
+        dryRun: opts.dryRun,
+        defaultPermission,
+        aiAccess: opts.aiAccess,
+        visibility: opts.visibility,
+    };
+    try {
+        // Run page migration
+        const log = await runMigration(source, migrationOptions);
+        // After page migration, import databases
+        if (!opts.dryRun) {
+            const dbImports = source.getDatabaseImports();
+            if (dbImports.length > 0) {
+                console.log(`\nImporting ${dbImports.length} database(s)...`);
+                const client = new MoxnClient(migrationOptions);
+                for (const dbImport of dbImports) {
+                    try {
+                        await importNotionDatabase(client, dbImport, log, migrationOptions);
+                    }
+                    catch (error) {
+                        console.error(`  Error importing database "${dbImport.schema.name}": ${error instanceof Error ? error.message : error}`);
+                    }
+                }
+            }
+        }
+        // Cleanup temp files
+        await source.cleanup();
+        if (opts.json) {
+            console.log(JSON.stringify(log, null, 2));
+        }
+        else {
+            printSummary(log);
+        }
+        if (log.summary.failed > 0) {
+            process.exit(1);
+        }
+    }
+    catch (error) {
+        await source.cleanup().catch(() => { });
+        console.error('Migration failed:', error instanceof Error ? error.message : error);
+        process.exit(1);
+    }
+});
+/**
+ * Import a Notion database into Moxn KB.
+ *
+ * Creates the database, columns with tags, links entries, and assigns tag values.
+ */
+async function importNotionDatabase(client, dbImport, log, options) {
+    const { schema, entries } = dbImport;
+    console.log(`  Creating database: ${schema.name}`);
+    // 1. Create the database
+    const db = await client.createDatabase({
+        name: schema.name,
+        description: schema.description || undefined,
+    });
+    console.log(`    Database created: ${db.id}`);
+    // 2. Create columns with tags
+    // Map: column name → { columnId, optionTagMap: option name → tagId }
+    const columnMap = new Map();
+    for (const col of schema.mappedColumns) {
+        const tagBasePath = `/imported/${slugifyTagPath(schema.name)}/${slugifyTagPath(col.notionPropertyName)}`;
+        // Create tags for each option
+        const optionTagMap = new Map();
+        const tagIds = [];
+        for (const option of col.options) {
+            const tagPath = `${tagBasePath}/${slugifyTagPath(option.name)}`;
+            try {
+                const tag = await client.createTag({
+                    path: tagPath,
+                    color: notionColorToHex(option.color),
+                });
+                optionTagMap.set(option.name, tag.id);
+                tagIds.push(tag.id);
+            }
+            catch (error) {
+                console.warn(`    Warning: Failed to create tag "${tagPath}": ${error instanceof Error ? error.message : error}`);
+            }
+        }
+        // Create the column
+        try {
+            const column = await client.addDatabaseColumn(db.id, {
+                name: col.notionPropertyName,
+                type: col.moxnType,
+                optionTagIds: tagIds,
+                newOptionParentPath: tagBasePath,
+            });
+            columnMap.set(col.notionPropertyName, {
+                columnId: column.id,
+                optionTagMap,
+            });
+            console.log(`    Column "${col.notionPropertyName}" (${col.moxnType}) with ${tagIds.length} options`);
+        }
+        catch (error) {
+            console.warn(`    Warning: Failed to create column "${col.notionPropertyName}": ${error instanceof Error ? error.message : error}`);
+        }
+    }
+    // 3. Link entries and assign tags
+    // Build a map of KB path → { documentId, branchId } from migration results
+    const docByPath = new Map();
+    for (const result of log.results) {
+        if (result.documentId && result.branchId) {
+            docByPath.set(result.documentPath, {
+                documentId: result.documentId,
+                branchId: result.branchId,
+            });
+        }
+    }
+    let linkedCount = 0;
+    for (const entry of entries) {
+        const kbPath = '/' +
+            options.basePath.replace(/^\/+|\/+$/g, '') +
+            '/' +
+            entry.kbPath.replace(/^\/+/, '');
+        const docInfo = docByPath.get(kbPath);
+        if (!docInfo) {
+            // Document wasn't created (maybe skipped or failed)
+            continue;
+        }
+        // Add document to database
+        try {
+            await client.addDocumentToDatabase(db.id, docInfo.documentId);
+            linkedCount++;
+            // Parse and assign tag values
+            const { parseEntryValues } = await import('./sources/notion-databases.js');
+            const values = parseEntryValues(entry.page, schema);
+            for (const [colName, selectedOptions] of values.tagValues) {
+                const colInfo = columnMap.get(colName);
+                if (!colInfo)
+                    continue;
+                for (const optionName of selectedOptions) {
+                    const tagId = colInfo.optionTagMap.get(optionName);
+                    if (tagId) {
+                        try {
+                            await client.assignTag(docInfo.documentId, tagId, docInfo.branchId);
+                        }
+                        catch (error) {
+                            console.warn(`    Warning: Failed to assign tag "${optionName}" to ${docInfo.documentId}: ${error instanceof Error ? error.message : error}`);
+                        }
+                    }
+                }
+            }
+        }
+        catch (error) {
+            console.warn(`    Warning: Failed to link entry to database: ${error instanceof Error ? error.message : error}`);
+        }
+    }
+    console.log(`    Linked ${linkedCount}/${entries.length} entries`);
+}
+/**
+ * Slugify a string for use as a tag path segment.
+ * Similar to page slug but for the tag hierarchy.
+ */
+function slugifyTagPath(s) {
+    return (s
+        .toLowerCase()
+        .trim()
+        .replace(/[^\w\s-]/g, '')
+        .replace(/[\s_]+/g, '-')
+        .replace(/-+/g, '-')
+        .replace(/^-|-$/g, '') || 'untitled');
+}
 program.parse();

package/dist/sources/index.d.ts

@@ -3,3 +3,4 @@
  */
 export { MigrationSource, type SourceConfig } from './base.js';
 export { LocalSource, type LocalSourceConfig } from './local.js';
+export { NotionSource, type NotionSourceConfig } from './notion.js';

package/dist/sources/index.js

@@ -3,6 +3,4 @@
  */
 export { MigrationSource } from './base.js';
 export { LocalSource } from './local.js';
-// Future sources:
-// export { NotionSource, type NotionSourceConfig } from './notion.js';
-// export { GoogleDocsSource, type GoogleDocsSourceConfig } from './google-docs.js';
+export { NotionSource } from './notion.js';

package/dist/sources/local.d.ts

@@ -27,10 +27,14 @@ export declare class LocalSource extends MigrationSource<LocalSourceConfig> {
     private parseMarkdownSections;
     private nodesToContentBlocks;
     /**
-     * Extract image nodes from paragraph children, returning image blocks
-     * and whether non-image text content exists.
+     * Extract image nodes and CSV links from paragraph children, returning media blocks
+     * and whether non-media text content exists.
      */
     private extractImagesFromParagraph;
+    /**
+     * Convert a markdown link to a CSV block if it points to a local CSV file.
+     */
+    private linkToCSVBlock;
     private imageToBlock;
     private guessImageType;
     private extensionToMediaType;

package/dist/sources/local.js

@@ -4,6 +4,7 @@
  * Extracts documents from local markdown and text files.
  */
 import * as fs from 'fs/promises';
+import * as fsSync from 'fs';
 import * as path from 'path';
 import { glob } from 'glob';
 import { unified } from 'unified';
@@ -85,7 +86,6 @@ export class LocalSource extends MigrationSource {
     }
     async extractDocument(relativePath) {
         const fullPath = path.join(this.config.directory, relativePath);
-        const content = await fs.readFile(fullPath, 'utf-8');
         // Parse relative path to create document path
         const parsed = path.parse(relativePath);
         const dirParts = parsed.dir ? parsed.dir.split(path.sep) : [];
@@ -100,6 +100,7 @@ export class LocalSource extends MigrationSource {
             .split(/[-_]/)
             .map((word) => word.charAt(0).toUpperCase() + word.slice(1))
             .join(' ');
+        const content = await fs.readFile(fullPath, 'utf-8');
         // Parse into sections
         const sections = this.parseMarkdownSections(content, path.dirname(fullPath));
         if (sections.length === 0) {
@@ -218,8 +219,8 @@ export class LocalSource extends MigrationSource {
         return blocks;
     }
     /**
-     * Extract image nodes from paragraph children, returning image blocks
-     * and whether non-image text content exists.
+     * Extract image nodes and CSV links from paragraph children, returning media blocks
+     * and whether non-media text content exists.
      */
     extractImagesFromParagraph(children, baseDir) {
         const images = [];
@@ -231,6 +232,20 @@ export class LocalSource extends MigrationSource {
                     images.push(imageBlock);
                 }
             }
+            else if (child.type === 'link') {
+                // Check if this is a link to a CSV file
+                const csvBlock = this.linkToCSVBlock(child, baseDir);
+                if (csvBlock) {
+                    images.push(csvBlock);
+                }
+                else {
+                    // Regular link, treat as text
+                    const text = this.nodeToMarkdown(child).trim();
+                    if (text) {
+                        hasText = true;
+                    }
+                }
+            }
             else {
                 // Check if the child has any meaningful text
                 const text = this.nodeToMarkdown(child).trim();
@@ -241,6 +256,36 @@ export class LocalSource extends MigrationSource {
         }
         return { images, hasText };
     }
+    /**
+     * Convert a markdown link to a CSV block if it points to a local CSV file.
+     */
+    linkToCSVBlock(node, baseDir) {
+        const href = node.url;
+        // Only handle local CSV files
+        if (href.startsWith('http://') || href.startsWith('https://')) {
+            return null;
+        }
+        if (!href.toLowerCase().endsWith('.csv')) {
+            return null;
+        }
+        const csvPath = path.isAbsolute(href) ? href : path.join(baseDir, href);
+        if (!fsSync.existsSync(csvPath)) {
+            return null;
+        }
+        // Read CSV for metadata
+        const content = fsSync.readFileSync(csvPath, 'utf-8');
+        const lines = content.split('\n').filter((l) => l.trim());
+        const headers = lines[0]?.split(',').map((h) => h.trim()) || [];
+        return {
+            blockType: 'csv',
+            type: 'file',
+            path: csvPath,
+            mediaType: 'text/csv',
+            filename: path.basename(csvPath),
+            headers,
+            rowCount: Math.max(0, lines.length - 1),
+        };
+    }
     imageToBlock(node, baseDir) {
         const url = node.url;
         // Handle external URLs