@moxn/kb-migrate 0.1.1 → 0.2.0

package/dist/client.d.ts

@@ -1,17 +1,30 @@
 /**
  * API client for Moxn KB
  */
-import type { ExtractedDocument, MigrationOptions, MigrationResult } from './types.js';
+import type { ExtractedDocument, MigrationOptions, MigrationResult, DocumentListItem, DocumentDetail, ExportOptions } from './types.js';
 export declare class MoxnClient {
     private apiUrl;
     private apiKey;
     private defaultPermission?;
     private aiAccess?;
-    constructor(options: MigrationOptions);
+    constructor(options: MigrationOptions | ExportOptions);
     /**
      * Migrate a single document
      */
     migrateDocument(doc: ExtractedDocument, basePath: string, onConflict: 'skip' | 'update', dryRun: boolean): Promise<MigrationResult>;
+    /**
+     * List all documents, optionally filtered by path prefix.
+     * Handles pagination automatically.
+     */
+    listDocuments(pathPrefix?: string): Promise<DocumentListItem[]>;
+    /**
+     * Get full document detail with sections and content.
+     */
+    getDocument(documentId: string): Promise<DocumentDetail>;
+    /**
+     * Download a file from a URL to a local path.
+     */
+    downloadFile(url: string, destPath: string): Promise<void>;
     private buildPath;
     private processSections;
     private processContentBlocks;

package/dist/client.js

@@ -10,8 +10,12 @@ export class MoxnClient {
     constructor(options) {
         this.apiUrl = options.apiUrl.replace(/\/$/, '');
         this.apiKey = options.apiKey;
-        this.defaultPermission = options.defaultPermission;
-        this.aiAccess = options.aiAccess;
+        if ('defaultPermission' in options) {
+            this.defaultPermission = options.defaultPermission;
+        }
+        if ('aiAccess' in options) {
+            this.aiAccess = options.aiAccess;
+        }
     }
     /**
      * Migrate a single document
@@ -101,6 +105,64 @@ export class MoxnClient {
             };
         }
     }
+    // ──────────────────────────────────────────────
+    // Export methods
+    // ──────────────────────────────────────────────
+    /**
+     * List all documents, optionally filtered by path prefix.
+     * Handles pagination automatically.
+     */
+    async listDocuments(pathPrefix) {
+        const allDocs = [];
+        let offset = 0;
+        const limit = 100;
+        while (true) {
+            const params = new URLSearchParams({ limit: String(limit), offset: String(offset) });
+            if (pathPrefix) {
+                params.set('path', pathPrefix);
+            }
+            const response = await fetch(`${this.apiUrl}/api/v1/kb/documents?${params}`, {
+                headers: { 'x-api-key': this.apiKey },
+            });
+            if (!response.ok) {
+                const error = await response.text();
+                throw new Error(`Failed to list documents: ${response.status} ${error}`);
+            }
+            const data = await response.json();
+            allDocs.push(...data.documents);
+            if (!data.pagination.hasMore)
+                break;
+            offset += limit;
+        }
+        return allDocs;
+    }
+    /**
+     * Get full document detail with sections and content.
+     */
+    async getDocument(documentId) {
+        const response = await fetch(`${this.apiUrl}/api/v1/kb/documents/${documentId}`, {
+            headers: { 'x-api-key': this.apiKey },
+        });
+        if (!response.ok) {
+            const error = await response.text();
+            throw new Error(`Failed to get document ${documentId}: ${response.status} ${error}`);
+        }
+        return response.json();
+    }
+    /**
+     * Download a file from a URL to a local path.
+     */
+    async downloadFile(url, destPath) {
+        const response = await fetch(url);
+        if (!response.ok) {
+            throw new Error(`Failed to download ${url}: ${response.status}`);
+        }
+        const buffer = Buffer.from(await response.arrayBuffer());
+        await fs.writeFile(destPath, buffer);
+    }
+    // ──────────────────────────────────────────────
+    // Migration methods
+    // ──────────────────────────────────────────────
     buildPath(basePath, relativePath) {
         const base = basePath.replace(/^\/+|\/+$/g, '');
         const rel = relativePath.replace(/^\/+|\/+$/g, '');

package/dist/export.d.ts

@@ -0,0 +1,8 @@
+/**
+ * Export runner for Moxn KB
+ *
+ * Exports documents from Moxn Knowledge Base to local markdown files
+ * with downloaded media assets (images, PDFs, CSVs).
+ */
+import type { ExportOptions, ExportLog } from './types.js';
+export declare function runExport(outputDir: string, options: ExportOptions): Promise<ExportLog>;

package/dist/export.js

@@ -0,0 +1,269 @@
+/**
+ * Export runner for Moxn KB
+ *
+ * Exports documents from Moxn Knowledge Base to local markdown files
+ * with downloaded media assets (images, PDFs, CSVs).
+ */
+import * as fs from 'fs';
+import * as path from 'path';
+import { MoxnClient } from './client.js';
+// ──────────────────────────────────────────────
+// Utility functions
+// ──────────────────────────────────────────────
+/** Strip <moxn:comment> tags from text, preserving the inner text. */
+function stripCommentTags(text) {
+    return text.replace(/<moxn:comment[^>]*>([\s\S]*?)<\/moxn:comment>/g, '$1');
+}
+/** Derive a local filename from a storage key or URL. */
+function deriveFilename(storageKey, url, fallbackExt) {
+    if (storageKey) {
+        const parts = storageKey.split('/');
+        return parts[parts.length - 1];
+    }
+    try {
+        const urlPath = new URL(url).pathname;
+        const basename = urlPath.split('/').pop();
+        if (basename && basename.includes('.')) {
+            return basename;
+        }
+    }
+    catch {
+        // Not a valid URL
+    }
+    return `export-${Date.now()}${fallbackExt}`;
+}
+/** Get file extension from MIME type. */
+function mimeToExt(mimeType) {
+    const map = {
+        'image/png': '.png',
+        'image/jpeg': '.jpg',
+        'image/gif': '.gif',
+        'image/webp': '.webp',
+        'application/pdf': '.pdf',
+        'text/csv': '.csv',
+    };
+    return map[mimeType] || '';
+}
+/** Convert a document path to a local file path. */
+function docPathToFilePath(docPath) {
+    return `${docPath.replace(/^\//, '')}.md`;
+}
+// ──────────────────────────────────────────────
+// Markdown builder
+// ──────────────────────────────────────────────
+function buildMarkdown(doc, mediaMap, mdFilePath) {
+    const lines = [];
+    lines.push(`# ${doc.name}`);
+    lines.push('');
+    if (doc.description) {
+        lines.push(doc.description);
+        lines.push('');
+    }
+    for (const section of doc.sections) {
+        lines.push(`## ${section.name}`);
+        lines.push('');
+        for (const block of section.content) {
+            if (block.blockType === 'text' && block.text) {
+                lines.push(stripCommentTags(block.text));
+                lines.push('');
+            }
+            else if (block.blockType === 'image' && block.url) {
+                const key = block.storageKey || block.url;
+                const localPath = mediaMap.get(key);
+                if (localPath) {
+                    const relativePath = path.relative(path.dirname(mdFilePath), localPath);
+                    lines.push(`![${block.alt || ''}](${relativePath})`);
+                }
+                else {
+                    lines.push(`![${block.alt || ''}](${block.url})`);
+                }
+                lines.push('');
+            }
+            else if (block.blockType === 'document' && block.url) {
+                const key = block.storageKey || block.url;
+                const localPath = mediaMap.get(key);
+                if (localPath) {
+                    const relativePath = path.relative(path.dirname(mdFilePath), localPath);
+                    lines.push(`[${block.filename || 'document'}](${relativePath})`);
+                }
+                else {
+                    lines.push(`[${block.filename || 'document'}](${block.url})`);
+                }
+                lines.push('');
+            }
+            else if (block.blockType === 'csv' && block.url) {
+                const key = block.storageKey || block.url;
+                const localPath = mediaMap.get(key);
+                if (localPath) {
+                    const relativePath = path.relative(path.dirname(mdFilePath), localPath);
+                    lines.push(`[${block.filename || 'data.csv'}](${relativePath})`);
+                }
+                else {
+                    lines.push(`[${block.filename || 'data.csv'}](${block.url})`);
+                }
+                lines.push('');
+            }
+        }
+    }
+    // Trim trailing blank lines, end with single newline
+    while (lines.length > 0 && lines[lines.length - 1].trim() === '') {
+        lines.pop();
+    }
+    lines.push('');
+    return lines.join('\n');
+}
+// ──────────────────────────────────────────────
+// Media helpers
+// ──────────────────────────────────────────────
+function getMediaTarget(block, options) {
+    if (block.blockType === 'image') {
+        return { targetDir: options.imageDir, fallbackExt: mimeToExt(block.mimeType || 'image/png') };
+    }
+    else if (block.blockType === 'document') {
+        return {
+            targetDir: options.pdfDir,
+            fallbackExt: mimeToExt(block.mimeType || 'application/pdf'),
+        };
+    }
+    else if (block.blockType === 'csv') {
+        return { targetDir: options.csvDir, fallbackExt: '.csv' };
+    }
+    return null;
+}
+// ──────────────────────────────────────────────
+// Main export runner
+// ──────────────────────────────────────────────
+export async function runExport(outputDir, options) {
+    const startTime = Date.now();
+    const client = new MoxnClient(options);
+    // List documents
+    console.log('Fetching document list...');
+    const documents = await client.listDocuments(options.basePath || undefined);
+    console.log(`Found ${documents.length} documents`);
+    if (documents.length === 0) {
+        return {
+            timestamp: new Date().toISOString(),
+            sourceApi: options.apiUrl,
+            outputDir: path.resolve(outputDir),
+            basePath: options.basePath || '/',
+            options: {
+                dryRun: options.dryRun,
+                imageDir: options.imageDir,
+                pdfDir: options.pdfDir,
+                csvDir: options.csvDir,
+            },
+            results: [],
+            summary: {
+                total: 0,
+                exported: 0,
+                failed: 0,
+                mediaDownloaded: 0,
+                duration: Date.now() - startTime,
+            },
+        };
+    }
+    // Create output directories
+    if (!options.dryRun) {
+        fs.mkdirSync(outputDir, { recursive: true });
+        fs.mkdirSync(path.join(outputDir, options.imageDir), { recursive: true });
+        fs.mkdirSync(path.join(outputDir, options.pdfDir), { recursive: true });
+        fs.mkdirSync(path.join(outputDir, options.csvDir), { recursive: true });
+    }
+    const results = [];
+    let totalMediaDownloaded = 0;
+    for (const docItem of documents) {
+        const docStart = Date.now();
+        const mediaFiles = [];
+        console.log(`Processing: ${docItem.path}`);
+        try {
+            const doc = await client.getDocument(docItem.id);
+            const mdRelativePath = docPathToFilePath(doc.path);
+            const mdFullPath = path.join(outputDir, mdRelativePath);
+            const mediaMap = new Map();
+            // Download media
+            for (const section of doc.sections) {
+                for (const block of section.content) {
+                    if (block.blockType === 'text')
+                        continue;
+                    if (!block.url)
+                        continue;
+                    if (block.url.startsWith('data:'))
+                        continue;
+                    const target = getMediaTarget(block, options);
+                    if (!target)
+                        continue;
+                    const filename = deriveFilename(block.storageKey, block.url, target.fallbackExt);
+                    const localRelativePath = path.join(target.targetDir, filename);
+                    const localFullPath = path.join(outputDir, localRelativePath);
+                    const mapKey = block.storageKey || block.url;
+                    mediaMap.set(mapKey, path.join(outputDir, localRelativePath));
+                    if (!options.dryRun) {
+                        if (!fs.existsSync(localFullPath)) {
+                            try {
+                                await client.downloadFile(block.url, localFullPath);
+                                totalMediaDownloaded++;
+                            }
+                            catch (err) {
+                                const msg = err instanceof Error ? err.message : String(err);
+                                console.log(`  \u2717 Download failed: ${filename}: ${msg}`);
+                            }
+                        }
+                    }
+                    else {
+                        totalMediaDownloaded++;
+                    }
+                    mediaFiles.push(localRelativePath);
+                }
+            }
+            const markdown = buildMarkdown(doc, mediaMap, mdFullPath);
+            if (!options.dryRun) {
+                fs.mkdirSync(path.dirname(mdFullPath), { recursive: true });
+                fs.writeFileSync(mdFullPath, markdown, 'utf-8');
+            }
+            console.log(`  \u2713 ${mdRelativePath} (${doc.sections.length} sections, ${mediaFiles.length} media)`);
+            results.push({
+                documentId: doc.id,
+                documentPath: doc.path,
+                outputFile: mdRelativePath,
+                status: 'exported',
+                sectionsCount: doc.sections.length,
+                mediaFiles,
+                duration: Date.now() - docStart,
+            });
+        }
+        catch (error) {
+            const message = error instanceof Error ? error.message : String(error);
+            console.log(`  \u2717 Failed: ${message}`);
+            results.push({
+                documentId: docItem.id,
+                documentPath: docItem.path,
+                outputFile: docPathToFilePath(docItem.path),
+                status: 'failed',
+                sectionsCount: 0,
+                mediaFiles: [],
+                error: message,
+                duration: Date.now() - docStart,
+            });
+        }
+    }
+    return {
+        timestamp: new Date().toISOString(),
+        sourceApi: options.apiUrl,
+        outputDir: path.resolve(outputDir),
+        basePath: options.basePath || '/',
+        options: {
+            dryRun: options.dryRun,
+            imageDir: options.imageDir,
+            pdfDir: options.pdfDir,
+            csvDir: options.csvDir,
+        },
+        results,
+        summary: {
+            total: results.length,
+            exported: results.filter((r) => r.status === 'exported').length,
+            failed: results.filter((r) => r.status === 'failed').length,
+            mediaDownloaded: totalMediaDownloaded,
+            duration: Date.now() - startTime,
+        },
+    };
+}

package/dist/index.js

@@ -15,6 +15,7 @@
 import { Command } from 'commander';
 import { LocalSource } from './sources/local.js';
 import { MoxnClient } from './client.js';
+import { runExport } from './export.js';
 const DEFAULT_API_URL = 'https://moxn.dev';
 const DEFAULT_EXTENSIONS = ['.md', '.txt'];
 async function runMigration(source, options) {
@@ -91,6 +92,27 @@ function printSummary(log) {
         console.log('\n(Dry run - no changes made)');
     }
 }
+function printExportSummary(log) {
+    console.log('\n--- Export Summary ---');
+    console.log(`Source: ${log.sourceApi}`);
+    console.log(`Target: ${log.outputDir}`);
+    console.log(`Base path: ${log.basePath}`);
+    console.log(`Duration: ${(log.summary.duration / 1000).toFixed(1)}s`);
+    console.log('');
+    console.log(`Total:      ${log.summary.total}`);
+    console.log(`Exported:   ${log.summary.exported}`);
+    console.log(`Failed:     ${log.summary.failed}`);
+    console.log(`Media:      ${log.summary.mediaDownloaded}`);
+    if (log.summary.failed > 0) {
+        console.log('\nFailed documents:');
+        for (const f of log.results.filter((r) => r.status === 'failed')) {
+            console.log(`  - ${f.documentPath}: ${f.error}`);
+        }
+    }
+    if (log.options.dryRun) {
+        console.log('\n(Dry run - no changes made)');
+    }
+}
 const program = new Command();
 program
     .name('moxn-kb-migrate')
@@ -151,6 +173,49 @@ program
         process.exit(1);
     }
 });
+program
+    .command('export <directory>')
+    .description('Export documents from Moxn Knowledge Base to local files')
+    .option('--api-key <key>', 'API key (or set MOXN_API_KEY env var)')
+    .option('--api-url <url>', 'API base URL', DEFAULT_API_URL)
+    .option('--base-path <path>', 'Only export docs under this path prefix', '/')
+    .option('--image-dir <name>', 'Directory name for images', 'images')
+    .option('--pdf-dir <name>', 'Directory name for PDFs', 'pdfs')
+    .option('--csv-dir <name>', 'Directory name for CSVs', 'csvs')
+    .option('--dry-run', 'Preview without writing files', false)
+    .option('--json', 'Output results as JSON', false)
+    .action(async (directory, opts) => {
+    const apiKey = opts.apiKey || process.env.MOXN_API_KEY;
+    if (!apiKey) {
+        console.error('Error: API key required. Use --api-key or set MOXN_API_KEY env var.');
+        process.exit(1);
+    }
+    const exportOptions = {
+        apiUrl: opts.apiUrl,
+        apiKey,
+        basePath: opts.basePath,
+        imageDir: opts.imageDir,
+        pdfDir: opts.pdfDir,
+        csvDir: opts.csvDir,
+        dryRun: opts.dryRun,
+    };
+    try {
+        const log = await runExport(directory, exportOptions);
+        if (opts.json) {
+            console.log(JSON.stringify(log, null, 2));
+        }
+        else {
+            printExportSummary(log);
+        }
+        if (log.summary.failed > 0) {
+            process.exit(1);
+        }
+    }
+    catch (error) {
+        console.error('Export failed:', error instanceof Error ? error.message : error);
+        process.exit(1);
+    }
+});
 // Future: Add notion and google-docs commands here
 // program
 //   .command('notion')

package/dist/types.d.ts

@@ -107,3 +107,104 @@ export interface ConflictError {
     documentId: string;
     branchId: string;
 }
+/**
+ * A content block as returned by the KB API
+ */
+export interface ExportContentBlock {
+    blockType: 'text' | 'image' | 'document' | 'csv';
+    text?: string;
+    url?: string;
+    mimeType?: string;
+    alt?: string;
+    filename?: string;
+    storageKey?: string;
+}
+/**
+ * A section within a document (API response)
+ */
+export interface DocumentSection {
+    id: string;
+    name: string;
+    position: number;
+    content: ExportContentBlock[];
+}
+/**
+ * A document in the list response
+ */
+export interface DocumentListItem {
+    id: string;
+    path: string;
+    name: string;
+    description: string | null;
+    createdAt: string;
+}
+/**
+ * Full document detail with sections
+ */
+export interface DocumentDetail {
+    id: string;
+    path: string;
+    name: string;
+    description: string | null;
+    sections: DocumentSection[];
+}
+/**
+ * Paginated list response from KB API
+ */
+export interface ListResponse {
+    documents: DocumentListItem[];
+    pagination: {
+        total: number;
+        limit: number;
+        offset: number;
+        hasMore: boolean;
+    };
+}
+/**
+ * Result of exporting a single document
+ */
+export interface ExportResult {
+    documentId: string;
+    documentPath: string;
+    outputFile: string;
+    status: 'exported' | 'failed';
+    sectionsCount: number;
+    mediaFiles: string[];
+    error?: string;
+    duration: number;
+}
+/**
+ * Complete export log for JSON output
+ */
+export interface ExportLog {
+    timestamp: string;
+    sourceApi: string;
+    outputDir: string;
+    basePath: string;
+    options: {
+        dryRun: boolean;
+        imageDir: string;
+        pdfDir: string;
+        csvDir: string;
+    };
+    results: ExportResult[];
+    summary: {
+        total: number;
+        exported: number;
+        failed: number;
+        mediaDownloaded: number;
+        duration: number;
+    };
+}
+/**
+ * Options for export
+ */
+export interface ExportOptions {
+    apiUrl: string;
+    apiKey: string;
+    basePath: string;
+    imageDir: string;
+    pdfDir: string;
+    csvDir: string;
+    dryRun: boolean;
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moxn/kb-migrate",
-  "version": "0.1.1",
+  "version": "0.2.0",
   "description": "Migration tool for importing documents into Moxn Knowledge Base from local files, Notion, Google Docs, and more",
   "type": "module",
   "main": "dist/index.js",