@moxn/kb-migrate 0.4.3 → 0.4.7

package/dist/index.js

@@ -18,6 +18,8 @@ import { NotionSource } from './sources/notion.js';
 import { notionColorToHex } from './sources/notion-api.js';
 import { MoxnClient } from './client.js';
 import { runExport } from './export.js';
+import { runNotionExport } from './export-notion.js';
+import { buildDateFilter } from './date-filter.js';
 const DEFAULT_API_URL = 'https://moxn.dev';
 const DEFAULT_EXTENSIONS = ['.md', '.txt'];
 async function runMigration(source, options) {
@@ -115,6 +117,29 @@ function printExportSummary(log) {
         console.log('\n(Dry run - no changes made)');
     }
 }
+function printNotionExportSummary(log) {
+    console.log('\n--- Export to Notion Summary ---');
+    console.log(`Source: ${log.sourceApi}`);
+    console.log(`Target: ${log.target.type} (${log.target.location})`);
+    console.log(`Base path: ${log.basePath}`);
+    console.log(`Conflict strategy: ${log.options.conflictStrategy}`);
+    console.log(`Duration: ${(log.summary.duration / 1000).toFixed(1)}s`);
+    console.log('');
+    console.log(`Total:   ${log.summary.total}`);
+    console.log(`Created: ${log.summary.created}`);
+    console.log(`Updated: ${log.summary.updated}`);
+    console.log(`Skipped: ${log.summary.skipped}`);
+    console.log(`Failed:  ${log.summary.failed}`);
+    if (log.summary.failed > 0) {
+        console.log('\nFailed documents:');
+        for (const f of log.results.filter((r) => r.status === 'failed')) {
+            console.log(`  - ${f.documentPath}: ${f.error}`);
+        }
+    }
+    if (log.options.dryRun) {
+        console.log('\n(Dry run - no changes made)');
+    }
+}
 const program = new Command();
 program
     .name('moxn-kb-migrate')
@@ -130,6 +155,10 @@ program
     .option('--on-conflict <action>', 'Action on conflict: skip or update', 'skip')
     .option('--default-permission <perm>', 'Default permission: edit, read, or none')
     .option('--ai-access <perm>', 'AI access permission: edit, read, or none')
+    .option('--created-after <date>', 'Only include docs created after this date (ISO 8601)')
+    .option('--created-before <date>', 'Only include docs created before this date (ISO 8601)')
+    .option('--modified-after <date>', 'Only include docs modified after this date (ISO 8601)')
+    .option('--modified-before <date>', 'Only include docs modified before this date (ISO 8601)')
     .option('--dry-run', 'Preview without making changes', false)
     .option('--json', 'Output results as JSON', false)
     .action(async (directory, opts) => {
@@ -144,9 +173,16 @@ program
         console.error('Error: --on-conflict must be "skip" or "update"');
         process.exit(1);
     }
+    const dateFilter = buildDateFilter({
+        createdAfter: opts.createdAfter,
+        createdBefore: opts.createdBefore,
+        modifiedAfter: opts.modifiedAfter,
+        modifiedBefore: opts.modifiedBefore,
+    });
     const source = new LocalSource({
         directory,
         extensions,
+        dateFilter,
     });
     const migrationOptions = {
         apiUrl: opts.apiUrl,
@@ -156,6 +192,7 @@ program
         dryRun: opts.dryRun,
         defaultPermission: opts.defaultPermission,
         aiAccess: opts.aiAccess,
+        dateFilter,
     };
     try {
         const log = await runMigration(source, migrationOptions);
@@ -184,6 +221,10 @@ program
     .option('--image-dir <name>', 'Directory name for images', 'images')
     .option('--pdf-dir <name>', 'Directory name for PDFs', 'pdfs')
     .option('--csv-dir <name>', 'Directory name for CSVs', 'csvs')
+    .option('--created-after <date>', 'Only include docs created after this date (ISO 8601)')
+    .option('--created-before <date>', 'Only include docs created before this date (ISO 8601)')
+    .option('--modified-after <date>', 'Only include docs modified after this date (ISO 8601)')
+    .option('--modified-before <date>', 'Only include docs modified before this date (ISO 8601)')
     .option('--dry-run', 'Preview without writing files', false)
     .option('--json', 'Output results as JSON', false)
     .action(async (directory, opts) => {
@@ -192,6 +233,12 @@ program
         console.error('Error: API key required. Use --api-key or set MOXN_API_KEY env var.');
         process.exit(1);
     }
+    const dateFilter = buildDateFilter({
+        createdAfter: opts.createdAfter,
+        createdBefore: opts.createdBefore,
+        modifiedAfter: opts.modifiedAfter,
+        modifiedBefore: opts.modifiedBefore,
+    });
     const exportOptions = {
         apiUrl: opts.apiUrl,
         apiKey,
@@ -200,6 +247,7 @@ program
         pdfDir: opts.pdfDir,
         csvDir: opts.csvDir,
         dryRun: opts.dryRun,
+        dateFilter,
     };
     try {
         const log = await runExport(directory, exportOptions);
@@ -231,6 +279,10 @@ program
     .option('--default-permission <perm>', 'Default permission: edit, read, or none')
     .option('--ai-access <perm>', 'AI access permission: edit, read, or none')
     .option('--visibility <vis>', 'Convenience flag: team (read) or private (none)')
+    .option('--created-after <date>', 'Only include docs created after this date (ISO 8601)')
+    .option('--created-before <date>', 'Only include docs created before this date (ISO 8601)')
+    .option('--modified-after <date>', 'Only include docs modified after this date (ISO 8601)')
+    .option('--modified-before <date>', 'Only include docs modified before this date (ISO 8601)')
     .option('--dry-run', 'Preview without making changes', false)
     .option('--json', 'Output results as JSON', false)
     .action(async (opts) => {
@@ -254,10 +306,17 @@ program
     if (!defaultPermission && opts.visibility) {
         defaultPermission = opts.visibility === 'private' ? 'none' : 'read';
     }
+    const dateFilter = buildDateFilter({
+        createdAfter: opts.createdAfter,
+        createdBefore: opts.createdBefore,
+        modifiedAfter: opts.modifiedAfter,
+        modifiedBefore: opts.modifiedBefore,
+    });
     const source = new NotionSource({
         token,
         rootPageId: opts.rootPageId,
         maxDepth: opts.maxDepth ? parseInt(opts.maxDepth, 10) : undefined,
+        dateFilter,
     });
     const migrationOptions = {
         apiUrl: opts.apiUrl,
@@ -268,25 +327,58 @@ program
         defaultPermission,
         aiAccess: opts.aiAccess,
         visibility: opts.visibility,
+        dateFilter,
     };
     try {
-        // Run page migration
+        // Step 1: Validate source (discover pages + databases)
+        await source.validate();
+        // Step 2: Pre-create databases before document extraction so that
+        // child_database blocks can be converted to database_embed blocks.
+        const databaseIdMap = new Map();
+        const dbImports = source.getDatabaseImports();
+        if (!opts.dryRun && dbImports.length > 0) {
+            console.log(`\nPre-creating ${dbImports.length} database(s)...`);
+            const preClient = new MoxnClient(migrationOptions);
+            for (const dbImport of dbImports) {
+                try {
+                    const kbDbId = await preCreateNotionDatabase(preClient, dbImport);
+                    databaseIdMap.set(dbImport.notionDatabaseId, kbDbId);
+                }
+                catch (error) {
+                    console.error(`  Error pre-creating database "${dbImport.schema.name}": ${error instanceof Error ? error.message : error}`);
+                }
+            }
+            // Make the map available during extraction
+            source.setDatabaseIdMap(databaseIdMap);
+            console.log(`  ${databaseIdMap.size} database(s) pre-created for embed resolution.\n`);
+        }
+        // Step 3: Run page migration (validate is idempotent, extract uses databaseIdMap)
         const log = await runMigration(source, migrationOptions);
-        // After page migration, import databases
-        if (!opts.dryRun) {
-            const dbImports = source.getDatabaseImports();
-            if (dbImports.length > 0) {
-                console.log(`\nImporting ${dbImports.length} database(s)...`);
-                const client = new MoxnClient(migrationOptions);
-                for (const dbImport of dbImports) {
-                    try {
-                        await importNotionDatabase(client, dbImport, log, migrationOptions);
+        // Step 4: Finalize databases — create columns, link entries, assign tags.
+        // Database records already exist from pre-create; this step adds schema + data.
+        if (!opts.dryRun && dbImports.length > 0) {
+            console.log(`\nFinalizing ${dbImports.length} database(s)...`);
+            const client = new MoxnClient(migrationOptions);
+            let totalEntries = 0;
+            const allPropertyTypes = new Set();
+            for (const dbImport of dbImports) {
+                try {
+                    const preCreatedId = databaseIdMap.get(dbImport.notionDatabaseId);
+                    await importNotionDatabase(client, dbImport, log, migrationOptions, preCreatedId);
+                    totalEntries += dbImport.entries.length;
+                    // Track property types encountered
+                    for (const col of dbImport.schema.mappedColumns) {
+                        allPropertyTypes.add(col.notionType);
                     }
-                    catch (error) {
-                        console.error(`  Error importing database "${dbImport.schema.name}": ${error instanceof Error ? error.message : error}`);
+                    for (const col of dbImport.schema.unmappedColumns) {
+                        allPropertyTypes.add(col.notionType);
                     }
                 }
+                catch (error) {
+                    console.error(`  Error finalizing database "${dbImport.schema.name}": ${error instanceof Error ? error.message : error}`);
+                }
             }
+            console.log(`\nImported ${dbImports.length} databases, ${totalEntries} entries, ${allPropertyTypes.size} property types`);
         }
         // Cleanup temp files
         await source.cleanup();
@@ -306,20 +398,128 @@ program
         process.exit(1);
     }
 });
+program
+    .command('export-notion')
+    .description('Export documents from Moxn Knowledge Base to Notion')
+    .option('--api-key <key>', 'Moxn API key (or set MOXN_API_KEY env var)')
+    .option('--api-url <url>', 'Moxn API base URL', DEFAULT_API_URL)
+    .option('--notion-token <token>', 'Notion integration token (or set NOTION_TOKEN env var)')
+    .option('--parent-page-id <id>', 'Notion parent page under which to create pages')
+    .option('--base-path <path>', 'Only export docs under this path prefix', '/')
+    .option('--conflict-strategy <strategy>', 'How to handle existing pages: skip or update', 'skip')
+    .option('--modified-after <date>', 'Only include docs modified after this date (ISO 8601)')
+    .option('--modified-before <date>', 'Only include docs modified before this date (ISO 8601)')
+    .option('--created-after <date>', 'Only include docs created after this date (ISO 8601)')
+    .option('--created-before <date>', 'Only include docs created before this date (ISO 8601)')
+    .option('--dry-run', 'Preview without making changes', false)
+    .option('--json', 'Output results as JSON', false)
+    .action(async (opts) => {
+    const apiKey = opts.apiKey || process.env.MOXN_API_KEY;
+    if (!apiKey) {
+        console.error('Error: Moxn API key required. Use --api-key or set MOXN_API_KEY env var.');
+        process.exit(1);
+    }
+    const notionToken = opts.notionToken || process.env.NOTION_TOKEN;
+    if (!notionToken) {
+        console.error('Error: Notion token required. Use --notion-token or set NOTION_TOKEN env var.');
+        process.exit(1);
+    }
+    if (!opts.parentPageId) {
+        console.error('Error: --parent-page-id is required.');
+        process.exit(1);
+    }
+    const conflictStrategy = opts.conflictStrategy;
+    if (!['skip', 'update'].includes(conflictStrategy)) {
+        console.error('Error: --conflict-strategy must be "skip" or "update"');
+        process.exit(1);
+    }
+    const dateFilter = buildDateFilter({
+        createdAfter: opts.createdAfter,
+        createdBefore: opts.createdBefore,
+        modifiedAfter: opts.modifiedAfter,
+        modifiedBefore: opts.modifiedBefore,
+    });
+    try {
+        const log = await runNotionExport({
+            apiUrl: opts.apiUrl,
+            apiKey,
+            notionToken,
+            parentPageId: opts.parentPageId,
+            basePath: opts.basePath,
+            conflictStrategy,
+            dryRun: opts.dryRun,
+            dateFilter,
+        });
+        if (opts.json) {
+            console.log(JSON.stringify(log, null, 2));
+        }
+        else {
+            printNotionExportSummary(log);
+        }
+        if (log.summary.failed > 0) {
+            process.exit(1);
+        }
+    }
+    catch (error) {
+        console.error('Export to Notion failed:', error instanceof Error ? error.message : error);
+        process.exit(1);
+    }
+});
 /**
  * Import a Notion database into Moxn KB.
  *
  * Creates the database, columns with tags, links entries, and assigns tag values.
  */
-async function importNotionDatabase(client, dbImport, log, options) {
-    const { schema, entries } = dbImport;
-    console.log(`  Creating database: ${schema.name}`);
-    // 1. Create the database
+/**
+ * Pre-create a KB database and store its Notion mapping.
+ * Called before document extraction so child_database blocks can resolve.
+ * Returns the KB database ID.
+ */
+async function preCreateNotionDatabase(client, dbImport) {
+    const { schema } = dbImport;
+    console.log(`  Pre-creating database: ${schema.name}`);
     const db = await client.createDatabase({
         name: schema.name,
         description: schema.description || undefined,
     });
     console.log(`    Database created: ${db.id}`);
+    // Store Notion → KB database mapping
+    try {
+        await client.createNotionDatabaseMapping({
+            kbDatabaseId: db.id,
+            notionDatabaseId: dbImport.notionDatabaseId,
+            notionDatabaseTitle: schema.name,
+        });
+        console.log(`    Database mapping stored: ${dbImport.notionDatabaseId} → ${db.id}`);
+    }
+    catch (error) {
+        console.warn(`    Warning: Failed to store database mapping: ${error instanceof Error ? error.message : error}`);
+    }
+    return db.id;
+}
+/**
+ * Import a Notion database into Moxn KB.
+ *
+ * Creates columns with tags, links entries, and assigns tag values.
+ * If preCreatedDbId is provided, skips database creation (already done in pre-create step).
+ */
+async function importNotionDatabase(client, dbImport, log, options, preCreatedDbId) {
+    const { schema, entries } = dbImport;
+    let dbId;
+    if (preCreatedDbId) {
+        dbId = preCreatedDbId;
+        console.log(`  Finalizing database: ${schema.name} (${dbId})`);
+    }
+    else {
+        // Fallback: create database now (non-Notion sources or if pre-create was skipped)
+        console.log(`  Creating database: ${schema.name}`);
+        const db = await client.createDatabase({
+            name: schema.name,
+            description: schema.description || undefined,
+        });
+        dbId = db.id;
+        console.log(`    Database created: ${dbId}`);
+    }
     // 2. Create columns with tags
     // Map: column name → { columnId, optionTagMap: option name → tagId }
     const columnMap = new Map();
@@ -344,7 +544,7 @@ async function importNotionDatabase(client, dbImport, log, options) {
         }
         // Create the column
         try {
-            const column = await client.addDatabaseColumn(db.id, {
+            const column = await client.addDatabaseColumn(dbId, {
                 name: col.notionPropertyName,
                 type: col.moxnType,
                 optionTagIds: tagIds,
@@ -384,7 +584,7 @@ async function importNotionDatabase(client, dbImport, log, options) {
         }
         // Add document to database
         try {
-            await client.addDocumentToDatabase(db.id, docInfo.documentId);
+            await client.addDocumentToDatabase(dbId, docInfo.documentId);
             linkedCount++;
             // Parse and assign tag values
             const { parseEntryValues } = await import('./sources/notion-databases.js');

package/dist/sources/local.d.ts

@@ -5,11 +5,14 @@
  */
 import { MigrationSource, type SourceConfig } from './base.js';
 import type { ExtractedDocument } from '../types.js';
+import type { DateFilter } from '../date-filter.js';
 export interface LocalSourceConfig extends SourceConfig {
     /** Directory path to scan for documents */
     directory: string;
     /** File extensions to include (default: ['.md', '.txt']) */
     extensions: string[];
+    /** Date filter for source files */
+    dateFilter?: DateFilter;
 }
 /**
  * Local filesystem migration source

package/dist/sources/local.js

@@ -60,7 +60,37 @@ export class LocalSource extends MigrationSource {
             allFiles.push(...matches);
         }
         // Deduplicate and sort
-        const uniqueFiles = [...new Set(allFiles)].sort();
+        let uniqueFiles = [...new Set(allFiles)].sort();
+        // Apply date filter if configured
+        if (this.config.dateFilter) {
+            const { matchesDateFilter } = await import('../date-filter.js');
+            const filtered = [];
+            let birthtimeWarned = false;
+            for (const file of uniqueFiles) {
+                const fullPath = path.join(this.config.directory, file);
+                const stat = await fs.stat(fullPath);
+                // Warn if birthtime is unreliable (equals ctime on some systems)
+                if (!birthtimeWarned &&
+                    (this.config.dateFilter.createdAfter ||
+                        this.config.dateFilter.createdBefore) &&
+                    stat.birthtime.getTime() === stat.ctime.getTime()) {
+                    console.warn('  \u26a0 File creation time (birthtime) may be unreliable on this filesystem.');
+                    console.warn('    --created-after/--created-before results may be inaccurate.');
+                    birthtimeWarned = true;
+                }
+                if (matchesDateFilter(this.config.dateFilter, {
+                    createdAt: stat.birthtime,
+                    modifiedAt: stat.mtime,
+                })) {
+                    filtered.push(file);
+                }
+            }
+            const skipped = uniqueFiles.length - filtered.length;
+            if (skipped > 0) {
+                console.log(`  Filtered ${skipped} files by date criteria`);
+            }
+            uniqueFiles = filtered;
+        }
         // Detect KB path collisions (e.g., doc.md and doc.mdx both map to /doc)
         const pathToFiles = new Map();
         for (const file of uniqueFiles) {

package/dist/sources/notion-blocks.d.ts

@@ -17,6 +17,8 @@ export type PagePathMap = Map<string, string>;
 export declare function blocksToSections(blocks: NotionBlock[], client: NotionApiClient, pagePathMap: PagePathMap, options?: {
     /** Track synced block IDs to detect cycles. */
     visitedSyncedBlocks?: Set<string>;
+    /** Map of Notion database IDs to KB database IDs (for child_database blocks). */
+    databaseIdMap?: Map<string, string>;
 }): Promise<SectionInput[]>;
 /** Convert rich text array to markdown string. */
 export declare function richTextToMarkdown(richText: NotionRichText[]): string;