@moxn/kb-migrate 0.3.0 → 0.4.0

package/dist/sources/notion-databases.js

@@ -0,0 +1,266 @@
+/**
+ * Parses Notion database schemas and entries for import into Moxn KB.
+ *
+ * Mapping strategy:
+ * - select, multi_select, status → Moxn select/multi_select columns (backed by tags)
+ * - title → document name (not a column)
+ * - All other property types → rendered as markdown in a "Properties" section
+ */
+import { richTextToPlain, richTextToMarkdown, getPageTitle } from './notion-blocks.js';
+// ============================================
+// Schema Parsing
+// ============================================
+/**
+ * Parse a Notion database schema into mapped and unmapped columns.
+ */
+export function parseDatabaseSchema(db) {
+    const name = richTextToPlain(db.title) || 'Untitled Database';
+    const description = richTextToPlain(db.description);
+    const mappedColumns = [];
+    const unmappedColumns = [];
+    for (const [propName, prop] of Object.entries(db.properties)) {
+        // Skip title — it becomes the document name
+        if (prop.type === 'title')
+            continue;
+        if (prop.type === 'select' && prop.select) {
+            mappedColumns.push({
+                notionPropertyId: prop.id,
+                notionPropertyName: propName,
+                notionType: 'select',
+                moxnType: 'select',
+                options: prop.select.options.map((o) => ({
+                    notionId: o.id,
+                    name: o.name,
+                    color: o.color,
+                })),
+            });
+        }
+        else if (prop.type === 'multi_select' && prop.multi_select) {
+            mappedColumns.push({
+                notionPropertyId: prop.id,
+                notionPropertyName: propName,
+                notionType: 'multi_select',
+                moxnType: 'multi_select',
+                options: prop.multi_select.options.map((o) => ({
+                    notionId: o.id,
+                    name: o.name,
+                    color: o.color,
+                })),
+            });
+        }
+        else if (prop.type === 'status' && prop.status) {
+            // Status is structurally identical to select
+            mappedColumns.push({
+                notionPropertyId: prop.id,
+                notionPropertyName: propName,
+                notionType: 'status',
+                moxnType: 'select',
+                options: prop.status.options.map((o) => ({
+                    notionId: o.id,
+                    name: o.name,
+                    color: o.color,
+                })),
+            });
+        }
+        else {
+            unmappedColumns.push({
+                notionPropertyId: prop.id,
+                notionPropertyName: propName,
+                notionType: prop.type,
+            });
+        }
+    }
+    return { name, description, mappedColumns, unmappedColumns };
+}
+// ============================================
+// Entry Parsing
+// ============================================
+/**
+ * Parse a Notion database entry's property values.
+ */
+export function parseEntryValues(page, schema) {
+    const title = getPageTitle(page);
+    const tagValues = new Map();
+    const textValues = new Map();
+    const fileBlocks = [];
+    // Build lookup by property name
+    const mappedByName = new Map(schema.mappedColumns.map((c) => [c.notionPropertyName, c]));
+    const unmappedByName = new Set(schema.unmappedColumns.map((c) => c.notionPropertyName));
+    for (const [propName, propValue] of Object.entries(page.properties)) {
+        if (propValue.type === 'title')
+            continue; // Already extracted
+        // Check if this is a mapped column
+        const mapped = mappedByName.get(propName);
+        if (mapped) {
+            const values = extractSelectValues(propValue, mapped.notionType);
+            if (values.length > 0) {
+                tagValues.set(propName, values);
+            }
+            continue;
+        }
+        // Check if unmapped
+        if (unmappedByName.has(propName)) {
+            const rendered = renderPropertyValue(propValue);
+            if (rendered) {
+                textValues.set(propName, rendered);
+            }
+            // Extract file attachments
+            if (propValue.type === 'files' && propValue.files) {
+                for (const file of propValue.files) {
+                    const url = file.type === 'file' ? file.file?.url : file.external?.url;
+                    if (!url)
+                        continue;
+                    const isPdf = file.name.toLowerCase().endsWith('.pdf');
+                    const isCsv = file.name.toLowerCase().endsWith('.csv');
+                    if (isPdf) {
+                        fileBlocks.push({
+                            blockType: 'document',
+                            type: 'url',
+                            url,
+                            mediaType: 'application/pdf',
+                            filename: file.name,
+                        });
+                    }
+                    else if (isCsv) {
+                        fileBlocks.push({
+                            blockType: 'csv',
+                            type: 'url',
+                            url,
+                            mediaType: 'text/csv',
+                            filename: file.name,
+                        });
+                    }
+                    else if (isImageFilename(file.name)) {
+                        fileBlocks.push({
+                            blockType: 'image',
+                            type: 'url',
+                            url,
+                            mediaType: guessImageMime(file.name),
+                            alt: file.name,
+                        });
+                    }
+                }
+            }
+        }
+    }
+    return { title, tagValues, textValues, fileBlocks };
+}
+/**
+ * Render unmapped properties as a markdown table section.
+ * Returns null if there are no unmapped values to render.
+ */
+export function renderPropertiesSection(values) {
+    if (values.textValues.size === 0 && values.fileBlocks.length === 0) {
+        return null;
+    }
+    const content = [];
+    // Render text properties as a markdown table
+    if (values.textValues.size > 0) {
+        const lines = ['| Property | Value |', '| --- | --- |'];
+        for (const [name, value] of values.textValues) {
+            // Escape pipes in values
+            const escaped = value.replace(/\|/g, '\\|').replace(/\n/g, ' ');
+            lines.push(`| ${name} | ${escaped} |`);
+        }
+        content.push({ blockType: 'text', text: lines.join('\n') });
+    }
+    // Add file blocks
+    content.push(...values.fileBlocks);
+    return { name: 'Properties', content };
+}
+// ============================================
+// Value extractors
+// ============================================
+function extractSelectValues(propValue, notionType) {
+    switch (notionType) {
+        case 'select':
+            return propValue.select ? [propValue.select.name] : [];
+        case 'multi_select':
+            return (propValue.multi_select ?? []).map((s) => s.name);
+        case 'status':
+            return propValue.status ? [propValue.status.name] : [];
+        default:
+            return [];
+    }
+}
+function renderPropertyValue(prop) {
+    switch (prop.type) {
+        case 'rich_text':
+            return richTextToMarkdown(prop.rich_text ?? []) || null;
+        case 'number':
+            return prop.number != null ? String(prop.number) : null;
+        case 'date':
+            if (!prop.date)
+                return null;
+            return prop.date.end ? `${prop.date.start} → ${prop.date.end}` : prop.date.start;
+        case 'checkbox':
+            return prop.checkbox != null ? (prop.checkbox ? 'Yes' : 'No') : null;
+        case 'url':
+            return prop.url ? `[${prop.url}](${prop.url})` : null;
+        case 'email':
+            return prop.email ?? null;
+        case 'phone_number':
+            return prop.phone_number ?? null;
+        case 'people':
+            if (!prop.people || prop.people.length === 0)
+                return null;
+            return prop.people.map((p) => p.name ?? p.id).join(', ');
+        case 'relation':
+            if (!prop.relation || prop.relation.length === 0)
+                return null;
+            return prop.relation.map((r) => r.id).join(', ');
+        case 'formula':
+            if (!prop.formula)
+                return null;
+            if (prop.formula.string != null)
+                return prop.formula.string;
+            if (prop.formula.number != null)
+                return String(prop.formula.number);
+            if (prop.formula.boolean != null)
+                return prop.formula.boolean ? 'Yes' : 'No';
+            return null;
+        case 'rollup':
+            if (!prop.rollup)
+                return null;
+            if (prop.rollup.number != null)
+                return String(prop.rollup.number);
+            return null;
+        case 'files':
+            if (!prop.files || prop.files.length === 0)
+                return null;
+            return prop.files.map((f) => f.name).join(', ');
+        case 'created_time':
+            return prop.created_time ?? null;
+        case 'last_edited_time':
+            return prop.last_edited_time ?? null;
+        case 'created_by':
+            return (prop.created_by
+                ?.name ?? null);
+        case 'last_edited_by':
+            return (prop
+                .last_edited_by?.name ?? null);
+        default:
+            return null;
+    }
+}
+// ============================================
+// Helpers
+// ============================================
+function isImageFilename(name) {
+    const lower = name.toLowerCase();
+    return (lower.endsWith('.png') ||
+        lower.endsWith('.jpg') ||
+        lower.endsWith('.jpeg') ||
+        lower.endsWith('.gif') ||
+        lower.endsWith('.webp'));
+}
+function guessImageMime(name) {
+    const lower = name.toLowerCase();
+    if (lower.endsWith('.jpg') || lower.endsWith('.jpeg'))
+        return 'image/jpeg';
+    if (lower.endsWith('.gif'))
+        return 'image/gif';
+    if (lower.endsWith('.webp'))
+        return 'image/webp';
+    return 'image/png';
+}

package/dist/sources/notion-media.d.ts

@@ -0,0 +1,30 @@
+/**
+ * Downloads media files from Notion's signed URLs to a temp directory.
+ *
+ * Signed URLs expire in ~1 hour, so files must be downloaded
+ * immediately during extraction.
+ */
+import type { ContentBlock } from '../types.js';
+export declare class NotionMediaDownloader {
+    private tempDir;
+    private downloadedFiles;
+    /** Initialize the temp directory. */
+    init(): Promise<void>;
+    /** Clean up the temp directory and all downloaded files. */
+    cleanup(): Promise<void>;
+    /**
+     * Process content blocks: download any Notion-hosted files
+     * and convert from `type: 'url'` to `type: 'file'`.
+     *
+     * External URLs (non-Notion) are left as-is.
+     * Only Notion signed URLs (containing "secure.notion-static.com" or
+     * "prod-files-secure" or "s3.us-west-2.amazonaws.com") are downloaded.
+     */
+    processContentBlocks(blocks: ContentBlock[]): Promise<ContentBlock[]>;
+    /** Check if a content block has a Notion signed URL that needs downloading. */
+    private isNotionSignedUrl;
+    /** Download a single file and return the local path. */
+    private downloadFile;
+    /** Get file extension from URL or content block metadata. */
+    private getExtension;
+}

package/dist/sources/notion-media.js

@@ -0,0 +1,133 @@
+/**
+ * Downloads media files from Notion's signed URLs to a temp directory.
+ *
+ * Signed URLs expire in ~1 hour, so files must be downloaded
+ * immediately during extraction.
+ */
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import * as os from 'os';
+import * as crypto from 'crypto';
+export class NotionMediaDownloader {
+    tempDir = null;
+    downloadedFiles = new Map(); // url → local path
+    /** Initialize the temp directory. */
+    async init() {
+        this.tempDir = await fs.mkdtemp(path.join(os.tmpdir(), 'moxn-notion-media-'));
+    }
+    /** Clean up the temp directory and all downloaded files. */
+    async cleanup() {
+        if (this.tempDir) {
+            await fs.rm(this.tempDir, { recursive: true, force: true }).catch(() => { });
+            this.tempDir = null;
+        }
+        this.downloadedFiles.clear();
+    }
+    /**
+     * Process content blocks: download any Notion-hosted files
+     * and convert from `type: 'url'` to `type: 'file'`.
+     *
+     * External URLs (non-Notion) are left as-is.
+     * Only Notion signed URLs (containing "secure.notion-static.com" or
+     * "prod-files-secure" or "s3.us-west-2.amazonaws.com") are downloaded.
+     */
+    async processContentBlocks(blocks) {
+        const results = [];
+        for (const block of blocks) {
+            if (this.isNotionSignedUrl(block)) {
+                try {
+                    const localPath = await this.downloadFile(block.url, block);
+                    results.push({
+                        ...block,
+                        type: 'file',
+                        path: localPath,
+                        url: undefined,
+                    });
+                }
+                catch (error) {
+                    console.warn(`  Warning: Failed to download ${block.blockType}: ${error instanceof Error ? error.message : error}`);
+                    // Fall back to URL (may expire)
+                    results.push(block);
+                }
+            }
+            else {
+                results.push(block);
+            }
+        }
+        return results;
+    }
+    /** Check if a content block has a Notion signed URL that needs downloading. */
+    isNotionSignedUrl(block) {
+        if (block.type !== 'url' || !block.url)
+            return false;
+        const url = block.url;
+        return (url.includes('secure.notion-static.com') ||
+            url.includes('prod-files-secure') ||
+            url.includes('s3.us-west-2.amazonaws.com') ||
+            url.includes('s3-us-west-2.amazonaws.com'));
+    }
+    /** Download a single file and return the local path. */
+    async downloadFile(url, block) {
+        // Check cache
+        const cached = this.downloadedFiles.get(url);
+        if (cached)
+            return cached;
+        if (!this.tempDir) {
+            throw new Error('NotionMediaDownloader not initialized. Call init() first.');
+        }
+        const response = await fetch(url);
+        if (!response.ok) {
+            throw new Error(`HTTP ${response.status}: ${response.statusText}`);
+        }
+        const data = Buffer.from(await response.arrayBuffer());
+        // Determine filename
+        const ext = this.getExtension(url, block);
+        const hash = crypto.createHash('md5').update(url).digest('hex').slice(0, 8);
+        const filename = `${hash}${ext}`;
+        const localPath = path.join(this.tempDir, filename);
+        await fs.writeFile(localPath, data);
+        this.downloadedFiles.set(url, localPath);
+        return localPath;
+    }
+    /** Get file extension from URL or content block metadata. */
+    getExtension(url, block) {
+        // Try from mediaType
+        if (block.mediaType) {
+            const ext = mimeToExtension(block.mediaType);
+            if (ext)
+                return ext;
+        }
+        // Try from URL path
+        try {
+            const pathname = new URL(url).pathname;
+            const match = pathname.match(/\.([a-zA-Z0-9]+)(?:\?|$)/);
+            if (match)
+                return '.' + match[1].toLowerCase();
+        }
+        catch {
+            // Invalid URL
+        }
+        // Default based on block type
+        switch (block.blockType) {
+            case 'image':
+                return '.png';
+            case 'document':
+                return '.pdf';
+            case 'csv':
+                return '.csv';
+            default:
+                return '.bin';
+        }
+    }
+}
+function mimeToExtension(mime) {
+    const map = {
+        'image/png': '.png',
+        'image/jpeg': '.jpg',
+        'image/gif': '.gif',
+        'image/webp': '.webp',
+        'application/pdf': '.pdf',
+        'text/csv': '.csv',
+    };
+    return map[mime] ?? null;
+}

package/dist/sources/notion.d.ts

@@ -0,0 +1,66 @@
+/**
+ * NotionSource — imports Notion workspace content into Moxn KB.
+ *
+ * Two-pass architecture:
+ *   Pass 1 (validate): Discover all pages/databases via search API, build tree, compute paths
+ *   Pass 2 (extract):  Walk tree depth-first, fetch blocks, convert to sections
+ *
+ * Databases are imported after all pages: creates KB database + columns, links entries.
+ */
+import type { ExtractedDocument } from '../types.js';
+import { MigrationSource, type SourceConfig } from './base.js';
+import type { NotionPage } from './notion-api.js';
+import { type ParsedDatabaseSchema } from './notion-databases.js';
+export interface NotionSourceConfig extends SourceConfig {
+    token: string;
+    rootPageId?: string;
+    maxDepth?: number;
+}
+/** Info yielded to the migration runner for database import (post-document pass). */
+export interface NotionDatabaseImportInfo {
+    notionDatabaseId: string;
+    schema: ParsedDatabaseSchema;
+    entries: Array<{
+        page: NotionPage;
+        /** The documentId already created for this entry page, if any. */
+        existingDocumentId?: string;
+        /** The branchId of the existing document. */
+        existingBranchId?: string;
+        kbPath: string;
+    }>;
+}
+export declare class NotionSource extends MigrationSource<NotionSourceConfig> {
+    private client;
+    private mediaDownloader;
+    private pageTree;
+    private allPages;
+    private pagePathMap;
+    private databases;
+    private databaseEntryPageIds;
+    private _documentCount;
+    constructor(config: NotionSourceConfig);
+    get sourceType(): string;
+    get sourceLocation(): string;
+    validate(): Promise<void>;
+    getDocumentCount(): Promise<number | undefined>;
+    /**
+     * Get database import info for the migration runner.
+     * Called after all pages are imported to create databases and link entries.
+     */
+    getDatabaseImports(): NotionDatabaseImportInfo[];
+    extract(): AsyncGenerator<ExtractedDocument, void, unknown>;
+    /** Clean up temp files after migration completes. */
+    cleanup(): Promise<void>;
+    private buildPageTree;
+    private getParentPageId;
+    private processDatabases;
+    private getDatabaseParentId;
+    private extractPage;
+    private extractDatabaseEntry;
+    private downloadSectionMedia;
+}
+/**
+ * Slugify a page title for use as a KB path segment.
+ * Lowercase, spaces→hyphens, strip special chars, ltree-compatible.
+ */
+export declare function slugify(title: string): string;