@moxn/kb-migrate 0.4.12 → 0.4.14

package/dist/client.js

@@ -227,6 +227,19 @@ export class MoxnClient {
                     rowCount: block.rowCount,
                 };
             }
+            // Handle generic files - upload to storage
+            if (block.blockType === 'file' && block.type === 'file' && block.path) {
+                const data = await fs.readFile(block.path);
+                const filename = block.filename || block.path.split('/').pop();
+                const { key } = await this.uploadFile(data, block.mediaType || 'application/octet-stream', filename);
+                return {
+                    blockType: block.blockType,
+                    type: 'storage',
+                    key,
+                    mediaType: block.mediaType || 'application/octet-stream',
+                    filename: block.filename,
+                };
+            }
             return block;
         }));
     }

package/dist/sources/notion-blocks.js

@@ -108,6 +108,9 @@ async function convertBlock(block, client, pagePathMap, visitedSyncedBlocks, dat
         case 'video':
             results.push(...convertVideo(block));
             break;
+        case 'audio':
+            results.push(...convertAudio(block));
+            break;
         case 'link_to_page':
             results.push(...convertLinkToPage(block, pagePathMap));
             break;
@@ -346,7 +349,7 @@ function convertFile(block) {
             },
         ];
     }
-    // CSV or other file — render as link
+    // CSV file
     if (filename.toLowerCase().endsWith('.csv')) {
         return [
             {
@@ -358,8 +361,16 @@ function convertFile(block) {
             },
         ];
     }
-    // Generic file — render as markdown link
-    return [textBlock(`[${filename}](${url})`)];
+    // Generic file — emit as file block
+    return [
+        {
+            blockType: 'file',
+            type: 'url',
+            url,
+            mediaType: guessFileMimeType(filename),
+            filename,
+        },
+    ];
 }
 function convertVideo(block) {
     const v = block;
@@ -367,7 +378,16 @@ function convertVideo(block) {
     if (!url)
         return [];
     const caption = richTextToPlain(v.video.caption ?? []);
-    return [textBlock(`[${caption || 'Video'}](${url})`)];
+    const filename = caption || extractFilename(url) || 'video';
+    return [
+        {
+            blockType: 'file',
+            type: 'url',
+            url,
+            mediaType: guessVideoMimeType(url),
+            filename,
+        },
+    ];
 }
 function convertLinkToPage(block, pagePathMap) {
     const ltp = block;
@@ -573,6 +593,74 @@ async function convertAndMergeChildren(children, client, pagePathMap, visitedSyn
     }
     return merged;
 }
+function convertAudio(block) {
+    const a = block;
+    const url = a.audio.type === 'external' ? a.audio.external?.url : a.audio.file?.url;
+    if (!url)
+        return [];
+    const caption = richTextToPlain(a.audio.caption ?? []);
+    const filename = caption || extractFilename(url) || 'audio';
+    return [
+        {
+            blockType: 'file',
+            type: 'url',
+            url,
+            mediaType: guessAudioMimeType(url),
+            filename,
+        },
+    ];
+}
+function guessFileMimeType(filename) {
+    const lower = filename.toLowerCase();
+    const ext = lower.split('.').pop();
+    const map = {
+        docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document',
+        xlsx: 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet',
+        pptx: 'application/vnd.openxmlformats-officedocument.presentationml.presentation',
+        doc: 'application/msword',
+        xls: 'application/vnd.ms-excel',
+        ppt: 'application/vnd.ms-powerpoint',
+        odt: 'application/vnd.oasis.opendocument.text',
+        ods: 'application/vnd.oasis.opendocument.spreadsheet',
+        rtf: 'application/rtf',
+        txt: 'text/plain',
+        md: 'text/markdown',
+        json: 'application/json',
+        xml: 'application/xml',
+        yaml: 'text/yaml',
+        yml: 'text/yaml',
+        html: 'text/html',
+        htm: 'text/html',
+        zip: 'application/zip',
+        gz: 'application/gzip',
+        tar: 'application/x-tar',
+        svg: 'image/svg+xml',
+        epub: 'application/epub+zip',
+    };
+    return (ext && map[ext]) || 'application/octet-stream';
+}
+function guessVideoMimeType(url) {
+    const lower = url.toLowerCase();
+    if (lower.includes('.mp4'))
+        return 'video/mp4';
+    if (lower.includes('.webm'))
+        return 'video/webm';
+    if (lower.includes('.mov'))
+        return 'video/quicktime';
+    return 'video/mp4';
+}
+function guessAudioMimeType(url) {
+    const lower = url.toLowerCase();
+    if (lower.includes('.mp3'))
+        return 'audio/mpeg';
+    if (lower.includes('.wav'))
+        return 'audio/wav';
+    if (lower.includes('.ogg'))
+        return 'audio/ogg';
+    if (lower.includes('.m4a'))
+        return 'audio/mp4';
+    return 'audio/mpeg';
+}
 function guessImageMediaType(url) {
     const lower = url.toLowerCase();
     if (lower.includes('.jpg') || lower.includes('.jpeg'))

package/dist/sources/notion-media.js

@@ -56,7 +56,7 @@ export class NotionMediaDownloader {
                             filename: urlBlock.filename,
                         });
                     }
-                    else {
+                    else if (urlBlock.blockType === 'csv') {
                         results.push({
                             blockType: 'csv',
                             type: 'file',
@@ -67,6 +67,16 @@ export class NotionMediaDownloader {
                             rowCount: urlBlock.rowCount,
                         });
                     }
+                    else {
+                        // file block
+                        results.push({
+                            blockType: 'file',
+                            type: 'file',
+                            path: localPath,
+                            mediaType: urlBlock.mediaType,
+                            filename: urlBlock.filename,
+                        });
+                    }
                 }
                 catch (error) {
                     console.warn(`  Warning: Failed to download ${block.blockType}: ${error instanceof Error ? error.message : error}`);
@@ -153,6 +163,23 @@ function mimeToExtension(mime) {
         'image/webp': '.webp',
         'application/pdf': '.pdf',
         'text/csv': '.csv',
+        'application/vnd.openxmlformats-officedocument.wordprocessingml.document': '.docx',
+        'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': '.xlsx',
+        'application/vnd.openxmlformats-officedocument.presentationml.presentation': '.pptx',
+        'application/msword': '.doc',
+        'application/vnd.ms-excel': '.xls',
+        'application/vnd.ms-powerpoint': '.ppt',
+        'text/plain': '.txt',
+        'text/markdown': '.md',
+        'application/json': '.json',
+        'application/xml': '.xml',
+        'text/yaml': '.yaml',
+        'text/html': '.html',
+        'application/zip': '.zip',
+        'application/gzip': '.gz',
+        'application/x-tar': '.tar',
+        'image/svg+xml': '.svg',
+        'application/epub+zip': '.epub',
     };
     return map[mime] ?? null;
 }

package/dist/targets/notion.js

@@ -62,14 +62,157 @@ function stripInvalidLinks(text) {
         return displayText;
     });
 }
-function sectionsToMarkdown(sections, options) {
-    const parts = [];
+function notionImageBlock(url, alt) {
+    return {
+        object: 'block',
+        type: 'image',
+        image: {
+            type: 'external',
+            external: { url },
+            ...(alt ? { caption: [{ type: 'text', text: { content: alt } }] } : {}),
+        },
+    };
+}
+function notionPdfBlock(url, caption) {
+    return {
+        object: 'block',
+        type: 'pdf',
+        pdf: {
+            type: 'external',
+            external: { url },
+            ...(caption ? { caption: [{ type: 'text', text: { content: caption } }] } : {}),
+        },
+    };
+}
+function notionFileBlock(url, caption) {
+    return {
+        object: 'block',
+        type: 'file',
+        file: {
+            type: 'external',
+            external: { url },
+            caption: [{ type: 'text', text: { content: caption || 'file' } }],
+        },
+    };
+}
+// ============================================
+// Notion file_upload block helpers
+// ============================================
+function notionImageUploadBlock(fileUploadId, alt) {
+    return {
+        object: 'block',
+        type: 'image',
+        image: {
+            type: 'file_upload',
+            file_upload: { id: fileUploadId },
+            ...(alt ? { caption: [{ type: 'text', text: { content: alt } }] } : {}),
+        },
+    };
+}
+function notionPdfUploadBlock(fileUploadId, caption) {
+    return {
+        object: 'block',
+        type: 'pdf',
+        pdf: {
+            type: 'file_upload',
+            file_upload: { id: fileUploadId },
+            ...(caption ? { caption: [{ type: 'text', text: { content: caption } }] } : {}),
+        },
+    };
+}
+function notionFileUploadBlock(fileUploadId, caption) {
+    return {
+        object: 'block',
+        type: 'file',
+        file: {
+            type: 'file_upload',
+            file_upload: { id: fileUploadId },
+            caption: [{ type: 'text', text: { content: caption || 'file' } }],
+        },
+    };
+}
+/**
+ * Derive a filename from a storage key and mime type.
+ * Falls back to a generic name if the key has no useful basename.
+ */
+function deriveFilename(storageKey, mimeType) {
+    const basename = storageKey.split('/').pop();
+    if (basename && basename.includes('.'))
+        return basename;
+    const ext = {
+        'image/png': '.png',
+        'image/jpeg': '.jpg',
+        'image/gif': '.gif',
+        'image/webp': '.webp',
+        'application/pdf': '.pdf',
+        'text/csv': '.csv',
+    };
+    return `file${ext[mimeType] || ''}`;
+}
+/**
+ * Upload a storage-backed file to Notion via the File Upload API.
+ *
+ * Downloads from signed URL, then uploads to Notion in single-part mode.
+ * Returns the file upload ID on success, or null on failure (graceful fallback).
+ */
+async function uploadFileToNotion(client, url, filename, contentType) {
+    try {
+        // Download file from signed URL
+        const response = await fetch(url);
+        if (!response.ok) {
+            console.warn(`  [upload] Download failed (${response.status}) for ${filename}`);
+            return null;
+        }
+        const buffer = Buffer.from(await response.arrayBuffer());
+        // Create file upload
+        await sleep(RATE_LIMIT_MS);
+        const upload = await client.fileUploads.create({
+            mode: 'single_part',
+            filename,
+            content_type: contentType,
+        });
+        // Send file data
+        await sleep(RATE_LIMIT_MS);
+        await client.fileUploads.send({
+            file_upload_id: upload.id,
+            file: { data: new Blob([buffer]), filename },
+        });
+        return upload.id;
+    }
+    catch (error) {
+        console.warn(`  [upload] Failed for ${filename}: ${error instanceof Error ? error.message : error}`);
+        return null;
+    }
+}
+/**
+ * Convert KB document sections to Notion blocks.
+ *
+ * Uses a block-by-block approach: text blocks go through martian for
+ * rich formatting, while media blocks (images, PDFs, files, CSVs) are
+ * converted directly to native Notion block types. This avoids the
+ * fragility of placeholder-based post-processing.
+ *
+ * Section names become H2 headings (mirrors the import convention).
+ */
+async function sectionsToNotionBlocks(sections, options) {
+    const allBlocks = [];
     const allReferences = [];
     const databaseIds = [];
-    const media = [];
     const extractRefs = options?.extractReferences ?? false;
+    // Accumulate contiguous text blocks into markdown, then flush through martian
+    let pendingMarkdown = [];
+    function flushText() {
+        if (pendingMarkdown.length === 0)
+            return;
+        const md = pendingMarkdown.join('\n').trim();
+        if (md) {
+            allBlocks.push(...markdownToBlocks(md));
+        }
+        pendingMarkdown = [];
+    }
     for (const section of sections) {
-        parts.push(`## ${section.name}\n`);
+        // Section heading — accumulate as markdown so martian handles it
+        pendingMarkdown.push(`## ${section.name}\n`);
         for (const block of section.content) {
             if (block.blockType === 'text' && block.text) {
                 let text = stripCommentTags(block.text);
@@ -78,93 +221,65 @@ function sectionsToMarkdown(sections, options) {
                     text = cleanedText;
                     allReferences.push(...references);
                 }
-                // Strip relative/internal links that aren't valid URLs
-                // (Notion rejects links without a protocol)
                 text = stripInvalidLinks(text);
-                parts.push(text);
-                parts.push('');
+                pendingMarkdown.push(text);
+                pendingMarkdown.push('');
             }
             else if (block.blockType === 'image' && block.url) {
-                const token = `MOXNMEDIA${media.length}PLACEHOLDER`;
-                media.push({ token, type: 'image', url: block.url, alt: block.alt });
-                parts.push(token);
-                parts.push('');
+                flushText();
+                if (block.storageKey && options?.notionClient) {
+                    const fname = block.filename || deriveFilename(block.storageKey, block.mimeType || 'image/png');
+                    const uploadId = await uploadFileToNotion(options.notionClient, block.url, fname, block.mimeType || 'image/png');
+                    allBlocks.push(uploadId ? notionImageUploadBlock(uploadId, block.alt) : notionImageBlock(block.url, block.alt));
+                }
+                else {
+                    allBlocks.push(notionImageBlock(block.url, block.alt));
+                }
             }
             else if (block.blockType === 'document' && block.url) {
-                const token = `MOXNMEDIA${media.length}PLACEHOLDER`;
-                media.push({ token, type: 'file', url: block.url, filename: block.filename });
-                parts.push(token);
-                parts.push('');
+                flushText();
+                if (block.storageKey && options?.notionClient) {
+                    const fname = block.filename || deriveFilename(block.storageKey, block.mimeType || 'application/octet-stream');
+                    const uploadId = await uploadFileToNotion(options.notionClient, block.url, fname, block.mimeType || 'application/octet-stream');
+                    if (uploadId) {
+                        allBlocks.push(block.mimeType === 'application/pdf' ? notionPdfUploadBlock(uploadId, block.filename) : notionFileUploadBlock(uploadId, block.filename));
+                    }
+                    else if (block.mimeType === 'application/pdf') {
+                        allBlocks.push(notionPdfBlock(block.url, block.filename));
+                    }
+                    else {
+                        allBlocks.push(notionFileBlock(block.url, block.filename));
+                    }
+                }
+                else if (block.mimeType === 'application/pdf') {
+                    allBlocks.push(notionPdfBlock(block.url, block.filename));
+                }
+                else {
+                    allBlocks.push(notionFileBlock(block.url, block.filename));
+                }
             }
             else if (block.blockType === 'csv' && block.url) {
-                const token = `MOXNMEDIA${media.length}PLACEHOLDER`;
-                media.push({ token, type: 'embed', url: block.url, filename: block.filename || 'data.csv' });
-                parts.push(token);
-                parts.push('');
+                flushText();
+                if (block.storageKey && options?.notionClient) {
+                    const fname = block.filename || deriveFilename(block.storageKey, 'text/csv');
+                    const uploadId = await uploadFileToNotion(options.notionClient, block.url, fname, 'text/csv');
+                    allBlocks.push(uploadId ? notionFileUploadBlock(uploadId, block.filename || 'data.csv') : notionFileBlock(block.url, block.filename || 'data.csv'));
+                }
+                else {
+                    allBlocks.push(notionFileBlock(block.url, block.filename || 'data.csv'));
+                }
             }
             else if (block.blockType === 'database_embed' && block.databaseId) {
-                // Collect database ID for Pass 1.5 export
                 databaseIds.push(block.databaseId);
-                // Add a placeholder in the markdown
-                parts.push(`> **[Database]** *(exported as inline database)*`);
-                parts.push('');
+                // Keep database placeholder as text (handled in Pass 1.5)
+                pendingMarkdown.push(`> **[Database]** *(exported as inline database)*`);
+                pendingMarkdown.push('');
             }
         }
     }
-    return { markdown: parts.join('\n').trim(), references: allReferences, databaseIds, media };
-}
-/**
- * Replace media placeholder paragraphs in Notion blocks with proper
- * image/file/embed blocks. Martian doesn't support images, so we
- * post-process the converted blocks.
- */
-function injectMediaBlocks(blocks, media) {
-    if (media.length === 0)
-        return blocks;
-    // Build a lookup from token to media info
-    const tokenMap = new Map(media.map((m) => [m.token, m]));
-    return blocks.map((block) => {
-        // Check if this is a paragraph containing a media placeholder
-        const b = block;
-        if (b.type !== 'paragraph' || !b.paragraph?.rich_text)
-            return block;
-        const text = b.paragraph.rich_text.map((rt) => rt.text?.content ?? '').join('').trim();
-        const mediaInfo = tokenMap.get(text);
-        if (!mediaInfo)
-            return block;
-        // Replace with proper Notion block
-        if (mediaInfo.type === 'image') {
-            return {
-                object: 'block',
-                type: 'image',
-                image: {
-                    type: 'external',
-                    external: { url: mediaInfo.url },
-                    ...(mediaInfo.alt ? { caption: [{ type: 'text', text: { content: mediaInfo.alt } }] } : {}),
-                },
-            };
-        }
-        if (mediaInfo.type === 'file') {
-            return {
-                object: 'block',
-                type: 'file',
-                file: {
-                    type: 'external',
-                    external: { url: mediaInfo.url },
-                    caption: [{ type: 'text', text: { content: mediaInfo.filename || 'document' } }],
-                },
-            };
-        }
-        // For CSV/embeds, use a bookmark block (Notion doesn't have native CSV embed)
-        return {
-            object: 'block',
-            type: 'bookmark',
-            bookmark: {
-                url: mediaInfo.url,
-                caption: [{ type: 'text', text: { content: mediaInfo.filename || 'file' } }],
-            },
-        };
-    });
+    // Flush remaining text
+    flushText();
+    return { blocks: allBlocks, references: allReferences, databaseIds };
 }
 // Max 100 blocks per API call
 const MAX_BLOCKS_PER_APPEND = 100;
@@ -242,8 +357,8 @@ export class NotionExportTarget extends ExportTarget {
      * Returns the number of references resolved.
      */
     async resolveAndAppendReferences(doc, notionPageId) {
-        // Extract references from section content
-        const { references } = sectionsToMarkdown(doc.sections, { extractReferences: true });
+        // Extract references from section content (no upload needed for reference pass)
+        const { references } = await sectionsToNotionBlocks(doc.sections, { extractReferences: true });
         if (references.length === 0) {
             return { resolved: 0, unresolved: 0 };
         }
@@ -413,8 +528,7 @@ export class NotionExportTarget extends ExportTarget {
     // Notion page creation / update
     // ============================================
     async createNotionPage(doc) {
-        const { markdown, media } = sectionsToMarkdown(doc.sections, { extractReferences: true });
-        const blocks = injectMediaBlocks(markdownToBlocks(markdown), media);
+        const { blocks } = await sectionsToNotionBlocks(doc.sections, { extractReferences: true, notionClient: this.client });
         // First batch: up to 100 blocks as children of the new page
         const firstBatch = blocks.slice(0, MAX_BLOCKS_PER_APPEND);
         const remainingBlocks = blocks.slice(MAX_BLOCKS_PER_APPEND);
@@ -446,8 +560,7 @@ export class NotionExportTarget extends ExportTarget {
             },
         });
         await this.clearPageContent(notionPageId);
-        const { markdown, media } = sectionsToMarkdown(doc.sections, { extractReferences: true });
-        const blocks = injectMediaBlocks(markdownToBlocks(markdown), media);
+        const { blocks } = await sectionsToNotionBlocks(doc.sections, { extractReferences: true, notionClient: this.client });
         await this.appendRemainingBlocks(notionPageId, blocks);
     }
     async clearPageContent(pageId) {

package/dist/types.d.ts

@@ -8,7 +8,7 @@
  * to kb-migrate (local filesystem paths) — MoxnClient converts them to
  * `type: 'storage'` before sending to the KB API.
  */
-export type ContentBlock = TextBlock | ImageRemoteBlock | ImageFileBlock | DocumentRemoteBlock | DocumentFileBlock | CsvRemoteBlock | CsvFileBlock | DatabaseEmbedBlock;
+export type ContentBlock = TextBlock | ImageRemoteBlock | ImageFileBlock | DocumentRemoteBlock | DocumentFileBlock | CsvRemoteBlock | CsvFileBlock | FileRemoteBlock | FileFileBlock | DatabaseEmbedBlock;
 export interface TextBlock {
     blockType: 'text';
     text: string;
@@ -65,6 +65,22 @@ export interface CsvFileBlock {
     headers?: string[];
     rowCount?: number;
 }
+export interface FileRemoteBlock {
+    blockType: 'file';
+    type: 'base64' | 'url' | 'storage';
+    mediaType: string;
+    base64?: string;
+    url?: string;
+    key?: string;
+    filename?: string;
+}
+export interface FileFileBlock {
+    blockType: 'file';
+    type: 'file';
+    path: string;
+    mediaType: string;
+    filename?: string;
+}
 export interface DatabaseEmbedBlock {
     blockType: 'database_embed';
     databaseId: string;
@@ -187,7 +203,7 @@ export interface ConflictError {
  * A content block as returned by the KB API
  */
 export interface ExportContentBlock {
-    blockType: 'text' | 'image' | 'document' | 'csv' | 'database_embed';
+    blockType: 'text' | 'image' | 'document' | 'csv' | 'file' | 'database_embed';
     text?: string;
     url?: string;
     mimeType?: string;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@moxn/kb-migrate",
-  "version": "0.4.12",
+  "version": "0.4.14",
   "description": "Migration tool for importing documents into Moxn Knowledge Base from local files, Notion, Google Docs, and more",
   "type": "module",
   "main": "dist/index.js",