@moxn/kb-migrate 0.4.29 → 0.4.31

package/dist/sources/onenote/onenote-html.js

@@ -0,0 +1,619 @@
+/**
+ * Convert OneNote page HTML into Moxn SectionInput[].
+ *
+ * Contract:
+ *   - Input: a page's HTML body as returned by GET /me/onenote/pages/{id}/content
+ *     (with ?includeIDs=true, so each element has a stable data-id).
+ *   - Output: zero or more SectionInput. Content before the first <h2> lands
+ *     in an "Introduction" section; each <h2> starts a new section.
+ *
+ * This is the highest-risk part of the import — OneNote HTML is idiosyncratic
+ * (absolute positioning, data-tag semantics, citation footers). The function
+ * is pure: no network, no filesystem. It consumes an ImageResolver callback
+ * that the fan-out task wires to a Supabase-upload pipeline.
+ *
+ * Images: the converter does NOT fetch images. It calls `resolveMedia` for
+ * every <img>/<object> and expects back a storage key. The resolver is free
+ * to return null to indicate "skip this embed".
+ */
+import { parse, NodeType } from 'node-html-parser';
+const INTRODUCTION_SECTION = 'Introduction';
+/** Map from OneNote note-tag semantic → emoji prefix. `to-do` is special-cased. */
+const NOTE_TAG_EMOJI = {
+    important: '⭐',
+    question: '❓',
+    definition: '📖',
+    highlight: '🖍️',
+    'contact-information': '📇',
+    address: '📍',
+    'phone-number': '📞',
+    'web-site-to-visit': '🔗',
+    idea: '💡',
+    password: '🔑',
+    critical: '🚨',
+    'project-a': '🅰️',
+    'project-b': '🅱️',
+    'remember-for-later': '🧠',
+    'movie-to-see': '🎬',
+    'book-to-read': '📚',
+    'music-to-listen-to': '🎵',
+    'source-for-article': '📰',
+    'remember-for-blog': '🖋️',
+    discuss: '💬',
+    'send-in-email': '📧',
+    schedule: '📅',
+    'call-back': '📞',
+    'to-do-priority-1': '❗',
+    'to-do-priority-2': '❕',
+    client: '👤',
+};
+export async function convertOneNoteHtmlToSections(html, options) {
+    // Extract title and strip it from the body.
+    const root = parse(html, {
+        lowerCaseTagName: false,
+        comment: false,
+    });
+    const title = extractTitle(root, options.titleOverride);
+    const body = resolveBodyRoot(root);
+    const flatTopLevel = flattenForReadingOrder(body);
+    const sections = [];
+    let currentName = INTRODUCTION_SECTION;
+    let currentBlocks = [];
+    /**
+     * True once we've seen the first H2. Before that, an "Introduction" with
+     * no content is dropped (matches Notion's behavior). After the first H2,
+     * every H2 pushes a section even if empty — the user's structure survives
+     * even when media resolution fails.
+     */
+    let headingSeen = false;
+    const refs = [];
+    const skippedReasons = [];
+    let mediaCount = 0;
+    let skippedItemCount = 0;
+    const flushSection = (preserveEmpty) => {
+        if (preserveEmpty || currentBlocks.length > 0) {
+            sections.push({ name: currentName, content: compact(currentBlocks) });
+        }
+        currentBlocks = [];
+    };
+    for (const node of flatTopLevel) {
+        if (isCitationFooter(node)) {
+            if (options.dropCitations)
+                continue;
+            const text = node.text.trim();
+            if (text) {
+                currentBlocks.push(textBlock('> ' + text));
+            }
+            continue;
+        }
+        if (isH1(node) && currentBlocks.length === 0 && sections.length === 0) {
+            // Leading H1 usually echoes the page title; skip.
+            continue;
+        }
+        if (isH2(node)) {
+            // First H2: flush Introduction only if it has content (no empty header).
+            // Subsequent H2s: flush even if empty so structure is preserved.
+            flushSection(headingSeen);
+            currentName = node.text.trim() || 'Untitled';
+            headingSeen = true;
+            continue;
+        }
+        const sectionIndex = sections.length;
+        const result = await convertTopLevelNode(node, {
+            sectionIndex,
+            resolveMedia: options.resolveMedia,
+        });
+        currentBlocks.push(...result.blocks);
+        refs.push(...result.refs);
+        mediaCount += result.mediaCount;
+        skippedItemCount += result.skippedItemCount;
+        skippedReasons.push(...result.skippedReasons);
+    }
+    flushSection(headingSeen);
+    if (sections.length === 0) {
+        // Empty page — surface a single empty section so downstream code has
+        // something to write. Callers typically suppress truly empty pages.
+        sections.push({ name: INTRODUCTION_SECTION, content: [] });
+    }
+    return {
+        title,
+        sections,
+        extractedReferences: refs,
+        mediaCount,
+        skippedItemCount,
+        skippedReasons,
+    };
+}
+// -----------------------------------------------------------------
+// Title + body extraction
+// -----------------------------------------------------------------
+function extractTitle(root, override) {
+    if (override && override.trim().length > 0)
+        return override.trim();
+    const titleEl = root.querySelector('title');
+    if (titleEl) {
+        const t = titleEl.text.trim();
+        if (t.length > 0)
+            return t;
+    }
+    const h1 = root.querySelector('h1');
+    if (h1) {
+        const t = h1.text.trim();
+        if (t.length > 0)
+            return t;
+    }
+    return 'Untitled';
+}
+function resolveBodyRoot(root) {
+    const body = root.querySelector('body');
+    if (body)
+        return body;
+    // Some responses come in as a naked fragment
+    return root;
+}
+/**
+ * OneNote pages authored as "canvases" wrap content in absolute-positioned
+ * <div> blocks. Reading order in the DOM may not reflect visual reading order.
+ *
+ * If the body has data-absolute-enabled="true", we sort direct children by
+ * (top, left) coordinates parsed from `style`. Otherwise DOM order is used.
+ *
+ * Returned nodes are the *direct children of body* (or a flattened version
+ * when the body is an absolute canvas).
+ */
+function flattenForReadingOrder(body) {
+    const isAbsoluteCanvas = body.getAttribute('data-absolute-enabled') === 'true';
+    const children = body.childNodes.filter(isElement);
+    if (!isAbsoluteCanvas) {
+        return children;
+    }
+    const withCoords = children.map((child) => ({
+        node: child,
+        top: parsePxStyle(child.getAttribute('style') ?? '', 'top'),
+        left: parsePxStyle(child.getAttribute('style') ?? '', 'left'),
+    }));
+    withCoords.sort((a, b) => {
+        if (a.top !== b.top)
+            return a.top - b.top;
+        return a.left - b.left;
+    });
+    // Each "canvas" div typically contains the actual content (p/h/img/etc.)
+    // Flatten one level of wrapping divs to expose the real elements.
+    const flat = [];
+    for (const { node } of withCoords) {
+        if (node.tagName === 'DIV') {
+            for (const grandchild of node.childNodes.filter(isElement)) {
+                flat.push(grandchild);
+            }
+        }
+        else {
+            flat.push(node);
+        }
+    }
+    return flat;
+}
+function parsePxStyle(style, prop) {
+    const match = style.match(new RegExp(`${prop}\\s*:\\s*(-?\\d+(?:\\.\\d+)?)px`));
+    return match ? parseFloat(match[1]) : 0;
+}
+function isElement(node) {
+    return node.nodeType === NodeType.ELEMENT_NODE;
+}
+async function convertTopLevelNode(node, ctx) {
+    const tag = node.tagName?.toUpperCase();
+    switch (tag) {
+        case 'H1':
+            return block(textBlock(`# ${node.text.trim()}`));
+        case 'H3':
+            return block(textBlock(`### ${extractInlineMarkdown(node, ctx).text}`));
+        case 'H4':
+            return block(textBlock(`#### ${extractInlineMarkdown(node, ctx).text}`));
+        case 'H5':
+            return block(textBlock(`##### ${extractInlineMarkdown(node, ctx).text}`));
+        case 'H6':
+            return block(textBlock(`###### ${extractInlineMarkdown(node, ctx).text}`));
+        case 'P':
+        case 'SPAN':
+        case 'DIV':
+            return convertParagraph(node, ctx);
+        case 'UL':
+            return convertList(node, ctx, 'bullet');
+        case 'OL':
+            return convertList(node, ctx, 'ordered');
+        case 'TABLE':
+            return convertTable(node, ctx);
+        case 'IMG':
+            return convertImage(node, ctx);
+        case 'OBJECT':
+            return convertObject(node, ctx);
+        case 'IFRAME':
+            return convertIframe(node);
+        case 'BLOCKQUOTE':
+            return convertBlockquote(node, ctx);
+        case 'PRE':
+            return convertPre(node);
+        case 'BR':
+            return empty();
+        default:
+            // Unknown top-level: render plain text if any
+            const text = node.text.trim();
+            return text ? block(textBlock(text)) : empty();
+    }
+}
+// -----------------------------------------------------------------
+// Block conversions
+// -----------------------------------------------------------------
+async function convertParagraph(node, ctx) {
+    const { refs, mediaBlocks, mediaCount } = await collectInlineMedia(node, ctx);
+    const noteTag = node.getAttribute('data-tag');
+    const md = extractInlineMarkdown(node, ctx);
+    const blocks = [];
+    if (noteTag === 'to-do' || noteTag === 'to-do:completed') {
+        const checked = noteTag === 'to-do:completed';
+        blocks.push(textBlock(`${checked ? '- [x]' : '- [ ]'} ${md.text}`));
+    }
+    else if (noteTag && NOTE_TAG_EMOJI[noteTag]) {
+        blocks.push(textBlock(`${NOTE_TAG_EMOJI[noteTag]} ${md.text}`));
+    }
+    else if (md.text.trim().length > 0) {
+        blocks.push(textBlock(md.text));
+    }
+    blocks.push(...mediaBlocks);
+    return {
+        blocks,
+        refs: [...md.refs, ...refs.map((r) => ({ ...r, sectionIndex: ctx.sectionIndex }))],
+        mediaCount,
+        skippedItemCount: 0,
+        skippedReasons: [],
+    };
+}
+async function convertList(node, ctx, kind) {
+    const lines = [];
+    const refs = [];
+    const mediaBlocks = [];
+    let mediaCount = 0;
+    let index = 1;
+    for (const child of node.childNodes.filter(isElement)) {
+        if (child.tagName !== 'LI')
+            continue;
+        const inline = extractInlineMarkdown(child, ctx);
+        refs.push(...inline.refs);
+        const bullet = kind === 'bullet' ? '-' : `${index++}.`;
+        const noteTag = child.getAttribute('data-tag');
+        let prefix = '';
+        if (noteTag === 'to-do')
+            prefix = '[ ] ';
+        else if (noteTag === 'to-do:completed')
+            prefix = '[x] ';
+        else if (noteTag && NOTE_TAG_EMOJI[noteTag])
+            prefix = `${NOTE_TAG_EMOJI[noteTag]} `;
+        lines.push(`${bullet} ${prefix}${inline.text.trim()}`);
+        const inner = await collectInlineMedia(child, ctx);
+        mediaBlocks.push(...inner.mediaBlocks);
+        mediaCount += inner.mediaCount;
+    }
+    return {
+        blocks: [textBlock(lines.join('\n')), ...mediaBlocks].filter((b) => !(b.blockType === 'text' && !b.text.trim())),
+        refs: refs.map((r) => ({ ...r, sectionIndex: ctx.sectionIndex })),
+        mediaCount,
+        skippedItemCount: 0,
+        skippedReasons: [],
+    };
+}
+async function convertTable(node, ctx) {
+    const rows = [];
+    for (const tr of node.querySelectorAll('tr')) {
+        const cells = [];
+        for (const cell of tr.childNodes.filter(isElement)) {
+            if (cell.tagName !== 'TD' && cell.tagName !== 'TH')
+                continue;
+            const inline = extractInlineMarkdown(cell, ctx);
+            cells.push(inline.text.trim().replace(/\|/g, '\\|') || ' ');
+        }
+        if (cells.length > 0)
+            rows.push(cells);
+    }
+    if (rows.length === 0)
+        return empty();
+    // Markdown table: synthesize a header row if the source didn't use <th>
+    const header = rows[0];
+    const sep = header.map(() => '---');
+    const rest = rows.slice(1);
+    const md = [
+        `| ${header.join(' | ')} |`,
+        `| ${sep.join(' | ')} |`,
+        ...rest.map((r) => `| ${r.join(' | ')} |`),
+    ].join('\n');
+    return block(textBlock(md));
+}
+async function convertImage(node, ctx) {
+    const src = node.getAttribute('src') ?? node.getAttribute('data-fullres-src');
+    if (!src)
+        return empty();
+    const alt = node.getAttribute('alt');
+    const declared = node.getAttribute('data-src-type') ?? undefined;
+    const resolved = await ctx.resolveMedia({
+        kind: 'image',
+        url: src,
+        declaredMimeType: declared,
+        alt: alt ?? undefined,
+    });
+    if (!resolved) {
+        return {
+            blocks: [],
+            refs: [],
+            mediaCount: 0,
+            skippedItemCount: 1,
+            skippedReasons: ['image skipped by media resolver'],
+        };
+    }
+    const blk = {
+        blockType: 'image',
+        type: 'storage',
+        key: resolved.key,
+        mediaType: resolved.mediaType,
+        alt: alt ?? undefined,
+    };
+    return {
+        blocks: [blk],
+        refs: [],
+        mediaCount: 1,
+        skippedItemCount: 0,
+        skippedReasons: [],
+    };
+}
+async function convertObject(node, ctx) {
+    const data = node.getAttribute('data');
+    if (!data)
+        return empty();
+    const mime = node.getAttribute('type') ?? undefined;
+    const filename = node.getAttribute('data-attachment') ?? undefined;
+    const resolved = await ctx.resolveMedia({
+        kind: 'file',
+        url: data,
+        declaredMimeType: mime,
+        declaredFilename: filename,
+    });
+    if (!resolved) {
+        return {
+            blocks: [],
+            refs: [],
+            mediaCount: 0,
+            skippedItemCount: 1,
+            skippedReasons: [`attachment skipped: ${filename ?? data}`],
+        };
+    }
+    // Pick the right block type based on MIME
+    let blk;
+    const mediaType = resolved.mediaType;
+    if (mediaType === 'application/pdf') {
+        blk = {
+            blockType: 'document',
+            type: 'storage',
+            key: resolved.key,
+            mediaType,
+            filename: resolved.filename ?? filename,
+        };
+    }
+    else if (mediaType === 'text/csv') {
+        blk = {
+            blockType: 'csv',
+            type: 'storage',
+            key: resolved.key,
+            mediaType,
+            filename: resolved.filename ?? filename,
+        };
+    }
+    else {
+        blk = {
+            blockType: 'file',
+            type: 'storage',
+            key: resolved.key,
+            mediaType,
+            filename: resolved.filename ?? filename,
+        };
+    }
+    return {
+        blocks: [blk],
+        refs: [],
+        mediaCount: 1,
+        skippedItemCount: 0,
+        skippedReasons: [],
+    };
+}
+async function convertIframe(node) {
+    const src = node.getAttribute('data-original-src') ?? node.getAttribute('src') ?? '';
+    if (!src)
+        return empty();
+    return block(textBlock(`[Embed](${src})`));
+}
+async function convertBlockquote(node, ctx) {
+    const md = extractInlineMarkdown(node, ctx);
+    const quoted = md.text
+        .split('\n')
+        .map((l) => (l.length ? `> ${l}` : '>'))
+        .join('\n');
+    return {
+        blocks: [textBlock(quoted)],
+        refs: md.refs.map((r) => ({ ...r, sectionIndex: ctx.sectionIndex })),
+        mediaCount: 0,
+        skippedItemCount: 0,
+        skippedReasons: [],
+    };
+}
+async function convertPre(node) {
+    const text = node.text;
+    return block(textBlock('```\n' + text + '\n```'));
+}
+function extractInlineMarkdown(node, ctx) {
+    const refs = [];
+    const text = walkInline(node, refs, ctx);
+    return { text, refs };
+}
+function walkInline(node, refs, ctx) {
+    const tag = node.tagName?.toUpperCase();
+    const style = node.getAttribute('style') ?? '';
+    const children = node.childNodes
+        .map((c) => renderChild(c, refs, ctx))
+        .join('');
+    switch (tag) {
+        case 'B':
+        case 'STRONG':
+            return wrap(children, '**');
+        case 'I':
+        case 'EM':
+            return wrap(children, '*');
+        case 'U':
+            // Markdown has no native underline — render as HTML so downstream
+            // converters (martian, remark) can preserve semantics.
+            return `<u>${children}</u>`;
+        case 'STRIKE':
+        case 'S':
+        case 'DEL':
+            return wrap(children, '~~');
+        case 'CODE':
+            return `\`${children}\``;
+        case 'SUB':
+            return `<sub>${children}</sub>`;
+        case 'SUP':
+            return `<sup>${children}</sup>`;
+        case 'A': {
+            const href = node.getAttribute('href') ?? '';
+            const ref = detectOneNoteRef(href);
+            if (ref) {
+                refs.push({
+                    sectionIndex: ctx.sectionIndex,
+                    targetOnenotePageId: ref,
+                    displayText: children,
+                });
+            }
+            if (!children)
+                return '';
+            return `[${children}](${href})`;
+        }
+        case 'BR':
+            return '  \n';
+        case 'SPAN': {
+            // OneNote likes to wrap text in <span style="...">. Preserve bold/italic
+            // via style when HTML tags weren't used.
+            const isBold = /font-weight\s*:\s*(bold|700|800|900)/i.test(style);
+            const isItalic = /font-style\s*:\s*italic/i.test(style);
+            let out = children;
+            if (isBold)
+                out = wrap(out, '**');
+            if (isItalic)
+                out = wrap(out, '*');
+            return out;
+        }
+        default:
+            return children;
+    }
+}
+function renderChild(node, refs, ctx) {
+    if (node.nodeType === NodeType.TEXT_NODE) {
+        return collapseWhitespace(node.text);
+    }
+    if (isElement(node)) {
+        return walkInline(node, refs, ctx);
+    }
+    return '';
+}
+function wrap(text, marker) {
+    if (!text.trim())
+        return text;
+    // Avoid marker collision by only wrapping when inner doesn't end with same marker
+    return `${marker}${text}${marker}`;
+}
+function collapseWhitespace(s) {
+    return s.replace(/\s+/g, ' ');
+}
+/**
+ * Detect OneNote-style cross-page references. We support:
+ *   - onenote:https://.../section/page.one#pageId={guid}&end
+ *   - query string containing pageId=...
+ *   - bare Graph API page URLs with /pages/{id}/
+ */
+function detectOneNoteRef(href) {
+    if (!href)
+        return null;
+    // 1) onenote:// URI with pageId
+    const pageIdMatch = href.match(/pageid=([^&]+)/i);
+    if (pageIdMatch)
+        return decodeURIComponent(pageIdMatch[1]);
+    // 2) Graph API URL /pages/{id}
+    const graphMatch = href.match(/\/pages\/([^/?#]+)/);
+    if (graphMatch)
+        return decodeURIComponent(graphMatch[1]);
+    return null;
+}
+async function collectInlineMedia(node, ctx) {
+    const mediaBlocks = [];
+    let mediaCount = 0;
+    const refs = [];
+    for (const img of node.querySelectorAll('img')) {
+        const result = await convertImage(img, ctx);
+        mediaBlocks.push(...result.blocks);
+        mediaCount += result.mediaCount;
+        refs.push(...result.refs);
+    }
+    for (const obj of node.querySelectorAll('object')) {
+        const result = await convertObject(obj, ctx);
+        mediaBlocks.push(...result.blocks);
+        mediaCount += result.mediaCount;
+        refs.push(...result.refs);
+    }
+    return { mediaBlocks, mediaCount, refs };
+}
+// -----------------------------------------------------------------
+// Citation footer detection
+// -----------------------------------------------------------------
+function isCitationFooter(node) {
+    if (node.tagName !== 'CITE' && node.tagName !== 'FOOTER') {
+        // OneNote typically wraps the citation in a <p> or <cite> with a known text
+        const txt = node.text.trim().toLowerCase();
+        if (txt.startsWith('from <') || txt.startsWith('copied from'))
+            return true;
+        // Graph also sometimes emits a <p> with data-citation-visible="true"
+        if (node.getAttribute('data-citation-visible') === 'true')
+            return true;
+        return false;
+    }
+    return true;
+}
+function isH1(node) {
+    return node.tagName === 'H1';
+}
+function isH2(node) {
+    return node.tagName === 'H2';
+}
+// -----------------------------------------------------------------
+// Helpers
+// -----------------------------------------------------------------
+function textBlock(text) {
+    return { blockType: 'text', text };
+}
+function block(b) {
+    return { blocks: [b], refs: [], mediaCount: 0, skippedItemCount: 0, skippedReasons: [] };
+}
+function empty() {
+    return { blocks: [], refs: [], mediaCount: 0, skippedItemCount: 0, skippedReasons: [] };
+}
+/**
+ * Collapse consecutive text blocks with two newlines between; drop empties.
+ */
+function compact(blocks) {
+    const out = [];
+    for (const b of blocks) {
+        if (b.blockType === 'text') {
+            if (!b.text.trim())
+                continue;
+            const last = out[out.length - 1];
+            if (last && last.blockType === 'text') {
+                last.text = `${last.text}\n\n${b.text}`;
+                continue;
+            }
+        }
+        out.push(b);
+    }
+    return out;
+}

package/dist/sources/onenote/onenote-tree.d.ts

@@ -0,0 +1,38 @@
+/**
+ * Discovery: walk Microsoft Graph's OneNote hierarchy and flatten into a
+ * list of pages with their computed KB path.
+ *
+ * The hierarchy is: Notebook → SectionGroup* → Section → Page.
+ * SectionGroups can nest; OneNote caps at 4 levels of nesting in practice
+ * and we cap recursion at 4 to avoid accidental cycles.
+ *
+ * Selection semantics: callers pass a set of selected Graph IDs (any mix of
+ * notebook / sectionGroup / section / page). A page is included iff its
+ * own ID or any ancestor's ID is in the set. If the set is empty, every
+ * page is included.
+ */
+import type { OneNoteApiClient } from './onenote-api.js';
+import type { DiscoveredOneNotePage } from './types.js';
+export interface DiscoverOptions {
+    /** Selected Graph IDs (any level). Empty / undefined = include everything. */
+    selectedIds?: Set<string>;
+    /** Path prefix prepended to every kbPath; defaults to `imported/onenote` */
+    pathPrefix?: string;
+    modifiedAfter?: Date;
+    modifiedBefore?: Date;
+}
+/**
+ * Normalize any human text into a ltree-safe slug segment.
+ * - lowercases
+ * - replaces spaces/underscores/periods with hyphens (ltree forbids _ and .)
+ * - drops anything outside [a-z0-9-]
+ * - collapses runs of hyphens; trims leading/trailing hyphens
+ * Falls back to `untitled` when the result is empty.
+ */
+export declare function slugifyPathSegment(raw: string): string;
+export declare function discoverOneNotePages(client: OneNoteApiClient, opts?: DiscoverOptions): Promise<DiscoveredOneNotePage[]>;
+/**
+ * Join kbPath segments into `/a/b/c` — public helper used by the orchestrator
+ * when it needs to derive a path outside the discovery walk.
+ */
+export declare function joinKbPath(...segments: Array<string | undefined>): string;