officeparser 7.2.2 → 7.3.0

package/dist/parsers/HtmlParser.js

@@ -1,6 +1,7 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.parseHtml = void 0;
+const types_js_1 = require("../types.js");
 const astUtils_js_1 = require("../utils/astUtils.js");
 const errorUtils_js_1 = require("../utils/errorUtils.js");
 const parseAttributes = (attrString) => {
@@ -36,14 +37,16 @@ const parseHtmlTree = (html) => {
             cursor = commentEnd !== -1 ? commentEnd + 3 : html.length;
             continue;
         }
-        const tagEndMatch = html.substring(tagStart).match(/>/);
-        if (!tagEndMatch) {
+        // indexOf (not substring().match) so scanning for the tag end is O(1) in
+        // allocation — a document with many "<" chars would otherwise be O(n^2).
+        const tagEndIdx = html.indexOf('>', tagStart);
+        if (tagEndIdx === -1) {
             const text = html.substring(tagStart);
             current.children.push({ type: 'text', text, children: [], parent: current });
             break;
         }
-        const tagContent = html.substring(tagStart + 1, tagStart + tagEndMatch.index);
-        cursor = tagStart + tagEndMatch.index + 1;
+        const tagContent = html.substring(tagStart + 1, tagEndIdx);
+        cursor = tagEndIdx + 1;
         const isClosing = tagContent.startsWith('/');
         const isSelfClosing = tagContent.endsWith('/');
         const tagCore = tagContent.replace(/^\/|\/$/g, '').trim();
@@ -77,16 +80,20 @@ const parseHtmlTree = (html) => {
             if (!isSelfClosing && !voidElements.has(tagName)) {
                 current = node;
                 if (tagName === 'script' || tagName === 'style') {
-                    const closeTag = `</${tagName}>`;
-                    const closeIdx = html.toLowerCase().indexOf(closeTag, cursor);
-                    if (closeIdx !== -1) {
+                    // Case-insensitive search from `cursor` via a sticky-ish regex, instead of
+                    // lower-casing the whole document on every <script>/<style> (was O(n^2)).
+                    // tagName is validated to /^[a-z0-9-]+$/ above, so it's safe to interpolate.
+                    const closeRe = new RegExp(`</${tagName}>`, 'gi');
+                    closeRe.lastIndex = cursor;
+                    const closeMatch = closeRe.exec(html);
+                    if (closeMatch) {
                         node.children.push({
                             type: 'text',
-                            text: html.substring(cursor, closeIdx),
+                            text: html.substring(cursor, closeMatch.index),
                             children: [],
                             parent: node
                         });
-                        cursor = closeIdx + closeTag.length;
+                        cursor = closeMatch.index + closeMatch[0].length;
                         current = node.parent;
                     }
                 }
@@ -183,7 +190,30 @@ const parseHtml = async (buffer, config) => {
     }
     const content = [];
     let htmlListIdCounter = 1;
-    const parseNode = (node, currentFormatting = {}, listContext) => {
+    // Finds the checked state from a nested <input type="checkbox"> (GFM task-list items
+    // nest it inside a <label>, so it isn't a direct child of the <li>).
+    const findNestedCheckboxChecked = (n) => {
+        if (n.tagName === 'input' && (n.attributes?.type || '').toLowerCase() === 'checkbox') {
+            return 'checked' in (n.attributes || {});
+        }
+        for (const child of n.children) {
+            const found = findNestedCheckboxChecked(child);
+            if (found !== undefined)
+                return found;
+        }
+        return undefined;
+    };
+    // Populated from a <section data-footnotes> block (found and parsed before the main
+    // body loop, since references can appear anywhere earlier in the document) and
+    // consulted by parseChildren's <sup data-footnote-ref> handling below.
+    const footnoteDefinitions = new Map();
+    const parseNode = (node, currentFormatting = {}, listContext, depth = 0) => {
+        // Guard against a maliciously deep element tree (e.g. tens of thousands of
+        // nested <div>) recursing until the call stack overflows. Real documents
+        // nest only a few dozen levels; this trips well before a RangeError.
+        if (depth > 1000) {
+            throw (0, errorUtils_js_1.getOfficeError)(types_js_1.OfficeErrorType.MAX_NESTING_DEPTH_EXCEEDED);
+        }
         if (node.type === 'text') {
             let decodedText = (node.text || '')
                 .replace(/&nbsp;/g, ' ')
@@ -264,7 +294,29 @@ const parseHtml = async (buffer, config) => {
             const parseChildren = (n, fmt, lCtx) => {
                 const kids = [];
                 for (const child of n.children) {
-                    const parsed = parseNode(child, fmt, lCtx);
+                    // Footnote/endnote reference: attach as .notes on the preceding node
+                    // instead of inserting a visible node, matching WordParser's convention.
+                    if (child.type === 'element' && child.tagName === 'sup' && child.attributes?.['data-footnote-ref'] !== undefined) {
+                        const key = child.attributes['data-footnote-ref'];
+                        const definition = footnoteDefinitions.get(key);
+                        const noteNode = {
+                            type: 'note',
+                            text: (definition || []).map(d => d.text || '').join(''),
+                            children: definition || [],
+                            metadata: { noteType: 'footnote', noteId: key }
+                        };
+                        if (kids.length > 0) {
+                            const target = kids[kids.length - 1];
+                            if (!target.notes)
+                                target.notes = [];
+                            target.notes.push(noteNode);
+                        }
+                        else {
+                            kids.push({ type: 'text', text: '', notes: [noteNode] });
+                        }
+                        continue;
+                    }
+                    const parsed = parseNode(child, fmt, lCtx, depth + 1);
                     if (parsed) {
                         if (Array.isArray(parsed))
                             kids.push(...parsed);
@@ -274,6 +326,94 @@ const parseHtml = async (buffer, config) => {
                 }
                 return kids;
             };
+            // YouTube embeds: inscript-editor's Youtube node renders
+            // <div data-youtube-video="ID" data-width="…" data-align="…">…<iframe…></div>.
+            // Recognise both the wrapper div and a bare iframe so externally-authored HTML
+            // (and a saved-then-reopened .md that fell back to raw HTML) both round-trip.
+            if (tagName === 'div' && node.attributes?.['data-youtube-video'] !== undefined) {
+                const videoId = node.attributes['data-youtube-video'] || '';
+                const width = node.attributes?.['data-width'];
+                const embedAlignAttr = node.attributes?.['data-align'];
+                const embedAlign = ['left', 'center', 'right'].includes(embedAlignAttr) ? embedAlignAttr : undefined;
+                const embedUrl = videoId ? `https://www.youtube.com/watch?v=${videoId}` : undefined;
+                const embedNode = {
+                    type: 'embed',
+                    // Childless nodes need .text so generic AST consumers (toText, chunking)
+                    // don't silently drop them.
+                    text: embedUrl,
+                    metadata: {
+                        embedType: 'youtube',
+                        videoId,
+                        url: embedUrl,
+                        width,
+                        align: embedAlign
+                    }
+                };
+                if (config.includeRawContent)
+                    embedNode.rawContent = '<div data-youtube-video>...</div>';
+                return embedNode;
+            }
+            if (tagName === 'iframe') {
+                const src = node.attributes?.src || '';
+                const ytMatch = /youtube(?:-nocookie)?\.com/.test(src) ? src.match(/(?:embed\/|v=)([^&?/\s]+)/) : null;
+                if (ytMatch) {
+                    const embedUrl = `https://www.youtube.com/watch?v=${ytMatch[1]}`;
+                    const embedNode = {
+                        type: 'embed',
+                        text: embedUrl,
+                        metadata: { embedType: 'youtube', videoId: ytMatch[1], url: embedUrl }
+                    };
+                    if (config.includeRawContent)
+                        embedNode.rawContent = '<iframe>...</iframe>';
+                    return embedNode;
+                }
+                return null;
+            }
+            // Footnotes section: its definitions were already extracted up front (see
+            // footnoteDefinitions below), so skip it here wherever it appears in the tree -
+            // it isn't necessarily a direct child of <body> (e.g. it may be nested inside
+            // a non-standalone HtmlGenerator output's wrapping <div>).
+            if (tagName === 'section' && node.attributes?.['data-footnotes'] !== undefined) {
+                return null;
+            }
+            // Math: proposed contract (no editor node built yet) - HtmlGenerator emits
+            // <span/div class="math math-inline|math-block" data-math="inline|block">
+            // with the $-delimited LaTeX as the visible (escaped) text content.
+            if ((tagName === 'div' || tagName === 'span') && node.attributes?.['data-math'] !== undefined) {
+                const mathMode = node.attributes['data-math'] === 'block' ? 'block' : 'inline';
+                const rawText = node.children.map(c => c.text || '').join('')
+                    .replace(/&nbsp;/g, ' ')
+                    .replace(/&lt;/g, '<')
+                    .replace(/&gt;/g, '>')
+                    .replace(/&amp;/g, '&')
+                    .replace(/&quot;/g, '"')
+                    .replace(/&#39;/g, '\'');
+                const delimiter = mathMode === 'block' ? '$$' : '$';
+                const latex = rawText.startsWith(delimiter) && rawText.endsWith(delimiter)
+                    ? rawText.slice(delimiter.length, -delimiter.length)
+                    : rawText;
+                return {
+                    type: 'code',
+                    text: latex,
+                    metadata: { math: mathMode }
+                };
+            }
+            // Admonition: inscript-editor's Admonition node renders
+            // <div class="admonition admonition-note" data-type="note">…children…</div>.
+            if (tagName === 'div' && (node.attributes?.class || '').split(/\s+/).includes('admonition')) {
+                const admonitionTypeAttr = node.attributes?.['data-type'];
+                const admonitionType = ['note', 'tip', 'important', 'warning', 'caution'].includes(admonitionTypeAttr)
+                    ? admonitionTypeAttr
+                    : 'note';
+                const admonitionNode = {
+                    type: 'admonition',
+                    metadata: { admonitionType },
+                    children: parseChildren(node, newFormatting, listContext)
+                };
+                if (config.includeRawContent)
+                    admonitionNode.rawContent = '<div class="admonition">...</div>';
+                return admonitionNode;
+            }
             // Skip structural containers produced by HtmlGenerator to avoid deep AST nesting
             if (tagName === 'div' && (node.attributes?.class === 'container' ||
                 node.attributes?.class === 'spreadsheet-container' ||
@@ -296,7 +436,7 @@ const parseHtml = async (buffer, config) => {
             if (tagName === 'p' || tagName === 'div') {
                 const children = parseChildren(node, newFormatting, listContext);
                 // If it's a div and contains block elements, return children directly
-                const hasBlockElements = children.some(c => ['paragraph', 'table', 'heading', 'list', 'image', 'chart', 'code'].includes(c.type));
+                const hasBlockElements = children.some(c => ['paragraph', 'table', 'heading', 'list', 'image', 'chart', 'code', 'embed', 'admonition', 'definitionList'].includes(c.type));
                 if (tagName === 'div' && hasBlockElements) {
                     return children;
                 }
@@ -330,13 +470,53 @@ const parseHtml = async (buffer, config) => {
                 };
                 return hNode;
             }
+            if (tagName === 'dl') {
+                return {
+                    type: 'definitionList',
+                    children: parseChildren(node, newFormatting, listContext)
+                };
+            }
+            if (tagName === 'dt') {
+                return {
+                    type: 'definitionTerm',
+                    children: parseChildren(node, newFormatting, listContext)
+                };
+            }
+            if (tagName === 'dd') {
+                return {
+                    type: 'definitionDescription',
+                    children: parseChildren(node, newFormatting, listContext)
+                };
+            }
+            if (tagName === 'abbr') {
+                const title = node.attributes?.title;
+                const children = parseChildren(node, newFormatting, listContext);
+                if (title) {
+                    children.forEach(c => {
+                        if (c.type === 'text') {
+                            c.metadata = { ...c.metadata, abbreviationTitle: title };
+                        }
+                    });
+                }
+                return children;
+            }
+            if (tagName === 'cite' && node.attributes?.['data-citation-key'] !== undefined) {
+                const citationKey = node.attributes['data-citation-key'];
+                return {
+                    type: 'text',
+                    text: citationKey,
+                    formatting: Object.keys(newFormatting).length > 0 ? { ...newFormatting } : undefined,
+                    metadata: { citationKey }
+                };
+            }
             if (tagName === 'ul' || tagName === 'ol') {
                 const isNewTopLevel = !listContext;
                 const newListContext = {
                     listId: isNewTopLevel ? `html-list-${htmlListIdCounter++}` : listContext.listId,
                     type: tagName === 'ol' ? 'ordered' : 'unordered',
                     level: isNewTopLevel ? 0 : listContext.level + 1,
-                    counters: isNewTopLevel ? {} : { ...listContext.counters } // Clone to avoid side effects on parent levels
+                    counters: isNewTopLevel ? {} : { ...listContext.counters }, // Clone to avoid side effects on parent levels
+                    isTask: node.attributes?.['data-type'] === 'taskList'
                 };
                 // Initialize counter for this level
                 if (tagName === 'ol' && node.attributes?.start) {
@@ -362,6 +542,13 @@ const parseHtml = async (buffer, config) => {
                 const children = parseChildren(node, newFormatting, listContext);
                 const nestedLists = children.filter(c => c.type === 'list');
                 const selfChildren = children.filter(c => c.type !== 'list');
+                let isTask;
+                let checked;
+                if (listContext?.isTask) {
+                    isTask = true;
+                    const dataChecked = node.attributes?.['data-checked'];
+                    checked = dataChecked !== undefined ? dataChecked === 'true' : (findNestedCheckboxChecked(node) ?? false);
+                }
                 const selfNode = {
                     type: 'list',
                     text: selfChildren.map(c => c.text || '').join(''),
@@ -371,16 +558,21 @@ const parseHtml = async (buffer, config) => {
                         alignment: newFormatting.alignment || 'left',
                         listId: listContext?.listId || 'html-list-none',
                         itemIndex: (listContext?.counters[listContext.level] ?? 1) - 1,
-                        anchorIds: anchorIds.length > 0 ? anchorIds : undefined
+                        anchorIds: anchorIds.length > 0 ? anchorIds : undefined,
+                        isTask,
+                        checked
                     },
                     children: selfChildren
                 };
                 return [selfNode, ...nestedLists];
             }
             if (tagName === 'table') {
+                // CustomTable (inscript-editor) renders data-align on the <table> itself.
+                const tableAlignAttr = node.attributes?.['data-align'];
+                const tableAlign = ['left', 'center', 'right'].includes(tableAlignAttr) ? tableAlignAttr : undefined;
                 const tableNode = {
                     type: 'table',
-                    metadata: { anchorIds: anchorIds.length > 0 ? anchorIds : undefined },
+                    metadata: { anchorIds: anchorIds.length > 0 ? anchorIds : undefined, align: tableAlign },
                     children: parseChildren(node, newFormatting, listContext)
                 };
                 if (config.includeRawContent) {
@@ -399,8 +591,18 @@ const parseHtml = async (buffer, config) => {
                 return rowNode;
             }
             if (tagName === 'td' || tagName === 'th') {
+                // Merged cells: mirrors the colspan/rowspan reading already done in
+                // MarkdownParser's inline HTML-table handler.
+                const colSpanAttr = node.attributes?.colspan;
+                const rowSpanAttr = node.attributes?.rowspan;
+                const colSpan = colSpanAttr ? parseInt(colSpanAttr, 10) : undefined;
+                const rowSpan = rowSpanAttr ? parseInt(rowSpanAttr, 10) : undefined;
                 const cellNode = {
                     type: 'cell',
+                    metadata: {
+                        colSpan: colSpan && !isNaN(colSpan) ? colSpan : undefined,
+                        rowSpan: rowSpan && !isNaN(rowSpan) ? rowSpan : undefined
+                    },
                     children: parseChildren(node, newFormatting, listContext)
                 };
                 if (config.includeRawContent) {
@@ -411,6 +613,14 @@ const parseHtml = async (buffer, config) => {
             if (tagName === 'img') {
                 const src = node.attributes?.src;
                 const alt = node.attributes?.alt;
+                // CustomImage (inscript-editor) renders data-width/data-align, falling back to
+                // parsing the inline style for consumers that only emit the CSS.
+                const imgStyle = node.attributes?.style || '';
+                const width = node.attributes?.['data-width'] || imgStyle.match(/width:\s*([^;]+)/)?.[1]?.trim();
+                const alignAttr = node.attributes?.['data-align']
+                    || (imgStyle.includes('margin-left: 0') && !imgStyle.includes('margin-right: 0') ? 'left'
+                        : (imgStyle.includes('margin-right: 0') && !imgStyle.includes('margin-left: 0') ? 'right' : undefined));
+                const align = ['left', 'center', 'right'].includes(alignAttr) ? alignAttr : undefined;
                 let imageNode;
                 if (src?.startsWith('data:')) {
                     const match = src.match(/^data:([^;]+);base64,(.*)$/);
@@ -429,7 +639,9 @@ const parseHtml = async (buffer, config) => {
                             type: 'image',
                             metadata: {
                                 attachmentName: name,
-                                altText: alt
+                                altText: alt,
+                                width,
+                                align
                             }
                         };
                     }
@@ -438,7 +650,9 @@ const parseHtml = async (buffer, config) => {
                             type: 'image',
                             metadata: {
                                 url: src,
-                                altText: alt
+                                altText: alt,
+                                width,
+                                align
                             }
                         };
                     }
@@ -449,7 +663,9 @@ const parseHtml = async (buffer, config) => {
                         metadata: {
                             url: src,
                             altText: alt,
-                            anchorIds: anchorIds.length > 0 ? anchorIds : undefined
+                            anchorIds: anchorIds.length > 0 ? anchorIds : undefined,
+                            width,
+                            align
                         }
                     };
                 }
@@ -460,8 +676,16 @@ const parseHtml = async (buffer, config) => {
             }
             if (tagName === 'a') {
                 const href = node.attributes?.href;
+                const wikilinkPage = node.attributes?.['data-wikilink-page'];
                 const children = parseChildren(node, newFormatting, listContext);
-                if (href) {
+                if (wikilinkPage !== undefined) {
+                    children.forEach(c => {
+                        if (c.type === 'text') {
+                            c.metadata = { ...c.metadata, link: wikilinkPage, linkType: 'internal', wikilink: true };
+                        }
+                    });
+                }
+                else if (href) {
                     const linkType = href.startsWith('#') ? 'internal' : 'external';
                     children.forEach(c => {
                         if (c.type === 'text') {
@@ -509,6 +733,42 @@ const parseHtml = async (buffer, config) => {
         }
         return null;
     };
+    // Extract <section data-footnotes> up front so its definitions are available to
+    // <sup data-footnote-ref> references encountered anywhere earlier in the body.
+    const findFootnotesSection = (n) => {
+        if (n.tagName === 'section' && n.attributes?.['data-footnotes'] !== undefined)
+            return n;
+        for (const child of n.children) {
+            const found = findFootnotesSection(child);
+            if (found)
+                return found;
+        }
+        return undefined;
+    };
+    const footnotesSectionNode = findFootnotesSection(body);
+    if (footnotesSectionNode) {
+        for (const item of footnotesSectionNode.children) {
+            if (item.type !== 'element')
+                continue;
+            const key = item.attributes?.['data-footnote-id'];
+            if (!key)
+                continue;
+            // Strip the generated back-reference link ("↩") - it's round-trip plumbing,
+            // not part of the footnote's actual content.
+            const filteredChildren = item.children.filter(c => !(c.tagName === 'a' && (c.attributes?.href || '').startsWith('#footnote-ref-')));
+            const contentNodes = [];
+            for (const child of filteredChildren) {
+                const parsed = parseNode(child);
+                if (parsed) {
+                    if (Array.isArray(parsed))
+                        contentNodes.push(...parsed);
+                    else
+                        contentNodes.push(parsed);
+                }
+            }
+            footnoteDefinitions.set(key, contentNodes);
+        }
+    }
     for (const child of body.children) {
         const parsed = parseNode(child);
         if (parsed) {
@@ -539,8 +799,12 @@ const parseHtml = async (buffer, config) => {
                 return node.text || '';
             if (node.type === 'break')
                 return '\n';
+            // Childless nodes still carry meaningful text - fall back to it instead of
+            // silently vanishing from plain-text/RAG-chunk output.
+            if (node.type === 'embed')
+                return node.metadata?.url || '';
             if (node.children) {
-                const isBlock = ['table', 'row', 'list', 'sheet', 'slide'].includes(node.type);
+                const isBlock = ['table', 'row', 'list', 'sheet', 'slide', 'admonition', 'definitionList'].includes(node.type);
                 return node.children.map(getText).join(isBlock ? config.newlineDelimiter : '');
             }
             return '';