explorbot 0.0.1

package/dist/src/utils/html-diff.js

@@ -0,0 +1,734 @@
+import { parse, serialize } from 'parse5';
+import { TAILWIND_CLASS_PATTERNS, TRASH_HTML_CLASSES, minifyHtml } from "./html.js";
+import { isDynamicId, isGenericClass } from "./xpath.js";
+const IGNORED_PATHS = new Set(['html[1]', 'html[1]/head[1]', 'html[1]/body[1]']);
+/**
+ * Get text content from an element node.
+ */
+function getTextContent(element) {
+    let text = '';
+    function processNode(node) {
+        if (node.nodeName === '#text') {
+            text += node.value;
+        }
+        else if ('childNodes' in node) {
+            node.childNodes.forEach(processNode);
+        }
+    }
+    processNode(element);
+    return text;
+}
+/**
+ * Extract document title from a parsed HTML document.
+ */
+function getDocumentTitle(document) {
+    if (!document.childNodes)
+        return null;
+    const htmlElement = document.childNodes.find((node) => 'tagName' in node && node.tagName?.toLowerCase() === 'html');
+    if (!htmlElement || !htmlElement.childNodes) {
+        return null;
+    }
+    const headElement = htmlElement.childNodes.find((node) => 'tagName' in node && node.tagName?.toLowerCase() === 'head');
+    if (!headElement || !headElement.childNodes) {
+        return null;
+    }
+    const titleElement = headElement.childNodes.find((node) => 'tagName' in node && node.tagName?.toLowerCase() === 'title');
+    if (!titleElement) {
+        return null;
+    }
+    const text = getTextContent(titleElement).trim();
+    return text.length > 0 ? text : null;
+}
+/**
+ * Ensure document has a title element in head section.
+ */
+function ensureDocumentTitle(document, titleText) {
+    if (!titleText || !document.childNodes) {
+        return;
+    }
+    const htmlElement = document.childNodes.find((node) => 'tagName' in node && node.tagName?.toLowerCase() === 'html');
+    if (!htmlElement) {
+        return;
+    }
+    const namespace = htmlElement.namespaceURI || 'http://www.w3.org/1999/xhtml';
+    let headElement = htmlElement.childNodes.find((node) => 'tagName' in node && node.tagName?.toLowerCase() === 'head');
+    if (!headElement) {
+        headElement = {
+            nodeName: 'head',
+            tagName: 'head',
+            attrs: [],
+            namespaceURI: namespace,
+            childNodes: [],
+            parentNode: htmlElement,
+        };
+        // Insert head before body if possible, otherwise prepend
+        const bodyIndex = htmlElement.childNodes.findIndex((node) => 'tagName' in node && node.tagName?.toLowerCase() === 'body');
+        if (bodyIndex === -1) {
+            htmlElement.childNodes.push(headElement);
+        }
+        else {
+            htmlElement.childNodes.splice(bodyIndex, 0, headElement);
+        }
+    }
+    else {
+        headElement.childNodes = [];
+    }
+    const titleElement = {
+        nodeName: 'title',
+        tagName: 'title',
+        attrs: [],
+        namespaceURI: namespace,
+        childNodes: [],
+        parentNode: headElement,
+    };
+    const textNode = {
+        nodeName: '#text',
+        value: titleText,
+    };
+    textNode.parentNode = titleElement;
+    titleElement.childNodes.push(textNode);
+    headElement.childNodes.push(titleElement);
+}
+/**
+ * Check if a node has child nodes.
+ */
+function hasChildNodes(node) {
+    return typeof node === 'object' && node !== null && 'childNodes' in node && Array.isArray(node.childNodes);
+}
+/**
+ * Remove elements with specified tags from a node tree.
+ */
+function stripElementsByTag(node, tagsToRemove) {
+    if (!node.childNodes)
+        return;
+    for (let i = node.childNodes.length - 1; i >= 0; i--) {
+        const child = node.childNodes[i];
+        // Remove comments
+        if (child.nodeName === '#comment') {
+            node.childNodes.splice(i, 1);
+            continue;
+        }
+        // Remove elements with matching tags
+        if ('tagName' in child && child.tagName) {
+            const tagName = child.tagName.toLowerCase();
+            if (tagsToRemove.has(tagName)) {
+                node.childNodes.splice(i, 1);
+                continue;
+            }
+            // Recursively process child elements
+            stripElementsByTag(child, tagsToRemove);
+        }
+        else if (hasChildNodes(child)) {
+            // Recursively process non-element nodes that have children
+            stripElementsByTag(child, tagsToRemove);
+        }
+    }
+}
+export function computeHtmlFingerprint(html) {
+    const document = parseDocument(html);
+    const root = getRootNodeForFlatten(document);
+    return flattenHtml(root);
+}
+/**
+ * Compares two HTML documents and returns differences along with a diff subtree.
+ */
+export async function htmlDiff(originalHtml, modifiedHtml, htmlConfig) {
+    const originalDocument = parseDocument(originalHtml, htmlConfig);
+    const modifiedDocument = parseDocument(modifiedHtml, htmlConfig);
+    const originalRoot = getRootNodeForFlatten(originalDocument);
+    const modifiedRoot = getRootNodeForFlatten(modifiedDocument);
+    const originalLines = flattenHtml(originalRoot);
+    const modifiedLines = flattenHtml(modifiedRoot);
+    const similarity = calculateSimilarity(originalLines, modifiedLines);
+    const { added, removed } = findDifferences(originalLines, modifiedLines);
+    const parts = await buildDiffParts(originalDocument, modifiedDocument);
+    const structuralAdditions = parts.flatMap((p) => p.added.filter((a) => a.startsWith('ELEMENT:')));
+    const allAdded = [...added, ...structuralAdditions];
+    const totalChanges = allAdded.length + removed.length;
+    const summary = totalChanges === 0 && parts.length > 0 ? 'Structural additions detected' : generateSummary(allAdded, removed, similarity);
+    return {
+        parts,
+        added: allAdded,
+        removed,
+        similarity,
+        summary,
+    };
+}
+/**
+ * Parse HTML into a document, wrapping fragments with html/body for consistency.
+ * Uses custom sanitization that removes iframes for diff purposes.
+ */
+function parseDocument(html, htmlConfig) {
+    const document = parse(html);
+    const documentTitle = getDocumentTitle(document);
+    sanitizeDocumentTreeForDiff(document);
+    ensureDocumentTitle(document, documentTitle);
+    return document;
+}
+/**
+ * Custom sanitization for html-diff that removes iframes in addition to standard non-semantic tags.
+ */
+function sanitizeDocumentTreeForDiff(document) {
+    // Remove standard non-semantic tags
+    stripElementsByTag(document, new Set([...NON_SEMANTIC_TAGS, 'iframe']));
+    pruneDocumentHeadForDiff(document);
+}
+/**
+ * Prune non-essential elements from document head for diff purposes.
+ * Only keeps title element, removes everything else.
+ */
+function pruneDocumentHeadForDiff(document) {
+    if (!document.childNodes)
+        return;
+    const htmlElement = document.childNodes.find((node) => 'tagName' in node && node.tagName?.toLowerCase() === 'html');
+    if (!htmlElement || !htmlElement.childNodes) {
+        return;
+    }
+    const headElement = htmlElement.childNodes.find((node) => 'tagName' in node && node.tagName?.toLowerCase() === 'head');
+    if (!headElement || !headElement.childNodes) {
+        return;
+    }
+    for (let i = headElement.childNodes.length - 1; i >= 0; i--) {
+        const child = headElement.childNodes[i];
+        if ('tagName' in child && child.tagName) {
+            const tagName = child.tagName.toLowerCase();
+            if (tagName !== 'title') {
+                headElement.childNodes.splice(i, 1);
+            }
+            continue;
+        }
+        // Remove non-element nodes (like text, comments)
+        headElement.childNodes.splice(i, 1);
+    }
+}
+// Copy NON_SEMANTIC_TAGS from html.ts to avoid circular imports
+const NON_SEMANTIC_TAGS = new Set([
+    'style',
+    'script',
+    'link',
+    'meta',
+    'base',
+    'template',
+    'slot',
+    'noscript',
+    'frame',
+    'frameset',
+    'object',
+    'embed',
+    'path',
+    'polygon',
+    'polyline',
+    'circle',
+    'ellipse',
+    'line',
+    'rect',
+    'defs',
+    'g',
+    'symbol',
+    'use',
+    'mask',
+    'pattern',
+    'clippath',
+    'animate',
+    'animatetransform',
+    'animatecolor',
+]);
+/**
+ * Returns the body (preferred) or html element converted into HtmlNode for flattening.
+ */
+function getRootNodeForFlatten(document) {
+    const body = findBodyElement(document);
+    if (body) {
+        return convertNode(body);
+    }
+    const htmlElement = findHtmlElement(document);
+    if (htmlElement) {
+        return convertNode(htmlElement);
+    }
+    return {
+        type: 'element',
+        tagName: 'document',
+        children: [],
+    };
+}
+const attributesDiffer = (current, previous) => {
+    const currentAttrs = current.attrs ?? [];
+    const previousAttrs = previous.attrs ?? [];
+    if (currentAttrs.length !== previousAttrs.length) {
+        return true;
+    }
+    const previousMap = new Map(previousAttrs.map((attr) => [attr.name, attr.value]));
+    for (const attr of currentAttrs) {
+        if (previousMap.get(attr.name) !== attr.value) {
+            return true;
+        }
+    }
+    return false;
+};
+const getDirectTextValues = (element) => {
+    if (!element.childNodes || element.childNodes.length === 0) {
+        return [];
+    }
+    const values = [];
+    for (const child of element.childNodes) {
+        if (child.nodeName === '#text') {
+            const value = child.value.trim();
+            if (value.length > 0) {
+                values.push(value);
+            }
+        }
+    }
+    return values;
+};
+const directTextDiffer = (current, previous) => {
+    const currentText = getDirectTextValues(current);
+    const previousText = getDirectTextValues(previous);
+    if (currentText.length !== previousText.length) {
+        return true;
+    }
+    for (let i = 0; i < currentText.length; i++) {
+        if (currentText[i] !== previousText[i]) {
+            return true;
+        }
+    }
+    return false;
+};
+const trashHtmlClasses = TRASH_HTML_CLASSES;
+function filterContainerClasses(classes) {
+    return classes
+        .filter((cls) => !/\d/.test(cls))
+        .filter((cls) => !isGenericClass(cls))
+        .filter((cls) => !trashHtmlClasses.test(cls))
+        .filter((cls) => !/(:|__)/.test(cls))
+        .filter((cls) => !TAILWIND_CLASS_PATTERNS.some((pattern) => pattern.test(cls)));
+}
+function buildContainerSelector(element, allElements) {
+    const tag = element.tagName?.toLowerCase();
+    if (!tag)
+        return null;
+    const attrs = element.attrs ?? [];
+    const id = attrs.find((a) => a.name === 'id')?.value;
+    if (id && !isDynamicId(id))
+        return `#${id}`;
+    const classAttr = attrs.find((a) => a.name === 'class')?.value;
+    if (!classAttr)
+        return null;
+    const meaningful = filterContainerClasses(classAttr.split(/\s+/).filter(Boolean));
+    if (meaningful.length === 0)
+        return null;
+    const selector = `${tag}.${meaningful.join('.')}`;
+    let matchCount = 0;
+    for (const el of allElements.values()) {
+        if (el.tagName?.toLowerCase() !== tag)
+            continue;
+        const elClass = (el.attrs ?? []).find((a) => a.name === 'class')?.value || '';
+        const elClasses = elClass.split(/\s+/);
+        if (meaningful.every((cls) => elClasses.includes(cls))) {
+            matchCount++;
+            if (matchCount > 1)
+                return null;
+        }
+    }
+    return matchCount === 1 ? selector : null;
+}
+function pathToXPath(treePath) {
+    const parts = treePath.split('/');
+    const bodyIdx = parts.findIndex((p) => p.startsWith('body'));
+    if (bodyIdx === -1)
+        return `//${parts.join('/')}`;
+    return `//body/${parts.slice(bodyIdx + 1).join('/')}`;
+}
+function findStableContainer(topLevelPath, originalMap, modifiedMap) {
+    const segments = topLevelPath.split('/');
+    for (let i = segments.length - 2; i >= 1; i--) {
+        const candidatePath = segments.slice(0, i + 1).join('/');
+        if (IGNORED_PATHS.has(candidatePath))
+            continue;
+        if (!originalMap.has(candidatePath) || !modifiedMap.has(candidatePath))
+            continue;
+        const element = modifiedMap.get(candidatePath);
+        const css = buildContainerSelector(element, modifiedMap);
+        if (css)
+            return { path: candidatePath, selector: css };
+        return { path: candidatePath, selector: pathToXPath(candidatePath) };
+    }
+    return { path: 'html[1]/body[1]', selector: 'body' };
+}
+async function buildDiffParts(originalDocument, modifiedDocument) {
+    const originalMap = collectElementMap(originalDocument);
+    const modifiedMap = collectElementMap(modifiedDocument);
+    const addedPaths = [];
+    const changedPaths = [];
+    for (const [path, element] of modifiedMap.entries()) {
+        if (IGNORED_PATHS.has(path))
+            continue;
+        const originalElement = originalMap.get(path);
+        if (!originalElement) {
+            addedPaths.push(path);
+            continue;
+        }
+        if (attributesDiffer(element, originalElement)) {
+            changedPaths.push(path);
+        }
+    }
+    if (addedPaths.length === 0 && changedPaths.length === 0)
+        return [];
+    const addedTopLevel = filterTopLevelPaths(addedPaths);
+    const changedFiltered = changedPaths.filter((path) => !addedTopLevel.some((ancestor) => isSameOrAncestor(ancestor, path)));
+    const changedTopLevel = filterTopLevelPaths(changedFiltered);
+    const allTopLevel = [...addedTopLevel, ...changedTopLevel];
+    const grouped = new Map();
+    for (const path of allTopLevel) {
+        const { path: containerPath, selector } = findStableContainer(path, originalMap, modifiedMap);
+        const existing = grouped.get(containerPath);
+        if (existing) {
+            existing.paths.push(path);
+        }
+        else {
+            grouped.set(containerPath, { selector, paths: [path] });
+        }
+    }
+    const parts = [];
+    for (const [containerPath, { selector, paths }] of grouped) {
+        const containerElement = modifiedMap.get(containerPath);
+        if (!containerElement)
+            continue;
+        const rootClone = cloneElementShallow(containerElement);
+        const diffMap = new Map();
+        diffMap.set(containerPath, rootClone);
+        for (const path of paths) {
+            const pathSegments = path.split('/');
+            const containerDepth = containerPath.split('/').length;
+            for (let i = containerDepth + 1; i < pathSegments.length; i++) {
+                const prefix = pathSegments.slice(0, i).join('/');
+                if (diffMap.has(prefix))
+                    continue;
+                const sourceNode = modifiedMap.get(prefix);
+                const parentPath = pathSegments.slice(0, i - 1).join('/');
+                const parentNode = diffMap.get(parentPath);
+                if (!sourceNode || !parentNode)
+                    continue;
+                const clone = cloneElementShallow(sourceNode);
+                appendChild(parentNode, clone);
+                diffMap.set(prefix, clone);
+            }
+            const sourceElement = modifiedMap.get(path);
+            const parentPath = getParentPath(path);
+            const parentNode = diffMap.get(parentPath);
+            if (!sourceElement || !parentNode)
+                continue;
+            appendChild(parentNode, cloneElementDeep(sourceElement));
+        }
+        const serialized = serialize({ childNodes: [rootClone], nodeName: '#document-fragment' }).trim();
+        const subtree = serialized ? await minifyHtml(serialized) : '';
+        if (!subtree)
+            continue;
+        const addedLines = paths.filter((p) => addedTopLevel.includes(p)).map((p) => `ELEMENT:${p}`);
+        const removedLines = [];
+        parts.push({ container: selector, subtree, added: addedLines, removed: removedLines });
+    }
+    return parts;
+}
+/**
+ * Collect a map of element paths to nodes using nth-of-type indexing.
+ */
+function collectElementMap(document) {
+    const map = new Map();
+    const htmlElement = findHtmlElement(document);
+    if (!htmlElement) {
+        return map;
+    }
+    traverse(htmlElement, 'html[1]');
+    return map;
+    function traverse(element, currentPath) {
+        map.set(currentPath, element);
+        if (!element.childNodes || element.childNodes.length === 0) {
+            return;
+        }
+        const counts = new Map();
+        for (const child of element.childNodes) {
+            if ('tagName' in child && child.tagName) {
+                const tagName = child.tagName.toLowerCase();
+                const index = (counts.get(tagName) ?? 0) + 1;
+                counts.set(tagName, index);
+                const childPath = `${currentPath}/${tagName}[${index}]`;
+                traverse(child, childPath);
+            }
+        }
+    }
+}
+function filterTopLevelPaths(paths) {
+    const uniquePaths = Array.from(new Set(paths));
+    uniquePaths.sort((a, b) => a.length - b.length);
+    const result = [];
+    for (const path of uniquePaths) {
+        if (result.some((existing) => isSameOrAncestor(existing, path))) {
+            continue;
+        }
+        result.push(path);
+    }
+    return result;
+}
+function getParentPath(path) {
+    const lastSeparator = path.lastIndexOf('/');
+    if (lastSeparator === -1) {
+        return '';
+    }
+    return path.slice(0, lastSeparator);
+}
+function isSameOrAncestor(ancestor, path) {
+    return ancestor === path || path.startsWith(`${ancestor}/`);
+}
+function appendChild(parent, child) {
+    if (!parent.childNodes) {
+        parent.childNodes = [];
+    }
+    parent.childNodes.push(child);
+    child.parentNode = parent;
+}
+function cloneElementShallow(element) {
+    return {
+        nodeName: element.nodeName,
+        tagName: element.tagName,
+        attrs: element.attrs ? element.attrs.map((attr) => ({ ...attr })) : [],
+        childNodes: [],
+        namespaceURI: element.namespaceURI,
+    };
+}
+function cloneElementDeep(element) {
+    const clone = cloneElementShallow(element);
+    if (!element.childNodes || element.childNodes.length === 0) {
+        return clone;
+    }
+    for (const child of element.childNodes) {
+        const clonedChild = cloneNodeDeep(child);
+        appendChild(clone, clonedChild);
+    }
+    return clone;
+}
+function cloneNodeDeep(node) {
+    if ('tagName' in node && node.tagName) {
+        return cloneElementDeep(node);
+    }
+    if (node.nodeName === '#text') {
+        const textNode = node;
+        return {
+            nodeName: '#text',
+            value: textNode.value,
+        };
+    }
+    if (node.nodeName === '#comment') {
+        const commentNode = node;
+        return {
+            nodeName: '#comment',
+            data: commentNode.data,
+        };
+    }
+    return { ...node };
+}
+function findHtmlElement(document) {
+    if (!document.childNodes) {
+        return null;
+    }
+    for (const node of document.childNodes) {
+        if ('tagName' in node && node.tagName && node.tagName.toLowerCase() === 'html') {
+            return node;
+        }
+    }
+    return null;
+}
+function findHeadElement(document) {
+    const html = findHtmlElement(document);
+    if (!html || !html.childNodes) {
+        return null;
+    }
+    for (const node of html.childNodes) {
+        if ('tagName' in node && node.tagName && node.tagName.toLowerCase() === 'head') {
+            return node;
+        }
+    }
+    return null;
+}
+function findBodyElement(document) {
+    const html = findHtmlElement(document);
+    if (!html || !html.childNodes) {
+        return null;
+    }
+    for (const node of html.childNodes) {
+        if ('tagName' in node && node.tagName && node.tagName.toLowerCase() === 'body') {
+            return node;
+        }
+    }
+    return null;
+}
+/**
+ * Convert parse5 node to simplified representation for textual diffing.
+ */
+function convertNode(node) {
+    if (node.nodeName === '#text') {
+        const textNode = node;
+        return {
+            type: 'text',
+            content: textNode.value.trim(),
+        };
+    }
+    if (node.nodeName === '#comment') {
+        return {
+            type: 'comment',
+            content: node.data,
+        };
+    }
+    if ('tagName' in node && node.tagName) {
+        const element = node;
+        const htmlNode = {
+            type: 'element',
+            tagName: element.tagName.toLowerCase(),
+        };
+        if (element.attrs && element.attrs.length > 0) {
+            htmlNode.attrs = element.attrs.map((attr) => ({
+                name: attr.name,
+                value: attr.value,
+            }));
+        }
+        if (element.childNodes && element.childNodes.length > 0) {
+            const children = [];
+            for (const child of element.childNodes) {
+                if (child.nodeName === '#text') {
+                    const text = child.value.trim();
+                    if (text.length === 0) {
+                        continue;
+                    }
+                }
+                children.push(convertNode(child));
+            }
+            if (children.length > 0) {
+                htmlNode.children = children;
+            }
+        }
+        if (htmlNode.children && htmlNode.children.length === 1 && htmlNode.children[0].type === 'text') {
+            htmlNode.content = htmlNode.children[0].content;
+            htmlNode.children = undefined;
+        }
+        return htmlNode;
+    }
+    return {
+        type: 'text',
+        content: '',
+    };
+}
+/**
+ * Flatten HTML structure into lines for comparison.
+ */
+function flattenHtml(node) {
+    const lines = [];
+    function process(n) {
+        if (n.type === 'text' && n.content) {
+            if (n.content.length >= 5) {
+                lines.push(`TEXT:${n.content}`);
+            }
+            return;
+        }
+        if (n.type === 'comment') {
+            return;
+        }
+        if (n.type === 'element' && n.tagName) {
+            if (isInteractiveElement(n)) {
+                const content = getElementContent(n);
+                if (content) {
+                    lines.push(`${n.tagName.toUpperCase()}:${content}`);
+                }
+                else {
+                    lines.push(`${n.tagName.toUpperCase()}`);
+                }
+                return;
+            }
+            if (n.content && n.content.length >= 5) {
+                lines.push(`TEXT:${n.content}`);
+            }
+            if (n.children) {
+                n.children.forEach((child) => process(child));
+            }
+        }
+    }
+    process(node);
+    return lines;
+}
+function isInteractiveElement(node) {
+    if (!node.tagName) {
+        return false;
+    }
+    const interactiveTags = new Set(['a', 'button', 'input', 'select', 'textarea', 'details', 'summary']);
+    if (interactiveTags.has(node.tagName)) {
+        return true;
+    }
+    if (node.attrs) {
+        const role = node.attrs.find((attr) => attr.name === 'role');
+        if (role && ['button', 'link', 'checkbox', 'radio', 'combobox', 'listbox', 'textbox', 'switch', 'tab'].includes(role.value)) {
+            return true;
+        }
+    }
+    return false;
+}
+function getElementContent(node) {
+    if (node.content) {
+        return node.content;
+    }
+    if (node.attrs) {
+        if (node.tagName === 'input') {
+            const placeholder = node.attrs.find((attr) => attr.name === 'placeholder');
+            if (placeholder) {
+                return placeholder.value;
+            }
+            const value = node.attrs.find((attr) => attr.name === 'value');
+            if (value) {
+                return value.value;
+            }
+            const name = node.attrs.find((attr) => attr.name === 'name');
+            if (name) {
+                return name.value;
+            }
+        }
+        if (node.tagName === 'a') {
+            const href = node.attrs.find((attr) => attr.name === 'href');
+            if (href) {
+                return href.value;
+            }
+        }
+    }
+    return '';
+}
+function calculateSimilarity(lines1, lines2) {
+    const set1 = new Set(lines1);
+    const set2 = new Set(lines2);
+    const intersection = new Set([...set1].filter((x) => set2.has(x)));
+    const union = new Set([...set1, ...set2]);
+    if (union.size === 0) {
+        return 100;
+    }
+    return Math.round((intersection.size / union.size) * 100);
+}
+function findDifferences(lines1, lines2) {
+    const set1 = new Set(lines1);
+    const set2 = new Set(lines2);
+    const added = [...set2].filter((line) => !set1.has(line));
+    const removed = [...set1].filter((line) => !set2.has(line));
+    return { added, removed };
+}
+function generateSummary(added, removed, similarity) {
+    const totalChanges = added.length + removed.length;
+    if (totalChanges === 0) {
+        return 'No changes detected';
+    }
+    const parts = [];
+    if (similarity < 100) {
+        parts.push(`${similarity}% similar`);
+    }
+    if (added.length > 0) {
+        parts.push(`${added.length} addition${added.length > 1 ? 's' : ''}`);
+    }
+    if (removed.length > 0) {
+        parts.push(`${removed.length} removal${removed.length > 1 ? 's' : ''}`);
+    }
+    return parts.join(', ');
+}
+//# sourceMappingURL=html-diff.js.map