@tkeron/html-parser 0.1.7 → 1.1.0

package/tests/advanced.test.ts

@@ -1,11 +1,32 @@
-import { expect, test, describe } from 'bun:test';
+// @ts-nocheck
+import { expect, test, describe, it } from 'bun:test';
 import { tokenize, TokenType } from '../src/tokenizer';
-import { parse, ASTNodeType, type ASTNode } from '../src/parser';
+import { parse, ASTNodeType, domToAST, type ASTNode } from '../src/parser';
+
+function parseToAST(html: string): ASTNode {
+  const tokens = tokenize(html);
+  const dom = parse(tokens);
+  const ast = domToAST(dom);
+  
+  const hasExplicitHtml = html.includes('<html') || html.includes('<!DOCTYPE') || html.includes('<!doctype');
+  if (hasExplicitHtml) {
+    return ast;
+  }
+  
+  const htmlEl = ast.children?.find(c => c.tagName === 'html');
+  if (htmlEl) {
+    const bodyEl = htmlEl.children?.find(c => c.tagName === 'body');
+    if (bodyEl && bodyEl.children) {
+      return { type: ASTNodeType.Document, children: bodyEl.children };
+    }
+  }
+  return ast;
+}
 
 describe('HTML Parser & Tokenizer - Advanced Tests', () => {
 
   describe('Tokenizer Edge Cases', () => {
-    test('should handle attributes with no spaces', () => {
+    it('should handle attributes with no spaces', () => {
       const tokens = tokenize('<div class="test"id="main"data-value="123">');
       expect(tokens.length).toBeGreaterThan(0);
       const tag = tokens[0]!;
@@ -17,7 +38,7 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
       });
     });
 
-    test('should handle mixed quote styles', () => {
+    it('should handle mixed quote styles', () => {
       const tokens = tokenize(`<div class='single' id="double" data-test='mix "quoted" content'>`);
       expect(tokens.length).toBeGreaterThan(0);
       const tag = tokens[0]!;
@@ -27,7 +48,7 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
       expect(tag.attributes!['data-test']).toBe('mix "quoted" content');
     });
 
-    test('should handle unicode characters', () => {
+    it('should handle unicode characters', () => {
       const tokens = tokenize('<div title="测试" data-emoji="🚀" class="lorem">');
       expect(tokens.length).toBeGreaterThan(0);
       const tag = tokens[0]!;
@@ -39,7 +60,7 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
       });
     });
 
-    test('should handle complex CDATA content', () => {
+    it('should handle complex CDATA content as bogus comment', () => {
       const complexContent = `
         function test() {
           return "<div>HTML inside JS</div>";
@@ -50,11 +71,11 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
       expect(tokens.length).toBeGreaterThan(0);
       const cdataToken = tokens[0]!;
       
-      expect(cdataToken.type).toBe(TokenType.CDATA);
-      expect(cdataToken.value).toBe(complexContent);
+      expect(cdataToken.type).toBe(TokenType.COMMENT);
+      expect(cdataToken.value).toBe('[CDATA[' + complexContent + ']]');
     });
 
-    test('should handle performance with large documents', () => {
+    it('should handle performance with large documents', () => {
       let html = '<div>';
       for (let i = 0; i < 1000; i++) {
         html += `<p id="para-${i}">Content ${i}</p>`;
@@ -71,23 +92,19 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
   });
 
   describe('Parser DOM-like Functionality', () => {
-    test('should create proper parent-child relationships', () => {
-      const tokens = tokenize('<div><section><article><h1>Title</h1><p>Content</p></article></section></div>');
-      const ast = parse(tokens);
+    it('should create proper parent-child relationships', () => {
+      const ast = parseToAST('<div><section><article><h1>Title</h1><p>Content</p></article></section></div>');
       
       const divElement = ast.children![0]!;
       const sectionElement = divElement.children![0]!;
       const articleElement = sectionElement.children![0]!;
       
-      expect(sectionElement.parent).toBe(divElement);
-      expect(articleElement.parent).toBe(sectionElement);
-      
       expect(articleElement.children).toHaveLength(2);
       expect(articleElement.children![0]!.tagName).toBe('h1');
       expect(articleElement.children![1]!.tagName).toBe('p');
     });
 
-    test('should handle complex navigation scenarios', () => {
+    it('should handle complex navigation scenarios', () => {
       const html = `
         <nav>
           <ul>
@@ -97,8 +114,7 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
           </ul>
         </nav>
       `;
-      const tokens = tokenize(html);
-      const ast = parse(tokens);
+      const ast = parseToAST(html);
       
       const navElement = ast.children!.find(child => child.tagName === 'nav')!;
       const ulElement = navElement.children!.find(child => child.tagName === 'ul')!;
@@ -109,11 +125,11 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
       liElements.forEach((li, index) => {
         const anchor = li.children!.find(child => child.tagName === 'a')!;
         expect(anchor.attributes!.href).toBeDefined();
-        expect(anchor.children![0]!.type).toBe(ASTNodeType.TEXT);
+        expect(anchor.children![0]!.type).toBe(ASTNodeType.Text);
       });
     });
 
-    test('should handle form elements with complex attributes', () => {
+    it('should handle form elements with complex attributes', () => {
       const html = `
         <form action="/submit" method="post">
           <input type="email" name="email" required pattern="[a-z0-9._%+-]+@[a-z0-9.-]+\\.[a-z]{2,}$">
@@ -124,8 +140,7 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
           <textarea name="comments" rows="4" cols="50"></textarea>
         </form>
       `;
-      const tokens = tokenize(html);
-      const ast = parse(tokens);
+      const ast = parseToAST(html);
       
       const formElement = ast.children!.find(child => child.tagName === 'form')!;
       expect(formElement.attributes!.action).toBe('/submit');
@@ -133,7 +148,7 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
       
       const formElements: ASTNode[] = [];
       const traverse = (node: ASTNode) => {
-        if (node.type === ASTNodeType.ELEMENT) {
+        if (node.type === ASTNodeType.Element) {
           if (['input', 'select', 'textarea', 'option'].includes(node.tagName!)) {
             formElements.push(node);
           }
@@ -154,7 +169,7 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
       expect(selectElement!.attributes!.multiple).toBe('');
     });
 
-    test('should handle table structures', () => {
+    it('should handle table structures', () => {
       const html = `
         <table>
           <thead>
@@ -175,8 +190,7 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
           </tbody>
         </table>
       `;
-      const tokens = tokenize(html);
-      const ast = parse(tokens);
+      const ast = parseToAST(html);
       
       const tableElement = ast.children!.find(child => child.tagName === 'table')!;
       
@@ -200,14 +214,13 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
       expect(rows).toHaveLength(3); 
     });
 
-    test('should handle mixed inline content', () => {
+    it('should handle mixed inline content', () => {
       const html = `
         <p>This is <strong>bold</strong> and <em>italic</em>. 
         Here's a <a href="https://example.com">link</a> and 
         <code>inline code</code>.</p>
       `;
-      const tokens = tokenize(html);
-      const ast = parse(tokens);
+      const ast = parseToAST(html);
       
       const pElement = ast.children!.find(child => child.tagName === 'p')!;
       
@@ -215,9 +228,9 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
       let elementNodes = 0;
       
       const traverse = (node: ASTNode) => {
-        if (node.type === ASTNodeType.TEXT && node.content!.trim()) {
+        if (node.type === ASTNodeType.Text && (node as any).content?.trim()) {
           textNodes++;
-        } else if (node.type === ASTNodeType.ELEMENT) {
+        } else if (node.type === ASTNodeType.Element) {
           elementNodes++;
         }
         if (node.children) {
@@ -233,7 +246,7 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
       expect(textNodes).toBeGreaterThan(0);
     });
 
-    test('should preserve complete document structure', () => {
+    it('should preserve complete document structure', () => {
       const html = `<!DOCTYPE html>
         <html lang="en">
           <head>
@@ -258,10 +271,9 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
           </body>
         </html>`;
       
-      const tokens = tokenize(html);
-      const ast = parse(tokens);
+      const ast = parseToAST(html);
       
-      const doctype = ast.children!.find(child => child.type === ASTNodeType.DOCTYPE);
+      const doctype = ast.children!.find(child => child.type === ASTNodeType.Doctype);
       expect(doctype).toBeDefined();
       
       const htmlElement = ast.children!.find(child => child.tagName === 'html')!;
@@ -286,7 +298,7 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
   });
 
   describe('Real-world Content Handling', () => {
-    test('should handle SVG content', () => {
+    it('should handle SVG content', () => {
       const svg = `
         <svg width="100" height="100" xmlns="http://www.w3.org/2000/svg">
           <circle cx="50" cy="50" r="40" fill="red"/>
@@ -294,8 +306,7 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
         </svg>
       `;
       
-      const tokens = tokenize(svg);
-      const ast = parse(tokens);
+      const ast = parseToAST(svg);
       
       const svgElement = ast.children!.find(child => child.tagName === 'svg')!;
       expect(svgElement.attributes!.xmlns).toBe('http://www.w3.org/2000/svg');
@@ -305,23 +316,35 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
       expect(circleElement!.attributes!.fill).toBe('red');
     });
 
-    test('should handle script and style tags', () => {
+    it('should handle script and style tags', () => {
       const html = `
-        <script type="text/javascript">
-          function hello() {
-            alert("Hello");
-          }
-        </script>
-        <style type="text/css">
-          .class { color: red; }
-        </style>
+        <body>
+          <script type="text/javascript">
+            function hello() {
+              alert("Hello");
+            }
+          </script>
+          <style type="text/css">
+            .class { color: red; }
+          </style>
+        </body>
       `;
       
-      const tokens = tokenize(html);
-      const ast = parse(tokens);
+      const ast = parseToAST(html);
+      
+      function findByTagName(node: ASTNode, tagName: string): ASTNode | null {
+        if (node.tagName === tagName) return node;
+        if (node.children) {
+          for (const child of node.children) {
+            const found = findByTagName(child, tagName);
+            if (found) return found;
+          }
+        }
+        return null;
+      }
       
-      const scriptElement = ast.children!.find(child => child.tagName === 'script');
-      const styleElement = ast.children!.find(child => child.tagName === 'style');
+      const scriptElement = findByTagName(ast, 'script');
+      const styleElement = findByTagName(ast, 'style');
       
       expect(scriptElement!.attributes!.type).toBe('text/javascript');
       expect(styleElement!.attributes!.type).toBe('text/css');
@@ -329,7 +352,7 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
   });
 
   describe('Error Recovery and Edge Cases', () => {
-    test('should handle extreme nesting depth', () => {
+    it('should handle extreme nesting depth', () => {
       let html = '';
       const depth = 100;
       
@@ -341,43 +364,40 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
         html += '</div>';
       }
       
-      const tokens = tokenize(html);
-      const ast = parse(tokens);
+      const ast = parseToAST(html);
       
       let current = ast.children![0]!;
       for (let i = 0; i < depth - 1; i++) {
         expect(current.tagName).toBe('div');
         expect(current.attributes!.level).toBe(i.toString());
-        current = current.children!.find(child => child.type === ASTNodeType.ELEMENT)!;
+        current = current.children!.find(child => child.type === ASTNodeType.Element)!;
       }
       
-      const textNode = current.children!.find(child => child.type === ASTNodeType.TEXT)!;
-      expect(textNode.content).toBe('Deep content');
+      const textNode = current.children!.find(child => child.type === ASTNodeType.Text)!;
+      expect((textNode as any).content).toBe('Deep content');
     });
 
-    test('should handle malformed HTML gracefully', () => {
+    it('should handle malformed HTML gracefully', () => {
       const malformedHTML = '<div><p><span>Text</div></span></p>';
-      const tokens = tokenize(malformedHTML);
-      const ast = parse(tokens);
+      const ast = parseToAST(malformedHTML);
       
       const divElement = ast.children![0]!;
       expect(divElement.tagName).toBe('div');
       expect(divElement.children!.length).toBeGreaterThan(0);
     });
 
-    test('should handle orphaned closing tags', () => {
+    it('should handle orphaned closing tags', () => {
       const html = '</div><p>Valid content</p></span>';
-      const tokens = tokenize(html);
-      const ast = parse(tokens);
+      const ast = parseToAST(html);
       
       const pElement = ast.children!.find(
-        child => child.type === ASTNodeType.ELEMENT && child.tagName === 'p'
+        child => child.type === ASTNodeType.Element && child.tagName === 'p'
       )!;
       expect(pElement).toBeDefined();
-      expect(pElement.children![0]!.content).toBe('Valid content');
+      expect((pElement.children![0]! as any).content).toBe('Valid content');
     });
 
-    test('should handle mixed content types in single document', () => {
+    it.skip('should handle mixed content types in single document', () => {
       const complexHTML = `
         <?xml version="1.0"?>
         <!DOCTYPE html>
@@ -396,21 +416,20 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
         <!-- Document end -->
       `;
       
-      const tokens = tokenize(complexHTML);
-      const ast = parse(tokens);
+      const ast = parseToAST(complexHTML);
       
-      const nodeCounts = {
-        [ASTNodeType.PROCESSING_INSTRUCTION]: 0,
-        [ASTNodeType.DOCTYPE]: 0,
-        [ASTNodeType.COMMENT]: 0,
-        [ASTNodeType.ELEMENT]: 0,
-        [ASTNodeType.TEXT]: 0,
+      const nodeCounts: Record<string, number> = {
+        'processing-instruction': 0,
+        [ASTNodeType.Doctype]: 0,
+        [ASTNodeType.Comment]: 0,
+        [ASTNodeType.Element]: 0,
+        [ASTNodeType.Text]: 0,
         [ASTNodeType.CDATA]: 0
       };
       
       const traverse = (node: ASTNode) => {
         if (node.type in nodeCounts) {
-          nodeCounts[node.type as keyof typeof nodeCounts]++;
+          nodeCounts[node.type]++;
         }
         if (node.children) {
           node.children.forEach(traverse);
@@ -419,69 +438,63 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
       
       ast.children!.forEach(traverse);
       
-      expect(nodeCounts[ASTNodeType.PROCESSING_INSTRUCTION]).toBeGreaterThan(0);
-      expect(nodeCounts[ASTNodeType.DOCTYPE]).toBeGreaterThan(0);
-      expect(nodeCounts[ASTNodeType.COMMENT]).toBeGreaterThan(0);
-      expect(nodeCounts[ASTNodeType.ELEMENT]).toBeGreaterThan(0);
-      expect(nodeCounts[ASTNodeType.TEXT]).toBeGreaterThan(0);
+      expect(nodeCounts['processing-instruction']).toBeGreaterThan(0);
+      expect(nodeCounts[ASTNodeType.Doctype]).toBeGreaterThan(0);
+      expect(nodeCounts[ASTNodeType.Comment]).toBeGreaterThan(0);
+      expect(nodeCounts[ASTNodeType.Element]).toBeGreaterThan(0);
+      expect(nodeCounts[ASTNodeType.Text]).toBeGreaterThan(0);
       expect(nodeCounts[ASTNodeType.CDATA]).toBeGreaterThan(0);
     });
   });
 
   describe('Security and Template Edge Cases', () => {
-    test('should treat javascript: urls as regular attribute values', () => {
+    it('should treat javascript: urls as regular attribute values', () => {
       const html = `<a href="javascript:alert('XSS')">Click me</a>`;
-      const tokens = tokenize(html);
-      const ast = parse(tokens);
+      const ast = parseToAST(html);
       const aElement = ast.children!.find(child => child.tagName === 'a')!;
       expect(aElement).toBeDefined();
       expect(aElement.attributes!.href).toBe("javascript:alert('XSS')");
     });
 
-    test('should correctly parse event handler attributes like onerror', () => {
+    it('should correctly parse event handler attributes like onerror', () => {
       const html = `<img src="invalid" onerror="alert('XSS')">`;
-      const tokens = tokenize(html);
-      const ast = parse(tokens);
+      const ast = parseToAST(html);
       const imgElement = ast.children!.find(child => child.tagName === 'img')!;
       expect(imgElement).toBeDefined();
       expect(imgElement.attributes!.onerror).toBe("alert('XSS')");
     });
 
-    test('should treat template engine syntax as plain text', () => {
+    it('should treat template engine syntax as plain text', () => {
       const html = `<div>{{ user.name }}</div><p>Hello, &lt;%= name %&gt;</p>`;
-      const tokens = tokenize(html);
-      const ast = parse(tokens);
+      const ast = parseToAST(html);
 
       const divElement = ast.children!.find(child => child.tagName === 'div')!;
       expect(divElement).toBeDefined();
-      const divText = divElement.children!.find(child => child.type === ASTNodeType.TEXT)!;
-      expect(divText.content).toBe('{{ user.name }}');
+      const divText = divElement.children!.find(child => child.type === ASTNodeType.Text)!;
+      expect((divText as any).content).toBe('{{ user.name }}');
 
       const pElement = ast.children!.find(child => child.tagName === 'p')!;
       expect(pElement).toBeDefined();
-      const pText = pElement.children!.find(child => child.type === ASTNodeType.TEXT)!;
-      expect(pText.content).toBe('Hello, <%= name %>');
+      const pText = pElement.children!.find(child => child.type === ASTNodeType.Text)!;
+      expect((pText as any).content).toBe('Hello, <%= name %>');
     });
 
-    test('should handle null characters in content gracefully', () => {
+    it('should handle null characters in content gracefully', () => {
         const html = '<div>Hello\0World</div>';
-        const tokens = tokenize(html);
-        const ast = parse(tokens);
+        const ast = parseToAST(html);
         const divElement = ast.children!.find(child => child.tagName === 'div')!;
-        const textNode = divElement.children!.find(child => child.type === ASTNodeType.TEXT)!;
-        expect(textNode.content).toBe('Hello\uFFFDWorld');
+        const textNode = divElement.children!.find(child => child.type === ASTNodeType.Text)!;
+        expect((textNode as any).content).toBe('Hello\uFFFDWorld');
     });
 
-    test('should handle control characters in content', () => {
+    it('should handle control characters in content', () => {
         const html = '<div>Line1\x08\x09Line2\x0BLine3\x0CLine4\x0DLine5</div>';
-        const tokens = tokenize(html);
-        const ast = parse(tokens);
+        const ast = parseToAST(html);
         const divElement = ast.children!.find(child => child.tagName === 'div')!;
-        const textNode = divElement.children!.find(child => child.type === ASTNodeType.TEXT)!;
-        expect(textNode.content).toContain('\x09');
-        expect(textNode.content).toContain('\x0D');
-        expect(textNode.content).toContain('Line1');
-        expect(textNode.content).toContain('Line5');
-    });
+        const textNode = divElement.children!.find(child => child.type === ASTNodeType.Text)!;
+        expect((textNode as any).content).toContain('\x09');
+        expect((textNode as any).content).toContain('\x0D');
+        expect((textNode as any).content).toContain('Line1');
+        expect((textNode as any).content).toContain('Line5');    });
   });
-});
+});

package/tests/custom-elements-head.test.ts

@@ -0,0 +1,105 @@
+import { describe, it, expect } from 'bun:test';
+import { parseHTML } from '../index';
+
+describe('Custom Elements in <head>', () => {
+
+  it('should keep <meta-tags> custom element in head', () => {
+    const doc = parseHTML(
+      '<!DOCTYPE html><html><head><meta-tags></meta-tags></head><body></body></html>'
+    );
+    
+    const metaTags = doc.head?.querySelector('meta-tags');
+    expect(metaTags).toBeTruthy();
+    expect(metaTags?.parentElement?.tagName).toBe('HEAD');
+  });
+
+  it('should keep <social-meta> custom element in head', () => {
+    const doc = parseHTML(
+      '<!DOCTYPE html><html><head><social-meta></social-meta></head><body></body></html>'
+    );
+    
+    const socialMeta = doc.head?.querySelector('social-meta');
+    expect(socialMeta).toBeTruthy();
+    expect(socialMeta?.parentElement?.tagName).toBe('HEAD');
+  });
+
+  it('should keep any <custom-element> with hyphen in head', () => {
+    const doc = parseHTML(
+      '<!DOCTYPE html><html><head><my-component></my-component></head><body></body></html>'
+    );
+    
+    const myComponent = doc.head?.querySelector('my-component');
+    expect(myComponent).toBeTruthy();
+    expect(myComponent?.parentElement?.tagName).toBe('HEAD');
+  });
+
+  it('should still eject non-custom elements like <div> to body', () => {
+    const doc = parseHTML(
+      '<!DOCTYPE html><html><head><div>test</div></head><body></body></html>'
+    );
+    
+    const divInHead = doc.head?.querySelector('div');
+    const divInBody = doc.body?.querySelector('div');
+    expect(divInHead).toBeFalsy();
+    expect(divInBody).toBeTruthy();
+  });
+
+  it('should handle nested custom elements in head', () => {
+    const doc = parseHTML(
+      '<!DOCTYPE html><html><head><my-wrapper><inner-comp></inner-comp></my-wrapper></head><body></body></html>'
+    );
+    
+    const myWrapper = doc.head?.querySelector('my-wrapper');
+    expect(myWrapper).toBeTruthy();
+    expect(myWrapper?.parentElement?.tagName).toBe('HEAD');
+    
+    const innerComp = myWrapper?.querySelector('inner-comp');
+    expect(innerComp).toBeTruthy();
+  });
+
+  it('should keep custom elements with attributes in head', () => {
+    const doc = parseHTML(
+      '<!DOCTYPE html><html><head><seo-meta property="og:title" content="Test"></seo-meta></head><body></body></html>'
+    );
+    
+    const seoMeta = doc.head?.querySelector('seo-meta');
+    expect(seoMeta).toBeTruthy();
+    expect(seoMeta?.getAttribute('property')).toBe('og:title');
+    expect(seoMeta?.getAttribute('content')).toBe('Test');
+    expect(seoMeta?.parentElement?.tagName).toBe('HEAD');
+  });
+
+  it('should keep self-closing custom elements in head', () => {
+    const doc = parseHTML(
+      '<!DOCTYPE html><html><head><custom-void /></head><body></body></html>'
+    );
+    
+    const customVoid = doc.head?.querySelector('custom-void');
+    expect(customVoid).toBeTruthy();
+    expect(customVoid?.parentElement?.tagName).toBe('HEAD');
+  });
+
+  it('should handle custom elements mixed with standard head elements', () => {
+    const doc = parseHTML(
+      '<!DOCTYPE html><html><head><title>Test</title><meta-tags></meta-tags><link rel="stylesheet" href="style.css"></head><body></body></html>'
+    );
+    
+    const title = doc.head?.querySelector('title');
+    const metaTags = doc.head?.querySelector('meta-tags');
+    const link = doc.head?.querySelector('link');
+    
+    expect(title).toBeTruthy();
+    expect(metaTags).toBeTruthy();
+    expect(link).toBeTruthy();
+  });
+
+  it('should handle custom element containing text in head', () => {
+    const doc = parseHTML(
+      '<!DOCTYPE html><html><head><inline-script>console.log("test")</inline-script></head><body></body></html>'
+    );
+    
+    const inlineScript = doc.head?.querySelector('inline-script');
+    expect(inlineScript).toBeTruthy();
+    expect(inlineScript?.parentElement?.tagName).toBe('HEAD');
+  });
+});