@tkeron/html-parser 0.1.4 → 0.1.7

package/src/tokenizer.ts

@@ -44,9 +44,6 @@ const HTML_ENTITIES: Record<string, string> = {
   '&not;': '¬'
 };
 
-/**
- * Decode HTML entities in a string and handle null characters
- */
 function decodeEntities(text: string): string {
   let result = text.replace(/\u0000/g, '\uFFFD');
   
@@ -78,9 +75,6 @@ function decodeEntities(text: string): string {
   });
 }
 
-/**
- * Parse attributes from a tag string
- */
 function parseAttributes(attributeString: string): Record<string, string> {
   const attributes: Record<string, string> = {};
   
@@ -98,9 +92,6 @@ function parseAttributes(attributeString: string): Record<string, string> {
   return attributes;
 }
 
-/**
- * Calculate position in text
- */
 function calculatePosition(text: string, offset: number): Position {
   const lines = text.slice(0, offset).split('\n');
   return {
@@ -110,10 +101,6 @@ function calculatePosition(text: string, offset: number): Position {
   };
 }
 
-/**
- * Tokenize HTML using a combination of HTMLRewriter and manual parsing
- * HTMLRewriter is great for structured HTML but we need manual parsing for edge cases
- */
 export function tokenize(html: string): Token[] {
   const tokens: Token[] = [];
   let position = 0;
@@ -254,10 +241,8 @@ export function tokenize(html: string): Token[] {
     }
   }
   
-  // Sort tokens by position
   tokens.sort((a, b) => a.position.offset - b.position.offset);
   
-  // Add EOF token
   tokens.push({
     type: TokenType.EOF,
     value: '',
@@ -266,104 +251,3 @@ export function tokenize(html: string): Token[] {
   
   return tokens;
 }
-
-export function tokenizeWithRewriter(html: string): Token[] {
-  const tokens: Token[] = [];
-  let textBuffer = '';
-  let position = 0;
-  
-  const rewriter = new HTMLRewriter();
-  
-  rewriter.on('*', {
-    element(element) {
-      if (textBuffer.trim()) {
-        tokens.push({
-          type: TokenType.TEXT,
-          value: decodeEntities(textBuffer),
-          position: calculatePosition(html, position - textBuffer.length)
-        });
-        textBuffer = '';
-      }
-      
-      // Add opening tag
-      const attributes: Record<string, string> = {};
-      for (const [name, value] of element.attributes) {
-        attributes[name] = value;
-      }
-      
-      tokens.push({
-        type: TokenType.TAG_OPEN,
-        value: element.tagName.toLowerCase(),
-        position: calculatePosition(html, position),
-        attributes,
-        isSelfClosing: element.selfClosing
-      });
-      
-      if (!element.selfClosing) {
-        element.onEndTag((endTag) => {
-          tokens.push({
-            type: TokenType.TAG_CLOSE,
-            value: endTag.name.toLowerCase(),
-            position: calculatePosition(html, position),
-            isClosing: true
-          });
-        });
-      }
-    },
-    
-    text(text) {
-      textBuffer += text.text;
-    },
-    
-    comments(comment) {
-      tokens.push({
-        type: TokenType.COMMENT,
-        value: comment.text,
-        position: calculatePosition(html, position)
-      });
-    }
-  });
-  
-  try {
-    // Transform the HTML (this triggers the rewriter)
-    const response = new Response(html, {
-      headers: { 'Content-Type': 'text/html' }
-    });
-    
-    rewriter.transform(response);
-    
-    // Flush any remaining text
-    if (textBuffer.trim()) {
-      tokens.push({
-        type: TokenType.TEXT,
-        value: decodeEntities(textBuffer),
-        position: calculatePosition(html, position - textBuffer.length)
-      });
-    }
-    
-  } catch (error) {
-    // If HTMLRewriter fails, fall back to manual parsing
-    console.warn('HTMLRewriter failed, falling back to manual parsing:', error);
-    return tokenize(html);
-  }
-  
-  // Sort tokens by position and add EOF
-  tokens.sort((a, b) => a.position.offset - b.position.offset);
-  tokens.push({
-    type: TokenType.EOF,
-    value: '',
-    position: calculatePosition(html, html.length)
-  });
-  
-  return tokens;
-}
-
-export function smartTokenize(html: string): Token[] {
-  const hasSpecialContent = /<!DOCTYPE|<!--|\[CDATA\[|<\?/.test(html);
-  
-  if (hasSpecialContent || html.length < 1000) {
-    return tokenize(html);
-  } else {
-    return tokenizeWithRewriter(html);
-  }
-}

package/tests/advanced.test.ts

@@ -28,14 +28,14 @@ describe('HTML Parser & Tokenizer - Advanced Tests', () => {
     });
 
     test('should handle unicode characters', () => {
-      const tokens = tokenize('<div title="测试" data-emoji="🚀" class="café">');
+      const tokens = tokenize('<div title="测试" data-emoji="🚀" class="lorem">');
       expect(tokens.length).toBeGreaterThan(0);
       const tag = tokens[0]!;
       
       expect(tag.attributes).toEqual({
         title: '测试',
         'data-emoji': '🚀',
-        class: 'café'
+        class: 'lorem'
       });
     });
 

package/tests/cloneNode.test.ts

@@ -11,17 +11,17 @@ describe("cloneNode functionality", () => {
 
       const cloned = original.cloneNode(true);
 
-      // Verificar que el nodo clonado existe
+      
       expect(cloned).toBeTruthy();
       expect(cloned.nodeName).toBe("DIV");
 
-      // Verificar que los atributos se copian
+      
       expect(cloned.getAttribute("id")).toBe("original");
 
-      // Verificar que el contenido de texto se copia
+      
       expect(cloned.textContent).toBe("Hello World");
 
-      // Verificar que tiene los hijos correctos
+      
       expect(cloned.childNodes.length).toBeGreaterThan(0);
     });
 
@@ -38,20 +38,20 @@ describe("cloneNode functionality", () => {
 
       const cloned = parent.cloneNode(true);
 
-      // Verificar estructura básica
+      
       expect(cloned.nodeName).toBe("DIV");
       expect(cloned.getAttribute("id")).toBe("parent");
 
-      // Verificar que los hijos se clonaron
+      
       expect(cloned.childNodes.length).toBeGreaterThan(0);
       
-      // Verificar que el hijo div está presente
+      
       const childDiv = cloned.querySelector(".child");
       expect(childDiv).toBeTruthy();
       expect(childDiv?.nodeName).toBe("DIV");
       expect(childDiv?.getAttribute("class")).toBe("child");
 
-      // Verificar el nieto span
+      
       const span = cloned.querySelector("span");
       expect(span).toBeTruthy();
       expect(span?.textContent).toBe("Nested Text");
@@ -70,7 +70,7 @@ describe("cloneNode functionality", () => {
 
       const cloned = list.cloneNode(true);
 
-      // Verificar que todos los items se clonaron
+      
       const items = cloned.querySelectorAll("li");
       expect(items.length).toBe(3);
       expect(items[0]?.textContent).toBe("Item 1");
@@ -89,18 +89,18 @@ describe("cloneNode functionality", () => {
       const doc = parseHTML(html);
       const container = doc.querySelector("#container")!;
 
-      // Guardar innerHTML original
+      
       const originalInnerHTML = container.innerHTML;
       expect(originalInnerHTML).toBeTruthy();
       expect(originalInnerHTML.length).toBeGreaterThan(0);
 
       const cloned = container.cloneNode(true);
 
-      // Verificar que innerHTML del clon no esté vacío
+      
       expect(cloned.innerHTML).toBeTruthy();
       expect(cloned.innerHTML.length).toBeGreaterThan(0);
 
-      // Verificar que el contenido es similar
+      
       expect(cloned.innerHTML).toContain("<h1>Title</h1>");
       expect(cloned.innerHTML).toContain("<p>Paragraph 1</p>");
       expect(cloned.innerHTML).toContain("<p>Paragraph 2</p>");
@@ -111,19 +111,19 @@ describe("cloneNode functionality", () => {
       const doc = parseHTML(html);
       const mixed = doc.querySelector("#mixed")!;
 
-      // Verificar estructura original
+      
       const originalChildCount = mixed.childNodes.length;
       expect(originalChildCount).toBeGreaterThan(0);
 
       const cloned = mixed.cloneNode(true);
 
-      // Verificar que tiene la misma cantidad de hijos
+      
       expect(cloned.childNodes.length).toBe(originalChildCount);
 
-      // Verificar contenido completo
+      
       expect(cloned.textContent).toBe("Text beforebold textText after");
 
-      // Verificar que el elemento strong existe
+      
       const strong = cloned.querySelector("strong");
       expect(strong).toBeTruthy();
       expect(strong?.textContent).toBe("bold text");
@@ -136,7 +136,7 @@ describe("cloneNode functionality", () => {
 
       const cloned = element.cloneNode(true);
 
-      // Verificar todos los atributos
+      
       expect(cloned.getAttribute("id")).toBe("attrs");
       expect(cloned.getAttribute("class")).toBe("test");
       expect(cloned.getAttribute("data-value")).toBe("123");
@@ -161,11 +161,11 @@ describe("cloneNode functionality", () => {
 
       const cloned = article.cloneNode(true);
 
-      // Verificar estructura
+      
       expect(cloned.nodeName).toBe("ARTICLE");
       expect(cloned.getAttribute("id")).toBe("article");
 
-      // Verificar elementos
+      
       expect(cloned.querySelector("h2")?.textContent).toBe("Article Title");
       
       const paragraphs = cloned.querySelectorAll("p");
@@ -173,12 +173,12 @@ describe("cloneNode functionality", () => {
       expect(paragraphs[0]?.textContent).toBe("First paragraph");
       expect(paragraphs[1]?.textContent).toBe("Last paragraph");
 
-      // Verificar div anidado
+      
       const highlight = cloned.querySelector(".highlight");
       expect(highlight).toBeTruthy();
       expect(highlight?.querySelector("span")?.textContent).toBe("Highlighted");
 
-      // Verificar que se copió el comentario
+      
       const hasComment = Array.from(cloned.childNodes).some(
         (node: any) => node.nodeType === NodeType.COMMENT_NODE
       );
@@ -203,7 +203,7 @@ describe("cloneNode functionality", () => {
 
       const cloned = section.cloneNode(true);
 
-      // Verificar outerHTML
+      
       expect(cloned.outerHTML).toBeTruthy();
       expect(cloned.outerHTML).toContain("section");
       expect(cloned.outerHTML).toContain("class=\"main\"");
@@ -220,7 +220,7 @@ describe("cloneNode functionality", () => {
 
       const cloned = parent.cloneNode(false);
 
-      // Debe copiar el elemento pero no los hijos
+      
       expect(cloned.nodeName).toBe("DIV");
       expect(cloned.getAttribute("id")).toBe("parent");
       expect(cloned.childNodes.length).toBe(0);
@@ -234,12 +234,12 @@ describe("cloneNode functionality", () => {
 
       const cloned = element.cloneNode(false);
 
-      // Atributos deben copiarse
+      
       expect(cloned.getAttribute("id")).toBe("test");
       expect(cloned.getAttribute("class")).toBe("container");
       expect(cloned.getAttribute("data-value")).toBe("123");
 
-      // Hijos NO deben copiarse
+      
       expect(cloned.childNodes.length).toBe(0);
       expect(cloned.innerHTML).toBe("");
     });
@@ -253,7 +253,7 @@ describe("cloneNode functionality", () => {
 
       const cloned = original.cloneNode(true);
 
-      // Modificar el clon no debe afectar el original
+      
       cloned.setAttribute("id", "cloned");
       cloned.setAttribute("data-modified", "true");
 
@@ -273,10 +273,10 @@ describe("cloneNode functionality", () => {
       
       expect(clonedChild).toBeTruthy();
       
-      // Modificar hijo del clon
+      
       clonedChild?.setAttribute("data-cloned", "yes");
 
-      // El hijo original no debe verse afectado
+      
       const originalChild = parent.querySelector("#child");
       expect(originalChild?.hasAttribute("data-cloned")).toBe(false);
     });
@@ -324,7 +324,7 @@ describe("cloneNode functionality", () => {
 
       const cloned = level1.cloneNode(true);
 
-      // Verificar todos los niveles
+      
       expect(cloned.querySelector("#level2")).toBeTruthy();
       expect(cloned.querySelector("#level3")).toBeTruthy();
       expect(cloned.querySelector("#level4")).toBeTruthy();
@@ -340,22 +340,22 @@ describe("cloneNode functionality", () => {
       const doc = parseHTML(html);
       const container = doc.querySelector("#container")!;
 
-      // Acceder a innerHTML para asegurar que _internalInnerHTML esté establecido
+      
       const originalInnerHTML = container.innerHTML;
       expect(originalInnerHTML).toBeTruthy();
 
       const cloned = container.cloneNode(true);
 
-      // Verificar que innerHTML funciona en el clon
+      
       const clonedInnerHTML = cloned.innerHTML;
       expect(clonedInnerHTML).toBeTruthy();
       expect(clonedInnerHTML.length).toBeGreaterThan(0);
 
-      // Verificar que contiene el mismo contenido
+      
       expect(clonedInnerHTML).toContain("<p>Paragraph 1</p>");
       expect(clonedInnerHTML).toContain("<p>Paragraph 2</p>");
 
-      // Verificar que el accessor de innerHTML funciona correctamente
+      
       expect(typeof cloned.innerHTML).toBe("string");
     });
 
@@ -369,10 +369,10 @@ describe("cloneNode functionality", () => {
 
       const cloned = parent.cloneNode(true);
 
-      // Verificar que childNodes tiene la misma estructura
+      
       expect(cloned.childNodes.length).toBe(originalChildCount);
       
-      // Verificar que podemos acceder a cada hijo
+      
       for (let i = 0; i < cloned.childNodes.length; i++) {
         expect(cloned.childNodes[i]).toBeTruthy();
         expect(cloned.childNodes[i].nodeType).toBeDefined();
@@ -386,12 +386,12 @@ describe("cloneNode functionality", () => {
 
       const cloned = container.cloneNode(true);
 
-      // Verificar que el array children está correctamente poblado
+      
       expect(cloned.children).toBeTruthy();
       expect(Array.isArray(cloned.children)).toBe(true);
       expect(cloned.children.length).toBe(3);
 
-      // Verificar que todos son elementos
+      
       for (const child of cloned.children) {
         expect(child.nodeType).toBe(NodeType.ELEMENT_NODE);
       }
@@ -404,12 +404,12 @@ describe("cloneNode functionality", () => {
 
       const cloned = list.cloneNode(true);
 
-      // Verificar referencias firstChild y lastChild
+      
       expect(cloned.firstChild).toBeTruthy();
       expect(cloned.lastChild).toBeTruthy();
       
-      // En DOM real, firstChild puede ser un nodo de texto (whitespace)
-      // pero debemos asegurar que existen
+      
+      
       expect(cloned.firstElementChild).toBeTruthy();
       expect(cloned.lastElementChild).toBeTruthy();
       
@@ -427,12 +427,12 @@ describe("cloneNode functionality", () => {
       const doc = parseHTML(html);
       const dynamic = doc.querySelector("#dynamic")!;
 
-      // Modificar innerHTML antes de clonar
+      
       dynamic.innerHTML = "<p>Dynamic content</p><span>More content</span>";
 
       const cloned = dynamic.cloneNode(true);
 
-      // Verificar que el contenido modificado se clonó
+      
       expect(cloned.querySelector("p")).toBeTruthy();
       expect(cloned.querySelector("p")?.textContent).toBe("Dynamic content");
       expect(cloned.querySelector("span")).toBeTruthy();
@@ -446,17 +446,17 @@ describe("cloneNode functionality", () => {
 
       const cloned = original.cloneNode(true);
 
-      // Verificar contenido clonado
+      
       expect(cloned.querySelector("p")?.textContent).toBe("Original");
 
-      // Modificar innerHTML del clon
+      
       cloned.innerHTML = "<span>Modified</span>";
 
-      // Original no debe cambiar
+      
       expect(original.querySelector("p")?.textContent).toBe("Original");
       expect(original.querySelector("span")).toBeNull();
 
-      // Clon debe tener el nuevo contenido
+      
       expect(cloned.querySelector("span")?.textContent).toBe("Modified");
       expect(cloned.querySelector("p")).toBeNull();
     });
@@ -488,13 +488,13 @@ describe("cloneNode functionality", () => {
 
       const cloned = card.cloneNode(true);
 
-      // Verificar estructura completa
+      
       expect(cloned.getAttribute("data-id")).toBe("123");
       expect(cloned.querySelector(".card-header")).toBeTruthy();
       expect(cloned.querySelector(".card-body")).toBeTruthy();
       expect(cloned.querySelector(".card-footer")).toBeTruthy();
 
-      // Verificar contenido específico
+      
       expect(cloned.querySelector(".card-title")?.textContent).toBe("Card Title");
       expect(cloned.querySelector("strong")?.textContent).toBe("bold");
       
@@ -502,7 +502,7 @@ describe("cloneNode functionality", () => {
       expect(items.length).toBe(2);
       
       const buttons = cloned.querySelectorAll("button");
-      expect(buttons.length).toBe(3); // close, save, cancel
+      expect(buttons.length).toBe(3); 
     });
 
     it("should clone a form with various input types", () => {
@@ -522,7 +522,7 @@ describe("cloneNode functionality", () => {
 
       const cloned = form.cloneNode(true);
 
-      // Verificar que todos los inputs se clonaron
+      
       const textInput = cloned.querySelector('[name="username"]');
       expect(textInput).toBeTruthy();
       expect(textInput?.getAttribute("value")).toBe("john");

package/tests/custom-elements.test.ts

@@ -210,7 +210,7 @@ describe('Custom Elements Support', () => {
       const ast = parse(tokens);
 
       const element = ast.children![0]!;
-      // nodeName should also be uppercase
+      
       if (element.nodeName) {
         expect(element.nodeName.toUpperCase()).toBe('MY-COMP');
       }
@@ -382,11 +382,11 @@ describe('Custom Elements Support', () => {
       const tokens = tokenize(html);
       const ast = parse(tokens);
 
-      // Find first element (skip whitespace text nodes)
+      
       const userProfile = ast.children!.find(node => node.type === ASTNodeType.ELEMENT)!;
       expect(userProfile.tagName).toBe('user-profile');
       
-      // Should have proper nesting
+      
       expect(userProfile.children).toBeDefined();
       expect(userProfile.children!.length).toBeGreaterThan(0);
     });
@@ -412,7 +412,7 @@ describe('Custom Elements Support', () => {
       const tokens = tokenize(html);
       const ast = parse(tokens);
 
-      // Find first element (skip whitespace text nodes)
+      
       const appRoot = ast.children!.find(node => node.type === ASTNodeType.ELEMENT)!;
       expect(appRoot.tagName).toBe('app-root');
     });
@@ -471,12 +471,12 @@ describe('Custom Elements Support', () => {
     test('tokenizer should capture full custom element name', () => {
       const tokens = tokenize('<my-component-123></my-component-123>');
       
-      // Find the opening tag token
+      
       const openTag = tokens.find(t => t.type === 'TAG_OPEN');
       expect(openTag).toBeDefined();
       expect(openTag!.value).toBe('my-component-123');
       
-      // Find the closing tag token
+      
       const closeTag = tokens.find(t => t.type === 'TAG_CLOSE');
       expect(closeTag).toBeDefined();
       expect(closeTag!.value).toBe('my-component-123');
@@ -642,7 +642,7 @@ describe('Custom Elements Support', () => {
       const tokens = tokenize(html);
       const ast = parse(tokens);
 
-      // Should have comment, element, comment
+      
       const myComp = ast.children!.find(node => node.type === ASTNodeType.ELEMENT)!;
       expect(myComp.tagName).toBe('my-comp');
     });
@@ -700,7 +700,7 @@ describe('Custom Elements Support', () => {
       const tokens = tokenize('<table><tr><td><my-cell>content</my-cell></td></tr></table>');
       const ast = parse(tokens);
 
-      // Find the custom element
+      
       const table = ast.children![0]!;
       expect(table.tagName).toBe('table');
     });