@tkeron/html-parser 1.1.2 → 1.3.0

package/src/serializer/serialize-tokens.ts

@@ -0,0 +1,511 @@
+import { serializeAttributes } from "./attributes.js";
+import { escapeText } from "./escape.js";
+
+export const serializeTokens = (
+  tokens: any[],
+  options?: {
+    inject_meta_charset?: boolean;
+    encoding?: string;
+    quote_char?: string;
+    quote_attr_values?: boolean;
+    minimize_boolean_attributes?: boolean;
+    escape_lt_in_attrs?: boolean;
+    use_trailing_solidus?: boolean;
+    escape_rcdata?: boolean;
+    strip_whitespace?: boolean;
+  },
+): string => {
+  const encoding = options?.encoding || "utf-8";
+  let result = "";
+  let inScript = false;
+  let inPre = false;
+  let inTextarea = false;
+  let inStyle = false;
+  let serializingHead = true;
+
+  let processedTokens = tokens;
+  if (options?.inject_meta_charset) {
+    let hasCharset = false;
+    let modifiedTokens: any[] = [];
+    let inHead = false;
+
+    for (const token of tokens) {
+      const type = token[0];
+      if (type === "StartTag" && token[2] === "head") {
+        inHead = true;
+      } else if (type === "EndTag" && token[2] === "head") {
+        inHead = false;
+      } else if (inHead && type === "EmptyTag" && token[1] === "meta") {
+        const attrs = token[2];
+        if (attrs.some((attr: any) => attr.name === "charset")) {
+          hasCharset = true;
+        }
+        const hasHttpEquiv = attrs.some(
+          (attr: any) =>
+            attr.name === "http-equiv" && attr.value === "content-type",
+        );
+        if (hasHttpEquiv) {
+          const contentAttr = attrs.find(
+            (attr: any) => attr.name === "content",
+          );
+          if (contentAttr && contentAttr.value.includes("charset=")) {
+            hasCharset = true;
+          }
+        }
+      }
+    }
+
+    inHead = false;
+    for (const token of tokens) {
+      const type = token[0];
+      if (type === "StartTag" && token[2] === "head") {
+        inHead = true;
+        modifiedTokens.push(token);
+        if (!hasCharset && options?.encoding) {
+          modifiedTokens.push([
+            "EmptyTag",
+            "meta",
+            [{ name: "charset", value: encoding }],
+          ]);
+        }
+      } else if (type === "EndTag" && token[2] === "head") {
+        inHead = false;
+        modifiedTokens.push(token);
+      } else if (inHead && type === "EmptyTag" && token[1] === "meta") {
+        let newAttrs = token[2].slice();
+        let isHttpEquiv = false;
+        for (let i = 0; i < newAttrs.length; i++) {
+          const attr = newAttrs[i];
+          if (attr.name === "charset" && options?.encoding) {
+            newAttrs[i] = { name: "charset", value: encoding };
+          } else if (
+            attr.name === "http-equiv" &&
+            attr.value === "content-type"
+          ) {
+            isHttpEquiv = true;
+          } else if (
+            attr.name === "content" &&
+            isHttpEquiv &&
+            options?.encoding
+          ) {
+            newAttrs[i] = {
+              name: "content",
+              value: attr.value.replace(/charset=[^;]*/, "charset=" + encoding),
+            };
+          }
+        }
+        modifiedTokens.push([type, token[1], newAttrs]);
+      } else {
+        modifiedTokens.push(token);
+      }
+    }
+    processedTokens = modifiedTokens;
+  }
+
+  let omitHtml = false;
+  let omitHead = false;
+  let omitBody = false;
+  let omitColgroup = false;
+  let htmlStartIndex = -1;
+  let headStartIndex = -1;
+  let bodyStartIndex = -1;
+  let colgroupStartIndex = -1;
+  let tbodyCount = 0;
+  let colgroupCount = 0;
+  for (let i = 0; i < processedTokens.length; i++) {
+    const token = processedTokens[i];
+    const type = token[0];
+    if (type === "StartTag") {
+      const name = token[2];
+      if (name === "html") {
+        htmlStartIndex = i;
+      }
+      if (name === "head") {
+        headStartIndex = i;
+      }
+      if (name === "body") {
+        bodyStartIndex = i;
+      }
+      if (name === "colgroup") {
+        colgroupStartIndex = i;
+        colgroupCount++;
+      }
+      if (name === "tbody") {
+        tbodyCount++;
+      }
+    }
+  }
+  if (htmlStartIndex >= 0) {
+    const htmlToken = processedTokens[htmlStartIndex];
+    const attrs = htmlToken[3];
+    const hasAttributes = Array.isArray(attrs)
+      ? attrs.length > 0
+      : attrs
+        ? Object.keys(attrs).length > 0
+        : false;
+    if (hasAttributes) {
+      omitHtml = false;
+    } else {
+      let firstToken = null;
+      for (let j = htmlStartIndex + 1; j < processedTokens.length; j++) {
+        const t = processedTokens[j];
+        if (t[0] !== "Characters" || t[1].trim() !== "") {
+          firstToken = t;
+          break;
+        }
+      }
+      if (!firstToken) {
+        omitHtml = true;
+      } else if (firstToken[0] === "Comment") {
+        omitHtml = false;
+      } else if (firstToken[0] === "Characters") {
+        if (/^\s/.test(firstToken[1])) {
+          omitHtml = false;
+        } else {
+          omitHtml = true;
+        }
+      } else {
+        omitHtml = true;
+      }
+    }
+  }
+  if (headStartIndex >= 0) {
+    let firstToken = null;
+    for (let j = headStartIndex + 1; j < processedTokens.length; j++) {
+      const t = processedTokens[j];
+      if (t[0] !== "Characters" || t[1].trim() !== "") {
+        firstToken = t;
+        break;
+      }
+    }
+    omitHead = false;
+    if (firstToken) {
+      if (firstToken[0] === "StartTag") {
+        omitHead = true;
+      } else if (firstToken[0] === "EndTag" && firstToken[2] === "head") {
+        omitHead = true;
+      } else if (firstToken[0] === "EmptyTag") {
+        omitHead = true;
+      }
+    }
+  }
+  if (bodyStartIndex >= 0) {
+    let firstToken = null;
+    for (let j = bodyStartIndex + 1; j < processedTokens.length; j++) {
+      const t = processedTokens[j];
+      if (t[0] !== "Characters" || t[1].trim() !== "") {
+        firstToken = t;
+        break;
+      }
+    }
+    omitBody = false;
+    if (firstToken) {
+      if (firstToken[0] === "StartTag") {
+        omitBody = true;
+      } else if (firstToken[0] === "EndTag") {
+        omitBody = true;
+      } else if (firstToken[0] === "Characters" && !/^\s/.test(firstToken[1])) {
+        omitBody = true;
+      }
+    } else {
+      omitBody = true;
+    }
+  }
+  if (colgroupStartIndex >= 0) {
+    const colgroupToken = processedTokens[colgroupStartIndex];
+    const attrs = colgroupToken[3];
+    const hasAttributes = Array.isArray(attrs)
+      ? attrs.length > 0
+      : attrs
+        ? Object.keys(attrs).length > 0
+        : false;
+    let firstToken = null;
+    for (let j = colgroupStartIndex + 1; j < processedTokens.length; j++) {
+      const t = processedTokens[j];
+      if (t[0] !== "Characters" || t[1].trim() !== "") {
+        firstToken = t;
+        break;
+      }
+    }
+    omitColgroup =
+      !hasAttributes &&
+      firstToken &&
+      (firstToken[0] === "StartTag" || firstToken[0] === "EmptyTag") &&
+      (firstToken[0] === "StartTag" ? firstToken[2] : firstToken[1]) === "col";
+  }
+
+  for (let i = 0; i < processedTokens.length; i++) {
+    const token = processedTokens[i];
+    const nextToken = processedTokens[i + 1];
+    const type = token[0];
+    switch (type) {
+      case "StartTag":
+        const [, , name, attrs] = token;
+
+        let omitThisTbody = false;
+        if (name === "tbody") {
+          const hasAttributes = Array.isArray(attrs)
+            ? attrs.length > 0
+            : attrs
+              ? Object.keys(attrs).length > 0
+              : false;
+          if (!hasAttributes) {
+            let firstToken = null;
+            for (let j = i + 1; j < processedTokens.length; j++) {
+              const t = processedTokens[j];
+              if (t[0] !== "Characters" || t[1].trim() !== "") {
+                firstToken = t;
+                break;
+              }
+            }
+            const hasTrChild =
+              firstToken &&
+              (firstToken[0] === "StartTag" || firstToken[0] === "EmptyTag") &&
+              firstToken[2] === "tr";
+
+            if (hasTrChild) {
+              let isPreceded = false;
+              for (let j = 0; j < i; j++) {
+                const t = processedTokens[j];
+                if (t[0] === "Characters" && t[1].trim() === "") continue;
+                if (
+                  t[0] === "EndTag" &&
+                  ["tbody", "thead", "tfoot"].includes(t[2])
+                ) {
+                  isPreceded = true;
+                }
+                break;
+              }
+              omitThisTbody = !isPreceded;
+            }
+          }
+        }
+
+        if (name === "colgroup" && omitColgroup) continue;
+        if (name === "tbody" && omitThisTbody) continue;
+        if (name === "head" && omitHead) continue;
+        if (name === "body" && omitBody) continue;
+        if (name === "html" && omitHtml) continue;
+        if (name === "pre") inPre = true;
+        if (name === "textarea") inTextarea = true;
+        if (name === "script") inScript = true;
+        if (name === "style") inStyle = true;
+        if (name === "head") {
+          if (options?.inject_meta_charset) {
+            serializingHead = true;
+          } else {
+            result += "<" + name + serializeAttributes(attrs, options) + ">";
+          }
+        } else if (serializingHead) {
+          result += "<" + name + serializeAttributes(attrs, options) + ">";
+        }
+        break;
+      case "EmptyTag":
+        const [, name2, attrs2] = token;
+        result +=
+          "<" +
+          name2 +
+          serializeAttributes(attrs2, options) +
+          (options?.use_trailing_solidus ? " />" : ">");
+        break;
+      case "EndTag":
+        const [, , name3] = token;
+        let omitEndTag = false;
+        if (["html", "head", "body"].includes(name3)) {
+          if (
+            !nextToken ||
+            nextToken[0] === "StartTag" ||
+            nextToken[0] === "EndTag" ||
+            (nextToken[0] === "Characters" && !/^\s/.test(nextToken[1]))
+          ) {
+            omitEndTag = true;
+          }
+        } else if (nextToken) {
+          const nextType = nextToken[0];
+          let nextName = null;
+          if (nextType === "StartTag" || nextType === "EndTag") {
+            nextName = nextToken[2];
+          } else if (nextType === "EmptyTag") {
+            nextName = nextToken[1];
+          }
+          if (nextType === "EndTag") {
+            omitEndTag = [
+              "p",
+              "li",
+              "option",
+              "optgroup",
+              "tbody",
+              "tfoot",
+              "tr",
+              "td",
+              "th",
+              "colgroup",
+              "dd",
+            ].includes(name3);
+          } else if (nextType === "StartTag") {
+            if (
+              name3 === "p" &&
+              [
+                "address",
+                "article",
+                "aside",
+                "blockquote",
+                "datagrid",
+                "dialog",
+                "dir",
+                "div",
+                "dl",
+                "fieldset",
+                "footer",
+                "form",
+                "h1",
+                "h2",
+                "h3",
+                "h4",
+                "h5",
+                "h6",
+                "header",
+                "hr",
+                "menu",
+                "nav",
+                "ol",
+                "p",
+                "pre",
+                "section",
+                "table",
+                "ul",
+              ].includes(nextName)
+            ) {
+              omitEndTag = true;
+            } else if (name3 === "li" && nextName === "li") {
+              omitEndTag = true;
+            } else if (
+              (name3 === "dt" || name3 === "dd") &&
+              (nextName === "dt" || nextName === "dd")
+            ) {
+              omitEndTag = true;
+            } else if (
+              name3 === "option" &&
+              (nextName === "option" || nextName === "optgroup")
+            ) {
+              omitEndTag = true;
+            } else if (name3 === "optgroup" && nextName === "optgroup") {
+              omitEndTag = true;
+            } else if (
+              (name3 === "tbody" || name3 === "tfoot") &&
+              (nextName === "tbody" || nextName === "tfoot")
+            ) {
+              omitEndTag = true;
+            } else if (
+              name3 === "thead" &&
+              (nextName === "tbody" || nextName === "tfoot")
+            ) {
+              omitEndTag = true;
+            } else if (name3 === "tr" && nextName === "tr") {
+              omitEndTag = true;
+            } else if (
+              (name3 === "td" || name3 === "th") &&
+              (nextName === "td" || nextName === "th")
+            ) {
+              omitEndTag = true;
+            } else if (name3 === "colgroup" && nextName !== "colgroup") {
+              omitEndTag = true;
+            }
+            if (name3 === "p" && nextName === "hr") {
+              omitEndTag = true;
+            }
+          } else if (nextType === "EmptyTag") {
+            if (name3 === "p" && nextName === "hr") {
+              omitEndTag = true;
+            }
+          }
+          if (
+            name3 === "colgroup" &&
+            nextType === "Characters" &&
+            !/^\s/.test(nextToken[1])
+          ) {
+            omitEndTag = true;
+          }
+        } else {
+          omitEndTag = [
+            "p",
+            "li",
+            "option",
+            "optgroup",
+            "tbody",
+            "tfoot",
+            "tr",
+            "td",
+            "th",
+            "colgroup",
+            "dd",
+          ].includes(name3);
+        }
+        if (omitEndTag) continue;
+        if (name3 === "script") inScript = false;
+        if (name3 === "pre") inPre = false;
+        if (name3 === "textarea") inTextarea = false;
+        if (name3 === "style") inStyle = false;
+        if (name3 === "head") {
+          if (options?.inject_meta_charset) {
+            serializingHead = false;
+          } else {
+            result += "</" + name3 + ">";
+          }
+        } else if (serializingHead) {
+          result += "</" + name3 + ">";
+        }
+        break;
+      case "Characters":
+        if (serializingHead) {
+          let text = token[1];
+          if (
+            options?.strip_whitespace &&
+            !inPre &&
+            !inTextarea &&
+            !inScript &&
+            !inStyle
+          ) {
+            text = text.replace(/\s+/g, " ");
+          }
+          if (inScript) {
+            if (options?.escape_rcdata) {
+              result += escapeText(text);
+            } else {
+              result += text;
+            }
+          } else if (inTextarea) {
+            if (options?.escape_rcdata) {
+              result += escapeText(text);
+            } else {
+              result += text;
+            }
+          } else {
+            result += escapeText(text);
+          }
+        }
+        break;
+      case "Doctype":
+        if (serializingHead) {
+          result += "<!DOCTYPE " + token[1];
+          if (token[2]) {
+            result += ' PUBLIC "' + token[2] + '"';
+            if (token[3]) result += ' "' + token[3] + '"';
+          } else if (token[3]) {
+            result += ' SYSTEM "' + token[3] + '"';
+          }
+          result += ">";
+        }
+        break;
+      case "Comment":
+        if (serializingHead) {
+          result += "<!--" + token[1] + "-->";
+        }
+        break;
+      default:
+        break;
+    }
+  }
+
+  return result;
+};

package/src/tokenizer/calculate-position.ts

@@ -0,0 +1,10 @@
+import { Position } from "./types.js";
+
+export const calculatePosition = (text: string, offset: number): Position => {
+  const lines = text.slice(0, offset).split("\n");
+  return {
+    line: lines.length,
+    column: lines[lines.length - 1]?.length ?? 0,
+    offset,
+  };
+};

package/src/tokenizer/constants.ts

@@ -0,0 +1,11 @@
+export const RAW_TEXT_ELEMENTS = new Set([
+  "script",
+  "style",
+  "xmp",
+  "iframe",
+  "noembed",
+  "noframes",
+  "noscript",
+]);
+
+export const RCDATA_ELEMENTS = new Set(["textarea", "title"]);

package/src/tokenizer/decode-entities.ts

@@ -0,0 +1,64 @@
+import { HTML_ENTITIES } from "../html-entities.js";
+
+export const decodeEntities = (text: string): string => {
+  let result = "";
+  let i = 0;
+  while (i < text.length) {
+    if (text[i] === "&") {
+      let match = "";
+      let j = i + 1;
+      if (text[j] === "#") {
+        j++;
+        if (text[j] === "x" || text[j] === "X") {
+          j++;
+          while (j < text.length && /[0-9a-fA-F]/.test(text[j])) {
+            j++;
+          }
+        } else {
+          while (j < text.length && /[0-9]/.test(text[j])) {
+            j++;
+          }
+        }
+        if (text[j] === ";") {
+          j++;
+        }
+        match = text.substring(i, j);
+        const entity = match;
+        if (entity.startsWith("&#x") && entity.endsWith(";")) {
+          const hex = entity.slice(3, -1);
+          result += String.fromCharCode(parseInt(hex, 16));
+          i = j;
+          continue;
+        } else if (entity.startsWith("&#") && entity.endsWith(";")) {
+          const decimal = entity.slice(2, -1);
+          result += String.fromCharCode(parseInt(decimal, 10));
+          i = j;
+          continue;
+        }
+      } else {
+        while (j < text.length && /[a-zA-Z0-9]/.test(text[j])) {
+          j++;
+        }
+        const hasSemi = text[j] === ";";
+        if (hasSemi) {
+          j++;
+        }
+        match = text.substring(i, j);
+        const named = match.slice(1, hasSemi ? -1 : undefined);
+        if (HTML_ENTITIES[named]) {
+          if (hasSemi || (j < text.length && !/[a-zA-Z0-9]/.test(text[j]))) {
+            result += HTML_ENTITIES[named];
+            i = j;
+            continue;
+          }
+        }
+      }
+      result += text[i];
+      i++;
+    } else {
+      result += text[i];
+      i++;
+    }
+  }
+  return result.replace(/\u0000/g, "\uFFFD");
+};

package/src/tokenizer/index.ts

@@ -0,0 +1,2 @@
+export { TokenType, type Position, type Token } from "./types.js";
+export { tokenize } from "./tokenize.js";

package/src/tokenizer/parse-attributes.ts

@@ -0,0 +1,74 @@
+import { decodeEntities } from "./decode-entities.js";
+
+export const parseAttributes = (
+  attributeString: string,
+): Record<string, string> => {
+  const attributes: Record<string, string> = {};
+  let i = 0;
+
+  while (i < attributeString.length) {
+    while (i < attributeString.length && /\s/.test(attributeString[i])) {
+      i++;
+    }
+    if (
+      i >= attributeString.length ||
+      attributeString[i] === "/" ||
+      attributeString[i] === ">"
+    ) {
+      break;
+    }
+
+    let name = "";
+    while (i < attributeString.length && !/[\s=\/>]/.test(attributeString[i])) {
+      name += attributeString[i];
+      i++;
+    }
+
+    if (!name) {
+      i++;
+      continue;
+    }
+
+    while (i < attributeString.length && /\s/.test(attributeString[i])) {
+      i++;
+    }
+
+    let value = "";
+    if (i < attributeString.length && attributeString[i] === "=") {
+      i++;
+      while (i < attributeString.length && /\s/.test(attributeString[i])) {
+        i++;
+      }
+
+      if (i < attributeString.length) {
+        if (attributeString[i] === '"') {
+          i++;
+          while (i < attributeString.length && attributeString[i] !== '"') {
+            value += attributeString[i];
+            i++;
+          }
+          i++;
+        } else if (attributeString[i] === "'") {
+          i++;
+          while (i < attributeString.length && attributeString[i] !== "'") {
+            value += attributeString[i];
+            i++;
+          }
+          i++;
+        } else {
+          while (
+            i < attributeString.length &&
+            !/[\s>]/.test(attributeString[i])
+          ) {
+            value += attributeString[i];
+            i++;
+          }
+        }
+      }
+    }
+
+    attributes[name.toLowerCase()] = decodeEntities(value);
+  }
+
+  return attributes;
+};