libretto 0.3.1 → 0.4.0

package/dist/shared/condense-dom/condense-dom.cjs

@@ -0,0 +1,462 @@
+"use strict";
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var condense_dom_exports = {};
+__export(condense_dom_exports, {
+  condenseDom: () => condenseDom
+});
+module.exports = __toCommonJS(condense_dom_exports);
+const TEST_ATTRS = /* @__PURE__ */ new Set(["data-testid", "data-test", "data-qa", "data-cy"]);
+const TRUSTED_ATTRS = /* @__PURE__ */ new Set([
+  "id",
+  "name",
+  "for",
+  "tabindex",
+  "contenteditable",
+  "role",
+  "title",
+  "alt",
+  "type",
+  "value",
+  "placeholder",
+  "autocomplete",
+  "href",
+  "action",
+  "method",
+  "src"
+]);
+const STATE_ATTRS = /* @__PURE__ */ new Set([
+  "disabled",
+  "hidden",
+  "inert",
+  "readonly",
+  "required",
+  "checked",
+  "selected",
+  "open",
+  "multiple"
+]);
+const BOOLEAN_ATTRS = /* @__PURE__ */ new Set([
+  ...STATE_ATTRS,
+  "async",
+  "defer",
+  "nomodule"
+]);
+const EMPTY_VALUE_DROP_ATTRS = /* @__PURE__ */ new Set([
+  "alt",
+  "autocomplete",
+  "href",
+  "action",
+  "method",
+  "name",
+  "placeholder",
+  "src",
+  "tabindex",
+  "title",
+  "type"
+]);
+const URL_ATTRS = /* @__PURE__ */ new Set(["href", "src", "action"]);
+const SCRIPT_ATTRS = /* @__PURE__ */ new Set([
+  "src",
+  "type",
+  "id",
+  "defer",
+  "async",
+  "crossorigin",
+  "integrity",
+  "nomodule",
+  "referrerpolicy"
+]);
+const STYLE_TAG_ATTRS = /* @__PURE__ */ new Set(["media", "type", "nonce", "title"]);
+const INTERACTIVE_TAGS = /* @__PURE__ */ new Set([
+  "a",
+  "button",
+  "input",
+  "select",
+  "textarea",
+  "form",
+  "details",
+  "dialog",
+  "label"
+]);
+const INTERACTIVE_ROLES = /* @__PURE__ */ new Set([
+  "button",
+  "link",
+  "tab",
+  "menuitem",
+  "checkbox",
+  "radio",
+  "switch",
+  "slider",
+  "combobox"
+]);
+const OPEN_TAG_PATTERN = /<([a-zA-Z][\w:-]*)(\s(?:[^"'<>/]|"[^"]*"|'[^']*')*)?\s*(\/?)>/g;
+function condenseDom(html) {
+  const originalLength = html.length;
+  const reductions = {};
+  function track(label, before, after) {
+    const diff = before.length - after.length;
+    if (diff > 0) {
+      reductions[label] = (reductions[label] ?? 0) + diff;
+    }
+    return after;
+  }
+  let result = html;
+  result = track(
+    "noscript",
+    result,
+    result.replace(/<noscript\b[^>]*>[\s\S]*?<\/noscript>/gi, "")
+  );
+  result = track(
+    "comments",
+    result,
+    result.replace(/<!--[\s\S]*?(?:-->|$)/g, "")
+  );
+  result = track(
+    "scripts",
+    result,
+    result.replace(
+      /(<script\b[^>]*>)([\s\S]*?)(<\/script(?:\s[^>]*)?>)/gi,
+      (_match, open, content, close) => {
+        if (!content.trim()) return `${open}${close}`;
+        const isDataScript = /type\s*=\s*["']application\/(json|ld\+json)["']/i.test(open);
+        if (isDataScript) {
+          return `${open}[JSON data, ${content.length} chars]${close}`;
+        }
+        return `${open}[script, ${content.length} chars]${close}`;
+      }
+    )
+  );
+  result = track(
+    "styles",
+    result,
+    result.replace(
+      /(<style\b[^>]*>)([\s\S]*?)(<\/style(?:\s[^>]*)?>)/gi,
+      (_match, open, content, close) => {
+        if (!content.trim()) return `${open}${close}`;
+        return `${open}[CSS, ${content.length} chars]${close}`;
+      }
+    )
+  );
+  result = track(
+    "base64",
+    result,
+    result.replace(
+      /(src|href)\s*=\s*["'](data:[^;]+;base64,)[A-Za-z0-9+/=]{100,}["']/gi,
+      (_match, attr, prefix) => {
+        const mime = prefix.replace("data:", "").replace(";base64,", "");
+        return `${attr}="[base64 ${mime}]"`;
+      }
+    )
+  );
+  result = track("attribute-allowlist", result, rewriteTagAttributes(result));
+  const svgPattern = /<svg\b([^>]*)>((?:(?!<svg\b)[\s\S])*?)<\/svg>/gi;
+  result = track(
+    "svg-collapse",
+    result,
+    (() => {
+      let prev;
+      let current = result;
+      do {
+        prev = current;
+        current = current.replace(
+          svgPattern,
+          (_match, attrs, inner) => {
+            const keepAttrs = [];
+            const attrPatterns = [
+              "id",
+              "class",
+              "role",
+              "aria-label",
+              "aria-hidden",
+              "title",
+              "data-testid"
+            ];
+            for (const name of attrPatterns) {
+              const attrToken = findAttributeToken(attrs, name);
+              if (attrToken) keepAttrs.push(attrToken);
+            }
+            const hasAriaLabel = /aria-label\s*=/i.test(attrs);
+            if (!hasAriaLabel) {
+              const titleMatch = inner.match(
+                /<title[^>]*>([^<]+)<\/title>/i
+              );
+              const descMatch = inner.match(
+                /<desc[^>]*>([^<]+)<\/desc>/i
+              );
+              const labelText = titleMatch?.[1]?.trim() || descMatch?.[1]?.trim();
+              if (labelText) {
+                keepAttrs.push(
+                  `aria-label="${escapeHtmlAttribute(labelText)}"`
+                );
+              }
+            }
+            const attrStr = keepAttrs.length > 0 ? ` ${keepAttrs.join(" ")}` : "";
+            return `<svg${attrStr}><!-- [icon] --></svg>`;
+          }
+        );
+        svgPattern.lastIndex = 0;
+      } while (current !== prev);
+      return current;
+    })()
+  );
+  const layoutProps = /(?:^|;)\s*(?:display|visibility|opacity|pointer-events|position|z-index|overflow)(?:-[a-z]+)?\s*:[^;"]*/gi;
+  result = track(
+    "inline-styles",
+    result,
+    result.replace(
+      /\sstyle\s*=\s*["']([^"']*)["']/gi,
+      (_match, value) => {
+        const kept = [];
+        let propMatch;
+        layoutProps.lastIndex = 0;
+        while ((propMatch = layoutProps.exec(value)) !== null) {
+          kept.push(propMatch[0].replace(/^[;\s]+/, "").trim());
+        }
+        if (kept.length === 0) return "";
+        return ` style="${kept.join("; ")}"`;
+      }
+    )
+  );
+  result = track(
+    "obfuscated-classes",
+    result,
+    result.replace(
+      /\sclass\s*=\s*["']([^"']*)["']/gi,
+      (_match, value) => {
+        const filtered = filterSemanticClasses(value);
+        if (!filtered) return "";
+        return ` class="${filtered}"`;
+      }
+    )
+  );
+  const removableAttrs = /\s(?:xmlns(?::[a-z]+)?|xml:space|xml:lang|fill|stroke|stroke-width|stroke-linecap|stroke-linejoin|stroke-miterlimit|stroke-dasharray|stroke-dashoffset|stroke-opacity|fill-opacity|clip-rule|fill-rule|focusable)\s*=\s*["'][^"']*["']/gi;
+  result = track(
+    "framework-svg-attrs",
+    result,
+    result.replace(removableAttrs, "")
+  );
+  const preBlocks = [];
+  result = result.replace(
+    /(<pre\b[^>]*>)([\s\S]*?)(<\/pre>)/gi,
+    (_match, open, content, close) => {
+      const idx = preBlocks.length;
+      preBlocks.push(`${open}${content}${close}`);
+      return `__PRE_PLACEHOLDER_${idx}__`;
+    }
+  );
+  result = track(
+    "whitespace",
+    result,
+    result.replace(/[ \t]+/g, " ").replace(/\n\s*\n/g, "\n")
+  );
+  for (let i = 0; i < preBlocks.length; i++) {
+    const placeholder = `__PRE_PLACEHOLDER_${i}__`;
+    const preBlock = preBlocks[i];
+    result = result.replace(placeholder, () => preBlock);
+  }
+  return {
+    html: result,
+    originalLength,
+    condensedLength: result.length,
+    reductions
+  };
+}
+function rewriteTagAttributes(html) {
+  return html.replace(
+    OPEN_TAG_PATTERN,
+    (match, rawTagName, rawAttrs, selfClosing) => {
+      const tagName = rawTagName.toLowerCase();
+      if (!rawAttrs?.trim()) return match;
+      const attrs = parseAttributes(rawAttrs);
+      if (attrs.length === 0) return match;
+      const interactive = isInteractiveElement(tagName, attrs);
+      const kept = attrs.map((attr) => keepAttribute(tagName, attr, interactive)).filter((value) => value !== null);
+      const attrStr = kept.length > 0 ? ` ${kept.join(" ")}` : "";
+      const closing = selfClosing ? " /" : "";
+      return `<${rawTagName}${attrStr}${closing}>`;
+    }
+  );
+}
+function keepAttribute(tagName, attr, interactive) {
+  const name = attr.name.toLowerCase();
+  const value = attr.value;
+  if (name === "class") {
+    if (!value?.trim()) return null;
+    const filtered = filterSemanticClasses(value);
+    if (!filtered) return null;
+    return serializeAttribute(attr.name, filtered);
+  }
+  if (name === "style") {
+    if (!value?.trim()) return null;
+    return serializeAttribute(attr.name, value);
+  }
+  if (name.startsWith("aria-")) {
+    if (!value?.trim()) return null;
+    return attr.rawToken;
+  }
+  if (TEST_ATTRS.has(name)) {
+    if (!value?.trim()) return null;
+    return attr.rawToken;
+  }
+  if (tagName === "script" && SCRIPT_ATTRS.has(name)) {
+    return serializePreservedAttribute(attr);
+  }
+  if (tagName === "style" && STYLE_TAG_ATTRS.has(name)) {
+    if (!value?.trim()) return null;
+    return attr.rawToken;
+  }
+  if (STATE_ATTRS.has(name)) {
+    return serializePreservedAttribute(attr);
+  }
+  if (URL_ATTRS.has(name)) {
+    if (!value?.trim()) return null;
+    const normalized = normalizeUrlValue(value);
+    if (normalized === value) return attr.rawToken;
+    return serializeAttribute(attr.name, normalized);
+  }
+  if (TRUSTED_ATTRS.has(name)) {
+    if (shouldDropEmptyValue(name, value)) return null;
+    return serializePreservedAttribute(attr);
+  }
+  if (shouldKeepCustomDataAttribute(tagName, name, value, interactive)) {
+    return attr.rawToken;
+  }
+  return null;
+}
+function serializePreservedAttribute(attr) {
+  if (BOOLEAN_ATTRS.has(attr.name.toLowerCase())) {
+    return attr.rawToken;
+  }
+  if (attr.value === null) return attr.rawToken;
+  return attr.rawToken;
+}
+function shouldDropEmptyValue(name, value) {
+  if (value === null) return false;
+  if (value.trim()) return false;
+  if (name.startsWith("aria-")) return true;
+  return EMPTY_VALUE_DROP_ATTRS.has(name);
+}
+function normalizeUrlValue(value) {
+  const loweredValue = value.trim().toLowerCase();
+  if (loweredValue.startsWith("blob:")) return "blob:[omitted]";
+  if (loweredValue.startsWith("javascript:")) return "javascript:[omitted]";
+  if (loweredValue.startsWith("vbscript:")) return "vbscript:[omitted]";
+  if (loweredValue.startsWith("data:")) return "data:[omitted]";
+  if (value.length <= 160) return value;
+  try {
+    const isAbsolute = /^[a-z][a-z0-9+.-]*:/i.test(value);
+    const parsed = isAbsolute ? new URL(value) : new URL(value, "https://condensed.local");
+    const prefix = isAbsolute ? `${parsed.protocol}//${parsed.host}${parsed.pathname}` : `${parsed.pathname}${parsed.hash}`;
+    const query = parsed.search ? "?[query omitted]" : "";
+    return `${prefix}${query}`;
+  } catch {
+    return `${value.slice(0, 96)}[omitted]`;
+  }
+}
+function filterSemanticClasses(value) {
+  const classes = value.split(/\s+/).filter(Boolean);
+  const kept = classes.filter((cls) => !isObfuscatedClass(cls));
+  return kept.join(" ");
+}
+function isObfuscatedClass(cls) {
+  if (cls.length > 80) return true;
+  if (/^_?[0-9a-f]{6,}$/i.test(cls)) return true;
+  if (/^[a-z]+_[0-9a-f]{4,}$/i.test(cls)) return true;
+  if (/^[a-z]{1,2}[0-9]{2,}$/i.test(cls)) return true;
+  const digits = (cls.match(/[0-9]/g) || []).length;
+  const letters = (cls.match(/[a-zA-Z]/g) || []).length;
+  if (cls.length >= 6 && digits >= letters * 0.5 && digits >= 2) return true;
+  return false;
+}
+function parseAttributes(rawAttrs) {
+  const attrs = [];
+  const attrPattern = /([^\s"'<>\/=]+)(?:\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'=<>`]+)))?/g;
+  let match;
+  while ((match = attrPattern.exec(rawAttrs)) !== null) {
+    const name = match[1];
+    if (!name) continue;
+    attrs.push({
+      name,
+      rawToken: match[0].trim(),
+      value: match[2] ?? match[3] ?? match[4] ?? null
+    });
+  }
+  return attrs;
+}
+function isInteractiveElement(tagName, attrs) {
+  if (INTERACTIVE_TAGS.has(tagName)) return true;
+  for (const attr of attrs) {
+    const name = attr.name.toLowerCase();
+    if (name === "tabindex" || name === "contenteditable") return true;
+    if (name !== "role") continue;
+    const role = attr.value?.trim().toLowerCase();
+    if (role && INTERACTIVE_ROLES.has(role)) {
+      return true;
+    }
+  }
+  return false;
+}
+function shouldKeepCustomDataAttribute(tagName, attrName, value, interactive) {
+  if (!interactive) return false;
+  if (!attrName.startsWith("data-")) return false;
+  if (TEST_ATTRS.has(attrName)) return false;
+  if (!value?.trim()) return false;
+  if (value.length > 80) return false;
+  if (tagName === "script" || tagName === "style") return false;
+  const key = attrName.slice("data-".length);
+  if (!looksMeaningfulToken(key)) return false;
+  if (!looksMeaningfulDataValue(value)) return false;
+  return true;
+}
+function looksMeaningfulToken(value) {
+  if (!/^[a-z][a-z0-9-]{1,40}$/i.test(value)) return false;
+  if (!/[a-z]{3}/i.test(value)) return false;
+  if (/(track|metric|telemetry|analytics|component|display|loaded|token|dps|color|screen|strict|rehydr|fetch)/i.test(value)) {
+    return false;
+  }
+  return true;
+}
+function looksMeaningfulDataValue(value) {
+  if (value.length > 80) return false;
+  if (/[<>]/.test(value)) return false;
+  if (/https?:\/\//i.test(value)) return false;
+  return /^[a-z0-9:_./ -]+$/i.test(value);
+}
+function findAttributeToken(attrs, name) {
+  const match = attrs.match(
+    new RegExp(
+      `(?:^|\\s)(${escapeRegExp(name)}(?:\\s*=\\s*(?:"[^"]*"|'[^']*'|[^\\s"'=<>\\x60]+))?)`,
+      "i"
+    )
+  );
+  return match?.[1] ?? null;
+}
+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+}
+function serializeAttribute(name, value) {
+  return `${name}="${escapeHtmlAttribute(value)}"`;
+}
+function escapeHtmlAttribute(value) {
+  return value.replace(/&/g, "&amp;").replace(/"/g, "&quot;").replace(/</g, "&lt;").replace(/>/g, "&gt;");
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  condenseDom
+});

package/dist/shared/condense-dom/condense-dom.d.cts

@@ -0,0 +1,34 @@
+/**
+ * DOM condensation — reduces serialized HTML for LLM consumption.
+ *
+ * All rules run unconditionally (no tiers). The function operates on
+ * already-serialized HTML strings (the output of `page.content()`),
+ * not a browser-side DOM walk or parsed DOM tree.
+ *
+ * Rules applied in order:
+ *   1.  Noscript blocks — remove entirely
+ *   2.  HTML comments — remove entirely
+ *   3.  Script contents — hollow out, keep tags + useful attributes
+ *   4.  Style contents — hollow out, keep tags + useful attributes
+ *   5.  Embedded binary data — replace base64 data URIs
+ *   6.  Attribute allowlist — keep trusted attrs, special-case class/style/URLs
+ *   7.  SVG elements — collapse to single tag, extract title/desc
+ *   8.  Inline style properties — keep only layout-relevant props
+ *   9.  Non-semantic class names — filter or delete class values
+ *  10.  (Cross-reference IDs — preserved, no action needed)
+ *  11.  Framework-internal and SVG visual attributes — remove
+ *  12.  Whitespace — collapse (preserve <pre> content)
+ */
+type CondenseDomResult = {
+    /** The condensed HTML string. Valid, parseable HTML. */
+    html: string;
+    /** Character count of the input. */
+    originalLength: number;
+    /** Character count of the output. */
+    condensedLength: number;
+    /** Characters removed, keyed by rule name. */
+    reductions: Record<string, number>;
+};
+declare function condenseDom(html: string): CondenseDomResult;
+
+export { type CondenseDomResult, condenseDom };

package/dist/shared/condense-dom/condense-dom.d.ts

@@ -0,0 +1,34 @@
+/**
+ * DOM condensation — reduces serialized HTML for LLM consumption.
+ *
+ * All rules run unconditionally (no tiers). The function operates on
+ * already-serialized HTML strings (the output of `page.content()`),
+ * not a browser-side DOM walk or parsed DOM tree.
+ *
+ * Rules applied in order:
+ *   1.  Noscript blocks — remove entirely
+ *   2.  HTML comments — remove entirely
+ *   3.  Script contents — hollow out, keep tags + useful attributes
+ *   4.  Style contents — hollow out, keep tags + useful attributes
+ *   5.  Embedded binary data — replace base64 data URIs
+ *   6.  Attribute allowlist — keep trusted attrs, special-case class/style/URLs
+ *   7.  SVG elements — collapse to single tag, extract title/desc
+ *   8.  Inline style properties — keep only layout-relevant props
+ *   9.  Non-semantic class names — filter or delete class values
+ *  10.  (Cross-reference IDs — preserved, no action needed)
+ *  11.  Framework-internal and SVG visual attributes — remove
+ *  12.  Whitespace — collapse (preserve <pre> content)
+ */
+type CondenseDomResult = {
+    /** The condensed HTML string. Valid, parseable HTML. */
+    html: string;
+    /** Character count of the input. */
+    originalLength: number;
+    /** Character count of the output. */
+    condensedLength: number;
+    /** Characters removed, keyed by rule name. */
+    reductions: Record<string, number>;
+};
+declare function condenseDom(html: string): CondenseDomResult;
+
+export { type CondenseDomResult, condenseDom };