npm - @letsrunit/playwright - Versions diffs - 0.3.6 → 0.3.7 - Mend

@letsrunit/playwright 0.3.6 → 0.3.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (5) hide show

package/dist/index.d.ts CHANGED Viewed

@@ -104,4 +104,36 @@ declare function waitAfterInteraction(page: Page, target: Locator, opts?: {
     quietMs?: number;
 }): Promise<void>;
-export { type PageInfo, type Snapshot, browse, createDateEngine, createFieldEngine, formatDate, formatDateForInput, formatHtml, getMonthNames, locator, screenshot, screenshotElement, scrollToCenter, setFieldValue, snapshot, suppressInterferences, waitAfterInteraction, waitForAnimationsToFinish, waitForDomIdle, waitForIdle, waitForMeta, waitForUrlChange, waitUntilEnabled };
+type ScrubHtmlOptions = {
+    /** Remove nodes that look hidden/inert by attributes. Default: true */
+    dropHidden?: boolean;
+    /** Remove the entire <head>. Default: true */
+    dropHead?: boolean;
+    /** Remove <svg> glyphs. Default: false */
+    dropSvg?: boolean;
+    /** Only keep <main> element. Default: false */
+    pickMain?: boolean;
+    /** Keep a conservative attribute allowlist (0=none, 1=normal, 2=aggressive). Default: 1 */
+    stripAttributes?: 0 | 1 | 2;
+    /** Normalize whitespace in text nodes (outside pre/code). Default: true */
+    normalizeWhitespace?: boolean;
+    /** Remove HTML comments. Default: true */
+    dropComments?: boolean;
+    /** Replace <br> within headings (h1–h6) with a space. Default: true */
+    replaceBrInHeadings?: boolean;
+    /** Limit lists to max items: -1 mean no limit. Default: -1 */
+    limitLists?: number;
+};
+declare function scrubHtml(page: {
+    html: string;
+    url: string;
+} | Page, opts?: ScrubHtmlOptions): Promise<string>;
+/**
+ * Scrub HTML conservatively for LLMs without destroying semantics.
+ */
+declare function realScrubHtml({ html, url }: {
+    html: string;
+    url: string;
+}, opts?: ScrubHtmlOptions): Promise<string>;
+export { type PageInfo, type ScrubHtmlOptions, type Snapshot, browse, createDateEngine, createFieldEngine, formatDate, formatDateForInput, formatHtml, getMonthNames, locator, realScrubHtml, screenshot, screenshotElement, scrollToCenter, scrubHtml, setFieldValue, snapshot, suppressInterferences, waitAfterInteraction, waitForAnimationsToFinish, waitForDomIdle, waitForIdle, waitForMeta, waitForUrlChange, waitUntilEnabled };

package/dist/index.js CHANGED Viewed

@@ -1,8 +1,10 @@
-import { getWeekNumber, sleep, chain, hashKey, isRange, isDate, isArray, diffArray, cartesian, uniqueItem } from '@letsrunit/utils';
+import { memoize, getWeekNumber, sleep, chain, hashKey, isRange, isDate, isArray, diffArray, cartesian, uniqueItem } from '@letsrunit/utils';
 import rehypeFormat from 'rehype-format';
 import rehypeParse from 'rehype-parse';
 import rehypeStringify from 'rehype-stringify';
 import { unified } from 'unified';
+import stringify from 'fast-json-stable-stringify';
+import { JSDOM } from 'jsdom';
 // src/browser.ts
 async function browse(browser, options = {}) {
@@ -3006,6 +3008,257 @@ async function suppressInterferences(page, opts = {}) {
   }
 }
-export { browse, createDateEngine, createFieldEngine, formatDate, formatDateForInput, formatHtml, getMonthNames, locator, screenshot, screenshotElement, scrollToCenter, setFieldValue, snapshot, suppressInterferences, waitAfterInteraction, waitForAnimationsToFinish, waitForDomIdle, waitForIdle, waitForMeta, waitForUrlChange, waitUntilEnabled };
+// src/utils/type-check.ts
+function isPage(page) {
+  return typeof page.content === "function" && typeof page.url === "function" && typeof page.screenshot === "function";
+}
+// src/scrub-html.ts
+var HTML_MIN_ATTR_THRESHOLD = 25e4;
+var HTML_LIMIT_LISTS_THRESHOLD = 4e5;
+var HTML_MAIN_ONLY_THRESHOLD = 6e5;
+function getDefaults(contentLength) {
+  return {
+    dropHidden: true,
+    dropHead: true,
+    dropSvg: false,
+    pickMain: contentLength >= HTML_MAIN_ONLY_THRESHOLD,
+    stripAttributes: contentLength >= HTML_MIN_ATTR_THRESHOLD ? 2 : 1,
+    normalizeWhitespace: true,
+    dropComments: true,
+    replaceBrInHeadings: true,
+    limitLists: contentLength >= HTML_LIMIT_LISTS_THRESHOLD ? 20 : -1
+  };
+}
+var ALLOWED_ATTRS = {
+  match: /* @__PURE__ */ new Set([
+    // identity/semantics
+    "id",
+    "class",
+    "role",
+    // internationalization
+    "lang",
+    "dir",
+    // anchors & media
+    "href",
+    "title",
+    "target",
+    "rel",
+    "src",
+    "alt",
+    "width",
+    "height",
+    "loading",
+    // tables
+    "scope",
+    "headers",
+    "colspan",
+    "rowspan",
+    // forms (pure semantics—doesn’t change structure)
+    "name",
+    "value",
+    "type",
+    "for",
+    "placeholder",
+    "checked",
+    "selected",
+    "multiple",
+    "method",
+    "action",
+    // time, figure, etc.
+    "datetime"
+  ]),
+  regexp: /^aria-[\w-]+|^data-[\w-]+$/i
+  // ARIA attributes & data-* attributes
+};
+var ALLOWED_ATTRS_AGGRESSIVE = {
+  match: /* @__PURE__ */ new Set([
+    // structuur / algemene selectors
+    "id",
+    "class",
+    "role",
+    // links / media
+    "href",
+    "src",
+    "alt",
+    "title",
+    // tables
+    "scope",
+    // forms / velden
+    "name",
+    "type",
+    "for",
+    "placeholder",
+    "value",
+    "checked",
+    "selected",
+    // ARIA voor Playwright getByRole/getByLabel
+    "aria-label",
+    "aria-labelledby",
+    "aria-describedby",
+    // veelgebruikte test selectors
+    "data-testid",
+    "data-test-id",
+    "data-cy",
+    "data-qa"
+  ]),
+  regexp: null
+};
+var HIDDEN_SELECTORS = [
+  "[hidden]",
+  "[inert]",
+  '[aria-hidden="true"]',
+  '[style*="display:none"]',
+  '[style*="visibility:hidden"]',
+  '[style*="opacity:0"]'
+].join(",");
+var ALWAYS_DROP = [
+  "script",
+  "style",
+  "template",
+  "noscript",
+  "slot",
+  "object",
+  "embed"
+];
+async function scrubHtml(page, opts = {}) {
+  if (isPage(page)) page = { html: await page.content(), url: page.url() };
+  return await memoizedScrubHtml(page, opts);
+}
+var memoizedScrubHtml = memoize(realScrubHtml, {
+  max: 16,
+  ttl: 10 * 6e4,
+  cacheKey: (args) => stringify({ html: args[0].html, url: args[0].url, ...args[1] })
+});
+async function realScrubHtml({ html, url }, opts = {}) {
+  const o = { ...getDefaults(html.length), ...opts };
+  const dom = new JSDOM(html, { url });
+  const doc = dom.window.document;
+  if (o.pickMain) pickMain(doc);
+  dropInfraAndSvg(doc, !!o.dropSvg);
+  if (o.dropHidden) dropHiddenTrees(doc);
+  if (o.stripAttributes) stripAttributesAndSanitize(doc, o.stripAttributes);
+  if (o.dropComments) dropHtmlComments(doc);
+  if (o.replaceBrInHeadings) replaceBrsInHeadings(doc);
+  if (o.limitLists >= 0) limitListsAndRows(doc, o.limitLists);
+  if (o.normalizeWhitespace) normalizeWhitespace(doc.body);
+  return doc.body.innerHTML;
+}
+function hasHiddenAncestor(el) {
+  let p = el.parentElement;
+  while (p) {
+    if (p.hasAttribute("hidden") || p.hasAttribute("inert") || p.getAttribute("aria-hidden") === "true") return true;
+    const style = p.getAttribute("style") || "";
+    if (/\bdisplay\s*:\s*none\b/i.test(style)) return true;
+    if (/\bvisibility\s*:\s*hidden\b/i.test(style)) return true;
+    if (/\bopacity\s*:\s*0(?:\D|$)/i.test(style)) return true;
+    p = p.parentElement;
+  }
+  return false;
+}
+function normalizeWhitespace(root) {
+  const preLike = /* @__PURE__ */ new Set(["PRE", "CODE", "SAMP", "KBD"]);
+  const doc = root.ownerDocument;
+  const walker = doc.createTreeWalker(
+    root,
+    4
+    /*NodeFilter.SHOW_TEXT*/
+  );
+  const changes = [];
+  let node;
+  while (node = walker.nextNode()) {
+    const text = node;
+    const parent = text.parentElement;
+    if (!parent) continue;
+    if (preLike.has(parent.tagName)) continue;
+    const v = text.nodeValue ?? "";
+    const collapsed = v.replace(/\s+/g, " ");
+    if (collapsed !== v) changes.push(text);
+  }
+  for (const t of changes) {
+    const parent = t.parentElement;
+    const isBlockish = /^(P|LI|DIV|SECTION|ARTICLE|ASIDE|HEADER|FOOTER|MAIN|NAV|H[1-6]|BLOCKQUOTE|FIGCAPTION|TD|TH)$/i.test(parent.tagName);
+    t.nodeValue = (t.nodeValue || "").replace(/\s+/g, " ");
+    if (isBlockish) t.nodeValue = (t.nodeValue || "").trim();
+  }
+}
+function pickMain(doc) {
+  const main = doc.querySelector("main");
+  if (!main) return false;
+  const clone = main.cloneNode(true);
+  doc.body.innerHTML = "";
+  doc.body.appendChild(clone);
+  return true;
+}
+function dropInfraAndSvg(doc, dropSvg) {
+  const toDrop = [...ALWAYS_DROP, dropSvg ? "svg" : ""].filter(Boolean).join(",");
+  if (!toDrop) return;
+  doc.querySelectorAll(toDrop).forEach((el) => el.remove());
+}
+function dropHiddenTrees(doc) {
+  doc.querySelectorAll(HIDDEN_SELECTORS).forEach((el) => el.remove());
+  const all = [...doc.body.querySelectorAll("*")];
+  for (const el of all) {
+    if (!el.isConnected) continue;
+    if (hasHiddenAncestor(el)) el.remove();
+  }
+}
+function stripAttributesAndSanitize(doc, level) {
+  if (!level) return;
+  const all = [...doc.body.querySelectorAll("*")];
+  for (const el of all) {
+    const isSvg = el.namespaceURI === "http://www.w3.org/2000/svg";
+    for (const { name } of [...el.attributes]) {
+      const lower = name.toLowerCase();
+      if (lower.startsWith("on")) {
+        el.removeAttribute(name);
+        continue;
+      }
+      if (lower === "style") {
+        el.removeAttribute(name);
+        continue;
+      }
+      if (isSvg) continue;
+      const allowed = level === 1 ? ALLOWED_ATTRS : ALLOWED_ATTRS_AGGRESSIVE;
+      if (!allowed.match.has(lower) && !allowed.regexp?.test(name)) {
+        el.removeAttribute(name);
+      }
+    }
+  }
+  doc.querySelectorAll("a[href]").forEach((a) => {
+    const href = a.getAttribute("href") || "";
+    if (/^\s*javascript:/i.test(href)) a.removeAttribute("href");
+  });
+}
+function dropHtmlComments(doc) {
+  const nf = doc.defaultView?.NodeFilter;
+  const SHOW_COMMENT = nf?.SHOW_COMMENT ?? 128;
+  const walker = doc.createTreeWalker(doc, SHOW_COMMENT);
+  const toRemove = [];
+  let n;
+  while (n = walker.nextNode()) toRemove.push(n);
+  toRemove.forEach((c) => c.parentNode?.removeChild(c));
+}
+function replaceBrsInHeadings(doc) {
+  doc.querySelectorAll("h1, h2, h3, h4, h5, h6").forEach((h) => {
+    h.querySelectorAll("br").forEach((br) => {
+      const space = doc.createTextNode(" ");
+      br.replaceWith(space);
+    });
+  });
+}
+function limitListsAndRows(doc, limit) {
+  doc.querySelectorAll("ul, ol").forEach((list) => {
+    const items = Array.from(list.children).filter((c) => c.tagName === "LI");
+    for (let i = limit; i < items.length; i++) items[i].remove();
+  });
+  const rowContainers = doc.querySelectorAll("table, thead, tbody, tfoot");
+  rowContainers.forEach((container) => {
+    const rows = Array.from(container.children).filter((c) => c.tagName === "TR");
+    for (let i = limit; i < rows.length; i++) rows[i].remove();
+  });
+}
+export { browse, createDateEngine, createFieldEngine, formatDate, formatDateForInput, formatHtml, getMonthNames, locator, realScrubHtml, screenshot, screenshotElement, scrollToCenter, scrubHtml, setFieldValue, snapshot, suppressInterferences, waitAfterInteraction, waitForAnimationsToFinish, waitForDomIdle, waitForIdle, waitForMeta, waitForUrlChange, waitUntilEnabled };
 //# sourceMappingURL=index.js.map
 //# sourceMappingURL=index.js.map