@letsrunit/playwright 0.1.0

package/src/field/types.ts

@@ -0,0 +1,16 @@
+import type { Range, Scalar } from '@letsrunit/utils';
+import type { Locator } from '@playwright/test';
+
+export interface SetOptions {
+  force?: boolean;
+  noWaitAfter?: boolean;
+  timeout?: number;
+}
+
+export type Value = Scalar | Scalar[] | Range | boolean | null;
+
+export interface Loc {
+  el: Locator;
+  tag: string;
+  type: string | null;
+}

package/src/format-html.ts

@@ -0,0 +1,17 @@
+import type { Page } from '@playwright/test';
+import rehypeFormat from 'rehype-format';
+import rehypeParse from 'rehype-parse';
+import rehypeStringify from 'rehype-stringify';
+import { unified } from 'unified';
+
+export async function formatHtml(page: string | Page) {
+  const html = typeof page === 'string' ? page : await page.content();
+
+  const file = await unified()
+    .use(rehypeParse, { fragment: true })
+    .use(rehypeFormat, { indent: 2 }) // collapses existing whitespace and formats nodes
+    .use(rehypeStringify)
+    .process(html);
+  
+  return String(file);
+}

package/src/index.ts

@@ -0,0 +1,12 @@
+export * from './browser';
+export * from './field/index';
+export * from './format-html';
+export * from './locator';
+export * from './selector';
+export * from './snapshot';
+export * from './screenshot';
+export * from './scroll';
+export * from './suppress-interferences';
+export * from './types';
+export * from './utils/date';
+export * from './wait';

package/src/locator.ts

@@ -0,0 +1,102 @@
+import { Locator, Page } from '@playwright/test';
+
+type LocatorOptions = Parameters<Page['locator']>[1];
+
+/**
+ * Locates an element using Playwright selectors, with fallbacks.
+ */
+export async function locator(page: Page, selector: string): Promise<Locator> {
+  const primary = page.locator(selector).first();
+  if (await primary.count()) return primary;
+
+  return await tryRelaxNameToHasText(page, selector)
+    || await tryTagInsteadOfRole(page, selector)
+    || await tryRoleNameProximity(page, selector)
+    || await tryFieldAlternative(page, selector)
+    || await tryAsField(page, selector)
+    || primary; // Nothing found, return the original locator (so caller can still wait/assert)
+}
+
+async function firstMatch(page: Page, sel: string | string[], opts: LocatorOptions = {}): Promise<Locator | null> {
+  for (const selector of Array.isArray(sel) ? sel : [sel]) {
+    const loc = page.locator(selector, opts).first();
+    if (await loc.count()) return loc;
+  }
+
+  return null;
+}
+
+// Preserve the selector but relax [name="..."] to [has-text="..."]
+// This keeps all other parts of the selector intact (prefix/suffix, additional filters).
+// Examples:
+//  - role=link[name="Foo"]            → role=link:has-text="Foo"
+//  - css=button[name="Save"i]:visible → css=button:visible:has-text="Save"
+//  - [name="Hello"]                   → :has-text="Hello"
+async function tryRelaxNameToHasText(page: Page, selector: string): Promise<Locator | null> {
+  const matchAnyNameFull = selector.match(/^(role=.*)\[name="([^"]+)"i?](.*)$/i);
+  if (!matchAnyNameFull) return null;
+  const [, pre, nameText, post] = matchAnyNameFull;
+  const containsSelector = `${pre}${post}`;
+  return firstMatch(page, containsSelector, { hasText: nameText });
+}
+
+// Try using the tag name for `link`, `button` and `option` instead fo the aria role.
+// This keeps all other parts of the selector intact (prefix/suffix, additional filters).
+// Examples:
+//  - role=button[name="Foo"] → css=button:has-text="Save"
+async function tryTagInsteadOfRole(page: Page, selector: string): Promise<Locator | null> {
+  const matchAnyNameFull = selector.match(/^role=(link|button|option)\s*\[name="([^"]+)"i?](.*)$/i);
+  if (!matchAnyNameFull) return null;
+  const [, role, nameText, post] = matchAnyNameFull;
+  const tag = role === 'link' ? 'a' : role;
+  const containsSelector = `css=${tag}${post}`;
+  return firstMatch(page, containsSelector, { hasText: nameText });
+}
+
+// If a role selector with a name filter fails, try proximity-based fallback while
+// preserving the role and any remainder of the selector.
+// Example: role=switch[name="Adres tonen"i] → text=Adres tonen >> .. >> role=switch
+async function tryRoleNameProximity(page: Page, selector: string): Promise<Locator | null> {
+  const matchRole = selector.match(/^role=(\w+)\s*\[name="([^"]+)"i?](.*)$/i);
+  if (!matchRole) return null;
+  const [, role, name, rest] = matchRole;
+  const proximitySelector = `text=${name} >> .. >> role=${role}${rest}`;
+  return firstMatch(page, proximitySelector);
+}
+
+// Try alternatives if field is not found
+// field="foo" → #foo > input
+async function tryFieldAlternative(page: Page, selector: string): Promise<Locator | null> {
+  const matchField = selector.match(/^field="([^"]+)"i?$/i);
+  if (!matchField) return null;
+  const [, field] = matchField;
+  return firstMatch(page, `#${field} > input`);
+}
+
+// Try matching using the field selector in case of role mismatch
+async function tryAsField(page: Page, selector: string): Promise<Locator | null> {
+  const matchRole = selector.match(/^role=(\w+)\s*\[name="([^"]+)"i?](.*)$/i);
+  if (!matchRole) return null;
+
+  const [, role, name, rest] = matchRole;
+
+  // Only allow ARIA roles that correspond to field-like controls
+  const fieldRoles = new Set([
+    'button', // Sometimes used for date pickers or checkboxes
+    'textbox',
+    'searchbox',
+    'combobox',
+    'spinbutton',
+    'slider',
+    'checkbox',
+    'switch',
+    'radio',
+    'radiogroup',
+    'listbox',
+    'option',
+  ]);
+
+  if (!fieldRoles.has(role.toLowerCase())) return null;
+
+  return firstMatch(page, `field=${name}${rest}`);
+}

package/src/page-info.ts

@@ -0,0 +1,33 @@
+import metascraper, { type MetascraperOptions } from 'metascraper';
+import metascraperDescription from 'metascraper-description';
+import metascraperImage from 'metascraper-image';
+import metascraperLang from 'metascraper-lang';
+import metascraperLogo from 'metascraper-logo';
+import metascraperLogoFavicon from 'metascraper-logo-favicon';
+import metascraperTitle from 'metascraper-title';
+import metascraperUrl from 'metascraper-url';
+import type { PageInfo, Snapshot } from './types';
+
+const scrape = metascraper([
+  metascraperTitle(),
+  metascraperDescription(),
+  metascraperImage(),
+  metascraperLogo(),
+  metascraperLogoFavicon(),
+  metascraperLang(),
+  metascraperUrl(),
+]);
+
+export async function extractPageInfo(options: MetascraperOptions & Partial<Snapshot>): Promise<PageInfo> {
+  const meta = await scrape(options);
+
+  return {
+    url: meta.url || options.url,
+    name: meta.title || undefined,
+    description: meta.description || undefined,
+    image: meta.image || undefined,
+    favicon: meta.logo || undefined,
+    lang: meta.lang || undefined,
+    screenshot: options.screenshot,
+  };
+}

package/src/screenshot.ts

@@ -0,0 +1,84 @@
+import { hashKey } from '@letsrunit/utils';
+import type { Page, PageScreenshotOptions } from '@playwright/test';
+
+export async function screenshot(page: Page, options?: PageScreenshotOptions): Promise<File> {
+  const buffer = options?.mask?.length ? await screenshotWithMask(page, options) : await page.screenshot(options);
+  const filename = await hashKey(`screenshot-{hash}.png`, buffer);
+
+  return new File([new Uint8Array(buffer)], filename, { type: 'image/png' });
+}
+
+async function screenshotWithMask(page: Page, options: PageScreenshotOptions): Promise<Buffer> {
+  const { mask: locators, ...otherOptions } = options;
+
+  if (!locators?.length) throw new Error('No locators specified');
+
+  // 1. Inject CSS + overlay
+  await page.evaluate(() => {
+    const doc = document;
+
+    // Inject CSS once
+    if (!doc.getElementById('lri-mask-style')) {
+      const style = doc.createElement('style');
+      style.id = 'lri-mask-style';
+      style.textContent = `
+        .lri-mask-overlay {
+          position: fixed;
+          inset: 0;
+          background: rgba(0, 0, 0, 0.65);
+          pointer-events: none;
+          z-index: 2147483646;
+        }
+        .lri-mask-highlight {
+          position: relative !important;
+          z-index: 2147483647 !important;
+          box-shadow: 0 0 0 2px rgba(255, 255, 255, 0.95);
+          border-radius: 4px;
+        }
+      `;
+      doc.head.appendChild(style);
+    }
+
+    if (!doc.getElementById('lri-mask-overlay')) {
+      const overlay = doc.createElement('div');
+      overlay.id = 'lri-mask-overlay';
+      overlay.className = 'lri-mask-overlay';
+      doc.body.appendChild(overlay);
+    }
+  });
+
+  // 2. Resolve locators to element handles and add highlight class
+  const handleGroups = await Promise.all(locators.map((loc) => loc.elementHandles()));
+  const handles = handleGroups.flat();
+
+  await Promise.all(
+    handles.map((h) =>
+      h.evaluate((el) => {
+        (el as HTMLElement).classList.add('lri-mask-highlight');
+      }),
+    ),
+  );
+
+  try {
+    // 3. Take screenshot with overlay in place
+    return await page.screenshot(otherOptions);
+  } finally {
+    // 4. Cleanup: remove classes and overlay (keep CSS optional)
+    await Promise.all(
+      handles.map((h) =>
+        h.evaluate((el) => {
+          (el as HTMLElement).classList.remove('lri-mask-highlight');
+        }),
+      ),
+    );
+
+    await page.evaluate(() => {
+      const overlay = document.getElementById('lri-mask-overlay');
+      if (overlay) overlay.remove();
+
+      // Optional: remove style as well if you want zero residue
+      const style = document.getElementById('lri-mask-style');
+      if (style) style.remove();
+    });
+  }
+}

package/src/scroll.ts

@@ -0,0 +1,10 @@
+import type { Locator } from '@playwright/test';
+
+export async function scrollToCenter(locator: Locator) {
+  const count = await locator.count();
+  if (!count) return;
+
+  await locator.evaluate((el) => {
+    el.scrollIntoView({ block: 'center', inline: 'center', behavior: 'instant' });
+  });
+}

package/src/scrub-html.ts

@@ -0,0 +1,333 @@
+// Conservatively scrub HTML for LLMs while preserving semantics.
+// - Keeps structural tags (header/nav/main/section/article/aside/footer, headings, lists, tables, forms, etc.)
+// - Removes <head>, <script>, <style>, <template>, <noscript>, comments
+// - Drops nodes that are clearly hidden/inert via attributes (hidden, aria-hidden="true", display:none, visibility:hidden, opacity:0)
+// - Strips event handlers (on*) and inline styles, keeps semantic attrs (id, class, role, aria-*, data-*, href, src, alt, etc.)
+// - Collapses whitespace in text nodes (but NOT inside <pre>/<code>)
+// - Does NOT change tag names, does NOT unwrap containers, does NOT reorder content
+
+import { memoize } from '@letsrunit/utils/src/memoize';
+import type { Page } from '@playwright/test';
+import stringify from 'fast-json-stable-stringify';
+import { JSDOM } from 'jsdom';
+import { isPage } from './utils/type-check';
+
+export type ScrubHtmlOptions = {
+  /** Remove nodes that look hidden/inert by attributes. Default: true */
+  dropHidden?: boolean;
+  /** Remove the entire <head>. Default: true */
+  dropHead?: boolean;
+  /** Remove <svg> glyphs. Default: false */
+  dropSvg?: boolean;
+  /** Only keep <main> element. Default: false */
+  pickMain?: boolean;
+  /** Keep a conservative attribute allowlist (0=none, 1=normal, 2=aggressive). Default: 1 */
+  stripAttributes?: 0 | 1 | 2;
+  /** Normalize whitespace in text nodes (outside pre/code). Default: true */
+  normalizeWhitespace?: boolean;
+  /** Remove HTML comments. Default: true */
+  dropComments?: boolean;
+  /** Replace <br> within headings (h1–h6) with a space. Default: true */
+  replaceBrInHeadings?: boolean;
+  /** Limit lists to max items: -1 mean no limit. Default: -1 */
+  limitLists?: number;
+};
+
+const HTML_MIN_ATTR_THRESHOLD = 250_000;    // ~70k tokens
+const HTML_LIMIT_LISTS_THRESHOLD = 400_000; // ~110k tokens
+const HTML_MAIN_ONLY_THRESHOLD = 600_000;   // ~170k tokens
+
+function getDefaults(contentLength: number): Required<ScrubHtmlOptions> {
+  return {
+    dropHidden: true,
+    dropHead: true,
+    dropSvg: false,
+    pickMain: contentLength >= HTML_MAIN_ONLY_THRESHOLD,
+    stripAttributes: contentLength >= HTML_MIN_ATTR_THRESHOLD ? 2 : 1,
+    normalizeWhitespace: true,
+    dropComments: true,
+    replaceBrInHeadings: true,
+    limitLists: contentLength >= HTML_LIMIT_LISTS_THRESHOLD ? 20 : -1,
+  };
+}
+
+// Attributes we keep to preserve semantics and minimal usefulness
+const ALLOWED_ATTRS = {
+  match: new Set([
+    // identity/semantics
+    'id',
+    'class',
+    'role',
+    // internationalization
+    'lang',
+    'dir',
+
+    // anchors & media
+    'href',
+    'title',
+    'target',
+    'rel',
+    'src',
+    'alt',
+    'width',
+    'height',
+    'loading',
+
+    // tables
+    'scope',
+    'headers',
+    'colspan',
+    'rowspan',
+
+    // forms (pure semantics—doesn’t change structure)
+    'name',
+    'value',
+    'type',
+    'for',
+    'placeholder',
+    'checked',
+    'selected',
+    'multiple',
+    'method',
+    'action',
+
+    // time, figure, etc.
+    'datetime',
+  ]),
+  regexp:/^aria-[\w-]+|^data-[\w-]+$/i, // ARIA attributes & data-* attributes
+};
+
+const ALLOWED_ATTRS_AGGRESSIVE = {
+  match: new Set([
+    // structuur / algemene selectors
+    'id',
+    'class',
+    'role',
+
+    // links / media
+    'href',
+    'src',
+    'alt',
+    'title',
+
+    // tables
+    'scope',
+
+    // forms / velden
+    'name',
+    'type',
+    'for',
+    'placeholder',
+    'value',
+    'checked',
+    'selected',
+
+    // ARIA voor Playwright getByRole/getByLabel
+    'aria-label',
+    'aria-labelledby',
+    'aria-describedby',
+
+    // veelgebruikte test selectors
+    'data-testid',
+    'data-test-id',
+    'data-cy',
+    'data-qa',
+  ]),
+  regexp: null,
+};
+
+const HIDDEN_SELECTORS = [
+  '[hidden]',
+  '[inert]',
+  '[aria-hidden="true"]',
+  '[style*="display:none"]',
+  '[style*="visibility:hidden"]',
+  '[style*="opacity:0"]'
+].join(',');
+
+// Tags that are always removed (infra/noise)
+const ALWAYS_DROP = [
+  'script', 'style', 'template', 'noscript', 'slot', 'object', 'embed'
+];
+
+export async function scrubHtml(
+  page: { html: string, url: string } | Page,
+  opts: ScrubHtmlOptions = {},
+): Promise<string> {
+  if (isPage(page)) page = { html: await page.content(), url: page.url() };
+  return await memoizedScrubHtml(page, opts);
+}
+
+const memoizedScrubHtml = memoize(realScrubHtml, {
+  max: 16,
+  ttl: 10 * 60_000,
+  cacheKey: (args) => stringify({ html: args[0].html, url: args[0].url, ...args[1] }),
+});
+
+/**
+ * Scrub HTML conservatively for LLMs without destroying semantics.
+ */
+export async function realScrubHtml(
+  { html, url }: { html: string, url: string },
+  opts: ScrubHtmlOptions = {},
+): Promise<string> {
+  const o = { ...getDefaults(html.length), ...opts };
+
+  const dom = new JSDOM(html, { url });
+  const doc = dom.window.document;
+
+  if (o.pickMain) pickMain(doc);
+  dropInfraAndSvg(doc, !!o.dropSvg);
+  if (o.dropHidden) dropHiddenTrees(doc);
+  if (o.stripAttributes) stripAttributesAndSanitize(doc, o.stripAttributes);
+  if (o.dropComments) dropHtmlComments(doc);
+  if (o.replaceBrInHeadings) replaceBrsInHeadings(doc);
+  if (o.limitLists >= 0) limitListsAndRows(doc, o.limitLists);
+  if (o.normalizeWhitespace) normalizeWhitespace(doc.body);
+
+  return doc.body.innerHTML;
+}
+
+/* ---------------- helpers ---------------- */
+
+function hasHiddenAncestor(el: Element): boolean {
+  let p: Element | null = el.parentElement;
+  while (p) {
+    if (
+      p.hasAttribute('hidden') ||
+      p.hasAttribute('inert') ||
+      p.getAttribute('aria-hidden') === 'true'
+    ) return true;
+
+    const style = p.getAttribute('style') || '';
+    if (/\bdisplay\s*:\s*none\b/i.test(style)) return true;
+    if (/\bvisibility\s*:\s*hidden\b/i.test(style)) return true;
+    if (/\bopacity\s*:\s*0(?:\D|$)/i.test(style)) return true;
+
+    p = p.parentElement;
+  }
+  return false;
+}
+
+function normalizeWhitespace(root: Element) {
+  const preLike = new Set(['PRE', 'CODE', 'SAMP', 'KBD']);
+  const doc = root.ownerDocument!;
+  const walker = doc.createTreeWalker(root, 4 /*NodeFilter.SHOW_TEXT*/);
+  const changes: Text[] = [];
+
+  let node: Node | null;
+
+  while ((node = walker.nextNode())) {
+    const text = node as Text;
+    const parent = text.parentElement;
+    if (!parent) continue;
+    if (preLike.has(parent.tagName)) continue; // don't touch pre/code
+
+    const v = text.nodeValue ?? '';
+    const collapsed = v.replace(/\s+/g, ' ');
+    if (collapsed !== v) changes.push(text);
+  }
+
+  for (const t of changes) {
+    // extra trim around block-ish elements
+    const parent = t.parentElement!;
+    const isBlockish = /^(P|LI|DIV|SECTION|ARTICLE|ASIDE|HEADER|FOOTER|MAIN|NAV|H[1-6]|BLOCKQUOTE|FIGCAPTION|TD|TH)$/i.test(parent.tagName);
+    t.nodeValue = (t.nodeValue || '').replace(/\s+/g, ' ');
+    if (isBlockish) t.nodeValue = (t.nodeValue || '').trim();
+  }
+}
+
+// Split-out helper steps (top-level, no nested functions)
+
+function pickMain(doc: Document): boolean {
+  const main = doc.querySelector('main');
+  if (!main) return false;
+  const clone = main.cloneNode(true);
+  doc.body.innerHTML = '';
+  doc.body.appendChild(clone);
+  return true;
+}
+
+function dropInfraAndSvg(doc: Document, dropSvg: boolean) {
+  const toDrop = [...ALWAYS_DROP, dropSvg ? 'svg' : ''].filter(Boolean).join(',');
+  if (!toDrop) return;
+  doc.querySelectorAll(toDrop).forEach((el) => el.remove());
+}
+
+function dropHiddenTrees(doc: Document) {
+  doc.querySelectorAll<HTMLElement>(HIDDEN_SELECTORS).forEach((el) => el.remove());
+  const all = [...doc.body.querySelectorAll<HTMLElement>('*')];
+  for (const el of all) {
+    if (!el.isConnected) continue;
+    if (hasHiddenAncestor(el)) el.remove();
+  }
+}
+
+function stripAttributesAndSanitize(doc: Document, level: 0 | 1 | 2) {
+  if (!level) return;
+
+  const all = [...doc.body.querySelectorAll<HTMLElement>('*')];
+
+  for (const el of all) {
+    const isSvg = el.namespaceURI === 'http://www.w3.org/2000/svg';
+    for (const { name } of [...el.attributes]) {
+      const lower = name.toLowerCase();
+      if (lower.startsWith('on')) {
+        el.removeAttribute(name);
+        continue;
+      }
+      if (lower === 'style') {
+        el.removeAttribute(name);
+        continue;
+      }
+      if (isSvg) continue; // keep svg attrs
+      const allowed = level === 1 ? ALLOWED_ATTRS : ALLOWED_ATTRS_AGGRESSIVE;
+      if (!allowed.match.has(lower) && !allowed.regexp?.test(name)) {
+        el.removeAttribute(name);
+      }
+    }
+  }
+
+  // sanitize javascript: hrefs
+  doc.querySelectorAll('a[href]').forEach((a) => {
+    const href = a.getAttribute('href') || '';
+    if (/^\s*javascript:/i.test(href)) a.removeAttribute('href');
+  });
+}
+
+function dropHtmlComments(doc: Document) {
+  const nf = doc.defaultView?.NodeFilter;
+  const SHOW_COMMENT = nf?.SHOW_COMMENT ?? 128; // fallback constant
+
+  // createTreeWalker expects a NodeFilter mask
+  const walker = doc.createTreeWalker(doc, SHOW_COMMENT as unknown as number);
+  const toRemove: Comment[] = [];
+  let n: Comment | null;
+
+  while ((n = walker.nextNode() as Comment | null)) toRemove.push(n);
+  toRemove.forEach((c) => c.parentNode?.removeChild(c));
+}
+
+function replaceBrsInHeadings(doc: Document) {
+  doc.querySelectorAll('h1, h2, h3, h4, h5, h6').forEach((h) => {
+    h.querySelectorAll('br').forEach((br) => {
+      const space = doc.createTextNode(' ');
+      (br as Element).replaceWith(space);
+    });
+  });
+}
+
+function limitListsAndRows(doc: Document, limit: number) {
+  // lists
+  doc.querySelectorAll('ul, ol').forEach((list) => {
+    const items = Array.from(list.children).filter((c) => c.tagName === 'LI');
+    for (let i = limit; i < items.length; i++) items[i].remove();
+  });
+
+  // table rows
+  const rowContainers = doc.querySelectorAll('table, thead, tbody, tfoot');
+  rowContainers.forEach((container) => {
+    const rows = Array.from(container.children).filter((c) => c.tagName === 'TR');
+    for (let i = limit; i < rows.length; i++) rows[i].remove();
+  });
+}