page-analyzer 1.0.0

package/extractors/pt-selector-builder.js

@@ -0,0 +1,344 @@
+const BODY_SELECTOR = 'body';
+const FORM_ELEMENTS = new Set(['input', 'select', 'textarea', 'button']);
+
+const SELECTOR_V2_CONTAINER_TAGS = [
+  'DIV',
+  'SECTION',
+  'ARTICLE',
+  'MAIN',
+  'ASIDE',
+  'NAV',
+  'HEADER',
+  'FOOTER'
+];
+
+const SELECTOR_V2_EXCLUDE_TAGS = ['SCRIPT', 'STYLE', 'LINK', 'META', 'NOSCRIPT'];
+const SELECTOR_V2_EXCLUDE_IDS = ['web-pixels-manager-sandbox-container'];
+
+const selectorCache = new WeakMap();
+
+const selectorMetrics = {
+  totalCalls: 0,
+  cacheHits: 0,
+  averageTime: 0,
+  errors: 0,
+  updateMetrics(time) {
+    this.averageTime = (this.averageTime * this.totalCalls + time) / (this.totalCalls + 1);
+  }
+};
+
+function now() {
+  if (typeof performance !== 'undefined' && typeof performance.now === 'function') {
+    return performance.now();
+  }
+  return Date.now();
+}
+
+function isShadowDom(node) {
+  return Object.prototype.toString.call(node) === '[object ShadowRoot]';
+}
+
+function getShadowDomHost(shadowDom) {
+  return shadowDom?.host || null;
+}
+
+function hasShadowRoot(node) {
+  const shadowRoot = node?.shadowRoot;
+  return Boolean(shadowRoot);
+}
+
+function escapeCss(component) {
+  return String(component || '')
+    .replace(/([^a-zA-Z0-9_-])/g, '\\$1')
+    .replace(/^([0-9])/, '\\3$1 ')
+    .replace(/^(-[0-9-])/, '\\$1');
+}
+
+function getElementAttribute(node, attr) {
+  if (!node || !node.attribs || typeof node.attribs !== 'object') {
+    return null;
+  }
+  if (!(attr in node.attribs)) {
+    return null;
+  }
+  return String(node.attribs[attr] ?? '');
+}
+
+function getTagName(node) {
+  return String(node?.name || '').toLowerCase();
+}
+
+function getElementChildren($, node) {
+  if (!node) {
+    return [];
+  }
+  return $(node)
+    .children()
+    .toArray()
+    .filter((child) => typeof child?.name === 'string' && child.name.length > 0);
+}
+
+function getParentElement($, node) {
+  if (!node) {
+    return null;
+  }
+  const parent = $(node).parent();
+  if (!parent || parent.length === 0) {
+    return null;
+  }
+  const parentNode = parent[0];
+  if (!parentNode || typeof parentNode.name !== 'string') {
+    return null;
+  }
+  return parentNode;
+}
+
+function hasSameIdSibling($, node, parentNode, id) {
+  return getElementChildren($, parentNode).some((child) => {
+    if (child === node) {
+      return false;
+    }
+    return getElementAttribute(child, 'id') === id;
+  });
+}
+
+function isEmptyContainer($, node) {
+  const children = getElementChildren($, node);
+  if (children.length > 0) {
+    const hasValidChild = children.some((child) => {
+      const childTag = String(child?.name || '').toUpperCase();
+      if (SELECTOR_V2_EXCLUDE_TAGS.includes(childTag)) {
+        return false;
+      }
+      if (!SELECTOR_V2_CONTAINER_TAGS.includes(childTag)) {
+        return true;
+      }
+      return !isEmptyContainer($, child);
+    });
+    return !hasValidChild;
+  }
+
+  const textContent = String($(node).text() || '').trim();
+  return textContent.length === 0;
+}
+
+function isInvalidContainer($, node) {
+  const tagName = String(node?.name || '').toUpperCase();
+  if (SELECTOR_V2_EXCLUDE_TAGS.includes(tagName)) {
+    return true;
+  }
+
+  const elementId = getElementAttribute(node, 'id');
+  if (elementId && SELECTOR_V2_EXCLUDE_IDS.includes(elementId)) {
+    return true;
+  }
+
+  if (SELECTOR_V2_CONTAINER_TAGS.includes(tagName)) {
+    return !hasShadowRoot(node) && isEmptyContainer($, node);
+  }
+
+  return false;
+}
+
+function getBodyNode($) {
+  const body = $('body').first();
+  if (!body || body.length === 0) {
+    return null;
+  }
+  return body[0] || null;
+}
+
+function getValidBodyChildren($, selectorV2) {
+  try {
+    const body = getBodyNode($);
+    if (!body) {
+      return [];
+    }
+
+    const children = getElementChildren($, body);
+    if (!selectorV2) {
+      return children;
+    }
+
+    return children.filter((child) => !isInvalidContainer($, child));
+  } catch (error) {
+    console.warn('Error in getValidBodyChildren:', error);
+    return [];
+  }
+}
+
+function getBodyChildSelector($, node, parentNode, ignoreId, selectorV2) {
+  const domNodeName = getTagName(node);
+  const domId = ignoreId ? '' : getElementAttribute(node, 'id');
+
+  if (domId && !hasSameIdSibling($, node, parentNode, domId)) {
+    return `${BODY_SELECTOR}>#${escapeCss(domId)}`;
+  }
+
+  const validBodyChildren = getValidBodyChildren($, selectorV2);
+  const siblings = (validBodyChildren || getElementChildren($, parentNode)).filter(
+    (sibling) => getTagName(sibling) === domNodeName
+  );
+  const position = siblings.indexOf(node);
+  return `${BODY_SELECTOR}>${domNodeName}:eq(${position})`;
+}
+
+function getFormElementSelector($, node, parentNode, ignoreId, selectorV2, useCache) {
+  const domNodeName = getTagName(node);
+  if (!FORM_ELEMENTS.has(domNodeName)) {
+    return null;
+  }
+
+  const name = getElementAttribute(node, 'name');
+  if (!name) {
+    return null;
+  }
+
+  const formNodes = $(parentNode)
+    .find(`${domNodeName}[name='${name}']`)
+    .toArray()
+    .filter((candidate) => getTagName(candidate) === domNodeName);
+
+  if (formNodes.length > 1) {
+    for (let index = 0; index < formNodes.length; index += 1) {
+      if (formNodes[index] === node) {
+        return `${getSelector($, {
+          dom: parentNode,
+          ignoreId,
+          selectorV2,
+          useCache
+        })}>${domNodeName}:input[name='${name}']:eq(${index})`;
+      }
+    }
+  } else if (formNodes.length === 1) {
+    return `${getSelector($, {
+      dom: parentNode,
+      ignoreId,
+      selectorV2,
+      useCache
+    })}>${domNodeName}:input[name='${name}']`;
+  }
+
+  return null;
+}
+
+function getPositionalSelector($, node, parentNode, ignoreId, selectorV2, useCache) {
+  const domNodeName = getTagName(node);
+  const siblings = getElementChildren($, parentNode).filter((child) => getTagName(child) === domNodeName);
+
+  for (let index = 0; index < siblings.length; index += 1) {
+    if (siblings[index] === node) {
+      return `${getSelector($, {
+        dom: parentNode,
+        ignoreId,
+        selectorV2,
+        useCache
+      })}>${domNodeName}:eq(${index})`;
+    }
+  }
+
+  return '';
+}
+
+export function getSelector($, { dom, ignoreId = false, selectorV2 = false, useCache = true } = {}) {
+  if (!$ || !dom) {
+    return '';
+  }
+
+  const start = now();
+  selectorMetrics.totalCalls += 1;
+
+  try {
+    if (useCache) {
+      const cached = selectorCache.get(dom);
+      if (cached) {
+        selectorMetrics.cacheHits += 1;
+        return cached;
+      }
+    }
+
+    let result = '';
+    const domNodeName = getTagName(dom);
+    if (!domNodeName) {
+      return '';
+    }
+
+    if (domNodeName === BODY_SELECTOR || domNodeName === 'html') {
+      result = BODY_SELECTOR;
+      return result;
+    }
+
+    let parentNode = getParentElement($, dom);
+
+    if (isShadowDom(dom)) {
+      result = `${getSelector($, {
+        dom: getShadowDomHost(dom),
+        ignoreId,
+        selectorV2,
+        useCache
+      })}>shadowRoot`;
+      return cacheAndReturn(result);
+    }
+
+    if (!parentNode) {
+      return '';
+    }
+
+    if (getTagName(parentNode) === BODY_SELECTOR) {
+      result = getBodyChildSelector($, dom, parentNode, ignoreId, selectorV2);
+      return cacheAndReturn(result);
+    }
+
+    const domId = !ignoreId && getElementAttribute(dom, 'id');
+    if (parentNode && domId && !hasSameIdSibling($, dom, parentNode, domId)) {
+      result = `${getSelector($, {
+        dom: parentNode,
+        ignoreId,
+        selectorV2,
+        useCache
+      })}>#${escapeCss(domId)}`;
+      return cacheAndReturn(result);
+    }
+
+    if (FORM_ELEMENTS.has(domNodeName)) {
+      const formResult = getFormElementSelector($, dom, parentNode, ignoreId, selectorV2, useCache);
+      if (formResult) {
+        return cacheAndReturn(formResult);
+      }
+    }
+
+    result = getPositionalSelector($, dom, parentNode, ignoreId, selectorV2, useCache);
+    return cacheAndReturn(result);
+  } catch (error) {
+    console.warn('Failed to generate selector:', error);
+    selectorMetrics.errors += 1;
+    return '';
+  } finally {
+    selectorMetrics.updateMetrics(now() - start);
+  }
+
+  function cacheAndReturn(selector) {
+    if (useCache) {
+      selectorCache.set(dom, selector);
+    }
+    return selector;
+  }
+}
+
+export class PtSelectorBuilder {
+  static build($, elem, options = {}) {
+    return getSelector($, {
+      dom: elem,
+      ignoreId: options.ignoreId === true,
+      selectorV2: options.selectorV2 === true,
+      useCache: options.useCache !== false
+    });
+  }
+
+  static buildVariants($, elem) {
+    return {
+      ptSelector: this.build($, elem, { ignoreId: false, selectorV2: false }),
+      ptSelectorV2: this.build($, elem, { ignoreId: false, selectorV2: true })
+    };
+  }
+}

package/html-parser.js

@@ -0,0 +1,206 @@
+import * as cheerio from 'cheerio';
+import { normalizeUrl } from './utils/url-utils.js';
+import { ContextExtractor } from './extractors/context-extractor.js';
+import { PtSelectorBuilder } from './extractors/pt-selector-builder.js';
+
+/**
+ * Parse HTML content and extract important DOM elements with context
+ */
+export class HtmlParser {
+  constructor(config = {}) {
+    const contextLevel = String(config.contextLevel || '').trim().toLowerCase() === 'lean'
+      ? 'lean'
+      : 'full';
+
+    this.config = {
+      importantTags: config.importantTags || [
+        'a',
+        'button',
+        'form',
+        'input',
+        'select',
+        'textarea',
+        'h1',
+        'h2',
+        'h3',
+        'h4',
+        'h5',
+        'h6',
+        'img',
+        'nav'
+      ],
+      maxTextLength: config.maxTextLength || 200,
+      maxParentTextLength: config.maxParentTextLength || 100,
+      maxAncestorDepth: config.maxAncestorDepth || 5,
+      maxNearbyTexts: config.maxNearbyTexts || 5,
+      contextLevel
+    };
+
+    this.contextExtractor = new ContextExtractor(this.config);
+  }
+
+  /**
+   * Parse HTML and extract elements
+   * @param {string} html - Raw HTML content
+   * @param {string} baseUrl - Base URL for resolving relative links
+   * @returns {Object} Parsed page data { title, elements, links, metrics }
+   */
+  parse(html, baseUrl) {
+    const parseStartedAt = Date.now();
+    const $ = cheerio.load(html);
+    const elements = [];
+    const selectors = [...this.config.importantTags, '[onclick]', '[role="button"]'];
+    const seenElements = new Set();
+    let contextBuildMs = 0;
+
+    selectors.forEach(selector => {
+      $(selector).each((i, elem) => {
+        if (seenElements.has(elem)) {
+          return;
+        }
+        seenElements.add(elem);
+
+        const tag = elem?.name || selector;
+        const ptSelectors = PtSelectorBuilder.buildVariants($, elem);
+        const contextStartedAt = Date.now();
+        const context = this.contextExtractor.extract($, elem).toJSON();
+        contextBuildMs += Date.now() - contextStartedAt;
+        const element = {
+          tag,
+          text: $(elem).text().trim().substring(0, this.config.maxTextLength),
+          ...this.getAttributes(elem, tag, baseUrl, $),
+          ptSelector: ptSelectors.ptSelector || '',
+          ptSelectorV2: ptSelectors.ptSelectorV2 || '',
+          context
+        };
+        elements.push(element);
+      });
+    });
+
+    const links = elements.filter(e => e.tag === 'a' && e.href).map(e => ({
+      href: e.href,
+      text: e.text
+    }));
+    const parseMs = Date.now() - parseStartedAt;
+    const heapUsedMB = Number((process.memoryUsage().heapUsed / 1024 / 1024).toFixed(1));
+
+    return {
+      title: $('title').text().trim() || 'No Title',
+      elements,
+      links,
+      metrics: {
+        parseMs,
+        contextBuildMs,
+        elementsCount: elements.length,
+        linksCount: links.length,
+        heapUsedMB,
+        contextLevel: this.config.contextLevel
+      }
+    };
+  }
+
+  getAttributes(elem, tag, baseUrl, $) {
+    const $elem = $(elem);
+    const attrs = {};
+
+    switch (tag) {
+      case 'a':
+        const href = $elem.attr('href');
+        if (href) {
+          attrs.href = normalizeUrl(href, baseUrl);
+        }
+        attrs.title = $elem.attr('title') || '';
+        attrs.role = $elem.attr('role') || '';
+        attrs.ariaLabel = $elem.attr('aria-label') || '';
+        attrs.onclick = $elem.attr('onclick') || '';
+        break;
+
+      case 'img':
+        attrs.src = normalizeUrl($elem.attr('src') || '', baseUrl);
+        attrs.alt = $elem.attr('alt') || '';
+        attrs.role = $elem.attr('role') || '';
+        attrs.ariaLabel = $elem.attr('aria-label') || '';
+        attrs.onclick = $elem.attr('onclick') || '';
+        break;
+
+      case 'button':
+        attrs.type = $elem.attr('type') || 'button';
+        attrs.id = $elem.attr('id') || '';
+        attrs.name = $elem.attr('name') || '';
+        attrs.value = $elem.attr('value') || '';
+        attrs.role = $elem.attr('role') || '';
+        attrs.ariaLabel = $elem.attr('aria-label') || '';
+        attrs.onclick = $elem.attr('onclick') || '';
+        {
+          const formAction = $elem.attr('formaction');
+          if (formAction) {
+            attrs.formAction = normalizeUrl(formAction, baseUrl);
+          }
+        }
+        break;
+
+      case 'form':
+        attrs.id = $elem.attr('id') || '';
+        attrs.class = $elem.attr('class') || '';
+        attrs.name = $elem.attr('name') || '';
+        attrs.role = $elem.attr('role') || '';
+        attrs.ariaLabel = $elem.attr('aria-label') || '';
+        attrs.onsubmit = $elem.attr('onsubmit') || '';
+        attrs.method = ($elem.attr('method') || 'get').toLowerCase();
+        {
+          const action = $elem.attr('action');
+          if (action) {
+            attrs.action = normalizeUrl(action, baseUrl);
+          }
+        }
+        break;
+
+      case 'input':
+        attrs.type = ($elem.attr('type') || 'text').toLowerCase();
+        attrs.id = $elem.attr('id') || '';
+        attrs.class = $elem.attr('class') || '';
+        attrs.name = $elem.attr('name') || '';
+        attrs.value = $elem.attr('value') || '';
+        attrs.placeholder = $elem.attr('placeholder') || '';
+        attrs.role = $elem.attr('role') || '';
+        attrs.ariaLabel = $elem.attr('aria-label') || '';
+        attrs.onclick = $elem.attr('onclick') || '';
+        attrs.onchange = $elem.attr('onchange') || '';
+        {
+          const formAction = $elem.attr('formaction');
+          if (formAction) {
+            attrs.formAction = normalizeUrl(formAction, baseUrl);
+          }
+        }
+        break;
+
+      case 'select':
+        attrs.id = $elem.attr('id') || '';
+        attrs.class = $elem.attr('class') || '';
+        attrs.name = $elem.attr('name') || '';
+        attrs.role = $elem.attr('role') || '';
+        attrs.ariaLabel = $elem.attr('aria-label') || '';
+        attrs.onchange = $elem.attr('onchange') || '';
+        break;
+
+      case 'textarea':
+        attrs.id = $elem.attr('id') || '';
+        attrs.class = $elem.attr('class') || '';
+        attrs.name = $elem.attr('name') || '';
+        attrs.placeholder = $elem.attr('placeholder') || '';
+        attrs.role = $elem.attr('role') || '';
+        attrs.ariaLabel = $elem.attr('aria-label') || '';
+        attrs.onchange = $elem.attr('onchange') || '';
+        break;
+
+      default:
+        attrs.id = $elem.attr('id') || '';
+        attrs.class = $elem.attr('class') || '';
+        attrs.role = $elem.attr('role') || '';
+        attrs.ariaLabel = $elem.attr('aria-label') || '';
+        attrs.onclick = $elem.attr('onclick') || '';
+    }
+
+    return attrs;
+  }
+}