page-analyzer 1.0.0

package/csv-exporter.js

@@ -0,0 +1,192 @@
+/**
+ * CSV Exporter for crawled page elements
+ * Exports element data in CSV format with enhanced fields for analysis
+ */
+export class CsvExporter {
+  constructor(config = {}) {
+    const maxContextAncestorLevels = Number.parseInt(config.maxContextAncestorLevels, 10);
+    this.maxContextAncestorLevels = Number.isFinite(maxContextAncestorLevels) && maxContextAncestorLevels > 0
+      ? maxContextAncestorLevels
+      : 3;
+  }
+
+  extractLongestAncestorText(element) {
+    const elementText = element.text || element.content || '';
+
+    if (!elementText) {
+      return '';
+    }
+
+    const context = element?.context && typeof element.context === 'object'
+      ? element.context
+      : {};
+    const ancestorTrail = Array.isArray(context.ancestorTrail)
+      ? context.ancestorTrail
+      : [];
+    const ancestors = ancestorTrail.slice(0, this.maxContextAncestorLevels);
+
+    if (ancestors.length === 0) {
+      return '';
+    }
+
+    let longestText = '';
+    for (const ancestor of ancestors) {
+      if (ancestor.text &&
+        ancestor.text.includes(elementText) &&
+        ancestor.text.length > longestText.length) {
+        longestText = ancestor.text;
+      }
+    }
+
+    return longestText;
+  }
+
+  normalizeTag(element) {
+    return String(element?.tag || '').trim().toLowerCase();
+  }
+
+  normalizeInputType(element) {
+    return String(element?.type || '').trim().toLowerCase();
+  }
+
+  hasDomEventHandler(element) {
+    return Boolean(
+      element?.onclick ||
+      element?.onsubmit ||
+      element?.onchange
+    );
+  }
+
+  hasButtonRole(element) {
+    return String(element?.role || '').trim().toLowerCase() === 'button';
+  }
+
+  isInteractiveInputType(inputType) {
+    const interactiveTypes = new Set([
+      'button',
+      'submit',
+      'reset',
+      'image',
+      'checkbox',
+      'radio',
+      'file',
+      'range',
+      'color',
+      'date',
+      'datetime-local',
+      'month',
+      'time',
+      'week',
+      'number'
+    ]);
+
+    return interactiveTypes.has(inputType);
+  }
+
+  isInteractiveElement(element) {
+    const tag = this.normalizeTag(element);
+    const hasDomEvents = this.hasDomEventHandler(element);
+
+    if (tag === 'a') {
+      return Boolean(element?.href) || hasDomEvents || this.hasButtonRole(element);
+    }
+
+    if (tag === 'button' || tag === 'form' || tag === 'select' || tag === 'textarea') {
+      return true;
+    }
+
+    if (tag === 'input') {
+      const inputType = this.normalizeInputType(element);
+      if (inputType === 'hidden') {
+        return false;
+      }
+      return this.isInteractiveInputType(inputType) || hasDomEvents || Boolean(element?.formAction);
+    }
+
+    return hasDomEvents || this.hasButtonRole(element);
+  }
+
+  transformElement(element, idx) {
+    const text = element.text || element.content || '';
+
+    let context = '';
+    if (text) {
+      context = this.extractLongestAncestorText(element);
+    }
+
+    const imageAlt = element.context?.semanticAnchors?.imageAlt || '';
+
+    return {
+      idx: idx,
+      blockIdx: Number.isInteger(element?.blockIdx) ? element.blockIdx : -1,
+      tag: element.tag || '',
+      imageAlt: imageAlt,
+      text: text,
+      context: context,
+      href: element.href || ''
+    };
+  }
+
+  hasMeaningfulContent(transformed) {
+    return Boolean(
+      String(transformed?.imageAlt || '').trim() ||
+      String(transformed?.text || '').trim() ||
+      String(transformed?.context || '').trim() ||
+      String(transformed?.href || '').trim()
+    );
+  }
+
+  escapeCsvField(field) {
+    if (!field) return '';
+    let str = String(field);
+
+    str = str.replace(/\r?\n|\r/g, ' ');
+    str = str.replace(/\s+/g, ' ');
+    str = str.trim();
+
+    if (str.includes(',') || str.includes('"')) {
+      return `"${str.replace(/"/g, '""')}"`;
+    }
+
+    return str;
+  }
+
+  elementsToCsv(elements) {
+    const sourceElements = Array.isArray(elements) ? elements : [];
+    const rows = [];
+
+    rows.push('idx,blockIdx,tag,imageAlt,text,context,href');
+
+    for (let i = 0; i < sourceElements.length; i++) {
+      const element = sourceElements[i];
+      if (!this.isInteractiveElement(element)) {
+        continue;
+      }
+      const transformed = this.transformElement(element, i);
+      if (!this.hasMeaningfulContent(transformed)) {
+        continue;
+      }
+
+      const row = [
+        transformed.idx,
+        transformed.blockIdx,
+        this.escapeCsvField(transformed.tag),
+        this.escapeCsvField(transformed.imageAlt),
+        this.escapeCsvField(transformed.text),
+        this.escapeCsvField(transformed.context),
+        this.escapeCsvField(transformed.href)
+      ].join(',');
+
+      rows.push(row);
+    }
+
+    return rows.join('\n');
+  }
+
+  buildCsvContent(nodeId, elements) {
+    const sourceElements = Array.isArray(elements) ? elements : [];
+    const csvContent = this.elementsToCsv(sourceElements);
+    console.log(`[CSV] Built ${nodeId} (${sourceElements.length} elements)`);
+    return csvContent;
+  }
+}

package/extractors/block-assigner.js

@@ -0,0 +1,281 @@
+import { normalizeText } from '../utils/text-utils.js';
+import { normalizeUrl } from '../utils/url-utils.js';
+
+function toFiniteNumber(value, fallback = 0) {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+function normalizeHrefLike(value, baseUrl) {
+  const raw = String(value || '').trim();
+  if (!raw) {
+    return '';
+  }
+  return normalizeUrl(raw, baseUrl, {
+    skipFragments: true,
+    normalizeTrailingSlash: true
+  });
+}
+
+function buildElementSignature(element, baseUrl = '') {
+  const tag = String(element?.tag || '').trim().toLowerCase();
+  const href = normalizeHrefLike(
+    element?.href || element?.action || element?.formAction || '',
+    baseUrl
+  );
+  const text = normalizeText(element?.text || element?.content || '').slice(0, 240);
+  const name = String(element?.name || '').trim().toLowerCase();
+  const type = String(element?.type || '').trim().toLowerCase();
+  const ariaLabel = normalizeText(element?.ariaLabel || '').slice(0, 120);
+  return `${tag}|${href}|${text}|${name}|${type}|${ariaLabel}`;
+}
+
+function buildGeometrySignature(geometry, baseUrl = '') {
+  const tag = String(geometry?.tag || '').trim().toLowerCase();
+  const href = normalizeHrefLike(
+    geometry?.href || geometry?.action || geometry?.formAction || '',
+    baseUrl
+  );
+  const text = normalizeText(geometry?.text || '').slice(0, 240);
+  const name = String(geometry?.name || '').trim().toLowerCase();
+  const type = String(geometry?.type || '').trim().toLowerCase();
+  const ariaLabel = normalizeText(geometry?.ariaLabel || '').slice(0, 120);
+  return `${tag}|${href}|${text}|${name}|${type}|${ariaLabel}`;
+}
+
+function overlapArea(rectA, rectB) {
+  const left = Math.max(rectA.left, rectB.left);
+  const top = Math.max(rectA.top, rectB.top);
+  const right = Math.min(rectA.left + rectA.width, rectB.left + rectB.width);
+  const bottom = Math.min(rectA.top + rectA.height, rectB.top + rectB.height);
+  const width = right - left;
+  const height = bottom - top;
+  if (width <= 0 || height <= 0) {
+    return 0;
+  }
+  return width * height;
+}
+
+function mapRectToBlock(rect, blocks = []) {
+  const hasRect = rect && rect.width > 0 && rect.height > 0;
+  if (!hasRect || !Array.isArray(blocks) || blocks.length === 0) {
+    return -1;
+  }
+
+  let bestByOverlap = null;
+  let bestOverlap = 0;
+  const rectArea = rect.width * rect.height;
+  const centerX = rect.left + rect.width / 2;
+  const centerY = rect.top + rect.height / 2;
+
+  let nearestByDistance = null;
+  let nearestDistance = Number.POSITIVE_INFINITY;
+
+  for (const block of blocks) {
+    const width = toFiniteNumber(block?.width, 0);
+    const height = toFiniteNumber(block?.height, 0);
+    if (width <= 0 || height <= 0) {
+      continue;
+    }
+    const blockRect = {
+      top: toFiniteNumber(block?.top, 0),
+      left: toFiniteNumber(block?.left, 0),
+      width,
+      height
+    };
+
+    const overlap = overlapArea(rect, blockRect);
+    if (overlap > 0) {
+      const ratio = overlap / rectArea;
+      if (ratio > bestOverlap) {
+        bestOverlap = ratio;
+        bestByOverlap = block;
+      }
+    }
+
+    const blockCenterX = blockRect.left + blockRect.width / 2;
+    const blockCenterY = blockRect.top + blockRect.height / 2;
+    const distance = ((centerX - blockCenterX) ** 2) + ((centerY - blockCenterY) ** 2);
+    if (distance < nearestDistance) {
+      nearestDistance = distance;
+      nearestByDistance = block;
+    }
+  }
+
+  if (bestByOverlap) {
+    const idx = Number.parseInt(String(bestByOverlap?.blockIdx), 10);
+    return Number.isInteger(idx) ? idx : -1;
+  }
+
+  if (nearestByDistance) {
+    const idx = Number.parseInt(String(nearestByDistance?.blockIdx), 10);
+    return Number.isInteger(idx) ? idx : -1;
+  }
+
+  return -1;
+}
+
+function normalizeSelectorCandidates(element) {
+  const out = [];
+  const context = element?.context && typeof element.context === 'object'
+    ? element.context
+    : {};
+  const push = (value) => {
+    const text = String(value || '').trim();
+    if (!text || out.includes(text)) {
+      return;
+    }
+    out.push(text);
+  };
+
+  push(context.cssSelector);
+  for (const candidate of Array.isArray(context.selectorCandidates) ? context.selectorCandidates : []) {
+    push(candidate);
+  }
+  return out;
+}
+
+function collectBlockIndexByElements(elements = [], blocks = [], geometries = [], pageUrl = '') {
+  const blockList = Array.isArray(blocks) ? blocks : [];
+  const geometryList = Array.isArray(geometries) ? geometries : [];
+  const bySelector = new Map();
+  const bySignature = new Map();
+  const usedGeometryIndices = new Set();
+
+  const pushMap = (map, key, value) => {
+    const normalized = String(key || '').trim();
+    if (!normalized) {
+      return;
+    }
+    if (!map.has(normalized)) {
+      map.set(normalized, []);
+    }
+    map.get(normalized).push(value);
+  };
+
+  for (let index = 0; index < geometryList.length; index += 1) {
+    const geometry = geometryList[index];
+    const selectors = [
+      geometry?.selectorNthOfType,
+      geometry?.selectorNthChild,
+      geometry?.selectorById
+    ];
+    for (const selector of selectors) {
+      pushMap(bySelector, selector, index);
+    }
+    pushMap(bySignature, buildGeometrySignature(geometry, pageUrl), index);
+  }
+
+  const pickUnused = (indices = []) => {
+    for (const index of indices) {
+      if (!usedGeometryIndices.has(index)) {
+        return index;
+      }
+    }
+    return indices.length > 0 ? indices[0] : -1;
+  };
+
+  const resolveGeometryIndex = (element) => {
+    const selectors = normalizeSelectorCandidates(element);
+    for (const selector of selectors) {
+      const candidates = bySelector.get(selector);
+      if (!Array.isArray(candidates) || candidates.length === 0) {
+        continue;
+      }
+      const found = pickUnused(candidates);
+      if (found >= 0) {
+        return found;
+      }
+    }
+
+    const signature = buildElementSignature(element, pageUrl);
+    const signatureCandidates = bySignature.get(signature);
+    if (Array.isArray(signatureCandidates) && signatureCandidates.length > 0) {
+      return pickUnused(signatureCandidates);
+    }
+
+    const tag = String(element?.tag || '').trim().toLowerCase();
+    const href = normalizeHrefLike(
+      element?.href || element?.action || element?.formAction || '',
+      pageUrl
+    );
+    const text = normalizeText(element?.text || element?.content || '').slice(0, 200);
+    let best = -1;
+    let bestScore = Number.NEGATIVE_INFINITY;
+
+    for (let index = 0; index < geometryList.length; index += 1) {
+      if (usedGeometryIndices.has(index)) {
+        continue;
+      }
+      const geometry = geometryList[index];
+      const gTag = String(geometry?.tag || '').trim().toLowerCase();
+      if (tag && gTag && tag !== gTag) {
+        continue;
+      }
+      const gHref = normalizeHrefLike(
+        geometry?.href || geometry?.action || geometry?.formAction || '',
+        pageUrl
+      );
+      const gText = normalizeText(geometry?.text || '').slice(0, 200);
+
+      let score = 0;
+      if (href && gHref && href === gHref) {
+        score += 10;
+      }
+      if (text && gText) {
+        if (text === gText) {
+          score += 8;
+        } else if (text.includes(gText) || gText.includes(text)) {
+          score += 4;
+        }
+      }
+      if (!href && !text) {
+        score += 1;
+      }
+
+      if (score > bestScore) {
+        bestScore = score;
+        best = index;
+      }
+    }
+
+    return best;
+  };
+
+  const sourceElements = Array.isArray(elements) ? elements : [];
+  return sourceElements.map((element) => {
+    const geometryIndex = resolveGeometryIndex(element);
+    if (geometryIndex >= 0) {
+      usedGeometryIndices.add(geometryIndex);
+      const geometry = geometryList[geometryIndex];
+      const rect = {
+        top: toFiniteNumber(geometry?.top, 0),
+        left: toFiniteNumber(geometry?.left, 0),
+        width: toFiniteNumber(geometry?.width, 0),
+        height: toFiniteNumber(geometry?.height, 0)
+      };
+      return mapRectToBlock(rect, blockList);
+    }
+    return -1;
+  });
+}
+
+/**
+ * Assign block indices to elements based on geometry matching.
+ * Each element receives a `blockIdx` property (integer >= 0, or -1 if no block matched).
+ *
+ * @param {Array} elements - Parsed interactive elements
+ * @param {Array} blocks - Visual blocks from page extraction
+ * @param {Array} elementGeometries - Element geometry records with selectors and coordinates
+ * @param {string} pageUrl - Base URL for normalizing element signatures
+ * @returns {Array} The same elements array, mutated with blockIdx
+ */
+export function assignBlocksToElements(elements = [], blocks = [], elementGeometries = [], pageUrl = '') {
+  const source = Array.isArray(elements) ? elements : [];
+  const blockIndices = collectBlockIndexByElements(source, blocks, elementGeometries, pageUrl);
+  for (let index = 0; index < source.length; index += 1) {
+    const value = Number.parseInt(String(blockIndices[index]), 10);
+    source[index].blockIdx = Number.isInteger(value) ? value : -1;
+  }
+  return source;
+}