page-analyzer 1.0.0

package/llm/analyzers/event-analyzer/event-analyzer-input.js

@@ -0,0 +1,168 @@
+import { parse } from 'csv-parse/sync';
+import {
+  cleanText,
+  wrapInteractiveActionSegments,
+  toCsvIdValue,
+  parseIntegerOr,
+  escapeCsvField
+} from './event-analyzer-utils.js';
+
+function parseInputCsvRows(csvData) {
+  const raw = String(csvData || '').trim();
+  if (!raw) {
+    return [];
+  }
+
+  let records = [];
+  try {
+    records = parse(raw, {
+      columns: true,
+      skip_empty_lines: true,
+      trim: true,
+      relax_column_count: true,
+      relax_quotes: true
+    });
+  } catch (error) {
+    throw new Error(`Failed to parse input CSV: ${error.message}`);
+  }
+
+  const rows = [];
+  for (let index = 0; index < records.length; index += 1) {
+    const record = records[index];
+    const csvId = toCsvIdValue(record?.csv_id ?? record?.idx);
+    if (!csvId) {
+      continue;
+    }
+    rows.push({
+      order: index,
+      csv_id: csvId,
+      blockIdx: parseIntegerOr(record?.blockIdx ?? record?.blockidx, -1),
+      tag: String(record?.tag || '').trim(),
+      imageAlt: cleanText(record?.imageAlt || '', 300),
+      text: cleanText(record?.text || '', 300),
+      context: cleanText(record?.context || '', 300),
+      href: String(record?.href || '').trim()
+    });
+  }
+
+  return rows;
+}
+
+function normalizeBlockPosition(position = {}, fallback = {}) {
+  const source = position && typeof position === 'object' && !Array.isArray(position)
+    ? position
+    : {};
+  const toNumber = (value) => {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? parsed : 0;
+  };
+
+  return {
+    left: toNumber(source.left ?? fallback.left),
+    top: toNumber(source.top ?? fallback.top),
+    height: Math.max(0, toNumber(source.height ?? fallback.height)),
+    width: Math.max(0, toNumber(source.width ?? fallback.width))
+  };
+}
+
+function formatBlockPosition(position = {}) {
+  const normalized = normalizeBlockPosition(position);
+  return [
+    `left=${Math.round(normalized.left)}`,
+    `top=${Math.round(normalized.top)}`,
+    `height=${Math.round(normalized.height)}`,
+    `width=${Math.round(normalized.width)}`
+  ].join(';');
+}
+
+function normalizeBlocks(blocks = []) {
+  const source = Array.isArray(blocks) ? blocks : [];
+  return source.map((item, index) => {
+    const childInteractiveCount = Number.isInteger(item?.childInteractiveCount)
+      ? item.childInteractiveCount
+      : parseIntegerOr(item?.childInteractiveCount, 0);
+    const textPreview = cleanText(
+      wrapInteractiveActionSegments(item?.textPreview || '', childInteractiveCount),
+      1200
+    );
+    const blockPosition = normalizeBlockPosition(item?.blockPosition, item);
+
+    return {
+      blockIdx: Number.isInteger(item?.blockIdx) ? item.blockIdx : index,
+      branchPath: cleanText(item?.branchPath || '', 120),
+      depth: Number.isInteger(item?.depth) ? item.depth : parseIntegerOr(item?.depth, 0),
+      domOrder: Number.isInteger(item?.domOrder) ? item.domOrder : parseIntegerOr(item?.domOrder, index + 1),
+      tag: String(item?.tag || '').trim().toLowerCase(),
+      fixed: Boolean(item?.fixed),
+      top: Number(item?.top) || 0,
+      left: Number(item?.left) || 0,
+      width: Number(item?.width) || 0,
+      height: Number(item?.height) || 0,
+      blockCssPath: cleanText(item?.blockCssPath || item?.cssPath || '', 500),
+      blockPosition,
+      textPreview,
+      childInteractiveCount,
+      order: index
+    };
+  });
+}
+
+function serializeBlocksIndexCsv(blocks = []) {
+  const lines = ['blockIdx,branchPath,depth,domOrder,tag,fixed,top,left,width,height,blockCssPath,blockPosition,textPreview,childInteractiveCount'];
+  for (const block of blocks) {
+    lines.push([
+      block.blockIdx,
+      escapeCsvField(block.branchPath || ''),
+      Number.isInteger(block.depth) ? block.depth : 0,
+      Number.isInteger(block.domOrder) ? block.domOrder : 0,
+      escapeCsvField(block.tag),
+      block.fixed ? 1 : 0,
+      Math.round(Number(block.top) || 0),
+      Math.round(Number(block.left) || 0),
+      Math.round(Number(block.width) || 0),
+      Math.round(Number(block.height) || 0),
+      escapeCsvField(block.blockCssPath || ''),
+      escapeCsvField(formatBlockPosition(block.blockPosition || block)),
+      escapeCsvField(block.textPreview),
+      Number.isInteger(block.childInteractiveCount) ? block.childInteractiveCount : 0
+    ].join(','));
+  }
+  return lines.join('\n');
+}
+
+function serializeBlockAnalysisCsv(blockContexts = []) {
+  const lines = ['blockIdx,blockName,blockPossibleEvents'];
+  for (const block of blockContexts) {
+    lines.push([
+      escapeCsvField(String(block.blockIdxKey || block.blockIdx)),
+      escapeCsvField(block.blockName),
+      escapeCsvField((Array.isArray(block.possibleEvents) ? block.possibleEvents : []).join(','))
+    ].join(','));
+  }
+  return lines.join('\n');
+}
+
+function serializeDomCsvRows(rows = []) {
+  const lines = ['idx,blockIdx,blockSemantic,tag,imageAlt,text,context,href'];
+  for (const row of rows) {
+    lines.push([
+      row.csv_id,
+      escapeCsvField(String(row.logicalBlockIdxKey ?? row.blockIdx)),
+      escapeCsvField(row.blockSemantic || ''),
+      escapeCsvField(row.tag),
+      escapeCsvField(row.imageAlt),
+      escapeCsvField(row.text),
+      escapeCsvField(row.context),
+      escapeCsvField(row.href)
+    ].join(','));
+  }
+  return lines.join('\n');
+}
+
+export {
+  parseInputCsvRows,
+  normalizeBlocks,
+  serializeBlocksIndexCsv,
+  serializeBlockAnalysisCsv,
+  serializeDomCsvRows
+};

package/llm/analyzers/event-analyzer/event-analyzer-metadata.js

@@ -0,0 +1,15 @@
+export function buildRunContextMetadata(runContext = {}, extra = {}) {
+  const metadata = { ...extra };
+  const safeRunContext = runContext && typeof runContext === 'object' ? runContext : {};
+  const domain = String(safeRunContext?.domain || '').trim();
+  const nodeId = String(safeRunContext?.nodeId || '').trim();
+
+  if (domain) {
+    metadata.domain = domain;
+  }
+  if (nodeId) {
+    metadata.nodeId = nodeId;
+  }
+
+  return metadata;
+}

package/llm/analyzers/event-analyzer/event-analyzer-prompt.js

@@ -0,0 +1,71 @@
+import { DEFAULT_ATTRIBUTE_KEYS } from './event-analyzer-constants.js';
+
+function cleanText(value, maxLength = 2000) {
+  return String(value || '')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .slice(0, maxLength);
+}
+
+export function buildSpecialBlockPrompt({
+  template,
+  blocksIndexCsv
+}) {
+  let prompt = String(template || '');
+  prompt = prompt.replace('<<<BLOCKS_INDEX_CSV>>>', String(blocksIndexCsv || '').trim());
+  return prompt;
+}
+
+export function buildEventAnalysisPrompt({
+  template,
+  siteSummary,
+  blockAnalysisCsv,
+  domCsv
+}) {
+  let prompt = String(template || '');
+  prompt = prompt.replace('<<<SITE_SUMMARY>>>', cleanText(siteSummary, 500));
+  prompt = prompt.replace('<<<BLOCK_ANALYSIS_CSV>>>', String(blockAnalysisCsv || '').trim());
+  prompt = prompt.replace('<<<DOM_CSV>>>', String(domCsv || '').trim());
+  return prompt;
+}
+
+export function buildCsvRepairPrompt({
+  rawResponse,
+  knownEventTypes = [],
+  attributeKeys = [],
+  parseErrorMessage = ''
+}) {
+  const knownTypesJson = JSON.stringify(knownEventTypes || []);
+  const attributeHint = Array.isArray(attributeKeys) && attributeKeys.length > 0
+    ? attributeKeys.join(', ')
+    : DEFAULT_ATTRIBUTE_KEYS.join(', ');
+  const parseError = String(parseErrorMessage || '').trim();
+
+  const lines = [
+    'You are a strict CSV formatter.',
+    'Reformat the input as plain text CSV only, with no markdown, no prose, no code fence.',
+    'Output must contain section 1 (required): events_by_node CSV.',
+    'Section 1 header must be exactly: csv_id,event_type,attributes_kv',
+    'attributes_kv must use key=value&key2=value2 format.',
+    'event_type must be snake_case.',
+    'CSV quoting rules are mandatory:',
+    '- If a field contains comma, quote, or newline, wrap it in double quotes.',
+    '- Escape internal double quotes by doubling them ("").',
+    '- Never leave unmatched quotes in any row.',
+    'Section 2 (optional): new_event_types CSV with header: name,why_webmasters_care',
+    'If section 2 exists, separate sections with a standalone line containing exactly ---',
+    'If no section 2 rows, do not output ---.',
+    'Do not change semantic meaning. Only fix structure/format.',
+    `Known event types: ${knownTypesJson}`,
+    `Preferred attribute keys: ${attributeHint}`,
+    'Input to fix:',
+    rawResponse,
+    'Output only the corrected CSV response.'
+  ];
+
+  if (parseError) {
+    lines.splice(lines.length - 2, 0, `Parser error to fix: ${parseError}`);
+  }
+
+  return lines.join('\n');
+}

package/llm/analyzers/event-analyzer/event-analyzer-response.js

@@ -0,0 +1,290 @@
+import {
+  deriveEventTypeMetadata,
+  parseEventsFromCsv,
+  parseNewEventTypesFromCsv
+} from '../../utils/event-csv.js';
+import { DEFAULT_ATTRIBUTE_KEYS } from './event-analyzer-constants.js';
+
+class EventAnalyzerResponse {
+  constructor(config = {}) {
+    this.config = config;
+  }
+
+  parseCsvResponse(response, knownEventTypes = [], options = {}) {
+    const logger = options?.logger || console;
+    const tolerateNewEventTypesParseError = options?.tolerateNewEventTypesParseError !== false;
+    const { eventsSection, newEventTypesSection, hasSeparator } = this.splitCsvSections(response);
+    if (!eventsSection) {
+      throw new Error('events_by_node CSV section is empty');
+    }
+    if (this.looksLikeJsonPayload(response)) {
+      throw new Error('Response appears to be JSON instead of CSV sections');
+    }
+
+    const { events, invalidRows } = parseEventsFromCsv(eventsSection, {
+      sourceLabel: 'llm_events_by_node',
+      logger: console,
+      skipInvalidRows: true
+    });
+
+    if (invalidRows > 0) {
+      logger.warn(`[LLM][CSV] Skipped ${invalidRows} invalid events_by_node row(s)`);
+    }
+    if (events.length === 0 && invalidRows > 0) {
+      throw new Error('events_by_node CSV has no valid rows');
+    }
+
+    const {
+      eventTypesSummary,
+      newEventTypes: derivedNewEventTypes
+    } = deriveEventTypeMetadata(events, knownEventTypes);
+
+    let parsedNewEventTypes = [];
+    let newTypeInvalidRows = 0;
+    if (newEventTypesSection) {
+      try {
+        const parsedResult = parseNewEventTypesFromCsv(newEventTypesSection, {
+          sourceLabel: 'llm_new_event_types',
+          logger,
+          skipInvalidRows: true
+        });
+        parsedNewEventTypes = parsedResult.newEventTypes;
+        newTypeInvalidRows = parsedResult.invalidRows;
+
+        if (newTypeInvalidRows > 0) {
+          logger.warn(`[LLM][CSV] Skipped ${newTypeInvalidRows} invalid new_event_types row(s)`);
+        }
+      } catch (error) {
+        if (!tolerateNewEventTypesParseError) {
+          throw error;
+        }
+        logger.warn(
+          `[LLM][CSV] Failed to parse new_event_types section: ${error.message}. ` +
+          'Fallback to derived new_event_types.'
+        );
+      }
+    } else if (hasSeparator) {
+      logger.warn('[LLM][CSV] Separator found but new_event_types section is empty; using derived fallback');
+    }
+
+    const normalizedNewEventTypes = this.resolveNewEventTypes({
+      parsedNewEventTypes,
+      derivedNewEventTypes,
+      knownEventTypes
+    });
+
+    return {
+      events_by_node: events,
+      event_types_summary: eventTypesSummary,
+      new_event_types: normalizedNewEventTypes
+    };
+  }
+
+  resolveNewEventTypes({ parsedNewEventTypes = [], derivedNewEventTypes = [], knownEventTypes = [] } = {}) {
+    const knownSet = new Set(this.normalizeStringList(knownEventTypes, { eventType: true }));
+    const derivedByName = new Map();
+    for (const item of Array.isArray(derivedNewEventTypes) ? derivedNewEventTypes : []) {
+      const name = this.normalizeEventType(item?.name);
+      if (!name) {
+        continue;
+      }
+      derivedByName.set(name, item);
+    }
+
+    const mergedByName = new Map();
+    for (const item of Array.isArray(parsedNewEventTypes) ? parsedNewEventTypes : []) {
+      const name = this.normalizeEventType(item?.name);
+      if (!name || knownSet.has(name) || mergedByName.has(name)) {
+        continue;
+      }
+
+      const parsedReasons = this.normalizeReasons(item?.why_webmasters_care);
+      const derivedReasons = this.normalizeReasons(derivedByName.get(name)?.why_webmasters_care);
+      const whyWebmastersCare = parsedReasons.length > 0 ? parsedReasons : derivedReasons;
+      const normalizedItem = {
+        name,
+        why_webmasters_care: whyWebmastersCare
+      };
+      if (knownSet.size > 0) {
+        normalizedItem.not_in_known_event_types_because = `event_type "${name}" was not found in known_event_types`;
+      }
+
+      mergedByName.set(name, normalizedItem);
+    }
+
+    for (const item of Array.isArray(derivedNewEventTypes) ? derivedNewEventTypes : []) {
+      const name = this.normalizeEventType(item?.name);
+      if (!name || knownSet.has(name) || mergedByName.has(name)) {
+        continue;
+      }
+
+      const normalizedItem = {
+        name,
+        why_webmasters_care: this.normalizeReasons(item?.why_webmasters_care)
+      };
+      const note = String(item?.not_in_known_event_types_because || '').trim();
+      if (note) {
+        normalizedItem.not_in_known_event_types_because = note;
+      } else if (knownSet.size > 0) {
+        normalizedItem.not_in_known_event_types_because = `event_type "${name}" was not found in known_event_types`;
+      }
+
+      mergedByName.set(name, normalizedItem);
+    }
+
+    return Array.from(mergedByName.values());
+  }
+
+  splitCsvSections(response) {
+    const normalized = this.unwrapCsvResponseText(response);
+    if (!normalized) {
+      return {
+        eventsSection: '',
+        newEventTypesSection: '',
+        hasSeparator: false
+      };
+    }
+
+    const lines = normalized.replace(/\r\n/g, '\n').split('\n');
+    const separatorIndex = lines.findIndex((line) => line.trim() === '---');
+    if (separatorIndex === -1) {
+      return {
+        eventsSection: normalized.trim(),
+        newEventTypesSection: '',
+        hasSeparator: false
+      };
+    }
+
+    const eventsSection = lines.slice(0, separatorIndex).join('\n').trim();
+    const newEventTypesSection = lines.slice(separatorIndex + 1).join('\n').trim();
+
+    return {
+      eventsSection,
+      newEventTypesSection,
+      hasSeparator: true
+    };
+  }
+
+  unwrapCsvResponseText(response) {
+    const raw = String(response || '').trim();
+    if (!raw) {
+      return '';
+    }
+
+    const codeBlockRegex = /```(?:csv|text)?\s*\n?([\s\S]*?)\n?```/gi;
+    const segments = [];
+    let lastIndex = 0;
+    let foundCodeBlock = false;
+    let match = null;
+
+    const appendSeparators = (text) => {
+      const lines = String(text || '')
+        .replace(/\r\n/g, '\n')
+        .split('\n')
+        .map((line) => line.trim())
+        .filter((line) => line === '---');
+      if (lines.length > 0) {
+        segments.push(...lines);
+      }
+    };
+
+    while ((match = codeBlockRegex.exec(raw)) !== null) {
+      foundCodeBlock = true;
+      appendSeparators(raw.slice(lastIndex, match.index));
+
+      const block = String(match[1] || '').trim();
+      if (block) {
+        segments.push(block);
+      }
+
+      lastIndex = codeBlockRegex.lastIndex;
+    }
+
+    if (!foundCodeBlock) {
+      return raw;
+    }
+    appendSeparators(raw.slice(lastIndex));
+    return segments.join('\n').trim();
+  }
+
+  looksLikeJsonPayload(response) {
+    const raw = String(response || '').trim();
+    if (!raw) {
+      return false;
+    }
+
+    if ((raw.startsWith('{') && raw.endsWith('}')) || (raw.startsWith('[') && raw.endsWith(']'))) {
+      return true;
+    }
+
+    return /"events_by_node"\s*:/.test(raw);
+  }
+
+  normalizeEventType(value) {
+    return String(value || '')
+      .trim()
+      .toLowerCase()
+      .replace(/[^a-z0-9_]+/g, '_')
+      .replace(/_+/g, '_')
+      .replace(/^_+|_+$/g, '');
+  }
+
+  normalizeStringList(values, { eventType = false } = {}) {
+    if (!Array.isArray(values)) {
+      return [];
+    }
+
+    const seen = new Set();
+    const normalized = [];
+    for (const item of values) {
+      let text = String(item ?? '').trim();
+      if (!text) {
+        continue;
+      }
+
+      if (eventType) {
+        text = this.normalizeEventType(text);
+        if (!text) {
+          continue;
+        }
+      }
+
+      if (seen.has(text)) {
+        continue;
+      }
+      seen.add(text);
+      normalized.push(text);
+    }
+
+    return normalized;
+  }
+
+  getAttributeKeys() {
+    const configured = this.normalizeStringList(this.config?.attributeKeys);
+    if (configured.length > 0) {
+      return configured;
+    }
+
+    return [...DEFAULT_ATTRIBUTE_KEYS];
+  }
+
+  normalizeReasons(value) {
+    if (Array.isArray(value)) {
+      return value
+        .map(item => String(item || '').trim())
+        .filter(Boolean);
+    }
+
+    const text = String(value || '').trim();
+    return text ? [text] : [];
+  }
+
+  logNewEventTypes(result) {
+    if (result.new_event_types && result.new_event_types.length > 0) {
+      const newTypeNames = result.new_event_types.map(e => e.name).join(', ');
+      console.log(`[LLM] Discovered ${result.new_event_types.length} new event types: ${newTypeNames}`);
+    }
+  }
+}
+
+export { EventAnalyzerResponse };

package/llm/analyzers/event-analyzer/event-analyzer-utils.js

@@ -0,0 +1,96 @@
+function cleanText(value, maxLength = 240) {
+  return String(value || '')
+    .replace(/\s+/g, ' ')
+    .trim()
+    .slice(0, maxLength);
+}
+
+function wrapInteractiveActionSegments(textPreview, childInteractiveCount = 0) {
+  if (!Number.isInteger(childInteractiveCount) || childInteractiveCount <= 0) {
+    return String(textPreview || '');
+  }
+
+  const source = String(textPreview || '');
+  if (!source) {
+    return '';
+  }
+
+  return source.replace(/\b(?:formaction|action)\s*[:=]\s*[^\s|;,<>]+/gi, (match, offset, input) => {
+    const before = offset > 0 ? input[offset - 1] : '';
+    const afterIndex = offset + match.length;
+    const after = afterIndex < input.length ? input[afterIndex] : '';
+    if (before === '<' && after === '>') {
+      return match;
+    }
+    return `<${match}>`;
+  });
+}
+
+function toCsvIdValue(value) {
+  return String(value ?? '').trim();
+}
+
+function parseIntegerOr(value, fallback = -1) {
+  const parsed = Number.parseInt(String(value ?? '').trim(), 10);
+  return Number.isInteger(parsed) ? parsed : fallback;
+}
+
+function escapeCsvField(value) {
+  const text = String(value ?? '');
+  if (text.includes('"') || text.includes(',') || text.includes('\n') || text.includes('\r')) {
+    return `"${text.replace(/"/g, '""')}"`;
+  }
+  return text;
+}
+
+function normalizeSpecialBlockName(name, fallback = 'ContentBlock') {
+  const text = cleanText(name, 80);
+  return text || fallback;
+}
+
+function normalizeDestination(value) {
+  const href = String(value || '').trim();
+  if (!href) {
+    return '';
+  }
+
+  try {
+    const parsed = new URL(href);
+    const pathPart = `${parsed.pathname || '/'}${parsed.search || ''}`;
+    return cleanText(pathPart || parsed.href, 200);
+  } catch {
+    return cleanText(href, 200);
+  }
+}
+
+function parseBlockIdxList(value) {
+  const raw = String(value || '').trim();
+  if (!raw) {
+    return [];
+  }
+
+  const out = [];
+  const seen = new Set();
+  for (const part of raw.split('.')) {
+    const parsed = Number.parseInt(String(part || '').trim(), 10);
+    if (!Number.isInteger(parsed) || parsed < 0 || seen.has(parsed)) {
+      continue;
+    }
+    seen.add(parsed);
+    out.push(parsed);
+  }
+
+  out.sort((left, right) => left - right);
+  return out;
+}
+
+export {
+  cleanText,
+  wrapInteractiveActionSegments,
+  toCsvIdValue,
+  parseIntegerOr,
+  escapeCsvField,
+  normalizeSpecialBlockName,
+  normalizeDestination,
+  parseBlockIdxList
+};