page-analyzer 1.0.0

package/llm/analyzers/event-analyzer/event-analyzer-blocks.js

@@ -0,0 +1,553 @@
+import { parse } from 'csv-parse/sync';
+import {
+  LLM_GROUP_MAX_ROWS,
+  STANDALONE_BLOCK_MIN_ROWS
+} from './event-analyzer-constants.js';
+import {
+  cleanText,
+  normalizeSpecialBlockName,
+  parseBlockIdxList
+} from './event-analyzer-utils.js';
+
+function unwrapTextResponse(response) {
+  const raw = String(response || '').trim();
+  if (!raw) {
+    return '';
+  }
+
+  const codeBlockRegex = /```(?:csv|text)?\s*\n?([\s\S]*?)\n?```/gi;
+  const segments = [];
+  let lastIndex = 0;
+  let foundCodeBlock = false;
+  let match = null;
+
+  const appendSeparators = (text) => {
+    const lines = String(text || '')
+      .replace(/\r\n/g, '\n')
+      .split('\n')
+      .map((line) => line.trim())
+      .filter((line) => line === '---');
+    if (lines.length > 0) {
+      segments.push(...lines);
+    }
+  };
+
+  while ((match = codeBlockRegex.exec(raw)) !== null) {
+    foundCodeBlock = true;
+    appendSeparators(raw.slice(lastIndex, match.index));
+
+    const block = String(match[1] || '').trim();
+    if (block) {
+      segments.push(block);
+    }
+
+    lastIndex = codeBlockRegex.lastIndex;
+  }
+
+  if (!foundCodeBlock) {
+    return raw;
+  }
+  appendSeparators(raw.slice(lastIndex));
+  return segments.join('\n').trim();
+}
+
+function defaultBlockNameForTag(tag) {
+  const normalizedTag = String(tag || '').trim().toLowerCase();
+  if (normalizedTag === 'main' || normalizedTag === 'section') return 'ContentSection';
+  if (normalizedTag === 'article') return 'ContentSection';
+  return 'ContentBlock';
+}
+
+function toFiniteNumber(value, fallback = 0) {
+  const parsed = Number(value);
+  return Number.isFinite(parsed) ? parsed : fallback;
+}
+
+function normalizeBlockPosition(position = {}, fallback = {}) {
+  const source = position && typeof position === 'object' && !Array.isArray(position)
+    ? position
+    : {};
+  return {
+    left: toFiniteNumber(source.left ?? fallback.left, 0),
+    top: toFiniteNumber(source.top ?? fallback.top, 0),
+    height: Math.max(0, toFiniteNumber(source.height ?? fallback.height, 0)),
+    width: Math.max(0, toFiniteNumber(source.width ?? fallback.width, 0))
+  };
+}
+
+function buildLogicalBlockPosition(sourceBlocks = []) {
+  const rects = [];
+  for (const block of Array.isArray(sourceBlocks) ? sourceBlocks : []) {
+    const position = normalizeBlockPosition(block?.blockPosition, block);
+    if (position.width <= 0 || position.height <= 0) {
+      continue;
+    }
+    rects.push(position);
+  }
+
+  if (rects.length === 0) {
+    return {
+      left: 0,
+      top: 0,
+      height: 0,
+      width: 0
+    };
+  }
+
+  let minLeft = Number.POSITIVE_INFINITY;
+  let minTop = Number.POSITIVE_INFINITY;
+  let maxRight = Number.NEGATIVE_INFINITY;
+  let maxBottom = Number.NEGATIVE_INFINITY;
+
+  for (const rect of rects) {
+    minLeft = Math.min(minLeft, rect.left);
+    minTop = Math.min(minTop, rect.top);
+    maxRight = Math.max(maxRight, rect.left + rect.width);
+    maxBottom = Math.max(maxBottom, rect.top + rect.height);
+  }
+
+  return {
+    left: minLeft,
+    top: minTop,
+    height: Math.max(0, maxBottom - minTop),
+    width: Math.max(0, maxRight - minLeft)
+  };
+}
+
+function resolveLogicalBlockCssPath(sourceBlocks = []) {
+  for (const block of Array.isArray(sourceBlocks) ? sourceBlocks : []) {
+    const path = cleanText(block?.blockCssPath || block?.cssPath || '', 500);
+    if (path) {
+      return path;
+    }
+  }
+  return '';
+}
+
+function normalizePossibleEvents(responseHelper, value) {
+  const raw = String(value || '').trim();
+  if (!raw) {
+    return [];
+  }
+
+  const out = [];
+  const seen = new Set();
+  for (const item of raw.split(/[,.，]/)) {
+    const eventType = responseHelper.normalizeEventType(item);
+    if (!eventType || seen.has(eventType)) {
+      continue;
+    }
+    seen.add(eventType);
+    out.push(eventType);
+    if (out.length >= 3) {
+      break;
+    }
+  }
+  return out;
+}
+
+function toSnakeCaseSegment(value) {
+  return String(value || '')
+    .replace(/([a-z0-9])([A-Z])/g, '$1_$2')
+    .replace(/[^a-zA-Z0-9]+/g, '_')
+    .replace(/^_+|_+$/g, '')
+    .toLowerCase();
+}
+
+function resolveStructuralSingleEventType(blockName, possibleEvents = []) {
+  const normalizedEvents = Array.isArray(possibleEvents)
+    ? possibleEvents.filter((eventType) => String(eventType || '').trim())
+    : [];
+  if (normalizedEvents.length !== 1) {
+    return normalizedEvents;
+  }
+
+  const normalizedBlockName = normalizeSpecialBlockName(blockName, '');
+  if (normalizedBlockName === 'Header') {
+    return ['click_header'];
+  }
+  if (normalizedBlockName === 'Footer') {
+    return ['click_footer'];
+  }
+  if (normalizedBlockName.endsWith('Navigation')) {
+    const segment = toSnakeCaseSegment(
+      normalizedBlockName.slice(0, normalizedBlockName.length - 'Navigation'.length)
+    );
+    return [segment ? `click_${segment}_nav` : 'click_nav'];
+  }
+
+  return normalizedEvents;
+}
+
+function splitResponseSections(response) {
+  const text = unwrapTextResponse(response);
+  const lines = String(text || '')
+    .replace(/\r\n/g, '\n')
+    .split('\n');
+  const separatorIndex = lines.findIndex((line) => String(line || '').trim() === '---');
+
+  if (separatorIndex === -1) {
+    return {
+      primarySection: text,
+      semanticSection: ''
+    };
+  }
+
+  return {
+    primarySection: lines.slice(0, separatorIndex).join('\n').trim(),
+    semanticSection: lines.slice(separatorIndex + 1).join('\n').trim()
+  };
+}
+
+function findHeaderIndex(lines, expectedHeaders = []) {
+  return lines.findIndex((line) => {
+    const normalized = String(line || '').replace(/\s+/g, '').toLowerCase();
+    return expectedHeaders.includes(normalized);
+  });
+}
+
+function parseSpecialBlockResponse(responseHelper, response) {
+  const { primarySection, semanticSection } = splitResponseSections(response);
+  const lines = String(primarySection || '')
+    .replace(/\r\n/g, '\n')
+    .split('\n');
+
+  const headerIndex = findHeaderIndex(lines, [
+    'blockidxs,blockname,blockdescription,blockpossibleevents',
+    'blockidx,blockname,blockdescription,blockpossibleevents',
+    'blockidxs,blockname,blockpossibleevents',
+    'blockidx,blockname,blockpossibleevents'
+  ]);
+  if (headerIndex < 0) {
+    throw new Error('special-block-confirmation CSV header not found');
+  }
+  const summaryLine = lines
+    .slice(0, headerIndex)
+    .find((line) => String(line || '').trim());
+  const siteSummary = cleanText(summaryLine || '', 240);
+
+  const csvText = lines.slice(headerIndex).join('\n').trim();
+  const records = parse(csvText, {
+    columns: true,
+    skip_empty_lines: true,
+    trim: true,
+    relax_column_count: true,
+    relax_quotes: true
+  });
+
+  const logicalBlocks = [];
+  for (const record of records) {
+    const blockIdxs = parseBlockIdxList(
+      record?.blockIdxs ?? record?.blockidxs ?? record?.blockIdx ?? record?.blockidx
+    );
+    if (blockIdxs.length === 0) {
+      continue;
+    }
+    const blockName = normalizeSpecialBlockName(record?.blockName, 'ContentBlock');
+    const blockDescription = cleanText(
+      record?.blockDescription ?? record?.blockdescription ?? record?.description,
+      300
+    );
+    const possibleEvents = normalizePossibleEvents(
+      responseHelper,
+      record?.blockPossibleEvents ?? record?.blockpossibleevents
+    );
+    logicalBlocks.push({
+      blockIdxs,
+      blockIdxKey: blockIdxs.join('.'),
+      blockName,
+      blockDescription,
+      possibleEvents
+    });
+  }
+
+  const semanticGroups = [];
+  const semanticLines = String(semanticSection || '')
+    .replace(/\r\n/g, '\n')
+    .split('\n');
+  const semanticHeaderIndex = findHeaderIndex(semanticLines, [
+    'blockidxs,blocksemantic',
+    'blockidx,blocksemantic'
+  ]);
+  if (semanticHeaderIndex >= 0) {
+    const semanticCsvText = semanticLines.slice(semanticHeaderIndex).join('\n').trim();
+    const semanticRecords = parse(semanticCsvText, {
+      columns: true,
+      skip_empty_lines: true,
+      trim: true,
+      relax_column_count: true,
+      relax_quotes: true
+    });
+
+    for (const record of semanticRecords) {
+      const blockIdxs = parseBlockIdxList(
+        record?.blockIdxs ?? record?.blockidxs ?? record?.blockIdx ?? record?.blockidx
+      );
+      const blockSemantic = cleanText(
+        record?.blockSemantic ?? record?.blocksemantic,
+        80
+      );
+      if (blockIdxs.length === 0 || !blockSemantic) {
+        continue;
+      }
+      semanticGroups.push({
+        blockIdxs,
+        blockSemantic
+      });
+    }
+  }
+
+  return {
+    siteSummary,
+    logicalBlocks,
+    semanticGroups
+  };
+}
+
+function buildBlockContexts(blocks, csvRows, specialResult) {
+  const contexts = [];
+  const blockByIdx = new Map();
+  for (const block of blocks) {
+    blockByIdx.set(block.blockIdx, block);
+  }
+  const mappedIndices = new Set();
+  const byPhysicalBlockIdx = new Map();
+  const semanticByPhysicalBlockIdx = new Map();
+
+  for (const group of Array.isArray(specialResult?.semanticGroups) ? specialResult.semanticGroups : []) {
+    const blockSemantic = cleanText(group?.blockSemantic || '', 80);
+    if (!blockSemantic) {
+      continue;
+    }
+    for (const blockIdx of Array.isArray(group?.blockIdxs) ? group.blockIdxs : []) {
+      if (Number.isInteger(blockIdx) && blockIdx >= 0) {
+        semanticByPhysicalBlockIdx.set(blockIdx, blockSemantic);
+      }
+    }
+  }
+
+  const createLogicalContext = (memberBlockIdxs, blockNameRaw, blockDescriptionRaw, possibleEventsRaw) => {
+    const uniqueMemberIdxs = Array.from(new Set(memberBlockIdxs)).sort((left, right) => left - right);
+    const sourceBlocks = uniqueMemberIdxs
+      .map((idx) => blockByIdx.get(idx))
+      .filter(Boolean)
+      .sort((left, right) => left.order - right.order);
+    if (sourceBlocks.length === 0) {
+      return null;
+    }
+
+    uniqueMemberIdxs.forEach((idx) => mappedIndices.add(idx));
+    const blockName = normalizeSpecialBlockName(
+      blockNameRaw,
+      defaultBlockNameForTag(sourceBlocks[0]?.tag || '')
+    );
+    const blockDescription = cleanText(blockDescriptionRaw || '', 300);
+    let possibleEvents = Array.isArray(possibleEventsRaw) ? [...possibleEventsRaw] : [];
+    if (sourceBlocks.every((block) => Number(block.childInteractiveCount || 0) <= 0)) {
+      possibleEvents = [];
+    } else {
+      possibleEvents = resolveStructuralSingleEventType(blockName, possibleEvents);
+    }
+    const semanticLabels = Array.from(new Set(
+      uniqueMemberIdxs
+        .map((idx) => semanticByPhysicalBlockIdx.get(idx))
+        .filter(Boolean)
+    ));
+    const semanticGroups = Array.from(
+      uniqueMemberIdxs.reduce((map, idx) => {
+        const blockSemantic = semanticByPhysicalBlockIdx.get(idx);
+        if (!blockSemantic) {
+          return map;
+        }
+        if (!map.has(blockSemantic)) {
+          map.set(blockSemantic, []);
+        }
+        map.get(blockSemantic).push(idx);
+        return map;
+      }, new Map())
+    ).map(([blockSemantic, blockIdxs]) => ({
+      blockIdxs: blockIdxs.join('.'),
+      blockSemantic
+    }));
+
+    const context = {
+      blockIdx: uniqueMemberIdxs[0],
+      blockIdxs: uniqueMemberIdxs,
+      blockIdxKey: uniqueMemberIdxs.join('.'),
+      order: Math.min(...sourceBlocks.map((block) => block.order)),
+      blockName,
+      blockDescription,
+      possibleEvents: possibleEvents.slice(0, 3),
+      semanticLabels,
+      semanticGroups,
+      rows: [],
+      sourceBlocks
+    };
+
+    for (const memberIdx of uniqueMemberIdxs) {
+      byPhysicalBlockIdx.set(memberIdx, context);
+    }
+
+    return context;
+  };
+
+  const logicalBlocks = Array.isArray(specialResult?.logicalBlocks) ? specialResult.logicalBlocks : [];
+  for (const logical of logicalBlocks) {
+    const context = createLogicalContext(
+      logical.blockIdxs,
+      logical.blockName,
+      logical.blockDescription,
+      logical.possibleEvents
+    );
+    if (context) {
+      contexts.push(context);
+    }
+  }
+
+  for (const block of blocks) {
+    if (mappedIndices.has(block.blockIdx)) {
+      continue;
+    }
+    const context = createLogicalContext([block.blockIdx], defaultBlockNameForTag(block.tag), '', []);
+    if (context) {
+      contexts.push(context);
+    }
+  }
+
+  let fallbackContext = byPhysicalBlockIdx.get(-1);
+  if (!fallbackContext) {
+    fallbackContext = {
+      blockIdx: -1,
+      blockIdxs: [-1],
+      blockIdxKey: '-1',
+      order: Number.MAX_SAFE_INTEGER,
+      blockName: 'ContentBlock',
+      blockDescription: '',
+      possibleEvents: ['item_click'],
+      semanticLabels: [],
+      semanticGroups: [],
+      rows: [],
+      sourceBlocks: []
+    };
+    contexts.push(fallbackContext);
+    byPhysicalBlockIdx.set(-1, fallbackContext);
+  }
+
+  for (const row of csvRows) {
+    const context = byPhysicalBlockIdx.get(row.blockIdx) || fallbackContext;
+    row.logicalBlockIdxKey = context.blockIdxKey;
+    row.blockSemantic = semanticByPhysicalBlockIdx.get(row.blockIdx) || '';
+    context.rows.push(row);
+  }
+
+  return contexts.sort((left, right) => left.order - right.order);
+}
+
+function buildLlmGroups(blockContexts = []) {
+  const targets = blockContexts
+    .filter((context) => context.possibleEvents.length > 1 && context.rows.length > 0)
+    .sort((left, right) => left.order - right.order);
+  const groups = [];
+  let currentBlocks = [];
+  let currentRows = 0;
+
+  const flush = () => {
+    if (currentBlocks.length === 0) {
+      return;
+    }
+    groups.push({
+      blocks: currentBlocks,
+      rowCount: currentRows
+    });
+    currentBlocks = [];
+    currentRows = 0;
+  };
+
+  for (const block of targets) {
+    const rowCount = block.rows.length;
+    const needsStandalone = block.possibleEvents.length > 2 && rowCount > STANDALONE_BLOCK_MIN_ROWS;
+    if (needsStandalone) {
+      flush();
+      groups.push({
+        blocks: [block],
+        rowCount
+      });
+      continue;
+    }
+
+    if (currentRows > 0 && currentRows + rowCount > LLM_GROUP_MAX_ROWS) {
+      flush();
+    }
+    currentBlocks.push(block);
+    currentRows += rowCount;
+  }
+
+  flush();
+  return groups;
+}
+
+function buildBlockAnalysisArtifact(siteSummary, blockContexts = [], llmGroups = []) {
+  const possibleEventTypes = [];
+  const seenEventTypes = new Set();
+  const blocks = [];
+  let skippedBlocks = 0;
+  let directBlocks = 0;
+  let llmBlocks = 0;
+
+  for (const context of blockContexts) {
+    if (context.blockIdx === -1 && context.rows.length === 0) {
+      continue;
+    }
+    const mode = context.possibleEvents.length === 0
+      ? 'skipped'
+      : (context.possibleEvents.length === 1 ? 'direct' : 'llm');
+    if (mode === 'skipped') skippedBlocks += 1;
+    if (mode === 'direct') directBlocks += 1;
+    if (mode === 'llm') llmBlocks += 1;
+
+    for (const eventType of context.possibleEvents) {
+      if (seenEventTypes.has(eventType)) {
+        continue;
+      }
+      seenEventTypes.add(eventType);
+      possibleEventTypes.push(eventType);
+    }
+
+    blocks.push({
+      blockIdxs: context.blockIdxKey,
+      blockName: context.blockName,
+      blockPossibleEvents: context.possibleEvents,
+      blockSemantics: context.semanticLabels,
+      blockSemanticGroups: context.semanticGroups,
+      blockDescription: context.blockDescription,
+      blockCssPath: resolveLogicalBlockCssPath(context.sourceBlocks),
+      blockPosition: buildLogicalBlockPosition(context.sourceBlocks),
+      rowCount: context.rows.length,
+      mode
+    });
+  }
+
+  return {
+    site_summary: cleanText(siteSummary, 240),
+    possible_event_types: possibleEventTypes.slice(0, 24),
+    blocks,
+    stats: {
+      total_blocks: blocks.length,
+      skipped_blocks: skippedBlocks,
+      direct_blocks: directBlocks,
+      llm_blocks: llmBlocks,
+      llm_group_count: llmGroups.length,
+      llm_group_rows: llmGroups.reduce((sum, group) => sum + (group.rowCount || 0), 0)
+    }
+  };
+}
+
+export {
+  unwrapTextResponse,
+  defaultBlockNameForTag,
+  normalizePossibleEvents,
+  parseSpecialBlockResponse,
+  buildBlockContexts,
+  buildLlmGroups,
+  buildBlockAnalysisArtifact
+};

package/llm/analyzers/event-analyzer/event-analyzer-constants.js

@@ -0,0 +1,22 @@
+const DEFAULT_ATTRIBUTE_KEYS = [
+  'text',
+  'page_area',
+  'content_category(producdt/support/company/legal)',
+  'is_external'
+];
+
+const CSV_REPAIR_ATTEMPTS = 3;
+const FULL_ANALYSIS_RETRY_ATTEMPTS = 1;
+const MAX_CSV_REPAIR_INPUT_CHARS = 20000;
+
+const LLM_GROUP_MAX_ROWS = 60;
+const STANDALONE_BLOCK_MIN_ROWS = 80;
+
+export {
+  DEFAULT_ATTRIBUTE_KEYS,
+  CSV_REPAIR_ATTEMPTS,
+  FULL_ANALYSIS_RETRY_ATTEMPTS,
+  MAX_CSV_REPAIR_INPUT_CHARS,
+  LLM_GROUP_MAX_ROWS,
+  STANDALONE_BLOCK_MIN_ROWS
+};

package/llm/analyzers/event-analyzer/event-analyzer-events.js

@@ -0,0 +1,97 @@
+import {
+  cleanText,
+  normalizeDestination
+} from './event-analyzer-utils.js';
+
+function buildDerivedAttributes(row, blockContext, options = {}) {
+  const attributes = {};
+  const text = cleanText(row?.text || '', 120);
+  if (text) {
+    attributes.text = text;
+  }
+  const blockName = cleanText(blockContext?.blockName || '', 80);
+  if (blockName) {
+    attributes.block = blockName;
+  }
+  const destination = normalizeDestination(row?.href || '');
+  if (destination) {
+    attributes.destination = destination;
+  }
+  const href = cleanText(row?.href || '', 240);
+  if (href) {
+    attributes.href = href;
+  }
+  const tag = cleanText(row?.tag || '', 40);
+  if (tag) {
+    attributes.tag = tag;
+  }
+  const context = cleanText(row?.context || '', 200);
+  if (context) {
+    attributes.context = context;
+  }
+
+  return attributes;
+}
+
+function expandEventAttributes(event, row, blockContext, options = {}) {
+  const attributes = event?.attributes && typeof event.attributes === 'object' && !Array.isArray(event.attributes)
+    ? { ...event.attributes }
+    : {};
+  const derivedAttributes = buildDerivedAttributes(row, blockContext, options);
+
+  for (const [key, value] of Object.entries(derivedAttributes)) {
+    const existingValue = attributes[key];
+    if (existingValue === undefined || existingValue === null || String(existingValue).trim() === '') {
+      attributes[key] = value;
+    }
+  }
+
+  return {
+    ...event,
+    attributes
+  };
+}
+
+function buildDirectEvent(row, blockContext) {
+  const eventType = String(blockContext?.possibleEvents?.[0] || '').trim();
+  if (!eventType) {
+    return null;
+  }
+
+  return {
+    csv_id: row.csv_id,
+    event_type: eventType,
+    attributes: buildDerivedAttributes(row, blockContext)
+  };
+}
+
+function enforceAllowedEventType(responseHelper, event, blockContext) {
+  const normalized = responseHelper.normalizeEventType(event?.event_type);
+  if (!normalized) {
+    return {
+      ...event,
+      event_type: String(blockContext?.possibleEvents?.[0] || '').trim()
+    };
+  }
+  const allowed = Array.isArray(blockContext?.possibleEvents)
+    ? blockContext.possibleEvents
+    : [];
+  if (allowed.length > 0 && !allowed.includes(normalized)) {
+    console.warn(
+      `[LLM] Event type "${normalized}" is out of block scope for blockIdx=${blockContext.blockIdxKey}. ` +
+      'Keeping analysis output.'
+    );
+  }
+
+  return {
+    ...event,
+    event_type: normalized
+  };
+}
+
+export {
+  buildDerivedAttributes,
+  expandEventAttributes,
+  buildDirectEvent,
+  enforceAllowedEventType
+};