page-analyzer 1.0.0

package/llm/analyzers/prompts/special-block-confirmation.txt

@@ -0,0 +1,127 @@
+You are a strict web layout analyzer.
+
+Task:
+From blocks_index.csv, output:
+1. one short site summary
+2. logical UI blocks
+3. possible click events
+4. an optional second CSV only when a merged block with the same blockName still contains clearly different meanings
+
+Be conservative. Do not guess without clear signals.
+
+Input columns:
+blockIdx,branchPath,depth,domOrder,tag,fixed,top,left,width,height,blockCssPath,blockPosition,textPreview,childInteractiveCount
+
+Use these signals together:
+- branchPath prefix
+- depth
+- domOrder
+- top/left/width/height
+- textPreview continuity
+
+Block names:
+Header, Footer, SideNavigation, Hero, TopNavigation, ContentSection, ArticleList, ProductGrid, FeatureCards, CTASection, Form, SearchBar, ImageGallery, ContentBlock
+
+Merge only when most signals align:
+- similar branchPath prefix
+- close domOrder
+- similar depth
+- small vertical gap
+- similar left and width
+- continuous or related textPreview
+
+Do not merge when:
+- different columns
+- large vertical gaps
+- unrelated branchPath
+- clearly different meanings
+
+Events:
+Use click-based, dot-separated labels only.
+
+Examples: logo_click nav_click article_click product_click card_click image_click cta_click form_submit_click search_click item_click
+
+Rules:
+- infer from textPreview, blockName, and childInteractiveCount
+- if childInteractiveCount = 0, leave events empty
+- if childInteractiveCount > 0, output all possible events.
+- if unclear, use item_click
+- any area may still have click events
+
+Second CSV:
+Output only if one merged first-CSV block:
+- contains multiple original blocks
+- has the same high-level blockName
+- still contains clearly different internal meanings worth splitting
+
+Use one short semantic label per split group.
+
+Examples: product_info review faq testimonial promo signup download comparison filter_bar sort_bar pagination breadcrumb
+
+Prefer the single most representative label. Do not assign both a parent label and its sub-aspect.
+
+Output format:
+Plain CSV only.
+
+First CSV:
+Line 1: site summary sentence
+Line 2: blockIdxs,blockName,blockDescription,blockPossibleEvents
+Line 3+: one row per logical block
+
+Rules:
+- blockIdxs must be dot-separated and sorted ascending
+- blockDescription must be a concise LLM-written description of the logical UI block, based on the block text, role, and layout signals
+- blockDescription must not contain commas
+- blockPossibleEvents must be dot-separated
+- no extra columns beyond blockIdxs, blockName, blockDescription, blockPossibleEvents
+- no markdown
+- no explanations
+- each original blockIdx must appear exactly once in the first CSV
+
+Optional second CSV:
+If needed, add a separator line exactly as:
+---
+
+Then output:
+blockIdxs,blockSemantic
+
+Rules:
+- group nearby sub-blocks with the same meaning when possible
+- use compact indexes such as 1.2, 3.4, 5
+- each blockIdx here must belong to a merged block from the first CSV
+- do not repeat the same blockIdx across rows
+- do not output this CSV if not needed
+
+Example 1:
+
+```text
+Marketing page with header, hero, feature cards, and footer
+blockIdxs,blockName,blockDescription,blockPossibleEvents
+0,Header,Global header with brand and navigation links,logo_click.nav_click
+1.2,Hero,Hero area introducing the offer with a primary CTA,cta_click
+3.4.5,FeatureCards,Feature card grid describing product benefits,card_click
+6,CTASection,Call-to-action section prompting conversion,cta_click
+7,Footer,Footer with secondary navigation links,nav_click
+```
+
+Example 2:
+
+```text
+Product page with navigation, product content, reviews, and footer
+blockIdxs,blockName,blockDescription,blockPossibleEvents
+0,Header,Global header with logo and navigation,logo_click.nav_click
+1.2.3.4.5,ContentBlock,Product detail content with reviews and FAQ links,product_click.cta_click
+6,CTASection,Conversion CTA section for the product,cta_click
+7,Footer,Footer with site navigation,nav_click
+---
+blockIdxs,blockSemantic
+1.2,product_info
+3.4,review
+5,faq
+```
+
+Input:
+blocks_index.csv
+```csv
+<<<BLOCKS_INDEX_CSV>>>
+```

package/llm/providers/base-provider.js

@@ -0,0 +1,64 @@
+/**
+ * Base class for LLM providers
+ * Defines the interface that all LLM providers must implement
+ */
+export class BaseLlmProvider {
+  constructor(config = {}) {
+    this.config = config;
+    this.apiKey = config.apiKey;
+    this.model = config.model;
+    this.interactionLogger = typeof config.interactionLogger === 'function'
+      ? config.interactionLogger
+      : null;
+    this.timeout = config.timeout || 600000;
+    this.maxRetries = config.maxRetries || 3;
+    this.maxTokens = Number.isInteger(config.maxTokens) && config.maxTokens > 0
+      ? config.maxTokens
+      : null;
+  }
+
+  async analyze(content, options = {}) {
+    throw new Error('analyze() must be implemented by subclass');
+  }
+
+  async makeRequestWithRetry(requestFn) {
+    let lastError;
+
+    for (let attempt = 1; attempt <= this.maxRetries; attempt++) {
+      try {
+        return await requestFn();
+      } catch (err) {
+        lastError = err;
+        console.warn(`LLM request attempt ${attempt} failed: ${err.message}`);
+
+        if (attempt < this.maxRetries) {
+          const delay = Math.min(1000 * Math.pow(2, attempt - 1), 10000);
+          await new Promise(resolve => setTimeout(resolve, delay));
+        }
+      }
+    }
+
+    throw lastError;
+  }
+
+  async emitInteractionLog(payload = {}) {
+    if (!this.interactionLogger) {
+      return;
+    }
+
+    try {
+      await this.interactionLogger(payload);
+    } catch (error) {
+      console.warn(`[LLM] Failed to persist interaction log: ${error.message}`);
+    }
+  }
+
+  validateConfig() {
+    if (!this.apiKey) {
+      throw new Error('API key is required');
+    }
+    if (!this.model) {
+      throw new Error('Model is required');
+    }
+  }
+}

package/llm/providers/openai-provider.js

@@ -0,0 +1,168 @@
+import { BaseLlmProvider } from './base-provider.js';
+
+/**
+ * OpenAI API provider implementation
+ */
+export class OpenAiProvider extends BaseLlmProvider {
+  constructor(config = {}) {
+    super(config);
+    this.apiEndpoint = config.apiEndpoint;
+    this.temperature = typeof config.temperature === 'number' ? config.temperature : 0;
+    if (!this.apiEndpoint) {
+      throw new Error('apiEndpoint is required');
+    }
+    this.validateConfig();
+  }
+
+  buildInputText(messages) {
+    if (!Array.isArray(messages)) {
+      return String(messages ?? '');
+    }
+
+    return messages
+      .map((message, index) => {
+        const role = String(message?.role || 'unknown').trim() || 'unknown';
+        const content = String(message?.content ?? '');
+        return `[${index + 1}] ${role}: ${content}`;
+      })
+      .join('\n\n');
+  }
+
+  normalizeInteger(value) {
+    const parsed = Number.parseInt(value, 10);
+    return Number.isFinite(parsed) ? parsed : null;
+  }
+
+  normalizeFloat(value) {
+    const parsed = Number.parseFloat(value);
+    return Number.isFinite(parsed) ? parsed : null;
+  }
+
+  resolveInteractionContext(metadata) {
+    const context = metadata && typeof metadata === 'object' ? metadata : {};
+    const domain = String(context.domain || '').trim();
+    const nodeId = String(context.nodeId || '').trim();
+    if (!domain || !nodeId) {
+      return null;
+    }
+
+    return {
+      domain,
+      nodeId,
+      operation: String(context.operation || 'analysis').trim() || 'analysis',
+      chunkLabel: String(context.chunkLabel || '').trim() || null
+    };
+  }
+
+  async makeRequest(messages, options = {}) {
+    const requestOptions = options && typeof options === 'object'
+      ? { ...options }
+      : {};
+    const metadata = requestOptions.metadata && typeof requestOptions.metadata === 'object'
+      ? requestOptions.metadata
+      : {};
+    delete requestOptions.metadata;
+
+    const payload = {
+      model: this.model,
+      messages,
+      temperature: this.model === 'gpt-5-mini' ? 1 : this.temperature,
+      ...requestOptions
+    };
+    if (payload.max_tokens === undefined && Number.isInteger(this.maxTokens) && this.maxTokens > 0) {
+      payload.max_tokens = this.maxTokens;
+    }
+
+    const interactionContext = this.resolveInteractionContext(metadata);
+    const inputText = this.buildInputText(messages);
+    let failureLogged = false;
+
+    try {
+      const response = await fetch(this.apiEndpoint, {
+        method: 'POST',
+        headers: {
+          'Content-Type': 'application/json',
+          'Authorization': `Bearer ${this.apiKey}`
+        },
+        body: JSON.stringify(payload),
+        signal: AbortSignal.timeout(this.timeout)
+      });
+
+      if (!response.ok) {
+        const error = await response.text();
+
+        if (interactionContext) {
+          await this.emitInteractionLog({
+            ...interactionContext,
+            provider: 'OpenAI',
+            model: this.model,
+            requestId: null,
+            inputText,
+            outputText: null,
+            requestPayload: payload,
+            responsePayload: null,
+            usagePromptTokens: null,
+            usageCompletionTokens: null,
+            usageReasoningTokens: null,
+            usageCost: null
+          });
+          failureLogged = true;
+        }
+
+        throw new Error(`OpenAI API error: ${response.status} - ${error}`);
+      }
+
+      const data = await response.json();
+      const outputText = String(data?.choices?.[0]?.message?.content ?? '');
+      const usage = data?.usage || {};
+
+      if (interactionContext) {
+        await this.emitInteractionLog({
+          ...interactionContext,
+          provider: 'OpenAI',
+          model: String(data?.model || this.model || ''),
+          requestId: data?.id || null,
+          inputText,
+          outputText,
+          requestPayload: payload,
+          responsePayload: data,
+          usagePromptTokens: this.normalizeInteger(usage?.prompt_tokens),
+          usageCompletionTokens: this.normalizeInteger(usage?.completion_tokens),
+          usageReasoningTokens: this.normalizeInteger(usage?.completion_tokens_details?.reasoning_tokens),
+          usageCost: this.normalizeFloat(usage?.cost)
+        });
+      }
+
+      return outputText;
+    } catch (error) {
+      if (!failureLogged && interactionContext) {
+        await this.emitInteractionLog({
+          ...interactionContext,
+          provider: 'OpenAI',
+          model: this.model,
+          requestId: null,
+          inputText,
+          outputText: null,
+          requestPayload: payload,
+          responsePayload: null,
+          usagePromptTokens: null,
+          usageCompletionTokens: null,
+          usageReasoningTokens: null,
+          usageCost: null
+        });
+      }
+      throw error;
+    }
+  }
+
+  async analyze(content, options = {}) {
+    const requestOptions = { ...options };
+    delete requestOptions.parseJson;
+
+    const messages = [
+      { role: 'user', content }
+    ];
+
+    return this.makeRequestWithRetry(() => this.makeRequest(messages, requestOptions));
+  }
+}

package/llm/utils/event-csv.js

@@ -0,0 +1,276 @@
+import { parse } from 'csv-parse/sync';
+
+export const EVENT_CSV_HEADERS = ['csv_id', 'event_type', 'attributes_kv'];
+export const NEW_EVENT_TYPES_CSV_HEADERS = ['name', 'why_webmasters_care'];
+
+function safeDecodeURIComponent(value) {
+  try {
+    return decodeURIComponent(value);
+  } catch {
+    return value;
+  }
+}
+
+function sanitizeResponseText(content) {
+  const raw = String(content || '').trim();
+  if (!raw) {
+    return '';
+  }
+
+  const codeBlockMatch = raw.match(/```(?:csv)?\s*\n?([\s\S]*?)\n?```/i);
+  if (codeBlockMatch) {
+    return codeBlockMatch[1].trim();
+  }
+
+  return raw;
+}
+
+export function decodeAttributesKv(attributesKv) {
+  const source = String(attributesKv || '').trim();
+  if (!source) {
+    return {};
+  }
+
+  const attributes = {};
+  const pairs = source.split('&');
+  for (const pair of pairs) {
+    const segment = pair.trim();
+    if (!segment) {
+      continue;
+    }
+
+    const equalsIndex = segment.indexOf('=');
+    const keyPart = equalsIndex === -1 ? segment : segment.slice(0, equalsIndex);
+    const valuePart = equalsIndex === -1 ? '' : segment.slice(equalsIndex + 1);
+    const key = safeDecodeURIComponent(keyPart).trim();
+    if (!key) {
+      continue;
+    }
+    attributes[key] = safeDecodeURIComponent(valuePart);
+  }
+
+  return attributes;
+}
+
+function normalizeEventType(value) {
+  return String(value || '')
+    .trim()
+    .toLowerCase()
+    .replace(/[^a-z0-9_]+/g, '_')
+    .replace(/_+/g, '_')
+    .replace(/^_+|_+$/g, '');
+}
+
+function normalizeReasonsPipe(value) {
+  const raw = String(value || '').trim();
+  if (!raw) {
+    return [];
+  }
+
+  return Array.from(new Set(
+    raw
+      .split('|')
+      .map((item) => String(item || '').trim())
+      .filter(Boolean)
+  ));
+}
+
+export function parseEventsFromCsv(content, options = {}) {
+  const logger = options.logger || console;
+  const sourceLabel = options.sourceLabel || 'llm_csv';
+  const skipInvalidRows = options.skipInvalidRows !== false;
+  const sanitized = sanitizeResponseText(content);
+  if (!sanitized) {
+    return { events: [], invalidRows: 0 };
+  }
+
+  const lines = sanitized.split(/\r?\n/).filter((line) => line.trim().length > 0);
+  if (lines.length === 0) {
+    return { events: [], invalidRows: 0 };
+  }
+
+  const firstLine = lines[0].trim().toLowerCase();
+  const hasHeader = firstLine.startsWith('csv_id,') && firstLine.includes('event_type');
+  const csvText = hasHeader
+    ? sanitized
+    : `${EVENT_CSV_HEADERS.join(',')}\n${sanitized}`;
+
+  let records;
+  try {
+    records = parse(csvText, {
+      columns: true,
+      skip_empty_lines: true,
+      trim: true,
+      relax_column_count: true,
+      relax_quotes: true
+    });
+  } catch (error) {
+    throw new Error(`Failed to parse CSV content from ${sourceLabel}: ${error.message}`);
+  }
+
+  const events = [];
+  let invalidRows = 0;
+
+  records.forEach((record, index) => {
+    const rowNumber = index + 2;
+    const csvId = String(record.csv_id ?? '').trim();
+    const eventType = normalizeEventType(record.event_type);
+
+    if (!csvId || !eventType) {
+      invalidRows++;
+      const message = `[LLM][CSV] Invalid row ${rowNumber} in ${sourceLabel}: missing csv_id or event_type`;
+      if (!skipInvalidRows) {
+        throw new Error(message);
+      }
+      logger.warn(message);
+      return;
+    }
+
+    let attributes;
+    try {
+      attributes = decodeAttributesKv(record.attributes_kv);
+    } catch (error) {
+      invalidRows++;
+      const message = `[LLM][CSV] Invalid attributes_kv in row ${rowNumber} (${sourceLabel}): ${error.message}`;
+      if (!skipInvalidRows) {
+        throw new Error(message);
+      }
+      logger.warn(message);
+      return;
+    }
+
+    events.push({
+      csv_id: csvId,
+      event_type: eventType,
+      attributes
+    });
+  });
+
+  return { events, invalidRows };
+}
+
+export function parseNewEventTypesFromCsv(content, options = {}) {
+  const logger = options.logger || console;
+  const sourceLabel = options.sourceLabel || 'llm_new_event_types_csv';
+  const skipInvalidRows = options.skipInvalidRows !== false;
+  const sanitized = sanitizeResponseText(content);
+  if (!sanitized) {
+    return { newEventTypes: [], invalidRows: 0 };
+  }
+
+  const lines = sanitized.split(/\r?\n/).filter((line) => line.trim().length > 0);
+  if (lines.length === 0) {
+    return { newEventTypes: [], invalidRows: 0 };
+  }
+
+  const firstLine = lines[0].trim().toLowerCase();
+  const hasHeader = firstLine.startsWith('name,') && firstLine.includes('why_webmasters_care');
+  const csvText = hasHeader
+    ? sanitized
+    : `${NEW_EVENT_TYPES_CSV_HEADERS.join(',')}\n${sanitized}`;
+
+  let records;
+  try {
+    records = parse(csvText, {
+      columns: true,
+      skip_empty_lines: true,
+      trim: true,
+      relax_column_count: true,
+      relax_quotes: true
+    });
+  } catch (error) {
+    throw new Error(`Failed to parse CSV content from ${sourceLabel}: ${error.message}`);
+  }
+
+  const byName = new Map();
+  let invalidRows = 0;
+
+  records.forEach((record, index) => {
+    const rowNumber = index + 2;
+    const name = normalizeEventType(record.name);
+    if (!name) {
+      invalidRows++;
+      const message = `[LLM][CSV] Invalid row ${rowNumber} in ${sourceLabel}: missing name`;
+      if (!skipInvalidRows) {
+        throw new Error(message);
+      }
+      logger.warn(message);
+      return;
+    }
+
+    const reasons = normalizeReasonsPipe(record.why_webmasters_care);
+    if (byName.has(name)) {
+      const existing = byName.get(name);
+      existing.why_webmasters_care = Array.from(new Set([
+        ...existing.why_webmasters_care,
+        ...reasons
+      ]));
+      return;
+    }
+
+    byName.set(name, {
+      name,
+      why_webmasters_care: reasons
+    });
+  });
+
+  return {
+    newEventTypes: Array.from(byName.values()),
+    invalidRows
+  };
+}
+
+export function deriveEventTypeMetadata(events, knownEventTypes = []) {
+  const knownSet = new Set(
+    (Array.isArray(knownEventTypes) ? knownEventTypes : [])
+      .map((value) => normalizeEventType(value))
+      .filter(Boolean)
+  );
+
+  const grouped = new Map();
+  for (const event of Array.isArray(events) ? events : []) {
+    const type = normalizeEventType(event?.event_type);
+    if (!type) {
+      continue;
+    }
+    if (!grouped.has(type)) {
+      grouped.set(type, {
+        csvIds: []
+      });
+    }
+    const group = grouped.get(type);
+    const csvId = String(event?.csv_id ?? '').trim();
+    if (csvId) {
+      group.csvIds.push(csvId);
+    }
+  }
+
+  const eventTypesSummary = [];
+  const newEventTypes = [];
+
+  for (const [name, group] of grouped.entries()) {
+    const coveredCsvIds = Array.from(new Set(group.csvIds));
+    const whyWebmastersCare = [];
+    eventTypesSummary.push({
+      name,
+      covered_csv_ids: coveredCsvIds,
+      why_webmasters_care: whyWebmastersCare
+    });
+
+    if (!knownSet.has(name)) {
+      const item = {
+        name,
+        why_webmasters_care: whyWebmastersCare
+      };
+      if (knownSet.size > 0) {
+        item.not_in_known_event_types_because = `event_type "${name}" was not found in known_event_types`;
+      }
+      newEventTypes.push(item);
+    }
+  }
+
+  return {
+    eventTypesSummary,
+    newEventTypes
+  };
+}

package/models/context.js

@@ -0,0 +1,44 @@
+/**
+ * Represents element context information
+ */
+export class Context {
+  constructor() {
+    this.contextVersion = 2;
+    this.parentTag = '';
+    this.parentText = '';
+    this.ancestorTrail = [];
+    this.cssSelector = '';
+    this.selectorCandidates = [];
+    this.nearbyText = [];
+    this.semanticAnchors = {
+      heading: null,
+      imageAlt: null
+    };
+    this.containerMeta = {
+      cssSelector: '',
+      attributes: {}
+    };
+  }
+
+  static fromObject(obj) {
+    const context = new Context();
+    if (obj && typeof obj === 'object') {
+      Object.assign(context, obj);
+    }
+    return context;
+  }
+
+  toJSON() {
+    return {
+      contextVersion: 2,
+      parentTag: this.parentTag,
+      parentText: this.parentText,
+      ancestorTrail: this.ancestorTrail,
+      cssSelector: this.cssSelector,
+      selectorCandidates: this.selectorCandidates,
+      nearbyText: this.nearbyText,
+      semanticAnchors: this.semanticAnchors,
+      containerMeta: this.containerMeta
+    };
+  }
+}

package/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "page-analyzer",
+  "version": "1.0.0",
+  "type": "module",
+  "description": "Standalone page analysis module.",
+  "main": "index.js",
+  "scripts": {
+    "test": "node test.js",
+    "analyze": "node test.js"
+  },
+  "dependencies": {
+    "cheerio": "^1.2.0",
+    "csv-parse": "^5.6.0",
+    "playwright": "^1.58.2"
+  }
+}