page-analyzer 1.0.0

package/index.js

@@ -0,0 +1,303 @@
+/**
+ * page-analyzer — Standalone module
+ *
+ * Simplest usage — just pass a URL:
+ *
+ *   import { analyzeUrl } from './page-analyzer/index.js';
+ *
+ *   const result = await analyzeUrl('https://example.com', {
+ *     llm: {
+ *       apiKey: 'sk-...',
+ *       apiEndpoint: 'https://api.openai.com/v1/chat/completions',
+ *       model: 'gpt-4',
+ *     },
+ *     showEvents: true
+ *   });
+ *
+ * Or step-by-step with analyzePageEvents() for pre-fetched data.
+ *
+ * npm dependencies: cheerio, csv-parse, playwright
+ */
+
+import { HtmlParser } from './html-parser.js';
+import { assignBlocksToElements } from './extractors/block-assigner.js';
+import { CsvExporter } from './csv-exporter.js';
+import { OpenAiProvider } from './llm/providers/openai-provider.js';
+import { EventAnalyzer } from './llm/analyzers/event-analyzer/event-analyzer.js';
+import { PageExtractor } from './page-extractor.js';
+
+function isObject(value) {
+  return value && typeof value === 'object' && !Array.isArray(value);
+}
+
+function normalizeDisplayOptions(options = {}) {
+  const showEvents = Boolean(options?.showEvents);
+  return {
+    showEvents,
+    showBlockIdx: showEvents || Boolean(options?.showBlockIdx)
+  };
+}
+
+function compactBlockAnalysisBlock(block, displayOptions) {
+  const source = isObject(block) ? block : {};
+  const out = {
+    blockName: source.blockName,
+    blockDescription: source.blockDescription,
+    blockSemantics: Array.isArray(source.blockSemantics) ? source.blockSemantics : [],
+    blockCssPath: source.blockCssPath,
+    blockPosition: source.blockPosition
+  };
+
+  if (displayOptions.showBlockIdx) {
+    out.blockIdxs = source.blockIdxs;
+    out.blockSemanticGroups = Array.isArray(source.blockSemanticGroups)
+      ? source.blockSemanticGroups
+      : [];
+    out.rowCount = source.rowCount;
+  }
+
+  if (displayOptions.showEvents) {
+    out.blockPossibleEvents = Array.isArray(source.blockPossibleEvents)
+      ? source.blockPossibleEvents
+      : [];
+    out.mode = source.mode;
+  }
+
+  return out;
+}
+
+function compactBlockAnalysis(blockAnalysis, displayOptions) {
+  const source = isObject(blockAnalysis) ? blockAnalysis : {};
+  const out = {
+    site_summary: source.site_summary,
+    blocks: Array.isArray(source.blocks)
+      ? source.blocks.map((block) => compactBlockAnalysisBlock(block, displayOptions))
+      : []
+  };
+
+  if (displayOptions.showEvents) {
+    out.possible_event_types = Array.isArray(source.possible_event_types)
+      ? source.possible_event_types
+      : [];
+  }
+
+  if (isObject(source.stats)) {
+    out.stats = displayOptions.showEvents
+      ? source.stats
+      : {
+        total_blocks: Number.parseInt(String(source.stats.total_blocks || 0), 10) || 0
+      };
+  }
+
+  return out;
+}
+
+function buildAnalysisResult(analysis, displayOptions) {
+  const source = isObject(analysis) ? analysis : {};
+  if (displayOptions.showEvents) {
+    return source;
+  }
+
+  const result = {};
+
+  for (const [key, value] of Object.entries(source)) {
+    if (
+      key === 'events_by_node' ||
+      key === 'event_types_summary' ||
+      key === 'new_event_types'
+    ) {
+      continue;
+    }
+    result[key] = key === 'block_analysis'
+      ? compactBlockAnalysis(value, displayOptions)
+      : value;
+  }
+
+  return result;
+}
+
+function buildPageAnalysisResult({
+  elements,
+  csvContent,
+  pageData,
+  analysis,
+  displayOptions
+}) {
+  const result = {
+    title: pageData.title,
+    parseMetrics: pageData.metrics,
+    analysis: buildAnalysisResult(analysis, displayOptions)
+  };
+
+  if (displayOptions.showEvents) {
+    result.elements = elements;
+    result.csvContent = csvContent;
+    result.links = pageData.links;
+  } else if (displayOptions.showBlockIdx) {
+    result.csvContent = csvContent;
+  }
+
+  return result;
+}
+
+/**
+ * One-call entry: pass a URL, get back everything.
+ *
+ * Playwright → HTML parse → block assign → CSV → LLM block/event analysis
+ *
+ * @param {string} url - URL to analyze
+ * @param {Object} options
+ * @param {Object} options.llm - { apiKey, apiEndpoint, model, ... }
+ * @param {Array}  [options.knownEventTypes] - Accumulated event types for consistency
+ * @param {Object} [options.parserConfig] - HtmlParser config overrides
+ * @param {Object} [options.extractorConfig] - PageExtractor config overrides
+ * @param {boolean} [options.showEvents=false] - Include event arrays and full event-related metadata.
+ *   Also enables node-level event classification.
+ * @param {boolean} [options.showBlockIdx=false] - Include CSV/block index alignment fields.
+ * @returns {Promise<Object>} Analysis result. Event and idx fields are omitted unless requested.
+ */
+export async function analyzeUrl(url, options = {}) {
+  const { llm: llmConfig, knownEventTypes, parserConfig, extractorConfig, showEvents, showBlockIdx } = options;
+
+  if (!url) throw new Error('url is required');
+  if (!llmConfig?.apiKey || !llmConfig?.apiEndpoint || !llmConfig?.model) {
+    throw new Error('options.llm.apiKey, apiEndpoint, and model are required');
+  }
+
+  // Step 0: Playwright extraction
+  console.log(`[page-analyzer] Extracting ${url} ...`);
+  const extractor = new PageExtractor(extractorConfig);
+  const bundle = await extractor.extract(url);
+  console.log(`[page-analyzer] Extracted: ${bundle.blocks.length} blocks, ${bundle.elementGeometries.length} geometries`);
+
+  // Derive domain from URL
+  let domain = '';
+  try { domain = new URL(url).hostname.replace(/^www\./, ''); } catch { /* ignore */ }
+
+  return analyzePageEvents({
+    html: bundle.html,
+    url,
+    blocks: bundle.blocks,
+    elementGeometries: bundle.elementGeometries,
+    llm: llmConfig,
+    knownEventTypes,
+    parserConfig,
+    showEvents,
+    showBlockIdx,
+    domain,
+    nodeId: `${domain}-root`
+  });
+}
+
+/**
+ * Run the full pipeline: HTML parse → block assign → CSV → LLM block/event analysis.
+ *
+ * @param {Object} input
+ * @param {string} input.html - Raw HTML of the crawled page
+ * @param {string} input.url - Page URL
+ * @param {Array}  input.blocks - Visual blocks from Playwright extraction
+ * @param {Array}  input.elementGeometries - Element geometry records
+ * @param {string} [input.markdown] - Markdown content (reserved for future use)
+ * @param {Object} input.llm - LLM provider config
+ * @param {string} input.llm.apiKey - API key
+ * @param {string} input.llm.apiEndpoint - API endpoint URL
+ * @param {string} input.llm.model - Model name
+ * @param {number} [input.llm.maxTokens] - Max tokens
+ * @param {number} [input.llm.temperature] - Temperature
+ * @param {number} [input.llm.timeout] - Request timeout ms
+ * @param {number} [input.llm.maxRetries] - Max retries
+ * @param {Array}  [input.llm.knownEventTypes] - Pre-configured known event types
+ * @param {Function} [input.llm.interactionLogger] - Optional interaction logger
+ * @param {Array}  [input.knownEventTypes] - Accumulated event types for consistency
+ * @param {Object} [input.parserConfig] - HtmlParser config overrides
+ * @param {boolean} [input.showEvents=false] - Include event arrays and full event-related metadata.
+ *   Also enables node-level event classification.
+ * @param {boolean} [input.showBlockIdx=false] - Include CSV/block index alignment fields.
+ * @param {string} [input.nodeId] - Node ID for logging context
+ * @param {string} [input.domain] - Domain for logging context
+ * @returns {Promise<Object>} Analysis result. Event and idx fields are omitted unless requested.
+ */
+export async function analyzePageEvents(input) {
+  const {
+    html,
+    url,
+    blocks = [],
+    elementGeometries = [],
+    markdown = '',
+    llm: llmConfig,
+    knownEventTypes = [],
+    parserConfig = {},
+    showEvents = false,
+    showBlockIdx = false,
+    nodeId = '',
+    domain = ''
+  } = input;
+  const displayOptions = normalizeDisplayOptions({ showEvents, showBlockIdx });
+
+  if (!html) {
+    throw new Error('html is required');
+  }
+  if (!url) {
+    throw new Error('url is required');
+  }
+  if (!llmConfig?.apiKey || !llmConfig?.apiEndpoint || !llmConfig?.model) {
+    throw new Error('llm.apiKey, llm.apiEndpoint, and llm.model are required');
+  }
+
+  // Step 1: Parse HTML → elements
+  const htmlParser = new HtmlParser(parserConfig);
+  const pageData = htmlParser.parse(html, url);
+  const elements = Array.isArray(pageData.elements) ? pageData.elements : [];
+
+  // Step 2: Assign blocks to elements
+  assignBlocksToElements(elements, blocks, elementGeometries, url);
+
+  // Step 3: Build CSV
+  const csvExporter = new CsvExporter();
+  const csvContent = csvExporter.buildCsvContent(nodeId || 'page', elements);
+
+  // Step 4: LLM event analysis
+  const provider = new OpenAiProvider({
+    apiKey: llmConfig.apiKey,
+    apiEndpoint: llmConfig.apiEndpoint,
+    model: llmConfig.model,
+    maxTokens: llmConfig.maxTokens,
+    temperature: llmConfig.temperature,
+    timeout: llmConfig.timeout,
+    maxRetries: llmConfig.maxRetries,
+    interactionLogger: llmConfig.interactionLogger
+  });
+
+  const eventAnalyzer = new EventAnalyzer(provider, llmConfig, {
+    domain,
+    nodeId,
+    url
+  });
+
+  const analysis = await eventAnalyzer.analyzeEvents(
+    csvContent,
+    markdown,
+    knownEventTypes,
+    {
+      blocks,
+      analyzeNodeEvents: displayOptions.showEvents
+    }
+  );
+
+  return buildPageAnalysisResult({
+    elements,
+    csvContent,
+    pageData,
+    analysis,
+    displayOptions
+  });
+}
+
+// Re-export individual components for advanced usage
+export { HtmlParser } from './html-parser.js';
+export { assignBlocksToElements } from './extractors/block-assigner.js';
+export { CsvExporter } from './csv-exporter.js';
+export { OpenAiProvider } from './llm/providers/openai-provider.js';
+export { BaseLlmProvider } from './llm/providers/base-provider.js';
+export { EventAnalyzer } from './llm/analyzers/event-analyzer/event-analyzer.js';
+export { PageExtractor } from './page-extractor.js';