page-analyzer 1.0.0 → 1.1.1

package/index.js

@@ -38,6 +38,131 @@ function normalizeDisplayOptions(options = {}) {
   };
 }
 
+function parseBlockIdxs(value) {
+  if (Array.isArray(value)) {
+    return value
+      .map((item) => Number.parseInt(String(item), 10))
+      .filter(Number.isInteger);
+  }
+
+  if (Number.isInteger(value)) {
+    return [value];
+  }
+
+  return String(value || '')
+    .split(/[.,\s]+/)
+    .map((item) => Number.parseInt(item, 10))
+    .filter(Number.isInteger);
+}
+
+function buildBlockScreenshotMap(screenshots) {
+  const map = new Map();
+  for (const item of Array.isArray(screenshots?.blocks) ? screenshots.blocks : []) {
+    const blockIdx = Number.isInteger(item?.blockIdx)
+      ? item.blockIdx
+      : Number.parseInt(String(item?.blockIdx), 10);
+    const screenshotPath = typeof item?.path === 'string' ? item.path : '';
+    if (Number.isInteger(blockIdx) && screenshotPath) {
+      map.set(blockIdx, screenshotPath);
+    }
+  }
+  return map;
+}
+
+function attachBlockScreenshotPaths(analysis, screenshots) {
+  const screenshotByBlockIdx = buildBlockScreenshotMap(screenshots);
+  if (screenshotByBlockIdx.size === 0 || !isObject(analysis?.block_analysis)) {
+    return analysis;
+  }
+
+  const sourceBlocks = analysis.block_analysis.blocks;
+  if (!Array.isArray(sourceBlocks)) {
+    return analysis;
+  }
+
+  const blocks = sourceBlocks.map((block) => {
+    const blockIdxs = parseBlockIdxs(block?.blockIdxs ?? block?.blockIdx);
+    const blockScreenshotPaths = blockIdxs
+      .map((blockIdx) => screenshotByBlockIdx.get(blockIdx))
+      .filter(Boolean);
+
+    if (blockScreenshotPaths.length === 0) {
+      return block;
+    }
+
+    return {
+      ...block,
+      blockScreenshotPaths
+    };
+  });
+
+  return {
+    ...analysis,
+    block_analysis: {
+      ...analysis.block_analysis,
+      blocks
+    }
+  };
+}
+
+function hasScreenshots(screenshots) {
+  return Boolean(
+    screenshots?.fullPage ||
+    (Array.isArray(screenshots?.blocks) && screenshots.blocks.length > 0)
+  );
+}
+
+function mergeScreenshots(primary, secondary) {
+  const merged = {};
+  if (primary?.fullPage) {
+    merged.fullPage = primary.fullPage;
+  }
+  if (secondary?.fullPage) {
+    merged.fullPage = secondary.fullPage;
+  }
+
+  const primaryBlocks = Array.isArray(primary?.blocks) ? primary.blocks : [];
+  const secondaryBlocks = Array.isArray(secondary?.blocks) ? secondary.blocks : [];
+  const blocks = secondaryBlocks.length > 0 ? secondaryBlocks : primaryBlocks;
+  if (blocks.length > 0) {
+    merged.blocks = blocks;
+  }
+
+  return hasScreenshots(merged) ? merged : null;
+}
+
+function attachLogicalBlockScreenshotPaths(result, screenshots) {
+  const blocks = result?.analysis?.block_analysis?.blocks;
+  if (!Array.isArray(blocks) || blocks.length === 0) {
+    return result;
+  }
+
+  const screenshotByLogicalIndex = buildBlockScreenshotMap(screenshots);
+  if (screenshotByLogicalIndex.size === 0) {
+    return result;
+  }
+
+  return {
+    ...result,
+    analysis: {
+      ...result.analysis,
+      block_analysis: {
+        ...result.analysis.block_analysis,
+        blocks: blocks.map((block, index) => {
+          const screenshotPath = screenshotByLogicalIndex.get(index);
+          if (!screenshotPath) {
+            return block;
+          }
+          return {
+            ...block,
+            blockScreenshotPaths: [screenshotPath]
+          };
+        })
+      }
+    }
+  };
+}
+
 function compactBlockAnalysisBlock(block, displayOptions) {
   const source = isObject(block) ? block : {};
   const out = {
@@ -63,6 +188,10 @@ function compactBlockAnalysisBlock(block, displayOptions) {
     out.mode = source.mode;
   }
 
+  if (Array.isArray(source.blockScreenshotPaths) && source.blockScreenshotPaths.length > 0) {
+    out.blockScreenshotPaths = source.blockScreenshotPaths;
+  }
+
   return out;
 }
 
@@ -121,14 +250,20 @@ function buildPageAnalysisResult({
   csvContent,
   pageData,
   analysis,
-  displayOptions
+  displayOptions,
+  screenshots
 }) {
+  const analysisWithScreenshots = attachBlockScreenshotPaths(analysis, screenshots);
   const result = {
     title: pageData.title,
     parseMetrics: pageData.metrics,
-    analysis: buildAnalysisResult(analysis, displayOptions)
+    analysis: buildAnalysisResult(analysisWithScreenshots, displayOptions)
   };
 
+  if (hasScreenshots(screenshots)) {
+    result.screenshots = screenshots;
+  }
+
   if (displayOptions.showEvents) {
     result.elements = elements;
     result.csvContent = csvContent;
@@ -154,19 +289,40 @@ function buildPageAnalysisResult({
  * @param {boolean} [options.showEvents=false] - Include event arrays and full event-related metadata.
  *   Also enables node-level event classification.
  * @param {boolean} [options.showBlockIdx=false] - Include CSV/block index alignment fields.
+ * @param {boolean} [options.fullPageScreenshot=false] - Save a full-page screenshot to snapshots/ and return its path.
+ * @param {boolean} [options.blockScreenshots=false] - Save one screenshot per merged logical block to snapshots/ and return their paths.
+ * @param {boolean} [options.waitForImagesLoaded=false] - Wait for page images before extracting and screenshotting.
  * @returns {Promise<Object>} Analysis result. Event and idx fields are omitted unless requested.
  */
 export async function analyzeUrl(url, options = {}) {
-  const { llm: llmConfig, knownEventTypes, parserConfig, extractorConfig, showEvents, showBlockIdx } = options;
+  const {
+    llm: llmConfig,
+    knownEventTypes,
+    parserConfig,
+    extractorConfig,
+    showEvents,
+    showBlockIdx,
+    fullPageScreenshot,
+    blockScreenshots,
+    waitForImagesLoaded
+  } = options;
 
   if (!url) throw new Error('url is required');
   if (!llmConfig?.apiKey || !llmConfig?.apiEndpoint || !llmConfig?.model) {
     throw new Error('options.llm.apiKey, apiEndpoint, and model are required');
   }
 
+  const shouldCaptureFullPage = fullPageScreenshot ?? extractorConfig?.fullPageScreenshot;
+  const shouldCaptureBlocks = blockScreenshots ?? extractorConfig?.blockScreenshots;
+
   // Step 0: Playwright extraction
   console.log(`[page-analyzer] Extracting ${url} ...`);
-  const extractor = new PageExtractor(extractorConfig);
+  const extractor = new PageExtractor({
+    ...extractorConfig,
+    fullPageScreenshot: shouldCaptureFullPage,
+    blockScreenshots: false,
+    waitForImagesLoaded: waitForImagesLoaded ?? extractorConfig?.waitForImagesLoaded
+  });
   const bundle = await extractor.extract(url);
   console.log(`[page-analyzer] Extracted: ${bundle.blocks.length} blocks, ${bundle.elementGeometries.length} geometries`);
 
@@ -174,7 +330,7 @@ export async function analyzeUrl(url, options = {}) {
   let domain = '';
   try { domain = new URL(url).hostname.replace(/^www\./, ''); } catch { /* ignore */ }
 
-  return analyzePageEvents({
+  let result = await analyzePageEvents({
     html: bundle.html,
     url,
     blocks: bundle.blocks,
@@ -184,9 +340,30 @@ export async function analyzeUrl(url, options = {}) {
     parserConfig,
     showEvents,
     showBlockIdx,
+    screenshots: bundle.screenshots,
     domain,
     nodeId: `${domain}-root`
   });
+
+  if (shouldCaptureBlocks) {
+    const logicalBlocks = Array.isArray(result?.analysis?.block_analysis?.blocks)
+      ? result.analysis.block_analysis.blocks
+      : [];
+    const blockScreenshotsBundle = await extractor.captureUrlScreenshots(url, logicalBlocks, {
+      fullPageScreenshot: false,
+      blockScreenshots: true
+    });
+    const screenshots = mergeScreenshots(result.screenshots, blockScreenshotsBundle);
+    result = attachLogicalBlockScreenshotPaths(
+      {
+        ...result,
+        ...(screenshots ? { screenshots } : {})
+      },
+      screenshots
+    );
+  }
+
+  return result;
 }
 
 /**
@@ -213,6 +390,7 @@ export async function analyzeUrl(url, options = {}) {
  * @param {boolean} [input.showEvents=false] - Include event arrays and full event-related metadata.
  *   Also enables node-level event classification.
  * @param {boolean} [input.showBlockIdx=false] - Include CSV/block index alignment fields.
+ * @param {Object} [input.screenshots] - Screenshot paths captured during extraction.
  * @param {string} [input.nodeId] - Node ID for logging context
  * @param {string} [input.domain] - Domain for logging context
  * @returns {Promise<Object>} Analysis result. Event and idx fields are omitted unless requested.
@@ -229,6 +407,7 @@ export async function analyzePageEvents(input) {
     parserConfig = {},
     showEvents = false,
     showBlockIdx = false,
+    screenshots = null,
     nodeId = '',
     domain = ''
   } = input;
@@ -289,7 +468,8 @@ export async function analyzePageEvents(input) {
     csvContent,
     pageData,
     analysis,
-    displayOptions
+    displayOptions,
+    screenshots
   });
 }
 

package/llm/analyzers/event-analyzer/event-analyzer-blocks.js

@@ -115,13 +115,34 @@ function buildLogicalBlockPosition(sourceBlocks = []) {
 }
 
 function resolveLogicalBlockCssPath(sourceBlocks = []) {
+  const paths = [];
   for (const block of Array.isArray(sourceBlocks) ? sourceBlocks : []) {
     const path = cleanText(block?.blockCssPath || block?.cssPath || '', 500);
     if (path) {
-      return path;
+      paths.push(path);
     }
   }
-  return '';
+
+  if (paths.length === 0) {
+    return '';
+  }
+  if (paths.length === 1) {
+    return paths[0];
+  }
+
+  const partsList = paths.map((path) => path.split('>').map((part) => part.trim()).filter(Boolean));
+  const commonParts = [];
+  const firstParts = partsList[0];
+  for (let index = 0; index < firstParts.length; index += 1) {
+    const part = firstParts[index];
+    if (partsList.every((parts) => parts[index] === part)) {
+      commonParts.push(part);
+      continue;
+    }
+    break;
+  }
+
+  return commonParts.length > 1 ? commonParts.join(' > ') : paths[0];
 }
 
 function normalizePossibleEvents(responseHelper, value) {

package/llm/analyzers/event-analyzer/event-analyzer-constants.js

@@ -1,7 +1,7 @@
 const DEFAULT_ATTRIBUTE_KEYS = [
   'text',
   'page_area',
-  'content_category(producdt/support/company/legal)',
+  'content_category(product/support/company/legal)',
   'is_external'
 ];
 

package/llm/analyzers/event-analyzer/event-analyzer.js

@@ -307,7 +307,7 @@ class EventAnalyzer {
   }
 
   async analyzeEvents(csvData, _mdData, knownEventTypes = [], options = {}) {
-    const analyzeNodeEvents = !options?.analyzeNodeEvents;
+    const analyzeNodeEvents = options?.analyzeNodeEvents === true;
     const configuredKnownEventTypes = this.response.normalizeStringList(
       this.config?.knownEventTypes,
       { eventType: true }

package/package.json

@@ -1,12 +1,14 @@
 {
   "name": "page-analyzer",
-  "version": "1.0.0",
+  "version": "1.1.1",
   "type": "module",
   "description": "Standalone page analysis module.",
+  "license": "MIT",
   "main": "index.js",
   "scripts": {
-    "test": "node test.js",
-    "analyze": "node test.js"
+    "test": "node test/smoke.test.js",
+    "analyze": "node scripts/analyze.js",
+    "viewer": "node scripts/serve-result-viewer.js"
   },
   "dependencies": {
     "cheerio": "^1.2.0",