@crawlith/core 0.1.0 → 0.1.1

package/src/analysis/analyze.ts

@@ -1,11 +1,10 @@
-import fs from 'node:fs/promises';
 import { crawl } from '../crawler/crawl.js';
 import { loadGraphFromSnapshot } from '../db/graphLoader.js';
 import { normalizeUrl } from '../crawler/normalize.js';
 import { calculateMetrics, Metrics } from '../graph/metrics.js';
 import { Graph, ClusterInfo } from '../graph/graph.js';
 import { analyzeContent, calculateThinContentScore } from './content.js';
-import { analyzeH1, analyzeMetaDescription, analyzeTitle, applyDuplicateStatuses, H1Analysis, TextFieldAnalysis } from './seo.js';
+import { analyzeH1, analyzeMetaDescription, analyzeTitle, H1Analysis, TextFieldAnalysis } from './seo.js';
 import { analyzeImageAlts, ImageAltAnalysis } from './images.js';
 import { analyzeLinks, LinkRatioAnalysis } from './links.js';
 import { analyzeStructuredData, StructuredDataResult } from './structuredData.js';
@@ -15,6 +14,8 @@ import { getDb } from '../db/index.js';
 import { SiteRepository } from '../db/repositories/SiteRepository.js';
 import { SnapshotRepository } from '../db/repositories/SnapshotRepository.js';
 import { PageRepository } from '../db/repositories/PageRepository.js';
+import { ANALYSIS_LIST_TEMPLATE, ANALYSIS_PAGE_TEMPLATE } from './templates.js';
+import { EngineContext } from '../events.js';
 
 export interface CrawlPage {
   url: string;
@@ -24,12 +25,11 @@ export interface CrawlPage {
   canonical?: string;
   noindex?: boolean;
   nofollow?: boolean;
+  crawlStatus?: string;
 }
 
 export interface AnalyzeOptions {
-  fromCrawl?: string;
   live?: boolean;
-  html?: boolean;
   seo?: boolean;
   content?: boolean;
   accessibility?: boolean;
@@ -40,6 +40,7 @@ export interface AnalyzeOptions {
   debug?: boolean;
   clusterThreshold?: number;
   minClusterSize?: number;
+  allPages?: boolean;
 }
 
 export interface PageAnalysis {
@@ -58,6 +59,7 @@ export interface PageAnalysis {
     canonical?: string;
     noindex?: boolean;
     nofollow?: boolean;
+    crawlStatus?: string;
   }
 }
 
@@ -77,45 +79,93 @@ export interface AnalysisResult {
     accessibility: boolean;
   };
   clusters?: ClusterInfo[];
+  snapshotId?: number;
+  crawledAt?: string;
 }
 
 interface CrawlData {
-  pages: CrawlPage[];
+  pages: Iterable<CrawlPage> | CrawlPage[];
   metrics: Metrics;
   graph: Graph;
+  snapshotId: number;
+  crawledAt?: string;
 }
 
-export async function analyzeSite(url: string, options: AnalyzeOptions): Promise<AnalysisResult> {
+/**
+ * Analyzes a site for SEO, content, and accessibility.
+ * Supports live crawling or loading from a database snapshot.
+ * Note: File-based data loading is not supported.
+ *
+ * @param url The root URL to analyze
+ * @param options Analysis options
+ * @param context Engine context for event emission
+ */
+export async function analyzeSite(url: string, options: AnalyzeOptions, context?: EngineContext): Promise<AnalysisResult> {
   const normalizedRoot = normalizeUrl(url, '', { stripQuery: false });
   if (!normalizedRoot) {
     throw new Error('Invalid URL for analysis');
   }
 
   let crawlData: CrawlData;
-
+  let robots: any = null;
+
+  // Always try to fetch robots.txt for the analysis session
+  // to ensure we have the latest rules for visibility reporting.
+  try {
+    const robotsUrl = new URL('/robots.txt', normalizedRoot).toString();
+    const robotsRes = await (new (await import('../crawler/fetcher.js')).Fetcher()).fetch(robotsUrl, { maxBytes: 500000 });
+    const status = robotsRes.status;
+    if (typeof status === 'number' && status >= 200 && status < 300) {
+      const robotsParserModule = await import('robots-parser');
+      const robotsParser = (robotsParserModule as any).default || robotsParserModule;
+      robots = (robotsParser as any)(robotsUrl, robotsRes.body);
+    }
+  } catch {
+    // Silence robots fetch errors, fallback to existing or none
+  }
   if (options.live) {
-    crawlData = await runLiveCrawl(normalizedRoot, options);
+    crawlData = await runLiveCrawl(normalizedRoot, options, context);
   } else {
     try {
-      crawlData = await loadCrawlData(normalizedRoot, options.fromCrawl);
+      crawlData = await loadCrawlData(normalizedRoot);
+
+      // Convert generator to array so it can be reused multiple times
+      const allPages = Array.from(crawlData.pages);
+      crawlData.pages = allPages;
+
+      // Check if the requested URL actually exists in this snapshot
+      const exists = allPages.some(p => p.url === normalizedRoot);
+      if (!exists) {
+        options.live = true; // Mark as live so the analysis knows to pick the first page if exact match fails
+        if (context) {
+          context.emit({ type: 'info', message: `URL ${normalizedRoot} not found in latest snapshot. Fetching live...` });
+        }
+        crawlData = await runLiveCrawl(normalizedRoot, options, context);
+      }
     } catch (error: any) {
       const isNotFound = error.code === 'ENOENT' ||
         error.message.includes('Crawl data not found') ||
         error.message.includes('No completed snapshot found') ||
         error.message.includes('not found in database');
-      if (isNotFound && !options.fromCrawl) {
-        console.log('No local crawl data found. Switching to live analysis mode...');
-        crawlData = await runLiveCrawl(normalizedRoot, options);
+      if (isNotFound) {
+        options.live = true; // Force live mode
+        if (context) {
+          context.emit({ type: 'info', message: 'No local crawl data found. Switching to live analysis mode...' });
+        }
+        crawlData = await runLiveCrawl(normalizedRoot, options, context);
       } else {
         throw error;
       }
     }
   }
 
+  const snapshotId = crawlData.snapshotId;
+  const crawledAt = crawlData.crawledAt;
+
   // Run clustering if requested or as default
   detectContentClusters(crawlData.graph, options.clusterThreshold, options.minClusterSize);
 
-  const pages = analyzePages(normalizedRoot, crawlData.pages);
+  const pages = analyzePages(normalizedRoot, crawlData.pages, robots);
 
   const activeModules = {
     seo: !!options.seo,
@@ -131,15 +181,21 @@ export async function analyzeSite(url: string, options: AnalyzeOptions): Promise
 
   // Filter to only the requested URL
   const targetPage = filteredPages.find(p => p.url === normalizedRoot);
-  const resultPages = targetPage ? [targetPage] : (options.live ? filteredPages.slice(0, 1) : filteredPages);
+  let resultPages: PageAnalysis[];
+
+  if (options.allPages) {
+    resultPages = filteredPages;
+  } else {
+    resultPages = targetPage ? [targetPage] : (options.live ? filteredPages.slice(0, 1) : []);
+  }
 
   const duplicateTitles = pages.filter((page) => page.title.status === 'duplicate').length;
   const thinPages = pages.filter((page) => page.thinScore >= 70).length;
-  const siteScores = aggregateSiteScore(crawlData.metrics, pages);
+  const siteScores = aggregateSiteScore(crawlData.metrics, resultPages.length === 1 ? resultPages : pages);
 
   return {
     site_summary: {
-      pages_analyzed: pages.length,
+      pages_analyzed: resultPages.length,
       avg_seo_score: siteScores.seoHealthScore,
       thin_pages: thinPages,
       duplicate_titles: duplicateTitles,
@@ -148,7 +204,9 @@ export async function analyzeSite(url: string, options: AnalyzeOptions): Promise
     site_scores: siteScores,
     pages: resultPages,
     active_modules: activeModules,
-    clusters: crawlData.graph.contentClusters
+    clusters: crawlData.graph.contentClusters,
+    snapshotId,
+    crawledAt
   };
 }
 
@@ -157,144 +215,54 @@ export function renderAnalysisHtml(result: AnalysisResult): string {
     return renderSinglePageHtml(result.pages[0]);
   }
   const rows = result.pages
-    .map((page) => `< tr > <td>${escapeHtml(page.url)} </td><td>${page.seoScore}</td > <td>${page.thinScore} </td><td>${page.title.status}</td > <td>${page.metaDescription.status} </td></tr > `)
+    .map((page) => `<tr><td>${escapeHtml(page.url)}</td><td>${page.seoScore}</td><td>${page.thinScore}</td><td>${page.title.status}</td><td>${page.metaDescription.status}</td></tr>`)
     .join('');
 
-  return `<!DOCTYPE html><html lang="en"><head><meta charset="utf-8" /><title>Crawlith Analysis Report</title></head><body><h1>Analysis</h1><p>Pages: ${result.site_summary.pages_analyzed}</p><p>Average SEO: ${result.site_summary.avg_seo_score}</p><table border="1" cellspacing="0" cellpadding="6"><thead><tr><th>URL</th><th>SEO Score</th><th>Thin Score</th><th>Title</th><th>Meta</th></tr></thead><tbody>${rows}</tbody></table></body></html>`;
+  return ANALYSIS_LIST_TEMPLATE
+    .replace('{{PAGES_ANALYZED}}', result.site_summary.pages_analyzed.toString())
+    .replace('{{AVG_SEO_SCORE}}', result.site_summary.avg_seo_score.toString())
+    .replace('{{ROWS}}', rows);
 }
 
 function renderSinglePageHtml(page: PageAnalysis): string {
-  return `<!DOCTYPE html>
-<html lang="en">
-  <head>
-    <meta charset="UTF-8">
-    <meta name="viewport" content="width=device-width, initial-scale=1.0">
-    <title>Analysis for ${escapeHtml(page.url)}</title>
-    <style>
-        body { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif; max-width: 800px; margin: 0 auto; padding: 20px; line-height: 1.6; color: #333; }
-        h1 { border-bottom: 2px solid #eee; padding-bottom: 10px; }
-        h2 { margin-top: 30px; border-bottom: 1px solid #eee; padding-bottom: 5px; }
-        .score-card { display: flex; gap: 20px; margin-bottom: 30px; }
-        .score-box { background: #f8f9fa; padding: 15px; border-radius: 8px; text-align: center; flex: 1; border: 1px solid #e1e4e8; }
-        .score-val { font-size: 24px; font-weight: bold; color: #0366d6; }
-        .status-ok { color: green; font-weight: bold; }
-        .status-warning { color: orange; font-weight: bold; }
-        .status-critical { color: red; font-weight: bold; }
-        .status-missing { color: red; font-weight: bold; }
-        .data-table { width: 100%; border-collapse: collapse; margin-top: 10px; }
-        .data-table th, .data-table td { text-align: left; padding: 8px; border-bottom: 1px solid #eee; }
-        .data-table th { width: 150px; color: #666; }
-        code { background: #f6f8fa; padding: 2px 4px; border-radius: 3px; font-size: 0.9em; }
-    </style>
-  </head>
-  <body>
-    <h1>Page Analysis</h1>
-    <p><strong>URL:</strong> <a href="${page.url}" target="_blank">${page.url}</a></p>
-
-    <div class="score-card">
-      <div class="score-box">
-        <div class="score-val">${page.seoScore}</div>
-        <div>SEO Score</div>
-      </div>
-      <div class="score-box">
-        <div class="score-val">${page.thinScore}</div>
-        <div>Thin Content Score</div>
-      </div>
-      <div class="score-box">
-        <div class="score-val">${page.status === 0 ? 'Pending/Limit' : page.status}</div>
-        <div>HTTP Status</div>
-      </div>
-    </div>
-
-    <h2>Meta Tags</h2>
-    <table class="data-table">
-      <tr>
-        <th>Title</th>
-        <td>
-          <div>${escapeHtml(page.title.value || '(missing)')}</div>
-          <small>Length: ${page.title.length} | Status: <span class="status-${page.title.status}">${page.title.status}</span></small>
-        </td>
-      </tr>
-      <tr>
-        <th>Description</th>
-        <td>
-          <div>${escapeHtml(page.metaDescription.value || '(missing)')}</div>
-          <small>Length: ${page.metaDescription.length} | Status: <span class="status-${page.metaDescription.status}">${page.metaDescription.status}</span></small>
-        </td>
-      </tr>
-      <tr>
-        <th>Canonical</th>
-        <td>${page.meta.canonical ? escapeHtml(page.meta.canonical) : '<em>(none)</em>'}</td>
-      </tr>
-      <tr>
-        <th>Robots</th>
-        <td>
-          Index: ${!page.meta.noindex},
-          Follow: ${!page.meta.nofollow}
-        </td>
-      </tr>
-    </table>
-
-    <h2>Content & Heading</h2>
-    <table class="data-table">
-      <tr>
-        <th>H1 Tag</th>
-        <td>
-          Status: <span class="status-${page.h1.status}">${page.h1.status}</span>
-          (${page.h1.count} detected)
-          ${page.h1.matchesTitle ? ' | Matches Title' : ''}
-        </td>
-      </tr>
-      <tr>
-        <th>Word Count</th>
-        <td>${page.content.wordCount} words</td>
-      </tr>
-      <tr>
-        <th>Unique Sentences</th>
-        <td>${page.content.uniqueSentenceCount}</td>
-      </tr>
-      <tr>
-        <th>Text / HTML Ratio</th>
-        <td>${(page.content.textHtmlRatio * 100).toFixed(2)}%</td>
-      </tr>
-    </table>
+  const structuredDataStatus = page.structuredData.present
+    ? (page.structuredData.valid ? '<span class="status-ok">Valid</span>' : '<span class="status-critical">Invalid JSON</span>')
+    : 'Not detected';
 
-    <h2>Links & Images</h2>
-    <table class="data-table">
-      <tr>
-        <th>Internal Links</th>
-        <td>${page.links.internalLinks}</td>
-      </tr>
-      <tr>
-        <th>External Links</th>
-        <td>${page.links.externalLinks} (${(page.links.externalRatio * 100).toFixed(1)}%)</td>
-      </tr>
-      <tr>
-        <th>Images</th>
-        <td>${page.images.totalImages} total (${page.images.missingAlt} missing alt text)</td>
-      </tr>
-    </table>
-
-    <h2>Structured Data</h2>
-    <table class="data-table">
-      <tr>
-        <th>Status</th>
-        <td>
-          ${page.structuredData.present
-      ? (page.structuredData.valid ? '<span class="status-ok">Valid</span>' : '<span class="status-critical">Invalid JSON</span>')
-      : 'Not detected'
-    }
-        </td>
-      </tr>
-      ${page.structuredData.present ? `
+  const structuredDataTypesRow = page.structuredData.present ? `
       <tr>
           <th>Types Found</th>
           <td>${page.structuredData.types.map(t => `<code>${t}</code>`).join(', ')}</td>
       </tr>
-      ` : ''}
-    </table>
-  </body>
-</html>`;
+      ` : '';
+
+  return ANALYSIS_PAGE_TEMPLATE
+    .replaceAll('{{URL}}', escapeHtml(page.url))
+    .replace('{{SEO_SCORE}}', page.seoScore.toString())
+    .replace('{{THIN_SCORE}}', page.thinScore.toString())
+    .replace('{{HTTP_STATUS}}', page.status === 0 ? 'Pending/Limit' : page.status.toString())
+    .replace('{{TITLE_VALUE}}', escapeHtml(page.title.value || '(missing)'))
+    .replace('{{TITLE_LENGTH}}', page.title.length.toString())
+    .replaceAll('{{TITLE_STATUS}}', page.title.status)
+    .replace('{{META_DESCRIPTION_VALUE}}', escapeHtml(page.metaDescription.value || '(missing)'))
+    .replace('{{META_DESCRIPTION_LENGTH}}', page.metaDescription.length.toString())
+    .replaceAll('{{META_DESCRIPTION_STATUS}}', page.metaDescription.status)
+    .replace('{{CANONICAL}}', page.meta.canonical ? escapeHtml(page.meta.canonical) : '<em>(none)</em>')
+    .replace('{{ROBOTS_INDEX}}', (!page.meta.noindex).toString())
+    .replace('{{ROBOTS_FOLLOW}}', (!page.meta.nofollow).toString())
+    .replaceAll('{{H1_STATUS}}', page.h1.status)
+    .replace('{{H1_COUNT}}', page.h1.count.toString())
+    .replace('{{H1_MATCHES_TITLE}}', page.h1.matchesTitle ? ' | Matches Title' : '')
+    .replace('{{WORD_COUNT}}', page.content.wordCount.toString())
+    .replace('{{UNIQUE_SENTENCES}}', page.content.uniqueSentenceCount.toString())
+    .replace('{{TEXT_HTML_RATIO}}', (page.content.textHtmlRatio * 100).toFixed(2))
+    .replace('{{INTERNAL_LINKS}}', page.links.internalLinks.toString())
+    .replace('{{EXTERNAL_LINKS}}', page.links.externalLinks.toString())
+    .replace('{{EXTERNAL_RATIO}}', (page.links.externalRatio * 100).toFixed(1))
+    .replace('{{TOTAL_IMAGES}}', page.images.totalImages.toString())
+    .replace('{{MISSING_ALT}}', page.images.missingAlt.toString())
+    .replace('{{STRUCTURED_DATA_STATUS}}', structuredDataStatus)
+    .replace('{{STRUCTURED_DATA_TYPES_ROW}}', structuredDataTypesRow);
 }
 
 export function renderAnalysisMarkdown(result: AnalysisResult): string {
@@ -347,52 +315,95 @@ function escapeHtml(value: string): string {
   return value.replaceAll('&', '&amp;').replaceAll('<', '&lt;').replaceAll('>', '&gt;');
 }
 
-function analyzePages(rootUrl: string, pages: CrawlPage[]): PageAnalysis[] {
-  const titleCandidates = pages.map((page) => analyzeTitle(page.html || ''));
-  const metaCandidates = pages.map((page) => analyzeMetaDescription(page.html || ''));
-  const titles = applyDuplicateStatuses(titleCandidates);
-  const metas = applyDuplicateStatuses(metaCandidates);
-
+export function analyzePages(rootUrl: string, pages: Iterable<CrawlPage> | CrawlPage[], robots?: any): PageAnalysis[] {
+  const titleCounts = new Map<string, number>();
+  const metaCounts = new Map<string, number>();
   const sentenceCountFrequency = new Map<number, number>();
-  const baseContent = pages.map((page) => analyzeContent(page.html || ''));
-  for (const item of baseContent) {
-    sentenceCountFrequency.set(item.uniqueSentenceCount, (sentenceCountFrequency.get(item.uniqueSentenceCount) || 0) + 1);
-  }
 
-  return pages.map((page, index) => {
+  const results: PageAnalysis[] = [];
+
+  for (const page of pages) {
     const html = page.html || '';
-    const title = titles[index];
-    const metaDescription = metas[index];
+
+    // 0. Update crawl status based on current robots rules
+    let crawlStatus = page.crawlStatus;
+    if (robots) {
+      const isBlocked = !robots.isAllowed(page.url, 'crawlith') ||
+        (!page.url.endsWith('/') && !robots.isAllowed(page.url + '/', 'crawlith'));
+      if (isBlocked) {
+        crawlStatus = 'blocked_by_robots';
+      }
+    }
+
+    // 1. Analyze Individual Components
+    const title = analyzeTitle(html);
+    const metaDescription = analyzeMetaDescription(html);
     const h1 = analyzeH1(html, title.value);
-    const content = baseContent[index];
-    const duplicationScore = (sentenceCountFrequency.get(content.uniqueSentenceCount) || 0) > 1 ? 100 : 0;
-    const thinScore = calculateThinContentScore(content, duplicationScore);
+    const content = analyzeContent(html);
     const images = analyzeImageAlts(html);
     const links = analyzeLinks(html, page.url, rootUrl);
     const structuredData = analyzeStructuredData(html);
 
-    const analysis: PageAnalysis = {
+    // 2. Accumulate Frequencies for Duplicates
+    if (title.value) {
+      const key = (title.value || '').trim().toLowerCase();
+      titleCounts.set(key, (titleCounts.get(key) || 0) + 1);
+    }
+    if (metaDescription.value) {
+      const key = (metaDescription.value || '').trim().toLowerCase();
+      metaCounts.set(key, (metaCounts.get(key) || 0) + 1);
+    }
+    sentenceCountFrequency.set(content.uniqueSentenceCount, (sentenceCountFrequency.get(content.uniqueSentenceCount) || 0) + 1);
+
+    // 3. Store Preliminary Result
+    results.push({
       url: page.url,
       status: page.status || 0,
       title,
       metaDescription,
       h1,
       content,
-      thinScore,
+      thinScore: 0, // Calculated in pass 2
       images,
       links,
       structuredData,
-      seoScore: 0,
+      seoScore: 0, // Calculated in pass 2
       meta: {
         canonical: page.canonical,
         noindex: page.noindex,
-        nofollow: page.nofollow
+        nofollow: page.nofollow,
+        crawlStatus
+      }
+    });
+  }
+
+  // 4. Finalize Statuses and Scores (Pass 2)
+  for (const analysis of results) {
+    // Check Title Duplicates
+    if (analysis.title.value) {
+      const key = (analysis.title.value || '').trim().toLowerCase();
+      if ((titleCounts.get(key) || 0) > 1) {
+        analysis.title.status = 'duplicate';
       }
-    };
+    }
 
+    // Check Meta Duplicates
+    if (analysis.metaDescription.value) {
+      const key = (analysis.metaDescription.value || '').trim().toLowerCase();
+      if ((metaCounts.get(key) || 0) > 1) {
+        analysis.metaDescription.status = 'duplicate';
+      }
+    }
+
+    // Check Content Duplication
+    const duplicationScore = (sentenceCountFrequency.get(analysis.content.uniqueSentenceCount) || 0) > 1 ? 100 : 0;
+    analysis.thinScore = calculateThinContentScore(analysis.content, duplicationScore);
+
+    // Calculate Final SEO Score
     analysis.seoScore = scorePageSeo(analysis);
-    return analysis;
-  });
+  }
+
+  return results;
 }
 
 function filterPageModules(
@@ -416,23 +427,7 @@ function filterPageModules(
   };
 }
 
-async function loadCrawlData(rootUrl: string, fromCrawl?: string): Promise<CrawlData> {
-  // If fromCrawl is provided, we could theoretically load JSON, but 
-  // we now default to DB fetching for all operations.
-
-  if (fromCrawl) {
-    try {
-      const content = await fs.readFile(fromCrawl, 'utf-8');
-      const raw = JSON.parse(content) as Record<string, unknown>;
-      const pages = parsePages(raw);
-      const graph = graphFromPages(rootUrl, pages, raw);
-      const metrics = calculateMetrics(graph, 5);
-      return { pages, metrics, graph };
-    } catch (_e) {
-      // Fallback downwards if file doesn't exist
-    }
-  }
-
+async function loadCrawlData(rootUrl: string): Promise<CrawlData> {
   const db = getDb();
   const siteRepo = new SiteRepository(db);
   const snapshotRepo = new SnapshotRepository(db);
@@ -442,107 +437,69 @@ async function loadCrawlData(rootUrl: string, fromCrawl?: string): Promise<Crawl
   const domain = urlObj.hostname.replace('www.', '');
   const site = siteRepo.firstOrCreateSite(domain);
 
-  const snapshot = snapshotRepo.getLatestSnapshot(site.id, 'completed');
-  if (!snapshot) {
-    throw new Error(`No completed snapshot found for ${rootUrl} in database.`);
+  let snapshot;
+  const page = pageRepo.getPage(site.id, rootUrl);
+  if (page && page.last_seen_snapshot_id) {
+    snapshot = snapshotRepo.getSnapshot(page.last_seen_snapshot_id);
   }
 
-  const graph = loadGraphFromSnapshot(snapshot.id);
-  const metrics = calculateMetrics(graph, 5);
-
-  // We also need the `pages` array for analysis. 
-  // It needs `html` which might not be fully available unless we look up from the DB or Graph.
-  // Wait, the Graph stores Node which doesn't contain HTML since we removed it from memory? 
-  // Actually, `loadGraphFromSnapshot` does NOT load actual raw HTML from nodes to save memory.
-  // We need HTML for `analyzeSite` module! So we must fetch it from `pageRepo`.
-
-  const dbPages = pageRepo.getPagesBySnapshot(snapshot.id);
-  const pages: CrawlPage[] = dbPages.map((p: any) => ({
-    url: p.normalized_url,
-    status: p.http_status || 0,
-    html: p.html || '',
-    depth: p.depth || 0
-  }));
-
-  return { pages, metrics, graph };
-}
-
-function parsePages(raw: Record<string, unknown>): CrawlPage[] {
-  if (Array.isArray(raw.pages)) {
-    return raw.pages.map((page) => {
-      const p = page as Record<string, unknown>;
-      return {
-        url: String(p.url || ''),
-        status: Number(p.status || 0),
-        html: typeof p.html === 'string' ? p.html : '',
-        depth: Number(p.depth || 0)
-      };
-    }).filter((page) => Boolean(page.url));
+  if (!snapshot) {
+    snapshot = snapshotRepo.getLatestSnapshot(site.id);
   }
 
-  if (Array.isArray(raw.nodes)) {
-    return raw.nodes.map((node) => {
-      const n = node as Record<string, unknown>;
-      return {
-        url: String(n.url || ''),
-        status: Number(n.status || 0),
-        html: typeof n.html === 'string' ? n.html : '',
-        depth: Number(n.depth || 0)
-      };
-    }).filter((page) => Boolean(page.url));
+  if (!snapshot) {
+    throw new Error(`No crawl data found for ${rootUrl} in database.`);
   }
 
-  return [];
-}
-
-function graphFromPages(rootUrl: string, pages: CrawlPage[], raw: Record<string, unknown>): Graph {
-  const graph = new Graph();
-
-  for (const page of pages) {
-    graph.addNode(page.url, page.depth || 0, page.status || 0);
-  }
+  const graph = loadGraphFromSnapshot(snapshot.id);
+  const metrics = calculateMetrics(graph, 5);
 
-  if (Array.isArray(raw.edges)) {
-    for (const edge of raw.edges) {
-      const e = edge as Record<string, unknown>;
-      if (typeof e.source === 'string' && typeof e.target === 'string') {
-        graph.addNode(e.source, 0, 0);
-        graph.addNode(e.target, 0, 0);
-        graph.addEdge(e.source, e.target);
-      }
+  // Use iterator to save memory
+  const dbPagesIterator = pageRepo.getPagesIteratorBySnapshot(snapshot.id);
+
+  // We need to map the DB pages to CrawlPage format lazily
+  const pagesGenerator = function* () {
+    for (const p of dbPagesIterator) {
+      yield {
+        url: p.normalized_url,
+        status: p.http_status || 0,
+        html: p.html || '',
+        depth: p.depth || 0,
+        canonical: p.canonical_url || undefined,
+        noindex: !!p.noindex,
+        nofollow: !!p.nofollow,
+        crawlStatus: graph.nodes.get(p.normalized_url)?.crawlStatus
+      } as CrawlPage;
     }
-    return graph;
-  }
-
-  for (const page of pages) {
-    if (!page.html) continue;
-    const linkAnalysis = analyzeLinks(page.html, page.url, rootUrl);
-    if (linkAnalysis.internalLinks === 0 && linkAnalysis.externalLinks === 0) continue;
-  }
+  };
 
-  return graph;
+  return { pages: pagesGenerator(), metrics, graph, snapshotId: snapshot.id, crawledAt: snapshot.created_at };
 }
 
-async function runLiveCrawl(url: string, options: AnalyzeOptions): Promise<CrawlData> {
+
+async function runLiveCrawl(url: string, options: AnalyzeOptions, context?: EngineContext): Promise<CrawlData> {
   const snapshotId = await crawl(url, {
-    limit: 1,
+    limit: 1, // Always limit to 1 for single page live analysis
     depth: 0,
     rate: options.rate,
     proxyUrl: options.proxyUrl,
     userAgent: options.userAgent,
     maxRedirects: options.maxRedirects,
-    debug: options.debug
-  });
+    debug: options.debug,
+    snapshotType: 'partial'
+  }, context) as number;
   const graph = loadGraphFromSnapshot(snapshotId);
   const pages = graph.getNodes().map((node) => ({
     url: node.url,
     status: node.status,
     html: node.html || '', // Include HTML
-    depth: node.depth
+    depth: node.depth,
+    crawlStatus: node.crawlStatus
   }));
   return {
     pages,
     metrics: calculateMetrics(graph, 1),
-    graph
+    graph,
+    snapshotId
   };
 }