@crawlith/core 0.1.0 → 0.1.1

package/src/crawler/metricsRunner.ts

@@ -6,37 +6,71 @@ import { PageRepository } from '../db/repositories/PageRepository.js';
 import { computePageRank } from '../graph/pagerank.js';
 import { calculateMetrics } from '../graph/metrics.js';
 import { computeHITS } from '../scoring/hits.js';
+import { EngineContext } from '../events.js';
+import { calculateHealthScore, collectCrawlIssues } from '../scoring/health.js';
 
-export function runPostCrawlMetrics(snapshotId: number, maxDepth: number, limitReached: boolean = false) {
+import { Graph } from '../graph/graph.js';
+
+export function runPostCrawlMetrics(snapshotId: number, maxDepth: number, context?: EngineContext, limitReached: boolean = false, graphInstance?: Graph) {
     const db = getDb();
     const metricsRepo = new MetricsRepository(db);
     const snapshotRepo = new SnapshotRepository(db);
     const pageRepo = new PageRepository(db);
 
+    const graph = graphInstance || loadGraphFromSnapshot(snapshotId);
+
+    // Fallback emitter
+    const emit = (event: any) => {
+        if (context) {
+            context.emit(event);
+        } else {
+            if (event.type === 'error') console.error(event.message);
+            else if (event.type !== 'debug') console.log(event.message || event.phase);
+        }
+    };
+
     const snapshot = snapshotRepo.getSnapshot(snapshotId);
     if (!snapshot) {
-        console.error(`Snapshot ${snapshotId} not found`);
+        emit({ type: 'error', message: `Snapshot ${snapshotId} not found` });
         return;
     }
 
-    console.log('Loading graph for metrics calculation...');
-    const graph = loadGraphFromSnapshot(snapshotId);
+    if (!graphInstance) {
+        emit({ type: 'metrics:start', phase: 'Loading graph' });
+    }
 
-    console.log('Computing PageRank...');
+    emit({ type: 'metrics:start', phase: 'Computing PageRank' });
     computePageRank(graph);
 
-    console.log('Computing HITS...');
+    emit({ type: 'metrics:start', phase: 'Computing HITS' });
     computeHITS(graph);
 
-    console.log('Updating metrics in DB...');
+    emit({ type: 'metrics:start', phase: 'Updating metrics in DB' });
     const nodes = graph.getNodes();
 
+    // Pre-fetch all page IDs to avoid N+1 queries
+    // Use getPagesIdentityBySnapshot to avoid loading full page content (HTML) into memory again
+    const pages = pageRepo.getPagesIdentityBySnapshot(snapshotId);
+    const urlToId = new Map<string, number>();
+    for (const p of pages) {
+        urlToId.set(p.normalized_url, p.id);
+    }
+
+    const clusterStmt = db.prepare(`
+        INSERT OR REPLACE INTO duplicate_clusters (id, snapshot_id, type, size, representative, severity)
+        VALUES (?, ?, ?, ?, ?, ?)
+    `);
+
+    const contentStmt = db.prepare(`
+        INSERT OR REPLACE INTO content_clusters (id, snapshot_id, count, primary_url, risk, shared_path_prefix)
+        VALUES (?, ?, ?, ?, ?, ?)
+    `);
+
     const tx = db.transaction(() => {
         for (const node of nodes) {
-            const pageId = pageRepo.getIdByUrl(snapshot.site_id, node.url);
+            const pageId = urlToId.get(node.url);
             if (!pageId) continue;
 
-            const existing = metricsRepo.getMetricsForPage(snapshotId, pageId);
 
             metricsRepo.insertMetrics({
                 snapshot_id: snapshotId,
@@ -46,11 +80,11 @@ export function runPostCrawlMetrics(snapshotId: number, maxDepth: number, limitR
                 pagerank: node.pageRank ?? null,
                 pagerank_score: node.pageRankScore ?? null,
                 link_role: node.linkRole ?? null,
-                crawl_status: existing?.crawl_status ?? null,
-                word_count: existing?.word_count ?? null,
-                thin_content_score: existing?.thin_content_score ?? null,
-                external_link_ratio: existing?.external_link_ratio ?? null,
-                orphan_score: existing?.orphan_score ?? null,
+                crawl_status: node.crawlStatus ?? null,
+                word_count: node.wordCount ?? null,
+                thin_content_score: node.thinContentScore ?? null,
+                external_link_ratio: node.externalLinkRatio ?? null,
+                orphan_score: node.orphanScore ?? null,
                 duplicate_cluster_id: node.duplicateClusterId ?? null,
                 duplicate_type: node.duplicateType ?? null,
                 is_cluster_primary: node.isClusterPrimary ? 1 : 0
@@ -72,53 +106,32 @@ export function runPostCrawlMetrics(snapshotId: number, maxDepth: number, limitR
         }
 
         // Save duplicate clusters
-        if (graph.duplicateClusters.length > 0) {
-            const clusterStmt = db.prepare(`
-                INSERT OR REPLACE INTO duplicate_clusters (id, snapshot_id, type, size, representative, severity)
-                VALUES (?, ?, ?, ?, ?, ?)
-            `);
-            for (const cluster of graph.duplicateClusters) {
-                clusterStmt.run(cluster.id, snapshotId, cluster.type, cluster.size, cluster.representative, cluster.severity);
-            }
+        for (const cluster of graph.duplicateClusters) {
+            clusterStmt.run(cluster.id, snapshotId, cluster.type, cluster.size, cluster.representative, cluster.severity);
         }
 
         // Save content clusters
-        if (graph.contentClusters.length > 0) {
-            const contentStmt = db.prepare(`
-                INSERT OR REPLACE INTO content_clusters (id, snapshot_id, count, primary_url, risk, shared_path_prefix)
-                VALUES (?, ?, ?, ?, ?, ?)
-            `);
-            for (const cluster of graph.contentClusters) {
-                contentStmt.run(cluster.id, snapshotId, cluster.count, cluster.primaryUrl, cluster.risk, cluster.sharedPathPrefix ?? null);
-            }
+        for (const cluster of graph.contentClusters) {
+            contentStmt.run(cluster.id, snapshotId, cluster.count, cluster.primaryUrl, cluster.risk, cluster.sharedPathPrefix ?? null);
         }
     });
     tx();
 
-    console.log('Computing aggregate stats...');
+    emit({ type: 'metrics:start', phase: 'Computing aggregate stats' });
     const metrics = calculateMetrics(graph, maxDepth);
 
-    let totalScore = 0;
-    let totalWeight = 0;
-    for (const node of nodes) {
-        const score = node.authorityScore || node.pageRankScore || 0;
-        const depth = node.depth;
-        const weight = 1 / (depth + 1);
-        totalScore += score * weight;
-        totalWeight += weight;
-    }
-    const healthScore = totalWeight > 0 ? (totalScore / totalWeight) * 100 : 0;
-
-    const thinCountRow = db.prepare('SELECT count(*) as count FROM metrics WHERE snapshot_id = ? AND thin_content_score >= 70').get(snapshotId) as { count: number };
+    // Calculate penalty-based health score (matches CLI)
+    const issues = collectCrawlIssues(graph, metrics);
+    const health = calculateHealthScore(metrics.totalPages, issues);
 
     snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', {
         node_count: metrics.totalPages,
         edge_count: metrics.totalEdges,
-        health_score: healthScore,
-        orphan_count: metrics.orphanPages.length,
-        thin_content_count: thinCountRow.count,
+        health_score: health.score,
+        orphan_count: issues.orphanPages,
+        thin_content_count: issues.thinContent,
         limit_reached: limitReached ? 1 : 0
     });
 
-    console.log('Metrics calculation complete.');
+    emit({ type: 'metrics:complete', durationMs: 0 });
 }

package/src/crawler/sitemap.ts

@@ -1,8 +1,11 @@
 import { request } from 'undici';
 import * as cheerio from 'cheerio';
 import { normalizeUrl } from './normalize.js';
+import { EngineContext } from '../events.js';
 
 export class Sitemap {
+  constructor(private context?: EngineContext) {}
+
   /**
    * Fetches and parses a sitemap (or sitemap index) to extract URLs.
    * Recursively handles sitemap indexes with loop detection and depth limits.
@@ -67,7 +70,7 @@ export class Sitemap {
         await res.body.dump();
       }
     } catch (e) {
-      console.warn(`Failed to fetch sitemap ${url}:`, e);
+      this.context?.emit({ type: 'warn', message: `Failed to fetch sitemap ${url}`, context: e });
     }
   }
 }

package/src/db/graphLoader.ts

@@ -12,8 +12,8 @@ export function loadGraphFromSnapshot(snapshotId: number): Graph {
     const metricsRepo = new MetricsRepository(db);
     const snapshotRepo = new SnapshotRepository(db);
 
-    const pages = pageRepo.getPagesBySnapshot(snapshotId);
-    const metrics = metricsRepo.getMetrics(snapshotId);
+    const pages = pageRepo.getPagesIteratorBySnapshot(snapshotId);
+    const metrics = metricsRepo.getMetricsIterator(snapshotId);
     const snapshot = snapshotRepo.getSnapshot(snapshotId);
     const metricsMap = new Map<number, DbMetrics>();
     for (const m of metrics) {
@@ -21,6 +21,10 @@ export function loadGraphFromSnapshot(snapshotId: number): Graph {
     }
 
     const graph = new Graph();
+    let pagesFetched = 0;
+    let pagesCached = 0;
+    let pagesSkipped = 0;
+
     if (snapshot) {
         graph.limitReached = !!snapshot.limit_reached;
     }
@@ -31,6 +35,18 @@ export function loadGraphFromSnapshot(snapshotId: number): Graph {
         graph.addNode(p.normalized_url, p.depth, p.http_status || 0);
 
         const m = metricsMap.get(p.id);
+        if (m) {
+            const isProcessed = m.crawl_status === 'fetched' ||
+                m.crawl_status === 'fetched_error' ||
+                m.crawl_status === 'network_error' ||
+                m.crawl_status === 'failed_after_retries' ||
+                m.crawl_status === 'blocked_by_robots';
+
+            if (isProcessed) pagesFetched++;
+            else if (m.crawl_status === 'cached') pagesCached++;
+            else if (m.crawl_status === 'skipped') pagesSkipped++;
+        }
+
         let incrementalStatus: 'new' | 'changed' | 'unchanged' | undefined;
         if (p.first_seen_snapshot_id === snapshotId) {
             incrementalStatus = 'new';
@@ -68,10 +84,16 @@ export function loadGraphFromSnapshot(snapshotId: number): Graph {
             duplicateClusterId: m?.duplicate_cluster_id ?? undefined,
             duplicateType: m?.duplicate_type ?? undefined,
             isClusterPrimary: m?.is_cluster_primary ? true : undefined,
+            // Additional metrics
+            crawlStatus: m?.crawl_status || undefined,
+            wordCount: m?.word_count != null ? m.word_count : undefined,
+            thinContentScore: m?.thin_content_score != null ? m.thin_content_score : undefined,
+            externalLinkRatio: m?.external_link_ratio != null ? m.external_link_ratio : undefined,
+            orphanScore: m?.orphan_score != null ? m.orphan_score : undefined,
         });
     }
 
-    const edges = edgeRepo.getEdgesBySnapshot(snapshotId);
+    const edges = edgeRepo.getEdgesIteratorBySnapshot(snapshotId);
 
     for (const e of edges) {
         const source = idMap.get(e.source_page_id);
@@ -101,5 +123,13 @@ export function loadGraphFromSnapshot(snapshotId: number): Graph {
         sharedPathPrefix: c.shared_path_prefix || undefined
     }));
 
+    // Set session stats
+    graph.sessionStats = {
+        pagesFetched,
+        pagesCached,
+        pagesSkipped,
+        totalFound: idMap.size
+    };
+
     return graph;
 }

package/src/db/index.ts

@@ -6,6 +6,10 @@ import { initSchema } from './schema.js';
 
 let dbInstance: Database.Database | null = null;
 
+export * from './repositories/SiteRepository.js';
+export * from './repositories/SnapshotRepository.js';
+export { initSchema } from './schema.js';
+
 export function getDbPath(): string {
   if (process.env.NODE_ENV === 'test') {
     return ':memory:';
@@ -52,6 +56,7 @@ export function getDb(): Database.Database {
   // Integrity check on startup
   const integrity = db.pragma('integrity_check', { simple: true });
   if (integrity !== 'ok') {
+    // Reverted to console.warn to avoid breaking change
     console.warn('Database integrity check failed:', integrity);
   }
 

package/src/db/repositories/EdgeRepository.ts

@@ -23,7 +23,21 @@ export class EdgeRepository {
     this.insertStmt.run(snapshotId, sourcePageId, targetPageId, weight, rel);
   }
 
+  insertEdges(edges: { snapshot_id: number; source_page_id: number; target_page_id: number; weight: number; rel: string }[]) {
+    if (edges.length === 0) return;
+    const tx = this.db.transaction((edgesBatch) => {
+      for (const edge of edgesBatch) {
+        this.insertStmt.run(edge.snapshot_id, edge.source_page_id, edge.target_page_id, edge.weight, edge.rel);
+      }
+    });
+    tx(edges);
+  }
+
   getEdgesBySnapshot(snapshotId: number): Edge[] {
     return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').all(snapshotId) as Edge[];
   }
+
+  getEdgesIteratorBySnapshot(snapshotId: number): IterableIterator<Edge> {
+    return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').iterate(snapshotId) as IterableIterator<Edge>;
+  }
 }

package/src/db/repositories/MetricsRepository.ts

@@ -20,8 +20,10 @@ export interface DbMetrics {
 
 export class MetricsRepository {
   private insertStmt;
+  private getByPageStmt;
 
   constructor(private db: Database) {
+    this.getByPageStmt = this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ? AND page_id = ?');
     this.insertStmt = this.db.prepare(`
       INSERT OR REPLACE INTO metrics (
         snapshot_id, page_id, authority_score, hub_score, pagerank, pagerank_score,
@@ -43,7 +45,19 @@ export class MetricsRepository {
     return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ?').all(snapshotId) as DbMetrics[];
   }
 
+  getMetricsIterator(snapshotId: number): IterableIterator<DbMetrics> {
+    return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ?').iterate(snapshotId) as IterableIterator<DbMetrics>;
+  }
+
   getMetricsForPage(snapshotId: number, pageId: number): DbMetrics | undefined {
-    return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ? AND page_id = ?').get(snapshotId, pageId) as DbMetrics | undefined;
+    return this.getByPageStmt.get(snapshotId, pageId) as DbMetrics | undefined;
+  }
+
+  insertMany(metricsList: DbMetrics[]) {
+    const insert = this.insertStmt;
+    const tx = this.db.transaction((items: DbMetrics[]) => {
+      for (const item of items) insert.run(item);
+    });
+    tx(metricsList);
   }
 }

package/src/db/repositories/PageRepository.ts

@@ -49,24 +49,24 @@ export class PageRepository {
       )
       ON CONFLICT(site_id, normalized_url) DO UPDATE SET
         last_seen_snapshot_id = excluded.last_seen_snapshot_id,
-        http_status = excluded.http_status,
-        canonical_url = excluded.canonical_url,
-        content_hash = excluded.content_hash,
-        simhash = excluded.simhash,
-        etag = excluded.etag,
-        last_modified = excluded.last_modified,
-        html = excluded.html,
-        soft404_score = excluded.soft404_score,
-        noindex = excluded.noindex,
-        nofollow = excluded.nofollow,
-        security_error = excluded.security_error,
-        retries = excluded.retries,
-        depth = excluded.depth,
-        redirect_chain = excluded.redirect_chain,
-        bytes_received = excluded.bytes_received,
-        crawl_trap_flag = excluded.crawl_trap_flag,
-        crawl_trap_risk = excluded.crawl_trap_risk,
-        trap_type = excluded.trap_type,
+        http_status = CASE WHEN excluded.http_status != 0 THEN excluded.http_status ELSE pages.http_status END,
+        canonical_url = COALESCE(excluded.canonical_url, pages.canonical_url),
+        content_hash = COALESCE(excluded.content_hash, pages.content_hash),
+        simhash = COALESCE(excluded.simhash, pages.simhash),
+        etag = COALESCE(excluded.etag, pages.etag),
+        last_modified = COALESCE(excluded.last_modified, pages.last_modified),
+        html = COALESCE(excluded.html, pages.html),
+        soft404_score = COALESCE(excluded.soft404_score, pages.soft404_score),
+        noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END,
+        nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END,
+        security_error = COALESCE(excluded.security_error, pages.security_error),
+        retries = MAX(pages.retries, excluded.retries),
+        depth = MIN(pages.depth, excluded.depth),
+        redirect_chain = COALESCE(excluded.redirect_chain, pages.redirect_chain),
+        bytes_received = COALESCE(excluded.bytes_received, pages.bytes_received),
+        crawl_trap_flag = MAX(pages.crawl_trap_flag, excluded.crawl_trap_flag),
+        crawl_trap_risk = COALESCE(excluded.crawl_trap_risk, pages.crawl_trap_risk),
+        trap_type = COALESCE(excluded.trap_type, pages.trap_type),
         updated_at = datetime('now')
     `);
 
@@ -117,8 +117,108 @@ export class PageRepository {
     return this.db.prepare('SELECT * FROM pages WHERE site_id = ? AND normalized_url = ?').get(siteId, url) as Page | undefined;
   }
 
+  getPagesByUrls(siteId: number, urls: string[]): Page[] {
+    if (urls.length === 0) return [];
+    const chunkSize = 900;
+    const results: Page[] = [];
+
+    for (let i = 0; i < urls.length; i += chunkSize) {
+      const chunk = urls.slice(i, i + chunkSize);
+      const placeholders = chunk.map(() => '?').join(',');
+      const chunkResults = this.db.prepare(`SELECT * FROM pages WHERE site_id = ? AND normalized_url IN (${placeholders})`).all(siteId, ...chunk) as Page[];
+      results.push(...chunkResults);
+    }
+
+    return results;
+  }
+
+  upsertMany(pages: (Partial<Page> & { site_id: number; normalized_url: string; last_seen_snapshot_id: number })[]): Map<string, number> {
+    if (pages.length === 0) return new Map();
+
+    const upsertStmtWithReturn = this.db.prepare(`
+      INSERT INTO pages (
+        site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id,
+        http_status, canonical_url, content_hash, simhash, etag, last_modified, html,
+        soft404_score, noindex, nofollow, security_error, retries, depth,
+        redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type,
+        updated_at
+      ) VALUES (
+        @site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id,
+        @http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html,
+        @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth,
+        @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
+        datetime('now')
+      )
+      ON CONFLICT(site_id, normalized_url) DO UPDATE SET
+        last_seen_snapshot_id = excluded.last_seen_snapshot_id,
+        http_status = CASE WHEN excluded.http_status != 0 THEN excluded.http_status ELSE pages.http_status END,
+        canonical_url = COALESCE(excluded.canonical_url, pages.canonical_url),
+        content_hash = COALESCE(excluded.content_hash, pages.content_hash),
+        simhash = COALESCE(excluded.simhash, pages.simhash),
+        etag = COALESCE(excluded.etag, pages.etag),
+        last_modified = COALESCE(excluded.last_modified, pages.last_modified),
+        html = COALESCE(excluded.html, pages.html),
+        soft404_score = COALESCE(excluded.soft404_score, pages.soft404_score),
+        noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END,
+        nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END,
+        security_error = COALESCE(excluded.security_error, pages.security_error),
+        retries = MAX(pages.retries, excluded.retries),
+        depth = MIN(pages.depth, excluded.depth),
+        redirect_chain = COALESCE(excluded.redirect_chain, pages.redirect_chain),
+        bytes_received = COALESCE(excluded.bytes_received, pages.bytes_received),
+        crawl_trap_flag = MAX(pages.crawl_trap_flag, excluded.crawl_trap_flag),
+        crawl_trap_risk = COALESCE(excluded.crawl_trap_risk, pages.crawl_trap_risk),
+        trap_type = COALESCE(excluded.trap_type, pages.trap_type),
+        updated_at = datetime('now')
+      RETURNING id
+    `);
+
+    const urlToId = new Map<string, number>();
+    const tx = this.db.transaction((pagesBatch) => {
+      for (const page of pagesBatch) {
+        const params = {
+          site_id: page.site_id,
+          normalized_url: page.normalized_url,
+          first_seen_snapshot_id: page.first_seen_snapshot_id ?? page.last_seen_snapshot_id,
+          last_seen_snapshot_id: page.last_seen_snapshot_id,
+          http_status: page.http_status ?? null,
+          canonical_url: page.canonical_url ?? null,
+          content_hash: page.content_hash ?? null,
+          simhash: page.simhash ?? null,
+          etag: page.etag ?? null,
+          last_modified: page.last_modified ?? null,
+          html: page.html ?? null,
+          soft404_score: page.soft404_score ?? null,
+          noindex: page.noindex ?? 0,
+          nofollow: page.nofollow ?? 0,
+          security_error: page.security_error ?? null,
+          retries: page.retries ?? 0,
+          depth: page.depth ?? 0,
+          redirect_chain: page.redirect_chain ?? null,
+          bytes_received: page.bytes_received ?? null,
+          crawl_trap_flag: page.crawl_trap_flag ?? 0,
+          crawl_trap_risk: page.crawl_trap_risk ?? null,
+          trap_type: page.trap_type ?? null,
+        };
+        const row = upsertStmtWithReturn.get(params) as { id: number };
+        urlToId.set(page.normalized_url, row.id);
+      }
+    });
+
+    tx(pages);
+    return urlToId;
+  }
+
   getPagesBySnapshot(snapshotId: number): Page[] {
-    return this.db.prepare('SELECT * FROM pages WHERE last_seen_snapshot_id = ?').all(snapshotId) as Page[];
+    return this.db.prepare('SELECT p.* FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND p.first_seen_snapshot_id <= ?').all(snapshotId, snapshotId) as Page[];
+  }
+
+  getPagesIdentityBySnapshot(snapshotId: number): { id: number; normalized_url: string }[] {
+    return this.db.prepare('SELECT p.id, p.normalized_url FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND p.first_seen_snapshot_id <= ?').all(snapshotId, snapshotId) as { id: number; normalized_url: string }[];
+  }
+
+  getPagesIteratorBySnapshot(snapshotId: number): IterableIterator<Page> {
+    return this.db.prepare('SELECT p.* FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND p.first_seen_snapshot_id <= ?').iterate(snapshotId, snapshotId) as IterableIterator<Page>;
   }
 
   getIdByUrl(siteId: number, url: string): number | undefined {

package/src/db/repositories/SiteRepository.ts

@@ -11,10 +11,18 @@ export interface Site {
 export class SiteRepository {
   constructor(private db: Database) { }
 
+  getSiteById(id: number): Site | undefined {
+    return this.db.prepare('SELECT * FROM sites WHERE id = ?').get(id) as Site | undefined;
+  }
+
   getSite(domain: string): Site | undefined {
     return this.db.prepare('SELECT * FROM sites WHERE domain = ?').get(domain) as Site | undefined;
   }
 
+  getAllSites(): Site[] {
+    return this.db.prepare('SELECT * FROM sites ORDER BY domain ASC').all() as Site[];
+  }
+
   createSite(domain: string): number {
     const stmt = this.db.prepare('INSERT INTO sites (domain) VALUES (?)');
     const info = stmt.run(domain);
@@ -29,4 +37,7 @@ export class SiteRepository {
     }
     return site!;
   }
+  deleteSite(id: number): void {
+    this.db.prepare('DELETE FROM sites WHERE id = ?').run(id);
+  }
 }

package/src/db/repositories/SnapshotRepository.ts

@@ -15,25 +15,35 @@ export interface Snapshot {
 }
 
 export class SnapshotRepository {
-  constructor(private db: Database) {}
+  constructor(private db: Database) { }
 
   createSnapshot(siteId: number, type: 'full' | 'partial' | 'incremental', status: 'running' | 'completed' | 'failed' = 'running'): number {
+    // Basic throttling or sleep if needed for tests, but generally SQLite is fast enough to have diff timestamps if not in same ms.
+    // However, if we run in memory, created_at is default current time.
+    // If two snapshots created in same second, ORDER BY created_at DESC is unstable or equal.
+    // We should rely on ID for stability if timestamps are equal, but the query uses created_at.
+    // Let's ensure we can also order by ID as tie-breaker.
     const stmt = this.db.prepare('INSERT INTO snapshots (site_id, type, status) VALUES (?, ?, ?)');
     const info = stmt.run(siteId, type, status);
     return info.lastInsertRowid as number;
   }
 
   getLatestSnapshot(siteId: number, status?: 'completed' | 'running' | 'failed'): Snapshot | undefined {
-    let sql = 'SELECT * FROM snapshots WHERE site_id = ?';
+    let sql = 'SELECT * FROM snapshots WHERE site_id = ? AND type != \'partial\'';
     const params: any[] = [siteId];
     if (status) {
       sql += ' AND status = ?';
       params.push(status);
     }
-    sql += ' ORDER BY created_at DESC LIMIT 1';
+    sql += ' ORDER BY created_at DESC, id DESC LIMIT 1';
     return this.db.prepare(sql).get(...params) as Snapshot | undefined;
   }
 
+  getSnapshotCount(siteId: number): number {
+    const result = this.db.prepare('SELECT COUNT(*) as count FROM snapshots WHERE site_id = ?').get(siteId) as { count: number };
+    return result.count;
+  }
+
   updateSnapshotStatus(id: number, status: 'completed' | 'failed', stats: Partial<Snapshot> = {}) {
     const sets: string[] = ['status = ?'];
     const params: any[] = [status];
@@ -71,4 +81,19 @@ export class SnapshotRepository {
   getSnapshot(id: number): Snapshot | undefined {
     return this.db.prepare('SELECT * FROM snapshots WHERE id = ?').get(id) as Snapshot | undefined;
   }
+
+  deleteSnapshot(id: number): void {
+    const tx = this.db.transaction(() => {
+      // Unlink pages from this snapshot to prevent FK constraint violations or data inconsistencies
+      this.db.prepare('UPDATE pages SET first_seen_snapshot_id = NULL WHERE first_seen_snapshot_id = ?').run(id);
+      this.db.prepare('UPDATE pages SET last_seen_snapshot_id = NULL WHERE last_seen_snapshot_id = ?').run(id);
+
+      // Cleanup: Delete pages that are no longer referenced by any snapshot
+      this.db.prepare('DELETE FROM pages WHERE first_seen_snapshot_id IS NULL AND last_seen_snapshot_id IS NULL').run();
+
+      // Delete the snapshot
+      this.db.prepare('DELETE FROM snapshots WHERE id = ?').run(id);
+    });
+    tx();
+  }
 }

package/src/events.ts

@@ -0,0 +1,16 @@
+export type CrawlEvent =
+  | { type: 'crawl:start'; url: string }
+  | { type: 'crawl:success'; url: string; status: number; durationMs: number; depth?: number }
+  | { type: 'crawl:error'; url: string; error: string; depth?: number }
+  | { type: 'crawl:limit-reached'; limit: number }
+  | { type: 'queue:enqueue'; url: string; depth: number }
+  | { type: 'metrics:start'; phase: string }
+  | { type: 'metrics:complete'; durationMs: number }
+  | { type: 'debug'; message: string; context?: unknown }
+  | { type: 'info'; message: string; context?: unknown }
+  | { type: 'warn'; message: string; context?: unknown }
+  | { type: 'error'; message: string; error?: unknown; context?: unknown };
+
+export interface EngineContext {
+  emit: (event: CrawlEvent) => void;
+}