@crawlith/core 0.1.0 → 0.1.1

package/dist/crawler/metricsRunner.js

@@ -6,30 +6,59 @@ import { PageRepository } from '../db/repositories/PageRepository.js';
 import { computePageRank } from '../graph/pagerank.js';
 import { calculateMetrics } from '../graph/metrics.js';
 import { computeHITS } from '../scoring/hits.js';
-export function runPostCrawlMetrics(snapshotId, maxDepth, limitReached = false) {
+import { calculateHealthScore, collectCrawlIssues } from '../scoring/health.js';
+export function runPostCrawlMetrics(snapshotId, maxDepth, context, limitReached = false, graphInstance) {
     const db = getDb();
     const metricsRepo = new MetricsRepository(db);
     const snapshotRepo = new SnapshotRepository(db);
     const pageRepo = new PageRepository(db);
+    const graph = graphInstance || loadGraphFromSnapshot(snapshotId);
+    // Fallback emitter
+    const emit = (event) => {
+        if (context) {
+            context.emit(event);
+        }
+        else {
+            if (event.type === 'error')
+                console.error(event.message);
+            else if (event.type !== 'debug')
+                console.log(event.message || event.phase);
+        }
+    };
     const snapshot = snapshotRepo.getSnapshot(snapshotId);
     if (!snapshot) {
-        console.error(`Snapshot ${snapshotId} not found`);
+        emit({ type: 'error', message: `Snapshot ${snapshotId} not found` });
         return;
     }
-    console.log('Loading graph for metrics calculation...');
-    const graph = loadGraphFromSnapshot(snapshotId);
-    console.log('Computing PageRank...');
+    if (!graphInstance) {
+        emit({ type: 'metrics:start', phase: 'Loading graph' });
+    }
+    emit({ type: 'metrics:start', phase: 'Computing PageRank' });
     computePageRank(graph);
-    console.log('Computing HITS...');
+    emit({ type: 'metrics:start', phase: 'Computing HITS' });
     computeHITS(graph);
-    console.log('Updating metrics in DB...');
+    emit({ type: 'metrics:start', phase: 'Updating metrics in DB' });
     const nodes = graph.getNodes();
+    // Pre-fetch all page IDs to avoid N+1 queries
+    // Use getPagesIdentityBySnapshot to avoid loading full page content (HTML) into memory again
+    const pages = pageRepo.getPagesIdentityBySnapshot(snapshotId);
+    const urlToId = new Map();
+    for (const p of pages) {
+        urlToId.set(p.normalized_url, p.id);
+    }
+    const clusterStmt = db.prepare(`
+        INSERT OR REPLACE INTO duplicate_clusters (id, snapshot_id, type, size, representative, severity)
+        VALUES (?, ?, ?, ?, ?, ?)
+    `);
+    const contentStmt = db.prepare(`
+        INSERT OR REPLACE INTO content_clusters (id, snapshot_id, count, primary_url, risk, shared_path_prefix)
+        VALUES (?, ?, ?, ?, ?, ?)
+    `);
     const tx = db.transaction(() => {
         for (const node of nodes) {
-            const pageId = pageRepo.getIdByUrl(snapshot.site_id, node.url);
+            const pageId = urlToId.get(node.url);
             if (!pageId)
                 continue;
-            const existing = metricsRepo.getMetricsForPage(snapshotId, pageId);
             metricsRepo.insertMetrics({
                 snapshot_id: snapshotId,
                 page_id: pageId,
@@ -38,11 +67,11 @@ export function runPostCrawlMetrics(snapshotId, maxDepth, limitReached = false)
                 pagerank: node.pageRank ?? null,
                 pagerank_score: node.pageRankScore ?? null,
                 link_role: node.linkRole ?? null,
-                crawl_status: existing?.crawl_status ?? null,
-                word_count: existing?.word_count ?? null,
-                thin_content_score: existing?.thin_content_score ?? null,
-                external_link_ratio: existing?.external_link_ratio ?? null,
-                orphan_score: existing?.orphan_score ?? null,
+                crawl_status: node.crawlStatus ?? null,
+                word_count: node.wordCount ?? null,
+                thin_content_score: node.thinContentScore ?? null,
+                external_link_ratio: node.externalLinkRatio ?? null,
+                orphan_score: node.orphanScore ?? null,
                 duplicate_cluster_id: node.duplicateClusterId ?? null,
                 duplicate_type: node.duplicateType ?? null,
                 is_cluster_primary: node.isClusterPrimary ? 1 : 0
@@ -62,47 +91,27 @@ export function runPostCrawlMetrics(snapshotId, maxDepth, limitReached = false)
             }
         }
         // Save duplicate clusters
-        if (graph.duplicateClusters.length > 0) {
-            const clusterStmt = db.prepare(`
-                INSERT OR REPLACE INTO duplicate_clusters (id, snapshot_id, type, size, representative, severity)
-                VALUES (?, ?, ?, ?, ?, ?)
-            `);
-            for (const cluster of graph.duplicateClusters) {
-                clusterStmt.run(cluster.id, snapshotId, cluster.type, cluster.size, cluster.representative, cluster.severity);
-            }
+        for (const cluster of graph.duplicateClusters) {
+            clusterStmt.run(cluster.id, snapshotId, cluster.type, cluster.size, cluster.representative, cluster.severity);
         }
         // Save content clusters
-        if (graph.contentClusters.length > 0) {
-            const contentStmt = db.prepare(`
-                INSERT OR REPLACE INTO content_clusters (id, snapshot_id, count, primary_url, risk, shared_path_prefix)
-                VALUES (?, ?, ?, ?, ?, ?)
-            `);
-            for (const cluster of graph.contentClusters) {
-                contentStmt.run(cluster.id, snapshotId, cluster.count, cluster.primaryUrl, cluster.risk, cluster.sharedPathPrefix ?? null);
-            }
+        for (const cluster of graph.contentClusters) {
+            contentStmt.run(cluster.id, snapshotId, cluster.count, cluster.primaryUrl, cluster.risk, cluster.sharedPathPrefix ?? null);
         }
     });
     tx();
-    console.log('Computing aggregate stats...');
+    emit({ type: 'metrics:start', phase: 'Computing aggregate stats' });
     const metrics = calculateMetrics(graph, maxDepth);
-    let totalScore = 0;
-    let totalWeight = 0;
-    for (const node of nodes) {
-        const score = node.authorityScore || node.pageRankScore || 0;
-        const depth = node.depth;
-        const weight = 1 / (depth + 1);
-        totalScore += score * weight;
-        totalWeight += weight;
-    }
-    const healthScore = totalWeight > 0 ? (totalScore / totalWeight) * 100 : 0;
-    const thinCountRow = db.prepare('SELECT count(*) as count FROM metrics WHERE snapshot_id = ? AND thin_content_score >= 70').get(snapshotId);
+    // Calculate penalty-based health score (matches CLI)
+    const issues = collectCrawlIssues(graph, metrics);
+    const health = calculateHealthScore(metrics.totalPages, issues);
     snapshotRepo.updateSnapshotStatus(snapshotId, 'completed', {
         node_count: metrics.totalPages,
         edge_count: metrics.totalEdges,
-        health_score: healthScore,
-        orphan_count: metrics.orphanPages.length,
-        thin_content_count: thinCountRow.count,
+        health_score: health.score,
+        orphan_count: issues.orphanPages,
+        thin_content_count: issues.thinContent,
         limit_reached: limitReached ? 1 : 0
     });
-    console.log('Metrics calculation complete.');
+    emit({ type: 'metrics:complete', durationMs: 0 });
 }

package/dist/crawler/sitemap.d.ts

@@ -1,4 +1,7 @@
+import { EngineContext } from '../events.js';
 export declare class Sitemap {
+    private context?;
+    constructor(context?: EngineContext | undefined);
     /**
      * Fetches and parses a sitemap (or sitemap index) to extract URLs.
      * Recursively handles sitemap indexes with loop detection and depth limits.

package/dist/crawler/sitemap.js

@@ -2,6 +2,10 @@ import { request } from 'undici';
 import * as cheerio from 'cheerio';
 import { normalizeUrl } from './normalize.js';
 export class Sitemap {
+    context;
+    constructor(context) {
+        this.context = context;
+    }
     /**
      * Fetches and parses a sitemap (or sitemap index) to extract URLs.
      * Recursively handles sitemap indexes with loop detection and depth limits.
@@ -64,7 +68,7 @@ export class Sitemap {
             }
         }
         catch (e) {
-            console.warn(`Failed to fetch sitemap ${url}:`, e);
+            this.context?.emit({ type: 'warn', message: `Failed to fetch sitemap ${url}`, context: e });
         }
     }
 }

package/dist/db/graphLoader.js

@@ -10,14 +10,17 @@ export function loadGraphFromSnapshot(snapshotId) {
     const edgeRepo = new EdgeRepository(db);
     const metricsRepo = new MetricsRepository(db);
     const snapshotRepo = new SnapshotRepository(db);
-    const pages = pageRepo.getPagesBySnapshot(snapshotId);
-    const metrics = metricsRepo.getMetrics(snapshotId);
+    const pages = pageRepo.getPagesIteratorBySnapshot(snapshotId);
+    const metrics = metricsRepo.getMetricsIterator(snapshotId);
     const snapshot = snapshotRepo.getSnapshot(snapshotId);
     const metricsMap = new Map();
     for (const m of metrics) {
         metricsMap.set(m.page_id, m);
     }
     const graph = new Graph();
+    let pagesFetched = 0;
+    let pagesCached = 0;
+    let pagesSkipped = 0;
     if (snapshot) {
         graph.limitReached = !!snapshot.limit_reached;
     }
@@ -26,6 +29,19 @@ export function loadGraphFromSnapshot(snapshotId) {
         idMap.set(p.id, p.normalized_url);
         graph.addNode(p.normalized_url, p.depth, p.http_status || 0);
         const m = metricsMap.get(p.id);
+        if (m) {
+            const isProcessed = m.crawl_status === 'fetched' ||
+                m.crawl_status === 'fetched_error' ||
+                m.crawl_status === 'network_error' ||
+                m.crawl_status === 'failed_after_retries' ||
+                m.crawl_status === 'blocked_by_robots';
+            if (isProcessed)
+                pagesFetched++;
+            else if (m.crawl_status === 'cached')
+                pagesCached++;
+            else if (m.crawl_status === 'skipped')
+                pagesSkipped++;
+        }
         let incrementalStatus;
         if (p.first_seen_snapshot_id === snapshotId) {
             incrementalStatus = 'new';
@@ -64,9 +80,15 @@ export function loadGraphFromSnapshot(snapshotId) {
             duplicateClusterId: m?.duplicate_cluster_id ?? undefined,
             duplicateType: m?.duplicate_type ?? undefined,
             isClusterPrimary: m?.is_cluster_primary ? true : undefined,
+            // Additional metrics
+            crawlStatus: m?.crawl_status || undefined,
+            wordCount: m?.word_count != null ? m.word_count : undefined,
+            thinContentScore: m?.thin_content_score != null ? m.thin_content_score : undefined,
+            externalLinkRatio: m?.external_link_ratio != null ? m.external_link_ratio : undefined,
+            orphanScore: m?.orphan_score != null ? m.orphan_score : undefined,
         });
     }
-    const edges = edgeRepo.getEdgesBySnapshot(snapshotId);
+    const edges = edgeRepo.getEdgesIteratorBySnapshot(snapshotId);
     for (const e of edges) {
         const source = idMap.get(e.source_page_id);
         const target = idMap.get(e.target_page_id);
@@ -92,5 +114,12 @@ export function loadGraphFromSnapshot(snapshotId) {
         risk: c.risk,
         sharedPathPrefix: c.shared_path_prefix || undefined
     }));
+    // Set session stats
+    graph.sessionStats = {
+        pagesFetched,
+        pagesCached,
+        pagesSkipped,
+        totalFound: idMap.size
+    };
     return graph;
 }

package/dist/db/index.d.ts

@@ -1,4 +1,7 @@
 import Database from 'better-sqlite3';
+export * from './repositories/SiteRepository.js';
+export * from './repositories/SnapshotRepository.js';
+export { initSchema } from './schema.js';
 export declare function getDbPath(): string;
 export declare function getDb(): Database.Database;
 export declare function closeDb(): void;

package/dist/db/index.js

@@ -4,6 +4,9 @@ import fs from 'node:fs';
 import os from 'node:os';
 import { initSchema } from './schema.js';
 let dbInstance = null;
+export * from './repositories/SiteRepository.js';
+export * from './repositories/SnapshotRepository.js';
+export { initSchema } from './schema.js';
 export function getDbPath() {
     if (process.env.NODE_ENV === 'test') {
         return ':memory:';
@@ -46,6 +49,7 @@ export function getDb() {
     // Integrity check on startup
     const integrity = db.pragma('integrity_check', { simple: true });
     if (integrity !== 'ok') {
+        // Reverted to console.warn to avoid breaking change
         console.warn('Database integrity check failed:', integrity);
     }
     // Initialize schema

package/dist/db/repositories/EdgeRepository.d.ts

@@ -12,5 +12,13 @@ export declare class EdgeRepository {
     private insertStmt;
     constructor(db: Database);
     insertEdge(snapshotId: number, sourcePageId: number, targetPageId: number, weight?: number, rel?: string): void;
+    insertEdges(edges: {
+        snapshot_id: number;
+        source_page_id: number;
+        target_page_id: number;
+        weight: number;
+        rel: string;
+    }[]): void;
     getEdgesBySnapshot(snapshotId: number): Edge[];
+    getEdgesIteratorBySnapshot(snapshotId: number): IterableIterator<Edge>;
 }

package/dist/db/repositories/EdgeRepository.js

@@ -11,7 +11,20 @@ export class EdgeRepository {
     insertEdge(snapshotId, sourcePageId, targetPageId, weight = 1.0, rel = 'internal') {
         this.insertStmt.run(snapshotId, sourcePageId, targetPageId, weight, rel);
     }
+    insertEdges(edges) {
+        if (edges.length === 0)
+            return;
+        const tx = this.db.transaction((edgesBatch) => {
+            for (const edge of edgesBatch) {
+                this.insertStmt.run(edge.snapshot_id, edge.source_page_id, edge.target_page_id, edge.weight, edge.rel);
+            }
+        });
+        tx(edges);
+    }
     getEdgesBySnapshot(snapshotId) {
         return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').all(snapshotId);
     }
+    getEdgesIteratorBySnapshot(snapshotId) {
+        return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').iterate(snapshotId);
+    }
 }

package/dist/db/repositories/MetricsRepository.d.ts

@@ -19,8 +19,11 @@ export interface DbMetrics {
 export declare class MetricsRepository {
     private db;
     private insertStmt;
+    private getByPageStmt;
     constructor(db: Database);
     insertMetrics(metrics: DbMetrics): void;
     getMetrics(snapshotId: number): DbMetrics[];
+    getMetricsIterator(snapshotId: number): IterableIterator<DbMetrics>;
     getMetricsForPage(snapshotId: number, pageId: number): DbMetrics | undefined;
+    insertMany(metricsList: DbMetrics[]): void;
 }

package/dist/db/repositories/MetricsRepository.js

@@ -1,8 +1,10 @@
 export class MetricsRepository {
     db;
     insertStmt;
+    getByPageStmt;
     constructor(db) {
         this.db = db;
+        this.getByPageStmt = this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ? AND page_id = ?');
         this.insertStmt = this.db.prepare(`
       INSERT OR REPLACE INTO metrics (
         snapshot_id, page_id, authority_score, hub_score, pagerank, pagerank_score,
@@ -21,7 +23,18 @@ export class MetricsRepository {
     getMetrics(snapshotId) {
         return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ?').all(snapshotId);
     }
+    getMetricsIterator(snapshotId) {
+        return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ?').iterate(snapshotId);
+    }
     getMetricsForPage(snapshotId, pageId) {
-        return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ? AND page_id = ?').get(snapshotId, pageId);
+        return this.getByPageStmt.get(snapshotId, pageId);
+    }
+    insertMany(metricsList) {
+        const insert = this.insertStmt;
+        const tx = this.db.transaction((items) => {
+            for (const item of items)
+                insert.run(item);
+        });
+        tx(metricsList);
     }
 }

package/dist/db/repositories/PageRepository.d.ts

@@ -42,6 +42,17 @@ export declare class PageRepository {
         last_seen_snapshot_id: number;
     }): number;
     getPage(siteId: number, url: string): Page | undefined;
+    getPagesByUrls(siteId: number, urls: string[]): Page[];
+    upsertMany(pages: (Partial<Page> & {
+        site_id: number;
+        normalized_url: string;
+        last_seen_snapshot_id: number;
+    })[]): Map<string, number>;
     getPagesBySnapshot(snapshotId: number): Page[];
+    getPagesIdentityBySnapshot(snapshotId: number): {
+        id: number;
+        normalized_url: string;
+    }[];
+    getPagesIteratorBySnapshot(snapshotId: number): IterableIterator<Page>;
     getIdByUrl(siteId: number, url: string): number | undefined;
 }

package/dist/db/repositories/PageRepository.js

@@ -20,24 +20,24 @@ export class PageRepository {
       )
       ON CONFLICT(site_id, normalized_url) DO UPDATE SET
         last_seen_snapshot_id = excluded.last_seen_snapshot_id,
-        http_status = excluded.http_status,
-        canonical_url = excluded.canonical_url,
-        content_hash = excluded.content_hash,
-        simhash = excluded.simhash,
-        etag = excluded.etag,
-        last_modified = excluded.last_modified,
-        html = excluded.html,
-        soft404_score = excluded.soft404_score,
-        noindex = excluded.noindex,
-        nofollow = excluded.nofollow,
-        security_error = excluded.security_error,
-        retries = excluded.retries,
-        depth = excluded.depth,
-        redirect_chain = excluded.redirect_chain,
-        bytes_received = excluded.bytes_received,
-        crawl_trap_flag = excluded.crawl_trap_flag,
-        crawl_trap_risk = excluded.crawl_trap_risk,
-        trap_type = excluded.trap_type,
+        http_status = CASE WHEN excluded.http_status != 0 THEN excluded.http_status ELSE pages.http_status END,
+        canonical_url = COALESCE(excluded.canonical_url, pages.canonical_url),
+        content_hash = COALESCE(excluded.content_hash, pages.content_hash),
+        simhash = COALESCE(excluded.simhash, pages.simhash),
+        etag = COALESCE(excluded.etag, pages.etag),
+        last_modified = COALESCE(excluded.last_modified, pages.last_modified),
+        html = COALESCE(excluded.html, pages.html),
+        soft404_score = COALESCE(excluded.soft404_score, pages.soft404_score),
+        noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END,
+        nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END,
+        security_error = COALESCE(excluded.security_error, pages.security_error),
+        retries = MAX(pages.retries, excluded.retries),
+        depth = MIN(pages.depth, excluded.depth),
+        redirect_chain = COALESCE(excluded.redirect_chain, pages.redirect_chain),
+        bytes_received = COALESCE(excluded.bytes_received, pages.bytes_received),
+        crawl_trap_flag = MAX(pages.crawl_trap_flag, excluded.crawl_trap_flag),
+        crawl_trap_risk = COALESCE(excluded.crawl_trap_risk, pages.crawl_trap_risk),
+        trap_type = COALESCE(excluded.trap_type, pages.trap_type),
         updated_at = datetime('now')
     `);
         this.getIdStmt = this.db.prepare('SELECT id FROM pages WHERE site_id = ? AND normalized_url = ?');
@@ -83,8 +83,101 @@ export class PageRepository {
     getPage(siteId, url) {
         return this.db.prepare('SELECT * FROM pages WHERE site_id = ? AND normalized_url = ?').get(siteId, url);
     }
+    getPagesByUrls(siteId, urls) {
+        if (urls.length === 0)
+            return [];
+        const chunkSize = 900;
+        const results = [];
+        for (let i = 0; i < urls.length; i += chunkSize) {
+            const chunk = urls.slice(i, i + chunkSize);
+            const placeholders = chunk.map(() => '?').join(',');
+            const chunkResults = this.db.prepare(`SELECT * FROM pages WHERE site_id = ? AND normalized_url IN (${placeholders})`).all(siteId, ...chunk);
+            results.push(...chunkResults);
+        }
+        return results;
+    }
+    upsertMany(pages) {
+        if (pages.length === 0)
+            return new Map();
+        const upsertStmtWithReturn = this.db.prepare(`
+      INSERT INTO pages (
+        site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id,
+        http_status, canonical_url, content_hash, simhash, etag, last_modified, html,
+        soft404_score, noindex, nofollow, security_error, retries, depth,
+        redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type,
+        updated_at
+      ) VALUES (
+        @site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id,
+        @http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html,
+        @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth,
+        @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
+        datetime('now')
+      )
+      ON CONFLICT(site_id, normalized_url) DO UPDATE SET
+        last_seen_snapshot_id = excluded.last_seen_snapshot_id,
+        http_status = CASE WHEN excluded.http_status != 0 THEN excluded.http_status ELSE pages.http_status END,
+        canonical_url = COALESCE(excluded.canonical_url, pages.canonical_url),
+        content_hash = COALESCE(excluded.content_hash, pages.content_hash),
+        simhash = COALESCE(excluded.simhash, pages.simhash),
+        etag = COALESCE(excluded.etag, pages.etag),
+        last_modified = COALESCE(excluded.last_modified, pages.last_modified),
+        html = COALESCE(excluded.html, pages.html),
+        soft404_score = COALESCE(excluded.soft404_score, pages.soft404_score),
+        noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END,
+        nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END,
+        security_error = COALESCE(excluded.security_error, pages.security_error),
+        retries = MAX(pages.retries, excluded.retries),
+        depth = MIN(pages.depth, excluded.depth),
+        redirect_chain = COALESCE(excluded.redirect_chain, pages.redirect_chain),
+        bytes_received = COALESCE(excluded.bytes_received, pages.bytes_received),
+        crawl_trap_flag = MAX(pages.crawl_trap_flag, excluded.crawl_trap_flag),
+        crawl_trap_risk = COALESCE(excluded.crawl_trap_risk, pages.crawl_trap_risk),
+        trap_type = COALESCE(excluded.trap_type, pages.trap_type),
+        updated_at = datetime('now')
+      RETURNING id
+    `);
+        const urlToId = new Map();
+        const tx = this.db.transaction((pagesBatch) => {
+            for (const page of pagesBatch) {
+                const params = {
+                    site_id: page.site_id,
+                    normalized_url: page.normalized_url,
+                    first_seen_snapshot_id: page.first_seen_snapshot_id ?? page.last_seen_snapshot_id,
+                    last_seen_snapshot_id: page.last_seen_snapshot_id,
+                    http_status: page.http_status ?? null,
+                    canonical_url: page.canonical_url ?? null,
+                    content_hash: page.content_hash ?? null,
+                    simhash: page.simhash ?? null,
+                    etag: page.etag ?? null,
+                    last_modified: page.last_modified ?? null,
+                    html: page.html ?? null,
+                    soft404_score: page.soft404_score ?? null,
+                    noindex: page.noindex ?? 0,
+                    nofollow: page.nofollow ?? 0,
+                    security_error: page.security_error ?? null,
+                    retries: page.retries ?? 0,
+                    depth: page.depth ?? 0,
+                    redirect_chain: page.redirect_chain ?? null,
+                    bytes_received: page.bytes_received ?? null,
+                    crawl_trap_flag: page.crawl_trap_flag ?? 0,
+                    crawl_trap_risk: page.crawl_trap_risk ?? null,
+                    trap_type: page.trap_type ?? null,
+                };
+                const row = upsertStmtWithReturn.get(params);
+                urlToId.set(page.normalized_url, row.id);
+            }
+        });
+        tx(pages);
+        return urlToId;
+    }
     getPagesBySnapshot(snapshotId) {
-        return this.db.prepare('SELECT * FROM pages WHERE last_seen_snapshot_id = ?').all(snapshotId);
+        return this.db.prepare('SELECT p.* FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND p.first_seen_snapshot_id <= ?').all(snapshotId, snapshotId);
+    }
+    getPagesIdentityBySnapshot(snapshotId) {
+        return this.db.prepare('SELECT p.id, p.normalized_url FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND p.first_seen_snapshot_id <= ?').all(snapshotId, snapshotId);
+    }
+    getPagesIteratorBySnapshot(snapshotId) {
+        return this.db.prepare('SELECT p.* FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND p.first_seen_snapshot_id <= ?').iterate(snapshotId, snapshotId);
     }
     getIdByUrl(siteId, url) {
         const row = this.getIdStmt.get(siteId, url);

package/dist/db/repositories/SiteRepository.d.ts

@@ -9,7 +9,10 @@ export interface Site {
 export declare class SiteRepository {
     private db;
     constructor(db: Database);
+    getSiteById(id: number): Site | undefined;
     getSite(domain: string): Site | undefined;
+    getAllSites(): Site[];
     createSite(domain: string): number;
     firstOrCreateSite(domain: string): Site;
+    deleteSite(id: number): void;
 }

package/dist/db/repositories/SiteRepository.js

@@ -3,9 +3,15 @@ export class SiteRepository {
     constructor(db) {
         this.db = db;
     }
+    getSiteById(id) {
+        return this.db.prepare('SELECT * FROM sites WHERE id = ?').get(id);
+    }
     getSite(domain) {
         return this.db.prepare('SELECT * FROM sites WHERE domain = ?').get(domain);
     }
+    getAllSites() {
+        return this.db.prepare('SELECT * FROM sites ORDER BY domain ASC').all();
+    }
     createSite(domain) {
         const stmt = this.db.prepare('INSERT INTO sites (domain) VALUES (?)');
         const info = stmt.run(domain);
@@ -19,4 +25,7 @@ export class SiteRepository {
         }
         return site;
     }
+    deleteSite(id) {
+        this.db.prepare('DELETE FROM sites WHERE id = ?').run(id);
+    }
 }

package/dist/db/repositories/SnapshotRepository.d.ts

@@ -17,6 +17,8 @@ export declare class SnapshotRepository {
     constructor(db: Database);
     createSnapshot(siteId: number, type: 'full' | 'partial' | 'incremental', status?: 'running' | 'completed' | 'failed'): number;
     getLatestSnapshot(siteId: number, status?: 'completed' | 'running' | 'failed'): Snapshot | undefined;
+    getSnapshotCount(siteId: number): number;
     updateSnapshotStatus(id: number, status: 'completed' | 'failed', stats?: Partial<Snapshot>): void;
     getSnapshot(id: number): Snapshot | undefined;
+    deleteSnapshot(id: number): void;
 }

package/dist/db/repositories/SnapshotRepository.js

@@ -4,20 +4,29 @@ export class SnapshotRepository {
         this.db = db;
     }
     createSnapshot(siteId, type, status = 'running') {
+        // Basic throttling or sleep if needed for tests, but generally SQLite is fast enough to have diff timestamps if not in same ms.
+        // However, if we run in memory, created_at is default current time.
+        // If two snapshots created in same second, ORDER BY created_at DESC is unstable or equal.
+        // We should rely on ID for stability if timestamps are equal, but the query uses created_at.
+        // Let's ensure we can also order by ID as tie-breaker.
         const stmt = this.db.prepare('INSERT INTO snapshots (site_id, type, status) VALUES (?, ?, ?)');
         const info = stmt.run(siteId, type, status);
         return info.lastInsertRowid;
     }
     getLatestSnapshot(siteId, status) {
-        let sql = 'SELECT * FROM snapshots WHERE site_id = ?';
+        let sql = 'SELECT * FROM snapshots WHERE site_id = ? AND type != \'partial\'';
         const params = [siteId];
         if (status) {
             sql += ' AND status = ?';
             params.push(status);
         }
-        sql += ' ORDER BY created_at DESC LIMIT 1';
+        sql += ' ORDER BY created_at DESC, id DESC LIMIT 1';
         return this.db.prepare(sql).get(...params);
     }
+    getSnapshotCount(siteId) {
+        const result = this.db.prepare('SELECT COUNT(*) as count FROM snapshots WHERE site_id = ?').get(siteId);
+        return result.count;
+    }
     updateSnapshotStatus(id, status, stats = {}) {
         const sets = ['status = ?'];
         const params = [status];
@@ -52,4 +61,16 @@ export class SnapshotRepository {
     getSnapshot(id) {
         return this.db.prepare('SELECT * FROM snapshots WHERE id = ?').get(id);
     }
+    deleteSnapshot(id) {
+        const tx = this.db.transaction(() => {
+            // Unlink pages from this snapshot to prevent FK constraint violations or data inconsistencies
+            this.db.prepare('UPDATE pages SET first_seen_snapshot_id = NULL WHERE first_seen_snapshot_id = ?').run(id);
+            this.db.prepare('UPDATE pages SET last_seen_snapshot_id = NULL WHERE last_seen_snapshot_id = ?').run(id);
+            // Cleanup: Delete pages that are no longer referenced by any snapshot
+            this.db.prepare('DELETE FROM pages WHERE first_seen_snapshot_id IS NULL AND last_seen_snapshot_id IS NULL').run();
+            // Delete the snapshot
+            this.db.prepare('DELETE FROM snapshots WHERE id = ?').run(id);
+        });
+        tx();
+    }
 }