@crawlith/core 0.1.0

package/dist/db/index.js

@@ -0,0 +1,61 @@
+import Database from 'better-sqlite3';
+import path from 'node:path';
+import fs from 'node:fs';
+import os from 'node:os';
+import { initSchema } from './schema.js';
+let dbInstance = null;
+export function getDbPath() {
+    if (process.env.NODE_ENV === 'test') {
+        return ':memory:';
+    }
+    if (process.env.CRAWLITH_DB_PATH) {
+        return process.env.CRAWLITH_DB_PATH;
+    }
+    const homeDir = os.homedir();
+    const crawlithDir = path.join(homeDir, '.crawlith');
+    if (!fs.existsSync(crawlithDir)) {
+        fs.mkdirSync(crawlithDir, { recursive: true });
+        // Set permissions to 700 (user only)
+        fs.chmodSync(crawlithDir, 0o700);
+    }
+    return path.join(crawlithDir, 'crawlith.db');
+}
+export function getDb() {
+    if (dbInstance) {
+        return dbInstance;
+    }
+    const dbPath = getDbPath();
+    const db = new Database(dbPath);
+    // Hardening & Performance Configuration
+    db.pragma('journal_mode = WAL');
+    db.pragma('synchronous = NORMAL');
+    db.pragma('foreign_keys = ON');
+    db.pragma('temp_store = MEMORY');
+    db.pragma('mmap_size = 30000000000');
+    db.pragma('cache_size = -20000');
+    db.pragma('busy_timeout = 5000');
+    // Security controls
+    // Ensure file permissions are 600 (user read/write only)
+    try {
+        fs.chmodSync(dbPath, 0o600);
+    }
+    catch (_e) {
+        // might fail on first creation if file doesn't exist yet, but better-sqlite3 creates it
+        // so we can try again or ignore if it's new
+    }
+    // Integrity check on startup
+    const integrity = db.pragma('integrity_check', { simple: true });
+    if (integrity !== 'ok') {
+        console.warn('Database integrity check failed:', integrity);
+    }
+    // Initialize schema
+    initSchema(db);
+    dbInstance = db;
+    return db;
+}
+export function closeDb() {
+    if (dbInstance) {
+        dbInstance.close();
+        dbInstance = null;
+    }
+}

package/dist/db/repositories/EdgeRepository.d.ts

@@ -0,0 +1,16 @@
+import { Database } from 'better-sqlite3';
+export interface Edge {
+    id: number;
+    snapshot_id: number;
+    source_page_id: number;
+    target_page_id: number;
+    weight: number;
+    rel: 'nofollow' | 'sponsored' | 'ugc' | 'internal' | 'external' | 'unknown';
+}
+export declare class EdgeRepository {
+    private db;
+    private insertStmt;
+    constructor(db: Database);
+    insertEdge(snapshotId: number, sourcePageId: number, targetPageId: number, weight?: number, rel?: string): void;
+    getEdgesBySnapshot(snapshotId: number): Edge[];
+}

package/dist/db/repositories/EdgeRepository.js

@@ -0,0 +1,17 @@
+export class EdgeRepository {
+    db;
+    insertStmt;
+    constructor(db) {
+        this.db = db;
+        this.insertStmt = this.db.prepare(`
+      INSERT INTO edges (snapshot_id, source_page_id, target_page_id, weight, rel)
+      VALUES (?, ?, ?, ?, ?)
+    `);
+    }
+    insertEdge(snapshotId, sourcePageId, targetPageId, weight = 1.0, rel = 'internal') {
+        this.insertStmt.run(snapshotId, sourcePageId, targetPageId, weight, rel);
+    }
+    getEdgesBySnapshot(snapshotId) {
+        return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').all(snapshotId);
+    }
+}

package/dist/db/repositories/MetricsRepository.d.ts

@@ -0,0 +1,26 @@
+import { Database } from 'better-sqlite3';
+export interface DbMetrics {
+    snapshot_id: number;
+    page_id: number;
+    authority_score: number | null;
+    hub_score: number | null;
+    pagerank: number | null;
+    pagerank_score: number | null;
+    link_role: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral' | null;
+    crawl_status: string | null;
+    word_count: number | null;
+    thin_content_score: number | null;
+    external_link_ratio: number | null;
+    orphan_score: number | null;
+    duplicate_cluster_id: string | null;
+    duplicate_type: 'exact' | 'near' | 'template_heavy' | 'none' | null;
+    is_cluster_primary: number;
+}
+export declare class MetricsRepository {
+    private db;
+    private insertStmt;
+    constructor(db: Database);
+    insertMetrics(metrics: DbMetrics): void;
+    getMetrics(snapshotId: number): DbMetrics[];
+    getMetricsForPage(snapshotId: number, pageId: number): DbMetrics | undefined;
+}

package/dist/db/repositories/MetricsRepository.js

@@ -0,0 +1,27 @@
+export class MetricsRepository {
+    db;
+    insertStmt;
+    constructor(db) {
+        this.db = db;
+        this.insertStmt = this.db.prepare(`
+      INSERT OR REPLACE INTO metrics (
+        snapshot_id, page_id, authority_score, hub_score, pagerank, pagerank_score,
+        link_role, crawl_status, word_count, thin_content_score, external_link_ratio,
+        orphan_score, duplicate_cluster_id, duplicate_type, is_cluster_primary
+      ) VALUES (
+        @snapshot_id, @page_id, @authority_score, @hub_score, @pagerank, @pagerank_score,
+        @link_role, @crawl_status, @word_count, @thin_content_score, @external_link_ratio,
+        @orphan_score, @duplicate_cluster_id, @duplicate_type, @is_cluster_primary
+      )
+    `);
+    }
+    insertMetrics(metrics) {
+        this.insertStmt.run(metrics);
+    }
+    getMetrics(snapshotId) {
+        return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ?').all(snapshotId);
+    }
+    getMetricsForPage(snapshotId, pageId) {
+        return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ? AND page_id = ?').get(snapshotId, pageId);
+    }
+}

package/dist/db/repositories/PageRepository.d.ts

@@ -0,0 +1,47 @@
+import { Database } from 'better-sqlite3';
+export interface Page {
+    id: number;
+    site_id: number;
+    normalized_url: string;
+    first_seen_snapshot_id: number | null;
+    last_seen_snapshot_id: number | null;
+    http_status: number | null;
+    canonical_url: string | null;
+    content_hash: string | null;
+    simhash: string | null;
+    etag: string | null;
+    last_modified: string | null;
+    html: string | null;
+    soft404_score: number | null;
+    noindex: number;
+    nofollow: number;
+    security_error: string | null;
+    retries: number;
+    depth: number;
+    redirect_chain: string | null;
+    bytes_received: number | null;
+    crawl_trap_flag: number;
+    crawl_trap_risk: number | null;
+    trap_type: string | null;
+    created_at: string;
+    updated_at: string;
+}
+export declare class PageRepository {
+    private db;
+    private upsertStmt;
+    private getIdStmt;
+    constructor(db: Database);
+    upsertPage(page: Partial<Page> & {
+        site_id: number;
+        normalized_url: string;
+        last_seen_snapshot_id: number;
+    }): import("better-sqlite3").RunResult;
+    upsertAndGetId(page: Partial<Page> & {
+        site_id: number;
+        normalized_url: string;
+        last_seen_snapshot_id: number;
+    }): number;
+    getPage(siteId: number, url: string): Page | undefined;
+    getPagesBySnapshot(snapshotId: number): Page[];
+    getIdByUrl(siteId: number, url: string): number | undefined;
+}

package/dist/db/repositories/PageRepository.js

@@ -0,0 +1,93 @@
+export class PageRepository {
+    db;
+    upsertStmt;
+    getIdStmt;
+    constructor(db) {
+        this.db = db;
+        this.upsertStmt = this.db.prepare(`
+      INSERT INTO pages (
+        site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id,
+        http_status, canonical_url, content_hash, simhash, etag, last_modified, html,
+        soft404_score, noindex, nofollow, security_error, retries, depth,
+        redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type,
+        updated_at
+      ) VALUES (
+        @site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id,
+        @http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html,
+        @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth,
+        @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
+        datetime('now')
+      )
+      ON CONFLICT(site_id, normalized_url) DO UPDATE SET
+        last_seen_snapshot_id = excluded.last_seen_snapshot_id,
+        http_status = excluded.http_status,
+        canonical_url = excluded.canonical_url,
+        content_hash = excluded.content_hash,
+        simhash = excluded.simhash,
+        etag = excluded.etag,
+        last_modified = excluded.last_modified,
+        html = excluded.html,
+        soft404_score = excluded.soft404_score,
+        noindex = excluded.noindex,
+        nofollow = excluded.nofollow,
+        security_error = excluded.security_error,
+        retries = excluded.retries,
+        depth = excluded.depth,
+        redirect_chain = excluded.redirect_chain,
+        bytes_received = excluded.bytes_received,
+        crawl_trap_flag = excluded.crawl_trap_flag,
+        crawl_trap_risk = excluded.crawl_trap_risk,
+        trap_type = excluded.trap_type,
+        updated_at = datetime('now')
+    `);
+        this.getIdStmt = this.db.prepare('SELECT id FROM pages WHERE site_id = ? AND normalized_url = ?');
+    }
+    upsertPage(page) {
+        const params = {
+            site_id: page.site_id,
+            normalized_url: page.normalized_url,
+            first_seen_snapshot_id: page.first_seen_snapshot_id ?? page.last_seen_snapshot_id,
+            last_seen_snapshot_id: page.last_seen_snapshot_id,
+            http_status: page.http_status ?? null,
+            canonical_url: page.canonical_url ?? null,
+            content_hash: page.content_hash ?? null,
+            simhash: page.simhash ?? null,
+            etag: page.etag ?? null,
+            last_modified: page.last_modified ?? null,
+            html: page.html ?? null,
+            soft404_score: page.soft404_score ?? null,
+            noindex: page.noindex ?? 0,
+            nofollow: page.nofollow ?? 0,
+            security_error: page.security_error ?? null,
+            retries: page.retries ?? 0,
+            depth: page.depth ?? 0,
+            redirect_chain: page.redirect_chain ?? null,
+            bytes_received: page.bytes_received ?? null,
+            crawl_trap_flag: page.crawl_trap_flag ?? 0,
+            crawl_trap_risk: page.crawl_trap_risk ?? null,
+            trap_type: page.trap_type ?? null,
+        };
+        const info = this.upsertStmt.run(params);
+        return info;
+    }
+    upsertAndGetId(page) {
+        const tx = this.db.transaction(() => {
+            this.upsertPage(page);
+            const row = this.getIdStmt.get(page.site_id, page.normalized_url);
+            if (!row)
+                throw new Error(`Failed to retrieve ID for upserted page: ${page.normalized_url}`);
+            return row.id;
+        });
+        return tx();
+    }
+    getPage(siteId, url) {
+        return this.db.prepare('SELECT * FROM pages WHERE site_id = ? AND normalized_url = ?').get(siteId, url);
+    }
+    getPagesBySnapshot(snapshotId) {
+        return this.db.prepare('SELECT * FROM pages WHERE last_seen_snapshot_id = ?').all(snapshotId);
+    }
+    getIdByUrl(siteId, url) {
+        const row = this.getIdStmt.get(siteId, url);
+        return row?.id;
+    }
+}

package/dist/db/repositories/SiteRepository.d.ts

@@ -0,0 +1,15 @@
+import { Database } from 'better-sqlite3';
+export interface Site {
+    id: number;
+    domain: string;
+    created_at: string;
+    settings_json: string | null;
+    is_active: number;
+}
+export declare class SiteRepository {
+    private db;
+    constructor(db: Database);
+    getSite(domain: string): Site | undefined;
+    createSite(domain: string): number;
+    firstOrCreateSite(domain: string): Site;
+}

package/dist/db/repositories/SiteRepository.js

@@ -0,0 +1,22 @@
+export class SiteRepository {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    getSite(domain) {
+        return this.db.prepare('SELECT * FROM sites WHERE domain = ?').get(domain);
+    }
+    createSite(domain) {
+        const stmt = this.db.prepare('INSERT INTO sites (domain) VALUES (?)');
+        const info = stmt.run(domain);
+        return info.lastInsertRowid;
+    }
+    firstOrCreateSite(domain) {
+        let site = this.getSite(domain);
+        if (!site) {
+            this.createSite(domain);
+            site = this.getSite(domain);
+        }
+        return site;
+    }
+}

package/dist/db/repositories/SnapshotRepository.d.ts

@@ -0,0 +1,22 @@
+import { Database } from 'better-sqlite3';
+export interface Snapshot {
+    id: number;
+    site_id: number;
+    type: 'full' | 'partial' | 'incremental';
+    created_at: string;
+    node_count: number;
+    edge_count: number;
+    status: 'running' | 'completed' | 'failed';
+    limit_reached: number;
+    health_score: number | null;
+    orphan_count: number | null;
+    thin_content_count: number | null;
+}
+export declare class SnapshotRepository {
+    private db;
+    constructor(db: Database);
+    createSnapshot(siteId: number, type: 'full' | 'partial' | 'incremental', status?: 'running' | 'completed' | 'failed'): number;
+    getLatestSnapshot(siteId: number, status?: 'completed' | 'running' | 'failed'): Snapshot | undefined;
+    updateSnapshotStatus(id: number, status: 'completed' | 'failed', stats?: Partial<Snapshot>): void;
+    getSnapshot(id: number): Snapshot | undefined;
+}

package/dist/db/repositories/SnapshotRepository.js

@@ -0,0 +1,55 @@
+export class SnapshotRepository {
+    db;
+    constructor(db) {
+        this.db = db;
+    }
+    createSnapshot(siteId, type, status = 'running') {
+        const stmt = this.db.prepare('INSERT INTO snapshots (site_id, type, status) VALUES (?, ?, ?)');
+        const info = stmt.run(siteId, type, status);
+        return info.lastInsertRowid;
+    }
+    getLatestSnapshot(siteId, status) {
+        let sql = 'SELECT * FROM snapshots WHERE site_id = ?';
+        const params = [siteId];
+        if (status) {
+            sql += ' AND status = ?';
+            params.push(status);
+        }
+        sql += ' ORDER BY created_at DESC LIMIT 1';
+        return this.db.prepare(sql).get(...params);
+    }
+    updateSnapshotStatus(id, status, stats = {}) {
+        const sets = ['status = ?'];
+        const params = [status];
+        if (stats.node_count !== undefined) {
+            sets.push('node_count = ?');
+            params.push(stats.node_count);
+        }
+        if (stats.edge_count !== undefined) {
+            sets.push('edge_count = ?');
+            params.push(stats.edge_count);
+        }
+        if (stats.limit_reached !== undefined) {
+            sets.push('limit_reached = ?');
+            params.push(stats.limit_reached);
+        }
+        if (stats.health_score !== undefined) {
+            sets.push('health_score = ?');
+            params.push(stats.health_score);
+        }
+        if (stats.orphan_count !== undefined) {
+            sets.push('orphan_count = ?');
+            params.push(stats.orphan_count);
+        }
+        if (stats.thin_content_count !== undefined) {
+            sets.push('thin_content_count = ?');
+            params.push(stats.thin_content_count);
+        }
+        params.push(id);
+        const sql = `UPDATE snapshots SET ${sets.join(', ')} WHERE id = ?`;
+        this.db.prepare(sql).run(...params);
+    }
+    getSnapshot(id) {
+        return this.db.prepare('SELECT * FROM snapshots WHERE id = ?').get(id);
+    }
+}

package/dist/db/schema.d.ts

@@ -0,0 +1,2 @@
+import { Database } from 'better-sqlite3';
+export declare function initSchema(db: Database): void;

package/dist/db/schema.js

@@ -0,0 +1,169 @@
+export function initSchema(db) {
+    // Sites Table
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS sites (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      domain TEXT UNIQUE NOT NULL,
+      created_at TEXT DEFAULT (datetime('now')),
+      settings_json TEXT,
+      is_active INTEGER DEFAULT 1
+    );
+  `);
+    // Snapshots Table
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS snapshots (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      site_id INTEGER NOT NULL,
+      type TEXT CHECK(type IN ('full', 'partial', 'incremental')) NOT NULL,
+      created_at TEXT DEFAULT (datetime('now')),
+      node_count INTEGER DEFAULT 0,
+      edge_count INTEGER DEFAULT 0,
+      status TEXT CHECK(status IN ('running', 'completed', 'failed')) DEFAULT 'running',
+      limit_reached INTEGER DEFAULT 0,
+      health_score REAL,
+      orphan_count INTEGER,
+      thin_content_count INTEGER,
+      FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE
+    );
+  `);
+    // Pages Table
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS pages (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      site_id INTEGER NOT NULL,
+      normalized_url TEXT NOT NULL,
+      first_seen_snapshot_id INTEGER,
+      last_seen_snapshot_id INTEGER,
+      http_status INTEGER,
+      canonical_url TEXT,
+      content_hash TEXT,
+      simhash TEXT,
+      etag TEXT,
+      last_modified TEXT,
+      html TEXT,
+      soft404_score REAL,
+      noindex INTEGER DEFAULT 0,
+      nofollow INTEGER DEFAULT 0,
+      security_error TEXT,
+      retries INTEGER DEFAULT 0,
+      depth INTEGER DEFAULT 0,
+      redirect_chain TEXT,
+      bytes_received INTEGER,
+      crawl_trap_flag INTEGER DEFAULT 0,
+      crawl_trap_risk REAL,
+      trap_type TEXT,
+      created_at TEXT DEFAULT (datetime('now')),
+      updated_at TEXT DEFAULT (datetime('now')),
+      FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE,
+      FOREIGN KEY(first_seen_snapshot_id) REFERENCES snapshots(id),
+      FOREIGN KEY(last_seen_snapshot_id) REFERENCES snapshots(id),
+      UNIQUE(site_id, normalized_url)
+    );
+  `);
+    // Index for Pages
+    db.exec(`CREATE INDEX IF NOT EXISTS idx_pages_site_last_seen ON pages(site_id, last_seen_snapshot_id);`);
+    // Edges Table
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS edges (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      snapshot_id INTEGER NOT NULL,
+      source_page_id INTEGER NOT NULL,
+      target_page_id INTEGER NOT NULL,
+      weight REAL DEFAULT 1.0,
+      rel TEXT CHECK(rel IN ('nofollow', 'sponsored', 'ugc', 'internal', 'external', 'unknown')) DEFAULT 'internal',
+      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
+      FOREIGN KEY(source_page_id) REFERENCES pages(id) ON DELETE CASCADE,
+      FOREIGN KEY(target_page_id) REFERENCES pages(id) ON DELETE CASCADE
+    );
+  `);
+    // Index for Edges
+    db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot_source ON edges(snapshot_id, source_page_id);`);
+    db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot ON edges(snapshot_id);`);
+    // Metrics Table
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS metrics (
+      snapshot_id INTEGER NOT NULL,
+      page_id INTEGER NOT NULL,
+      authority_score REAL,
+      hub_score REAL,
+      pagerank REAL,
+      pagerank_score REAL,
+      link_role TEXT CHECK(link_role IN ('hub', 'authority', 'power', 'balanced', 'peripheral')),
+      crawl_status TEXT,
+      word_count INTEGER,
+      thin_content_score REAL,
+      external_link_ratio REAL,
+      orphan_score INTEGER,
+      duplicate_cluster_id TEXT,
+      duplicate_type TEXT CHECK(duplicate_type IN ('exact', 'near', 'template_heavy', 'none')),
+      is_cluster_primary INTEGER DEFAULT 0,
+      PRIMARY KEY(snapshot_id, page_id),
+      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
+      FOREIGN KEY(page_id) REFERENCES pages(id) ON DELETE CASCADE
+    );
+  `);
+    db.exec(`CREATE INDEX IF NOT EXISTS idx_metrics_snapshot ON metrics(snapshot_id);`);
+    // Duplicate Clusters Table
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS duplicate_clusters (
+      id TEXT NOT NULL,
+      snapshot_id INTEGER NOT NULL,
+      type TEXT CHECK(type IN ('exact', 'near', 'template_heavy')) NOT NULL,
+      size INTEGER NOT NULL,
+      representative TEXT NOT NULL,
+      severity TEXT CHECK(severity IN ('low', 'medium', 'high')) NOT NULL,
+      PRIMARY KEY(snapshot_id, id),
+      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
+    );
+  `);
+    // Content Clusters Table
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS content_clusters (
+      id INTEGER NOT NULL,
+      snapshot_id INTEGER NOT NULL,
+      count INTEGER NOT NULL,
+      primary_url TEXT NOT NULL,
+      risk TEXT CHECK(risk IN ('low', 'medium', 'high')) NOT NULL,
+      shared_path_prefix TEXT,
+      PRIMARY KEY(snapshot_id, id),
+      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
+    );
+  `);
+    // Migration: add columns to existing DBs that were created before this update
+    migrateSchema(db);
+}
+function migrateSchema(db) {
+    // Add missing columns to pages (safe: ALTER TABLE ADD COLUMN is idempotent-safe with try/catch)
+    const pageColumns = [
+        ['redirect_chain', 'TEXT'],
+        ['bytes_received', 'INTEGER'],
+        ['crawl_trap_flag', 'INTEGER DEFAULT 0'],
+        ['crawl_trap_risk', 'REAL'],
+        ['trap_type', 'TEXT'],
+    ];
+    for (const [col, type] of pageColumns) {
+        try {
+            db.exec(`ALTER TABLE pages ADD COLUMN ${col} ${type}`);
+        }
+        catch { /* already exists */ }
+    }
+    // Add missing columns to edges
+    try {
+        db.exec('ALTER TABLE edges ADD COLUMN weight REAL DEFAULT 1.0');
+    }
+    catch { /* already exists */ }
+    // Add missing columns to metrics
+    const metricsColumns = [
+        ['pagerank_score', 'REAL'],
+        ['link_role', 'TEXT'],
+        ['duplicate_cluster_id', 'TEXT'],
+        ['duplicate_type', 'TEXT'],
+        ['is_cluster_primary', 'INTEGER DEFAULT 0'],
+    ];
+    for (const [col, type] of metricsColumns) {
+        try {
+            db.exec(`ALTER TABLE metrics ADD COLUMN ${col} ${type}`);
+        }
+        catch { /* already exists */ }
+    }
+}

package/dist/diff/compare.d.ts

@@ -0,0 +1,26 @@
+import { Graph } from '../graph/graph.js';
+export interface DiffResult {
+    addedUrls: string[];
+    removedUrls: string[];
+    changedStatus: {
+        url: string;
+        oldStatus: number;
+        newStatus: number;
+    }[];
+    changedCanonical: {
+        url: string;
+        oldCanonical: string | null;
+        newCanonical: string | null;
+    }[];
+    changedDuplicateGroup: {
+        url: string;
+        oldGroup: string | null;
+        newGroup: string | null;
+    }[];
+    metricDeltas: {
+        structuralEntropy: number;
+        orphanCount: number;
+        crawlEfficiency: number;
+    };
+}
+export declare function compareGraphs(oldGraph: Graph, newGraph: Graph): DiffResult;

package/dist/diff/compare.js

@@ -0,0 +1,64 @@
+import { calculateMetrics } from '../graph/metrics.js';
+export function compareGraphs(oldGraph, newGraph) {
+    const oldNodes = new Map(oldGraph.getNodes().map(n => [n.url, n]));
+    const newNodes = new Map(newGraph.getNodes().map(n => [n.url, n]));
+    const addedUrls = [];
+    const removedUrls = [];
+    const changedStatus = [];
+    const changedCanonical = [];
+    const changedDuplicateGroup = [];
+    // Added & Changed
+    for (const [url, newNode] of newNodes) {
+        const oldNode = oldNodes.get(url);
+        if (!oldNode) {
+            addedUrls.push(url);
+        }
+        else {
+            // Changed Status
+            if (oldNode.status !== newNode.status) {
+                changedStatus.push({ url, oldStatus: oldNode.status, newStatus: newNode.status });
+            }
+            // Changed Canonical
+            if (oldNode.canonical !== newNode.canonical) {
+                changedCanonical.push({
+                    url,
+                    oldCanonical: oldNode.canonical || null,
+                    newCanonical: newNode.canonical || null
+                });
+            }
+            // Changed Duplicate Group
+            const oldGroup = oldNode.duplicateClusterId || null;
+            const newGroup = newNode.duplicateClusterId || null;
+            if (oldGroup !== newGroup) {
+                changedDuplicateGroup.push({
+                    url,
+                    oldGroup,
+                    newGroup
+                });
+            }
+        }
+    }
+    // Removed
+    for (const url of oldNodes.keys()) {
+        if (!newNodes.has(url)) {
+            removedUrls.push(url);
+        }
+    }
+    // Metrics
+    // maxDepth is ignored by current calculateMetrics implementation but required by signature
+    const oldMetrics = calculateMetrics(oldGraph, 10);
+    const newMetrics = calculateMetrics(newGraph, 10);
+    const metricDeltas = {
+        structuralEntropy: newMetrics.structuralEntropy - oldMetrics.structuralEntropy,
+        orphanCount: newMetrics.orphanPages.length - oldMetrics.orphanPages.length,
+        crawlEfficiency: newMetrics.crawlEfficiencyScore - oldMetrics.crawlEfficiencyScore
+    };
+    return {
+        addedUrls,
+        removedUrls,
+        changedStatus,
+        changedCanonical,
+        changedDuplicateGroup,
+        metricDeltas
+    };
+}

package/dist/graph/cluster.d.ts

@@ -0,0 +1,6 @@
+import { Graph, ClusterInfo } from './graph.js';
+/**
+ * Detects content clusters using 64-bit SimHash and Hamming Distance.
+ * Uses band optimization to reduce O(n^2) comparisons.
+ */
+export declare function detectContentClusters(graph: Graph, threshold?: number, minSize?: number): ClusterInfo[];