@crawlith/core 0.1.1 → 0.1.2

package/dist/db/repositories/SiteRepository.d.ts

@@ -2,6 +2,8 @@ import { Database } from 'better-sqlite3';
 export interface Site {
     id: number;
     domain: string;
+    preferred_url: string | null;
+    ssl: number | null;
     created_at: string;
     settings_json: string | null;
     is_active: number;
@@ -13,6 +15,10 @@ export declare class SiteRepository {
     getSite(domain: string): Site | undefined;
     getAllSites(): Site[];
     createSite(domain: string): number;
+    updateSitePreference(id: number, prefs: {
+        preferred_url: string;
+        ssl: number;
+    }): void;
     firstOrCreateSite(domain: string): Site;
     deleteSite(id: number): void;
 }

package/dist/db/repositories/SiteRepository.js

@@ -17,6 +17,10 @@ export class SiteRepository {
         const info = stmt.run(domain);
         return info.lastInsertRowid;
     }
+    updateSitePreference(id, prefs) {
+        const stmt = this.db.prepare('UPDATE sites SET preferred_url = ?, ssl = ? WHERE id = ?');
+        stmt.run(prefs.preferred_url, prefs.ssl, id);
+    }
     firstOrCreateSite(domain) {
         let site = this.getSite(domain);
         if (!site) {

package/dist/db/repositories/SnapshotRepository.d.ts

@@ -2,11 +2,11 @@ import { Database } from 'better-sqlite3';
 export interface Snapshot {
     id: number;
     site_id: number;
-    type: 'full' | 'partial' | 'incremental';
+    run_type: 'completed' | 'incremental' | 'single';
     created_at: string;
     node_count: number;
     edge_count: number;
-    status: 'running' | 'completed' | 'failed';
+    status: 'queued' | 'running' | 'completed' | 'failed' | 'cancelled';
     limit_reached: number;
     health_score: number | null;
     orphan_count: number | null;
@@ -15,10 +15,17 @@ export interface Snapshot {
 export declare class SnapshotRepository {
     private db;
     constructor(db: Database);
-    createSnapshot(siteId: number, type: 'full' | 'partial' | 'incremental', status?: 'running' | 'completed' | 'failed'): number;
-    getLatestSnapshot(siteId: number, status?: 'completed' | 'running' | 'failed'): Snapshot | undefined;
+    createSnapshot(siteId: number, runType: Snapshot['run_type'], status?: Snapshot['status']): number;
+    getLatestSnapshot(siteId: number, status?: Snapshot['status'], includeSingle?: boolean): Snapshot | undefined;
+    touchSnapshot(id: number): void;
     getSnapshotCount(siteId: number): number;
-    updateSnapshotStatus(id: number, status: 'completed' | 'failed', stats?: Partial<Snapshot>): void;
+    /**
+     * Returns true if the site has ever had a completed full or incremental crawl.
+     * Single snapshots (from page --live) do NOT count as a "first crawl".
+     */
+    hasFullCrawl(siteId: number): boolean;
+    updateSnapshotStatus(id: number, status: Snapshot['status'], stats?: Partial<Snapshot>): void;
     getSnapshot(id: number): Snapshot | undefined;
     deleteSnapshot(id: number): void;
+    pruneSnapshots(siteId: number, maxSnapshots: number, maxSingleSnapshots: number, protectedSnapshotId?: number): void;
 }

package/dist/db/repositories/SnapshotRepository.js

@@ -3,18 +3,16 @@ export class SnapshotRepository {
     constructor(db) {
         this.db = db;
     }
-    createSnapshot(siteId, type, status = 'running') {
-        // Basic throttling or sleep if needed for tests, but generally SQLite is fast enough to have diff timestamps if not in same ms.
-        // However, if we run in memory, created_at is default current time.
-        // If two snapshots created in same second, ORDER BY created_at DESC is unstable or equal.
-        // We should rely on ID for stability if timestamps are equal, but the query uses created_at.
-        // Let's ensure we can also order by ID as tie-breaker.
-        const stmt = this.db.prepare('INSERT INTO snapshots (site_id, type, status) VALUES (?, ?, ?)');
-        const info = stmt.run(siteId, type, status);
+    createSnapshot(siteId, runType, status = 'running') {
+        const stmt = this.db.prepare('INSERT INTO snapshots (site_id, run_type, status) VALUES (?, ?, ?)');
+        const info = stmt.run(siteId, runType, status);
         return info.lastInsertRowid;
     }
-    getLatestSnapshot(siteId, status) {
-        let sql = 'SELECT * FROM snapshots WHERE site_id = ? AND type != \'partial\'';
+    getLatestSnapshot(siteId, status, includeSingle = false) {
+        let sql = 'SELECT * FROM snapshots WHERE site_id = ?';
+        if (!includeSingle) {
+            sql += ' AND run_type != \'single\'';
+        }
         const params = [siteId];
         if (status) {
             sql += ' AND status = ?';
@@ -23,10 +21,21 @@ export class SnapshotRepository {
         sql += ' ORDER BY created_at DESC, id DESC LIMIT 1';
         return this.db.prepare(sql).get(...params);
     }
+    touchSnapshot(id) {
+        this.db.prepare(`UPDATE snapshots SET created_at = datetime('now') WHERE id = ?`).run(id);
+    }
     getSnapshotCount(siteId) {
         const result = this.db.prepare('SELECT COUNT(*) as count FROM snapshots WHERE site_id = ?').get(siteId);
         return result.count;
     }
+    /**
+     * Returns true if the site has ever had a completed full or incremental crawl.
+     * Single snapshots (from page --live) do NOT count as a "first crawl".
+     */
+    hasFullCrawl(siteId) {
+        const result = this.db.prepare(`SELECT COUNT(*) as count FROM snapshots WHERE site_id = ? AND run_type IN ('completed', 'incremental') AND status = 'completed'`).get(siteId);
+        return result.count > 0;
+    }
     updateSnapshotStatus(id, status, stats = {}) {
         const sets = ['status = ?'];
         const params = [status];
@@ -73,4 +82,33 @@ export class SnapshotRepository {
         });
         tx();
     }
+    pruneSnapshots(siteId, maxSnapshots, maxSingleSnapshots, protectedSnapshotId) {
+        const tx = this.db.transaction(() => {
+            const singlesToDelete = this.db.prepare(`
+        SELECT id
+        FROM snapshots
+        WHERE site_id = ? AND run_type = 'single'
+        ORDER BY created_at DESC, id DESC
+        LIMIT -1 OFFSET ?
+      `).all(siteId, Math.max(0, maxSingleSnapshots));
+            const fullToDelete = this.db.prepare(`
+        SELECT id
+        FROM snapshots
+        WHERE site_id = ? AND run_type IN ('completed', 'incremental')
+        ORDER BY created_at DESC, id DESC
+        LIMIT -1 OFFSET ?
+      `).all(siteId, Math.max(0, maxSnapshots));
+            const ids = [...singlesToDelete, ...fullToDelete]
+                .map(r => r.id)
+                .filter(id => id !== protectedSnapshotId);
+            for (const id of ids) {
+                // Inline delete logic to keep operation inside this transaction.
+                this.db.prepare('UPDATE pages SET first_seen_snapshot_id = NULL WHERE first_seen_snapshot_id = ?').run(id);
+                this.db.prepare('UPDATE pages SET last_seen_snapshot_id = NULL WHERE last_seen_snapshot_id = ?').run(id);
+                this.db.prepare('DELETE FROM pages WHERE first_seen_snapshot_id IS NULL AND last_seen_snapshot_id IS NULL').run();
+                this.db.prepare('DELETE FROM snapshots WHERE id = ?').run(id);
+            }
+        });
+        tx();
+    }
 }

package/dist/db/reset.d.ts

@@ -0,0 +1,9 @@
+export interface ResetOptions {
+    reportsDir?: string;
+    dryRun?: boolean;
+}
+/**
+ * Completely resets the Crawlith state.
+ * Deletes the database, clears all locks, and optionally wipes the reports directory.
+ */
+export declare function resetCrawlith(options?: ResetOptions): Promise<void>;

package/dist/db/reset.js

@@ -0,0 +1,32 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+import { closeDb, getDb, getDbPath } from './index.js';
+import { LockManager } from '../lock/lockManager.js';
+/**
+ * Completely resets the Crawlith state.
+ * Deletes the database, clears all locks, and optionally wipes the reports directory.
+ */
+export async function resetCrawlith(options = {}) {
+    const { reportsDir, dryRun = false } = options;
+    if (dryRun) {
+        return;
+    }
+    // 1. Close database connection to release file handles
+    closeDb();
+    // 2. Clear all locks
+    await LockManager.clearAllLocks();
+    // 3. Remove the entire state directory (includes DB)
+    const dbPath = getDbPath();
+    if (dbPath !== ':memory:') {
+        const crawlithDir = path.join(os.homedir(), '.crawlith');
+        await fs.rm(crawlithDir, { recursive: true, force: true });
+    }
+    // 4. Remove reports directory if specified
+    if (reportsDir) {
+        const resolvedReportsDir = path.resolve(reportsDir);
+        await fs.rm(resolvedReportsDir, { recursive: true, force: true });
+    }
+    // 5. Re-initialize database to ensure schema is fresh for next use
+    getDb();
+}

package/dist/db/statements.d.ts

@@ -0,0 +1,12 @@
+import { Database, Statement } from 'better-sqlite3';
+export declare class Statements {
+    private db;
+    getPageIdByUrl: Statement;
+    insertPluginReport: Statement;
+    getPluginReport: Statement;
+    deleteSnapshotPlugins: Statement;
+    getSnapshot: Statement;
+    getMigration: Statement;
+    insertMigration: Statement;
+    constructor(db: Database);
+}

package/dist/db/statements.js

@@ -0,0 +1,40 @@
+export class Statements {
+    db;
+    getPageIdByUrl;
+    insertPluginReport;
+    getPluginReport;
+    deleteSnapshotPlugins;
+    getSnapshot;
+    getMigration;
+    insertMigration;
+    constructor(db) {
+        this.db = db;
+        this.getPageIdByUrl = this.db.prepare(`
+      SELECT id FROM pages 
+      WHERE site_id = (SELECT site_id FROM snapshots WHERE id = ?) 
+      AND normalized_url = ?
+    `);
+        this.insertPluginReport = this.db.prepare(`
+      INSERT OR REPLACE INTO plugin_reports 
+      (snapshot_id, plugin_name, data, total_score, score_count, score_weight_sum, score_calculated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `);
+        this.getPluginReport = this.db.prepare(`
+      SELECT data FROM plugin_reports 
+      WHERE snapshot_id = ? AND plugin_name = ?
+      ORDER BY created_at DESC LIMIT 1
+    `);
+        this.deleteSnapshotPlugins = this.db.prepare(`
+      DELETE FROM plugin_reports WHERE snapshot_id = ?
+    `);
+        this.getSnapshot = this.db.prepare(`
+      SELECT id FROM snapshots WHERE id = ?
+    `);
+        this.getMigration = this.db.prepare(`
+      SELECT plugin_name FROM plugin_migrations WHERE plugin_name = ?
+    `);
+        this.insertMigration = this.db.prepare(`
+      INSERT INTO plugin_migrations (plugin_name) VALUES (?)
+    `);
+    }
+}

package/dist/diff/compare.d.ts

@@ -12,11 +12,6 @@ export interface DiffResult {
         oldCanonical: string | null;
         newCanonical: string | null;
     }[];
-    changedDuplicateGroup: {
-        url: string;
-        oldGroup: string | null;
-        newGroup: string | null;
-    }[];
     metricDeltas: {
         structuralEntropy: number;
         orphanCount: number;

package/dist/diff/compare.js

@@ -6,7 +6,6 @@ export function compareGraphs(oldGraph, newGraph) {
     const removedUrls = [];
     const changedStatus = [];
     const changedCanonical = [];
-    const changedDuplicateGroup = [];
     // Added & Changed
     for (const [url, newNode] of newNodes) {
         const oldNode = oldNodes.get(url);
@@ -26,16 +25,6 @@ export function compareGraphs(oldGraph, newGraph) {
                     newCanonical: newNode.canonical || null
                 });
             }
-            // Changed Duplicate Group
-            const oldGroup = oldNode.duplicateClusterId || null;
-            const newGroup = newNode.duplicateClusterId || null;
-            if (oldGroup !== newGroup) {
-                changedDuplicateGroup.push({
-                    url,
-                    oldGroup,
-                    newGroup
-                });
-            }
         }
     }
     // Removed
@@ -58,7 +47,6 @@ export function compareGraphs(oldGraph, newGraph) {
         removedUrls,
         changedStatus,
         changedCanonical,
-        changedDuplicateGroup,
         metricDeltas
     };
 }

package/dist/diff/service.d.ts

@@ -0,0 +1,16 @@
+import { Graph } from '../graph/graph.js';
+export interface DiffOptions {
+    onlyCritical?: boolean;
+}
+export interface SnapshotDiff {
+    newPages: string[];
+    removedPages: string[];
+    changedPages: {
+        url: string;
+        changes: string[];
+        severity: 'low' | 'medium' | 'high';
+    }[];
+}
+export declare class DiffService {
+    compare(oldGraph: Graph | undefined, newGraph: Graph, _options?: DiffOptions): SnapshotDiff;
+}

package/dist/diff/service.js

@@ -0,0 +1,41 @@
+export class DiffService {
+    compare(oldGraph, newGraph, _options = {}) {
+        if (!oldGraph) {
+            return {
+                newPages: Array.from(newGraph.nodes.keys()),
+                removedPages: [],
+                changedPages: []
+            };
+        }
+        const oldUrls = new Set(oldGraph.nodes.keys());
+        const newUrls = new Set(newGraph.nodes.keys());
+        const newPages = Array.from(newUrls).filter(u => !oldUrls.has(u));
+        const removedPages = Array.from(oldUrls).filter(u => !newUrls.has(u));
+        const changedPages = [];
+        for (const url of newUrls) {
+            if (oldUrls.has(url)) {
+                const oldNode = oldGraph.nodes.get(url);
+                const newNode = newGraph.nodes.get(url);
+                const changes = [];
+                let severity = 'low';
+                if (oldNode.status !== newNode.status) {
+                    changes.push(`status: ${oldNode.status} -> ${newNode.status}`);
+                    severity = 'high';
+                }
+                if (oldNode.contentHash !== newNode.contentHash) {
+                    changes.push('content changed');
+                    if (severity !== 'high')
+                        severity = 'medium';
+                }
+                if (oldNode.noindex !== newNode.noindex) {
+                    changes.push(`noindex: ${oldNode.noindex} -> ${newNode.noindex}`);
+                    severity = 'high';
+                }
+                if (changes.length > 0) {
+                    changedPages.push({ url, changes, severity });
+                }
+            }
+        }
+        return { newPages, removedPages, changedPages };
+    }
+}

package/dist/domain/index.d.ts

@@ -0,0 +1,4 @@
+export * from '../graph/graph.js';
+export * from '../graph/metrics.js';
+export * from '../graph/simhash.js';
+export * from '../crawler/normalize.js';

package/dist/domain/index.js

@@ -0,0 +1,4 @@
+export * from '../graph/graph.js';
+export * from '../graph/metrics.js';
+export * from '../graph/simhash.js';
+export * from '../crawler/normalize.js';

package/dist/events.d.ts

@@ -15,6 +15,14 @@ export type CrawlEvent = {
 } | {
     type: 'crawl:limit-reached';
     limit: number;
+} | {
+    type: 'crawl:progress';
+    pagesCrawled: number;
+    queued: number;
+    active: number;
+    nodesFound: number;
+    edgesFound: number;
+    phase?: string;
 } | {
     type: 'queue:enqueue';
     url: string;

package/dist/graph/graph.d.ts

@@ -1,5 +1,6 @@
 export interface GraphNode {
     url: string;
+    isInternal?: boolean;
     depth: number;
     inLinks: number;
     outLinks: number;
@@ -9,50 +10,48 @@ export interface GraphNode {
     nofollow?: boolean;
     brokenLinks?: string[];
     redirectChain?: string[];
+    discoveredViaSitemap?: boolean;
     incrementalStatus?: 'new' | 'changed' | 'unchanged' | 'deleted';
     etag?: string;
     lastModified?: string;
     contentHash?: string;
     html?: string;
-    pageRank?: number;
-    pageRankScore?: number;
-    authorityScore?: number;
-    hubScore?: number;
-    duplicateClusterId?: string;
-    duplicateType?: 'exact' | 'near' | 'template_heavy' | 'none';
-    isClusterPrimary?: boolean;
-    isCollapsed?: boolean;
-    collapseInto?: string;
     simhash?: string;
     uniqueTokenRatio?: number;
-    soft404Score?: number;
-    soft404Signals?: string[];
     crawlTrapFlag?: boolean;
     crawlTrapRisk?: number;
     trapType?: string;
     securityError?: string;
     retries?: number;
-    clusterId?: number;
     bytesReceived?: number;
-    linkRole?: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral';
     crawlStatus?: string;
     wordCount?: number;
     thinContentScore?: number;
     externalLinkRatio?: number;
+    h1Count?: number;
+    h2Count?: number;
+    title?: string;
+    clusterId?: number;
+    duplicateClusterId?: string;
+    duplicateType?: 'exact' | 'near' | 'template_heavy';
+    pagerankScore?: number;
+    hubScore?: number;
+    authScore?: number;
+    linkRole?: string;
+    soft404Score?: number;
+    headingScore?: number;
     orphanScore?: number;
+    orphanType?: string;
+    impactLevel?: string;
+    headingData?: string;
+    isClusterPrimary?: boolean;
+    isCollapsed?: boolean;
 }
 export interface GraphEdge {
     source: string;
     target: string;
     weight: number;
 }
-export interface ClusterInfo {
-    id: number;
-    count: number;
-    primaryUrl: string;
-    risk: 'low' | 'medium' | 'high';
-    sharedPathPrefix?: string;
-}
 export interface CrawlStats {
     pagesFetched: number;
     pagesCached: number;
@@ -64,19 +63,6 @@ export declare class Graph {
     edges: Map<string, number>;
     limitReached: boolean;
     sessionStats: CrawlStats;
-    trapClusters: {
-        pattern: string;
-        type: string;
-        count: number;
-    }[];
-    duplicateClusters: {
-        id: string;
-        type: 'exact' | 'near' | 'template_heavy';
-        size: number;
-        representative: string;
-        severity: 'low' | 'medium' | 'high';
-    }[];
-    contentClusters: ClusterInfo[];
     /**
      * Generates a unique key for an edge.
      */
@@ -93,7 +79,7 @@ export declare class Graph {
      * If it exists, updates the status if the new status is non-zero (meaning we crawled it).
      * Depth is only set on creation (BFS guarantees shortest path first).
      */
-    addNode(url: string, depth: number, status?: number): void;
+    addNode(url: string, depth: number, status?: number, isInternal?: boolean): void;
     updateNodeData(url: string, data: Partial<GraphNode>): void;
     /**
      * Adds a directed edge between two nodes.
@@ -106,14 +92,6 @@ export declare class Graph {
     toJSON(): {
         nodes: GraphNode[];
         edges: GraphEdge[];
-        duplicateClusters: {
-            id: string;
-            type: "exact" | "near" | "template_heavy";
-            size: number;
-            representative: string;
-            severity: "low" | "medium" | "high";
-        }[];
-        contentClusters: ClusterInfo[];
     };
     static fromJSON(json: any): Graph;
 }

package/dist/graph/graph.js

@@ -9,32 +9,33 @@ export class Graph {
         pagesSkipped: 0,
         totalFound: 0
     };
-    trapClusters = [];
-    duplicateClusters = [];
-    contentClusters = [];
     /**
      * Generates a unique key for an edge.
      */
     static getEdgeKey(source, target) {
-        return JSON.stringify([source, target]);
+        return source + '\x00' + target;
     }
     /**
      * Parses an edge key back into source and target.
      */
     static parseEdgeKey(key) {
-        const [source, target] = JSON.parse(key);
-        return { source, target };
+        const splitIndex = key.indexOf('\x00');
+        return {
+            source: key.slice(0, splitIndex),
+            target: key.slice(splitIndex + 1)
+        };
     }
     /**
      * Adds a node to the graph if it doesn't exist.
      * If it exists, updates the status if the new status is non-zero (meaning we crawled it).
      * Depth is only set on creation (BFS guarantees shortest path first).
      */
-    addNode(url, depth, status = 0) {
+    addNode(url, depth, status = 0, isInternal = true) {
         const existing = this.nodes.get(url);
         if (!existing) {
             this.nodes.set(url, {
                 url,
+                isInternal,
                 depth,
                 status,
                 inLinks: 0,
@@ -46,6 +47,9 @@ export class Graph {
             if (status !== 0) {
                 existing.status = status;
             }
+            if (isInternal !== undefined) {
+                existing.isInternal = isInternal;
+            }
         }
     }
     updateNodeData(url, data) {
@@ -90,9 +94,7 @@ export class Graph {
     toJSON() {
         return {
             nodes: this.getNodes(),
-            edges: this.getEdges(),
-            duplicateClusters: this.duplicateClusters,
-            contentClusters: this.contentClusters
+            edges: this.getEdges()
         };
     }
     static fromJSON(json) {
@@ -108,12 +110,6 @@ export class Graph {
                 graph.edges.set(key, edge.weight || 1.0);
             }
         }
-        if (json.duplicateClusters) {
-            graph.duplicateClusters = json.duplicateClusters;
-        }
-        if (json.contentClusters) {
-            graph.contentClusters = json.contentClusters;
-        }
         return graph;
     }
 }

package/dist/graph/hits.d.ts

@@ -0,0 +1,23 @@
+import { Graph } from './graph.js';
+export type LinkRole = 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral';
+export interface HITSRow {
+    authority_score: number;
+    hub_score: number;
+    link_role: LinkRole;
+}
+export interface HITSOptions {
+    iterations?: number;
+}
+/**
+ * Service to compute Hub and Authority scores using the HITS algorithm.
+ * Operates purely on the internal link graph.
+ */
+export declare class HITSService {
+    /**
+     * Computes Hub and Authority scores using the HITS algorithm.
+     * @param {Graph} graph - The link graph to analyze.
+     * @param {HITSOptions} options - Algorithm options (e.g. number of iterations).
+     * @returns {Map<string, HITSRow>} A map of page URLs to their HITS results.
+     */
+    evaluate(graph: Graph, options?: HITSOptions): Map<string, HITSRow>;
+}