@crawlith/core 0.1.0 → 0.1.2

package/dist/db/repositories/PageRepository.js

@@ -8,36 +8,38 @@ export class PageRepository {
       INSERT INTO pages (
         site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id,
         http_status, canonical_url, content_hash, simhash, etag, last_modified, html,
-        soft404_score, noindex, nofollow, security_error, retries, depth,
-        redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type,
+        noindex, nofollow, security_error, retries, depth,
+        discovered_via_sitemap, redirect_chain, bytes_received, is_internal, crawl_trap_flag, crawl_trap_risk, trap_type,
         updated_at
       ) VALUES (
         @site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id,
         @http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html,
-        @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth,
-        @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
+        @noindex, @nofollow, @security_error, @retries, @depth,
+        @discovered_via_sitemap, @redirect_chain, @bytes_received, @is_internal, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
         datetime('now')
       )
       ON CONFLICT(site_id, normalized_url) DO UPDATE SET
+        first_seen_snapshot_id = COALESCE(pages.first_seen_snapshot_id, excluded.first_seen_snapshot_id),
         last_seen_snapshot_id = excluded.last_seen_snapshot_id,
-        http_status = excluded.http_status,
-        canonical_url = excluded.canonical_url,
-        content_hash = excluded.content_hash,
-        simhash = excluded.simhash,
-        etag = excluded.etag,
-        last_modified = excluded.last_modified,
-        html = excluded.html,
-        soft404_score = excluded.soft404_score,
-        noindex = excluded.noindex,
-        nofollow = excluded.nofollow,
-        security_error = excluded.security_error,
-        retries = excluded.retries,
-        depth = excluded.depth,
-        redirect_chain = excluded.redirect_chain,
-        bytes_received = excluded.bytes_received,
-        crawl_trap_flag = excluded.crawl_trap_flag,
-        crawl_trap_risk = excluded.crawl_trap_risk,
-        trap_type = excluded.trap_type,
+        http_status = CASE WHEN excluded.http_status != 0 THEN excluded.http_status ELSE pages.http_status END,
+        canonical_url = COALESCE(excluded.canonical_url, pages.canonical_url),
+        content_hash = COALESCE(excluded.content_hash, pages.content_hash),
+        simhash = COALESCE(excluded.simhash, pages.simhash),
+        etag = COALESCE(excluded.etag, pages.etag),
+        last_modified = COALESCE(excluded.last_modified, pages.last_modified),
+        html = COALESCE(excluded.html, pages.html),
+        noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END,
+        nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END,
+        security_error = COALESCE(excluded.security_error, pages.security_error),
+        retries = MAX(pages.retries, excluded.retries),
+        depth = MIN(pages.depth, excluded.depth),
+        discovered_via_sitemap = MAX(pages.discovered_via_sitemap, excluded.discovered_via_sitemap),
+        redirect_chain = COALESCE(excluded.redirect_chain, pages.redirect_chain),
+        bytes_received = COALESCE(excluded.bytes_received, pages.bytes_received),
+        is_internal = COALESCE(excluded.is_internal, pages.is_internal),
+        crawl_trap_flag = MAX(pages.crawl_trap_flag, excluded.crawl_trap_flag),
+        crawl_trap_risk = COALESCE(excluded.crawl_trap_risk, pages.crawl_trap_risk),
+        trap_type = COALESCE(excluded.trap_type, pages.trap_type),
         updated_at = datetime('now')
     `);
         this.getIdStmt = this.db.prepare('SELECT id FROM pages WHERE site_id = ? AND normalized_url = ?');
@@ -55,14 +57,15 @@ export class PageRepository {
             etag: page.etag ?? null,
             last_modified: page.last_modified ?? null,
             html: page.html ?? null,
-            soft404_score: page.soft404_score ?? null,
             noindex: page.noindex ?? 0,
             nofollow: page.nofollow ?? 0,
             security_error: page.security_error ?? null,
             retries: page.retries ?? 0,
             depth: page.depth ?? 0,
+            discovered_via_sitemap: page.discovered_via_sitemap ?? 0,
             redirect_chain: page.redirect_chain ?? null,
             bytes_received: page.bytes_received ?? null,
+            is_internal: page.is_internal ?? 1,
             crawl_trap_flag: page.crawl_trap_flag ?? 0,
             crawl_trap_risk: page.crawl_trap_risk ?? null,
             trap_type: page.trap_type ?? null,
@@ -83,11 +86,152 @@ export class PageRepository {
     getPage(siteId, url) {
         return this.db.prepare('SELECT * FROM pages WHERE site_id = ? AND normalized_url = ?').get(siteId, url);
     }
-    getPagesBySnapshot(snapshotId) {
-        return this.db.prepare('SELECT * FROM pages WHERE last_seen_snapshot_id = ?').all(snapshotId);
+    getPagesByUrls(siteId, urls) {
+        if (urls.length === 0)
+            return [];
+        const chunkSize = 900;
+        const results = [];
+        for (let i = 0; i < urls.length; i += chunkSize) {
+            const chunk = urls.slice(i, i + chunkSize);
+            const placeholders = chunk.map(() => '?').join(',');
+            const chunkResults = this.db.prepare(`SELECT * FROM pages WHERE site_id = ? AND normalized_url IN (${placeholders})`).all(siteId, ...chunk);
+            results.push(...chunkResults);
+        }
+        return results;
+    }
+    upsertMany(pages) {
+        if (pages.length === 0)
+            return new Map();
+        const upsertStmtWithReturn = this.db.prepare(`
+      INSERT INTO pages (
+        site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id,
+        http_status, canonical_url, content_hash, simhash, etag, last_modified, html,
+        noindex, nofollow, security_error, retries, depth,
+        discovered_via_sitemap, redirect_chain, bytes_received, is_internal, crawl_trap_flag, crawl_trap_risk, trap_type,
+        updated_at
+      ) VALUES (
+        @site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id,
+        @http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html,
+        @noindex, @nofollow, @security_error, @retries, @depth,
+        @discovered_via_sitemap, @redirect_chain, @bytes_received, @is_internal, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
+        datetime('now')
+      )
+      ON CONFLICT(site_id, normalized_url) DO UPDATE SET
+        first_seen_snapshot_id = COALESCE(pages.first_seen_snapshot_id, excluded.first_seen_snapshot_id),
+        last_seen_snapshot_id = excluded.last_seen_snapshot_id,
+        http_status = CASE WHEN excluded.http_status != 0 THEN excluded.http_status ELSE pages.http_status END,
+        canonical_url = COALESCE(excluded.canonical_url, pages.canonical_url),
+        content_hash = COALESCE(excluded.content_hash, pages.content_hash),
+        simhash = COALESCE(excluded.simhash, pages.simhash),
+        etag = COALESCE(excluded.etag, pages.etag),
+        last_modified = COALESCE(excluded.last_modified, pages.last_modified),
+        html = COALESCE(excluded.html, pages.html),
+        noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END,
+        nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END,
+        security_error = COALESCE(excluded.security_error, pages.security_error),
+        retries = MAX(pages.retries, excluded.retries),
+        depth = MIN(pages.depth, excluded.depth),
+        discovered_via_sitemap = MAX(pages.discovered_via_sitemap, excluded.discovered_via_sitemap),
+        redirect_chain = COALESCE(excluded.redirect_chain, pages.redirect_chain),
+        bytes_received = COALESCE(excluded.bytes_received, pages.bytes_received),
+        is_internal = COALESCE(excluded.is_internal, pages.is_internal),
+        crawl_trap_flag = MAX(pages.crawl_trap_flag, excluded.crawl_trap_flag),
+        crawl_trap_risk = COALESCE(excluded.crawl_trap_risk, pages.crawl_trap_risk),
+        trap_type = COALESCE(excluded.trap_type, pages.trap_type),
+        updated_at = datetime('now')
+      RETURNING id
+    `);
+        const urlToId = new Map();
+        const tx = this.db.transaction((pagesBatch) => {
+            for (const page of pagesBatch) {
+                const params = {
+                    site_id: page.site_id,
+                    normalized_url: page.normalized_url,
+                    first_seen_snapshot_id: page.first_seen_snapshot_id ?? page.last_seen_snapshot_id,
+                    last_seen_snapshot_id: page.last_seen_snapshot_id,
+                    http_status: page.http_status ?? null,
+                    canonical_url: page.canonical_url ?? null,
+                    content_hash: page.content_hash ?? null,
+                    simhash: page.simhash ?? null,
+                    etag: page.etag ?? null,
+                    last_modified: page.last_modified ?? null,
+                    html: page.html ?? null,
+                    noindex: page.noindex ?? 0,
+                    nofollow: page.nofollow ?? 0,
+                    security_error: page.security_error ?? null,
+                    retries: page.retries ?? 0,
+                    depth: page.depth ?? 0,
+                    discovered_via_sitemap: page.discovered_via_sitemap ?? 0,
+                    redirect_chain: page.redirect_chain ?? null,
+                    bytes_received: page.bytes_received ?? null,
+                    is_internal: page.is_internal ?? 1,
+                    crawl_trap_flag: page.crawl_trap_flag ?? 0,
+                    crawl_trap_risk: page.crawl_trap_risk ?? null,
+                    trap_type: page.trap_type ?? null,
+                };
+                const row = upsertStmtWithReturn.get(params);
+                urlToId.set(page.normalized_url, row.id);
+            }
+        });
+        tx(pages);
+        return urlToId;
+    }
+    getPagesBySnapshot(snapshotId, runType = 'completed') {
+        if (runType === 'single') {
+            return this.db.prepare('SELECT p.* FROM pages p JOIN metrics m ON p.id = m.page_id WHERE m.snapshot_id = ?').all(snapshotId);
+        }
+        return this.db.prepare('SELECT p.* FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND COALESCE(p.first_seen_snapshot_id, p.last_seen_snapshot_id) <= ?').all(snapshotId, snapshotId);
+    }
+    getPagesIdentityBySnapshot(snapshotId) {
+        // For identities, always loading all up to this point is fine for the crawler to map URLs to IDs.
+        return this.db.prepare('SELECT p.id, p.normalized_url FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND COALESCE(p.first_seen_snapshot_id, p.last_seen_snapshot_id) <= ?').all(snapshotId, snapshotId);
+    }
+    getPagesIteratorBySnapshot(snapshotId, runType = 'completed') {
+        if (runType === 'single') {
+            return this.db.prepare('SELECT p.* FROM pages p JOIN metrics m ON p.id = m.page_id WHERE m.snapshot_id = ?').iterate(snapshotId);
+        }
+        return this.db.prepare('SELECT p.* FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND COALESCE(p.first_seen_snapshot_id, p.last_seen_snapshot_id) <= ?').iterate(snapshotId, snapshotId);
     }
     getIdByUrl(siteId, url) {
         const row = this.getIdStmt.get(siteId, url);
         return row?.id;
     }
+    reconcileInternalUrls(siteId, siteOrigin) {
+        const origin = siteOrigin.replace(/\/+$/, '');
+        const tx = this.db.transaction(() => {
+            const rows = this.db
+                .prepare("SELECT id, normalized_url FROM pages WHERE site_id = ? AND (normalized_url LIKE 'http://%' OR normalized_url LIKE 'https://%')")
+                .all(siteId);
+            for (const row of rows) {
+                let parsed;
+                try {
+                    parsed = new URL(row.normalized_url);
+                }
+                catch {
+                    continue;
+                }
+                if (parsed.origin !== origin) {
+                    continue;
+                }
+                const targetPath = `${parsed.pathname}${parsed.search}`;
+                if (targetPath === row.normalized_url) {
+                    continue;
+                }
+                const existing = this.db
+                    .prepare('SELECT id FROM pages WHERE site_id = ? AND normalized_url = ?')
+                    .get(siteId, targetPath);
+                if (existing && existing.id !== row.id) {
+                    this.db.prepare('UPDATE edges SET source_page_id = ? WHERE source_page_id = ?').run(existing.id, row.id);
+                    this.db.prepare('UPDATE edges SET target_page_id = ? WHERE target_page_id = ?').run(existing.id, row.id);
+                    this.db.prepare('UPDATE OR IGNORE metrics SET page_id = ? WHERE page_id = ?').run(existing.id, row.id);
+                    this.db.prepare('DELETE FROM metrics WHERE page_id = ?').run(row.id);
+                    this.db.prepare('DELETE FROM pages WHERE id = ?').run(row.id);
+                }
+                else {
+                    this.db.prepare('UPDATE pages SET normalized_url = ? WHERE id = ?').run(targetPath, row.id);
+                }
+            }
+        });
+        tx();
+    }
 }

package/dist/db/repositories/SiteRepository.d.ts

@@ -2,6 +2,8 @@ import { Database } from 'better-sqlite3';
 export interface Site {
     id: number;
     domain: string;
+    preferred_url: string | null;
+    ssl: number | null;
     created_at: string;
     settings_json: string | null;
     is_active: number;
@@ -9,7 +11,14 @@ export interface Site {
 export declare class SiteRepository {
     private db;
     constructor(db: Database);
+    getSiteById(id: number): Site | undefined;
     getSite(domain: string): Site | undefined;
+    getAllSites(): Site[];
     createSite(domain: string): number;
+    updateSitePreference(id: number, prefs: {
+        preferred_url: string;
+        ssl: number;
+    }): void;
     firstOrCreateSite(domain: string): Site;
+    deleteSite(id: number): void;
 }

package/dist/db/repositories/SiteRepository.js

@@ -3,14 +3,24 @@ export class SiteRepository {
     constructor(db) {
         this.db = db;
     }
+    getSiteById(id) {
+        return this.db.prepare('SELECT * FROM sites WHERE id = ?').get(id);
+    }
     getSite(domain) {
         return this.db.prepare('SELECT * FROM sites WHERE domain = ?').get(domain);
     }
+    getAllSites() {
+        return this.db.prepare('SELECT * FROM sites ORDER BY domain ASC').all();
+    }
     createSite(domain) {
         const stmt = this.db.prepare('INSERT INTO sites (domain) VALUES (?)');
         const info = stmt.run(domain);
         return info.lastInsertRowid;
     }
+    updateSitePreference(id, prefs) {
+        const stmt = this.db.prepare('UPDATE sites SET preferred_url = ?, ssl = ? WHERE id = ?');
+        stmt.run(prefs.preferred_url, prefs.ssl, id);
+    }
     firstOrCreateSite(domain) {
         let site = this.getSite(domain);
         if (!site) {
@@ -19,4 +29,7 @@ export class SiteRepository {
         }
         return site;
     }
+    deleteSite(id) {
+        this.db.prepare('DELETE FROM sites WHERE id = ?').run(id);
+    }
 }

package/dist/db/repositories/SnapshotRepository.d.ts

@@ -2,11 +2,11 @@ import { Database } from 'better-sqlite3';
 export interface Snapshot {
     id: number;
     site_id: number;
-    type: 'full' | 'partial' | 'incremental';
+    run_type: 'completed' | 'incremental' | 'single';
     created_at: string;
     node_count: number;
     edge_count: number;
-    status: 'running' | 'completed' | 'failed';
+    status: 'queued' | 'running' | 'completed' | 'failed' | 'cancelled';
     limit_reached: number;
     health_score: number | null;
     orphan_count: number | null;
@@ -15,8 +15,17 @@ export interface Snapshot {
 export declare class SnapshotRepository {
     private db;
     constructor(db: Database);
-    createSnapshot(siteId: number, type: 'full' | 'partial' | 'incremental', status?: 'running' | 'completed' | 'failed'): number;
-    getLatestSnapshot(siteId: number, status?: 'completed' | 'running' | 'failed'): Snapshot | undefined;
-    updateSnapshotStatus(id: number, status: 'completed' | 'failed', stats?: Partial<Snapshot>): void;
+    createSnapshot(siteId: number, runType: Snapshot['run_type'], status?: Snapshot['status']): number;
+    getLatestSnapshot(siteId: number, status?: Snapshot['status'], includeSingle?: boolean): Snapshot | undefined;
+    touchSnapshot(id: number): void;
+    getSnapshotCount(siteId: number): number;
+    /**
+     * Returns true if the site has ever had a completed full or incremental crawl.
+     * Single snapshots (from page --live) do NOT count as a "first crawl".
+     */
+    hasFullCrawl(siteId: number): boolean;
+    updateSnapshotStatus(id: number, status: Snapshot['status'], stats?: Partial<Snapshot>): void;
     getSnapshot(id: number): Snapshot | undefined;
+    deleteSnapshot(id: number): void;
+    pruneSnapshots(siteId: number, maxSnapshots: number, maxSingleSnapshots: number, protectedSnapshotId?: number): void;
 }

package/dist/db/repositories/SnapshotRepository.js

@@ -3,21 +3,39 @@ export class SnapshotRepository {
     constructor(db) {
         this.db = db;
     }
-    createSnapshot(siteId, type, status = 'running') {
-        const stmt = this.db.prepare('INSERT INTO snapshots (site_id, type, status) VALUES (?, ?, ?)');
-        const info = stmt.run(siteId, type, status);
+    createSnapshot(siteId, runType, status = 'running') {
+        const stmt = this.db.prepare('INSERT INTO snapshots (site_id, run_type, status) VALUES (?, ?, ?)');
+        const info = stmt.run(siteId, runType, status);
         return info.lastInsertRowid;
     }
-    getLatestSnapshot(siteId, status) {
+    getLatestSnapshot(siteId, status, includeSingle = false) {
         let sql = 'SELECT * FROM snapshots WHERE site_id = ?';
+        if (!includeSingle) {
+            sql += ' AND run_type != \'single\'';
+        }
         const params = [siteId];
         if (status) {
             sql += ' AND status = ?';
             params.push(status);
         }
-        sql += ' ORDER BY created_at DESC LIMIT 1';
+        sql += ' ORDER BY created_at DESC, id DESC LIMIT 1';
         return this.db.prepare(sql).get(...params);
     }
+    touchSnapshot(id) {
+        this.db.prepare(`UPDATE snapshots SET created_at = datetime('now') WHERE id = ?`).run(id);
+    }
+    getSnapshotCount(siteId) {
+        const result = this.db.prepare('SELECT COUNT(*) as count FROM snapshots WHERE site_id = ?').get(siteId);
+        return result.count;
+    }
+    /**
+     * Returns true if the site has ever had a completed full or incremental crawl.
+     * Single snapshots (from page --live) do NOT count as a "first crawl".
+     */
+    hasFullCrawl(siteId) {
+        const result = this.db.prepare(`SELECT COUNT(*) as count FROM snapshots WHERE site_id = ? AND run_type IN ('completed', 'incremental') AND status = 'completed'`).get(siteId);
+        return result.count > 0;
+    }
     updateSnapshotStatus(id, status, stats = {}) {
         const sets = ['status = ?'];
         const params = [status];
@@ -52,4 +70,45 @@ export class SnapshotRepository {
     getSnapshot(id) {
         return this.db.prepare('SELECT * FROM snapshots WHERE id = ?').get(id);
     }
+    deleteSnapshot(id) {
+        const tx = this.db.transaction(() => {
+            // Unlink pages from this snapshot to prevent FK constraint violations or data inconsistencies
+            this.db.prepare('UPDATE pages SET first_seen_snapshot_id = NULL WHERE first_seen_snapshot_id = ?').run(id);
+            this.db.prepare('UPDATE pages SET last_seen_snapshot_id = NULL WHERE last_seen_snapshot_id = ?').run(id);
+            // Cleanup: Delete pages that are no longer referenced by any snapshot
+            this.db.prepare('DELETE FROM pages WHERE first_seen_snapshot_id IS NULL AND last_seen_snapshot_id IS NULL').run();
+            // Delete the snapshot
+            this.db.prepare('DELETE FROM snapshots WHERE id = ?').run(id);
+        });
+        tx();
+    }
+    pruneSnapshots(siteId, maxSnapshots, maxSingleSnapshots, protectedSnapshotId) {
+        const tx = this.db.transaction(() => {
+            const singlesToDelete = this.db.prepare(`
+        SELECT id
+        FROM snapshots
+        WHERE site_id = ? AND run_type = 'single'
+        ORDER BY created_at DESC, id DESC
+        LIMIT -1 OFFSET ?
+      `).all(siteId, Math.max(0, maxSingleSnapshots));
+            const fullToDelete = this.db.prepare(`
+        SELECT id
+        FROM snapshots
+        WHERE site_id = ? AND run_type IN ('completed', 'incremental')
+        ORDER BY created_at DESC, id DESC
+        LIMIT -1 OFFSET ?
+      `).all(siteId, Math.max(0, maxSnapshots));
+            const ids = [...singlesToDelete, ...fullToDelete]
+                .map(r => r.id)
+                .filter(id => id !== protectedSnapshotId);
+            for (const id of ids) {
+                // Inline delete logic to keep operation inside this transaction.
+                this.db.prepare('UPDATE pages SET first_seen_snapshot_id = NULL WHERE first_seen_snapshot_id = ?').run(id);
+                this.db.prepare('UPDATE pages SET last_seen_snapshot_id = NULL WHERE last_seen_snapshot_id = ?').run(id);
+                this.db.prepare('DELETE FROM pages WHERE first_seen_snapshot_id IS NULL AND last_seen_snapshot_id IS NULL').run();
+                this.db.prepare('DELETE FROM snapshots WHERE id = ?').run(id);
+            }
+        });
+        tx();
+    }
 }

package/dist/db/reset.d.ts

@@ -0,0 +1,9 @@
+export interface ResetOptions {
+    reportsDir?: string;
+    dryRun?: boolean;
+}
+/**
+ * Completely resets the Crawlith state.
+ * Deletes the database, clears all locks, and optionally wipes the reports directory.
+ */
+export declare function resetCrawlith(options?: ResetOptions): Promise<void>;

package/dist/db/reset.js

@@ -0,0 +1,32 @@
+import fs from 'node:fs/promises';
+import path from 'node:path';
+import os from 'node:os';
+import { closeDb, getDb, getDbPath } from './index.js';
+import { LockManager } from '../lock/lockManager.js';
+/**
+ * Completely resets the Crawlith state.
+ * Deletes the database, clears all locks, and optionally wipes the reports directory.
+ */
+export async function resetCrawlith(options = {}) {
+    const { reportsDir, dryRun = false } = options;
+    if (dryRun) {
+        return;
+    }
+    // 1. Close database connection to release file handles
+    closeDb();
+    // 2. Clear all locks
+    await LockManager.clearAllLocks();
+    // 3. Remove the entire state directory (includes DB)
+    const dbPath = getDbPath();
+    if (dbPath !== ':memory:') {
+        const crawlithDir = path.join(os.homedir(), '.crawlith');
+        await fs.rm(crawlithDir, { recursive: true, force: true });
+    }
+    // 4. Remove reports directory if specified
+    if (reportsDir) {
+        const resolvedReportsDir = path.resolve(reportsDir);
+        await fs.rm(resolvedReportsDir, { recursive: true, force: true });
+    }
+    // 5. Re-initialize database to ensure schema is fresh for next use
+    getDb();
+}

package/dist/db/statements.d.ts

@@ -0,0 +1,12 @@
+import { Database, Statement } from 'better-sqlite3';
+export declare class Statements {
+    private db;
+    getPageIdByUrl: Statement;
+    insertPluginReport: Statement;
+    getPluginReport: Statement;
+    deleteSnapshotPlugins: Statement;
+    getSnapshot: Statement;
+    getMigration: Statement;
+    insertMigration: Statement;
+    constructor(db: Database);
+}

package/dist/db/statements.js

@@ -0,0 +1,40 @@
+export class Statements {
+    db;
+    getPageIdByUrl;
+    insertPluginReport;
+    getPluginReport;
+    deleteSnapshotPlugins;
+    getSnapshot;
+    getMigration;
+    insertMigration;
+    constructor(db) {
+        this.db = db;
+        this.getPageIdByUrl = this.db.prepare(`
+      SELECT id FROM pages 
+      WHERE site_id = (SELECT site_id FROM snapshots WHERE id = ?) 
+      AND normalized_url = ?
+    `);
+        this.insertPluginReport = this.db.prepare(`
+      INSERT OR REPLACE INTO plugin_reports 
+      (snapshot_id, plugin_name, data, total_score, score_count, score_weight_sum, score_calculated_at)
+      VALUES (?, ?, ?, ?, ?, ?, ?)
+    `);
+        this.getPluginReport = this.db.prepare(`
+      SELECT data FROM plugin_reports 
+      WHERE snapshot_id = ? AND plugin_name = ?
+      ORDER BY created_at DESC LIMIT 1
+    `);
+        this.deleteSnapshotPlugins = this.db.prepare(`
+      DELETE FROM plugin_reports WHERE snapshot_id = ?
+    `);
+        this.getSnapshot = this.db.prepare(`
+      SELECT id FROM snapshots WHERE id = ?
+    `);
+        this.getMigration = this.db.prepare(`
+      SELECT plugin_name FROM plugin_migrations WHERE plugin_name = ?
+    `);
+        this.insertMigration = this.db.prepare(`
+      INSERT INTO plugin_migrations (plugin_name) VALUES (?)
+    `);
+    }
+}

package/dist/diff/compare.d.ts

@@ -12,11 +12,6 @@ export interface DiffResult {
         oldCanonical: string | null;
         newCanonical: string | null;
     }[];
-    changedDuplicateGroup: {
-        url: string;
-        oldGroup: string | null;
-        newGroup: string | null;
-    }[];
     metricDeltas: {
         structuralEntropy: number;
         orphanCount: number;

package/dist/diff/compare.js

@@ -6,7 +6,6 @@ export function compareGraphs(oldGraph, newGraph) {
     const removedUrls = [];
     const changedStatus = [];
     const changedCanonical = [];
-    const changedDuplicateGroup = [];
     // Added & Changed
     for (const [url, newNode] of newNodes) {
         const oldNode = oldNodes.get(url);
@@ -26,16 +25,6 @@ export function compareGraphs(oldGraph, newGraph) {
                     newCanonical: newNode.canonical || null
                 });
             }
-            // Changed Duplicate Group
-            const oldGroup = oldNode.duplicateClusterId || null;
-            const newGroup = newNode.duplicateClusterId || null;
-            if (oldGroup !== newGroup) {
-                changedDuplicateGroup.push({
-                    url,
-                    oldGroup,
-                    newGroup
-                });
-            }
         }
     }
     // Removed
@@ -58,7 +47,6 @@ export function compareGraphs(oldGraph, newGraph) {
         removedUrls,
         changedStatus,
         changedCanonical,
-        changedDuplicateGroup,
         metricDeltas
     };
 }

package/dist/diff/service.d.ts

@@ -0,0 +1,16 @@
+import { Graph } from '../graph/graph.js';
+export interface DiffOptions {
+    onlyCritical?: boolean;
+}
+export interface SnapshotDiff {
+    newPages: string[];
+    removedPages: string[];
+    changedPages: {
+        url: string;
+        changes: string[];
+        severity: 'low' | 'medium' | 'high';
+    }[];
+}
+export declare class DiffService {
+    compare(oldGraph: Graph | undefined, newGraph: Graph, _options?: DiffOptions): SnapshotDiff;
+}

package/dist/diff/service.js

@@ -0,0 +1,41 @@
+export class DiffService {
+    compare(oldGraph, newGraph, _options = {}) {
+        if (!oldGraph) {
+            return {
+                newPages: Array.from(newGraph.nodes.keys()),
+                removedPages: [],
+                changedPages: []
+            };
+        }
+        const oldUrls = new Set(oldGraph.nodes.keys());
+        const newUrls = new Set(newGraph.nodes.keys());
+        const newPages = Array.from(newUrls).filter(u => !oldUrls.has(u));
+        const removedPages = Array.from(oldUrls).filter(u => !newUrls.has(u));
+        const changedPages = [];
+        for (const url of newUrls) {
+            if (oldUrls.has(url)) {
+                const oldNode = oldGraph.nodes.get(url);
+                const newNode = newGraph.nodes.get(url);
+                const changes = [];
+                let severity = 'low';
+                if (oldNode.status !== newNode.status) {
+                    changes.push(`status: ${oldNode.status} -> ${newNode.status}`);
+                    severity = 'high';
+                }
+                if (oldNode.contentHash !== newNode.contentHash) {
+                    changes.push('content changed');
+                    if (severity !== 'high')
+                        severity = 'medium';
+                }
+                if (oldNode.noindex !== newNode.noindex) {
+                    changes.push(`noindex: ${oldNode.noindex} -> ${newNode.noindex}`);
+                    severity = 'high';
+                }
+                if (changes.length > 0) {
+                    changedPages.push({ url, changes, severity });
+                }
+            }
+        }
+        return { newPages, removedPages, changedPages };
+    }
+}

package/dist/domain/index.d.ts

@@ -0,0 +1,4 @@
+export * from '../graph/graph.js';
+export * from '../graph/metrics.js';
+export * from '../graph/simhash.js';
+export * from '../crawler/normalize.js';

package/dist/domain/index.js

@@ -0,0 +1,4 @@
+export * from '../graph/graph.js';
+export * from '../graph/metrics.js';
+export * from '../graph/simhash.js';
+export * from '../crawler/normalize.js';