@crawlith/core 0.1.1 → 0.1.2

package/dist/db/index.js

@@ -1,12 +1,12 @@
-import Database from 'better-sqlite3';
 import path from 'node:path';
 import fs from 'node:fs';
 import os from 'node:os';
-import { initSchema } from './schema.js';
+import { CrawlithDB } from './CrawlithDB.js';
 let dbInstance = null;
+let crawlithDbInstance = null;
 export * from './repositories/SiteRepository.js';
 export * from './repositories/SnapshotRepository.js';
-export { initSchema } from './schema.js';
+export * from './CrawlithDB.js';
 export function getDbPath() {
     if (process.env.NODE_ENV === 'test') {
         return ':memory:';
@@ -23,42 +23,50 @@ export function getDbPath() {
     }
     return path.join(crawlithDir, 'crawlith.db');
 }
-export function getDb() {
-    if (dbInstance) {
-        return dbInstance;
+/**
+ * Returns the higher-level CrawlithDB wrapper for plugins and new code.
+ */
+export function getCrawlithDB() {
+    if (crawlithDbInstance) {
+        return crawlithDbInstance;
     }
     const dbPath = getDbPath();
-    const db = new Database(dbPath);
-    // Hardening & Performance Configuration
-    db.pragma('journal_mode = WAL');
-    db.pragma('synchronous = NORMAL');
-    db.pragma('foreign_keys = ON');
-    db.pragma('temp_store = MEMORY');
-    db.pragma('mmap_size = 30000000000');
-    db.pragma('cache_size = -20000');
-    db.pragma('busy_timeout = 5000');
-    // Security controls
-    // Ensure file permissions are 600 (user read/write only)
+    crawlithDbInstance = new CrawlithDB(dbPath);
+    dbInstance = crawlithDbInstance.unsafeGetRawDb();
+    // Migrations for existing tables
     try {
-        fs.chmodSync(dbPath, 0o600);
+        dbInstance.exec(`ALTER TABLE pages ADD COLUMN discovered_via_sitemap INTEGER DEFAULT 0;`);
     }
-    catch (_e) {
-        // might fail on first creation if file doesn't exist yet, but better-sqlite3 creates it
-        // so we can try again or ignore if it's new
+    catch (_e) { /* ignore */ }
+    // Security controls: Ensure file permissions are 600 (user read/write only)
+    if (dbPath !== ':memory:') {
+        try {
+            fs.chmodSync(dbPath, 0o600);
+        }
+        catch (_e) {
+            // might fail if file doesn't exist yet but better-sqlite3 should have created it
+        }
     }
-    // Integrity check on startup
-    const integrity = db.pragma('integrity_check', { simple: true });
-    if (integrity !== 'ok') {
-        // Reverted to console.warn to avoid breaking change
-        console.warn('Database integrity check failed:', integrity);
+    return crawlithDbInstance;
+}
+/**
+ * Returns the raw better-sqlite3 Database instance for legacy repositories.
+ */
+export function getDb() {
+    if (dbInstance) {
+        return dbInstance;
     }
-    // Initialize schema
-    initSchema(db);
-    dbInstance = db;
-    return db;
+    // Initializing via getCrawlithDB ensures consistent configuration
+    getCrawlithDB();
+    return dbInstance;
 }
 export function closeDb() {
-    if (dbInstance) {
+    if (crawlithDbInstance) {
+        crawlithDbInstance.close();
+        crawlithDbInstance = null;
+        dbInstance = null;
+    }
+    else if (dbInstance) {
         dbInstance.close();
         dbInstance = null;
     }

package/dist/db/migrations.d.ts

@@ -0,0 +1,2 @@
+import { Database } from 'better-sqlite3';
+export declare function runBaseMigrations(db: Database): void;

package/dist/db/{schema.js → migrations.js}

@@ -1,9 +1,11 @@
-export function initSchema(db) {
+export function runBaseMigrations(db) {
     // Sites Table
     db.exec(`
     CREATE TABLE IF NOT EXISTS sites (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       domain TEXT UNIQUE NOT NULL,
+      preferred_url TEXT,
+      ssl INTEGER,
       created_at TEXT DEFAULT (datetime('now')),
       settings_json TEXT,
       is_active INTEGER DEFAULT 1
@@ -14,18 +16,36 @@ export function initSchema(db) {
     CREATE TABLE IF NOT EXISTS snapshots (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       site_id INTEGER NOT NULL,
-      type TEXT CHECK(type IN ('full', 'partial', 'incremental')) NOT NULL,
+      run_type TEXT CHECK(run_type IN ('completed', 'incremental', 'single')) NOT NULL DEFAULT 'completed',
       created_at TEXT DEFAULT (datetime('now')),
       node_count INTEGER DEFAULT 0,
       edge_count INTEGER DEFAULT 0,
-      status TEXT CHECK(status IN ('running', 'completed', 'failed')) DEFAULT 'running',
+      status TEXT CHECK(status IN ('queued', 'running', 'completed', 'failed', 'cancelled')) DEFAULT 'running',
       limit_reached INTEGER DEFAULT 0,
       health_score REAL,
       orphan_count INTEGER,
       thin_content_count INTEGER,
+      total_score REAL,
+      score_count INTEGER,
+      score_weight_sum REAL,
+      score_calculated_at TEXT,
       FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE
     );
   `);
+    // Migration for snapshots: run_type and status
+    try {
+        db.exec(`ALTER TABLE snapshots ADD COLUMN run_type TEXT CHECK(run_type IN ('completed', 'incremental', 'single')) NOT NULL DEFAULT 'completed';`);
+    }
+    catch (_e) { /* ignore */ }
+    try {
+        // If type column exists, populate run_type from it
+        db.exec(`UPDATE snapshots SET run_type = CASE WHEN type = 'partial' THEN 'single' ELSE 'completed' END WHERE run_type IS NULL OR run_type = 'full' OR run_type = 'completed';`);
+    }
+    catch (_e) { /* ignore */ }
+    try {
+        db.exec(`ALTER TABLE snapshots DROP COLUMN type;`);
+    }
+    catch (_e) { /* ignore */ }
     // Pages Table
     db.exec(`
     CREATE TABLE IF NOT EXISTS pages (
@@ -41,14 +61,15 @@ export function initSchema(db) {
       etag TEXT,
       last_modified TEXT,
       html TEXT,
-      soft404_score REAL,
       noindex INTEGER DEFAULT 0,
       nofollow INTEGER DEFAULT 0,
       security_error TEXT,
       retries INTEGER DEFAULT 0,
       depth INTEGER DEFAULT 0,
+      discovered_via_sitemap INTEGER DEFAULT 0,
       redirect_chain TEXT,
       bytes_received INTEGER,
+      is_internal INTEGER DEFAULT 1,
       crawl_trap_flag INTEGER DEFAULT 0,
       crawl_trap_risk REAL,
       trap_type TEXT,
@@ -60,7 +81,15 @@ export function initSchema(db) {
       UNIQUE(site_id, normalized_url)
     );
   `);
-    // Index for Pages
+    // Migrations for existing tables
+    try {
+        db.exec(`ALTER TABLE pages ADD COLUMN is_internal INTEGER DEFAULT 1;`);
+    }
+    catch (_e) { /* ignore */ }
+    try {
+        db.exec(`ALTER TABLE pages ADD COLUMN discovered_via_sitemap INTEGER DEFAULT 0;`);
+    }
+    catch (_e) { /* ignore */ }
     db.exec(`CREATE INDEX IF NOT EXISTS idx_pages_site_last_seen ON pages(site_id, last_seen_snapshot_id);`);
     // Edges Table
     db.exec(`
@@ -76,26 +105,29 @@ export function initSchema(db) {
       FOREIGN KEY(target_page_id) REFERENCES pages(id) ON DELETE CASCADE
     );
   `);
-    // Index for Edges
-    db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot_source ON edges(snapshot_id, source_page_id);`);
     db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot ON edges(snapshot_id);`);
     // Metrics Table
     db.exec(`
     CREATE TABLE IF NOT EXISTS metrics (
       snapshot_id INTEGER NOT NULL,
       page_id INTEGER NOT NULL,
-      authority_score REAL,
-      hub_score REAL,
-      pagerank REAL,
-      pagerank_score REAL,
-      link_role TEXT CHECK(link_role IN ('hub', 'authority', 'power', 'balanced', 'peripheral')),
       crawl_status TEXT,
       word_count INTEGER,
       thin_content_score REAL,
       external_link_ratio REAL,
       orphan_score INTEGER,
+      pagerank_score REAL,
+      hub_score REAL,
+      auth_score REAL,
+      link_role TEXT,
       duplicate_cluster_id TEXT,
-      duplicate_type TEXT CHECK(duplicate_type IN ('exact', 'near', 'template_heavy', 'none')),
+      duplicate_type TEXT,
+      cluster_id INTEGER,
+      soft404_score REAL,
+      heading_score REAL,
+      orphan_type TEXT,
+      impact_level TEXT,
+      heading_data TEXT,
       is_cluster_primary INTEGER DEFAULT 0,
       PRIMARY KEY(snapshot_id, page_id),
       FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
@@ -129,41 +161,56 @@ export function initSchema(db) {
       FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
     );
   `);
-    // Migration: add columns to existing DBs that were created before this update
-    migrateSchema(db);
-}
-function migrateSchema(db) {
-    // Add missing columns to pages (safe: ALTER TABLE ADD COLUMN is idempotent-safe with try/catch)
-    const pageColumns = [
-        ['redirect_chain', 'TEXT'],
-        ['bytes_received', 'INTEGER'],
-        ['crawl_trap_flag', 'INTEGER DEFAULT 0'],
-        ['crawl_trap_risk', 'REAL'],
-        ['trap_type', 'TEXT'],
-    ];
-    for (const [col, type] of pageColumns) {
-        try {
-            db.exec(`ALTER TABLE pages ADD COLUMN ${col} ${type}`);
-        }
-        catch { /* already exists */ }
-    }
-    // Add missing columns to edges
-    try {
-        db.exec('ALTER TABLE edges ADD COLUMN weight REAL DEFAULT 1.0');
-    }
-    catch { /* already exists */ }
-    // Add missing columns to metrics
-    const metricsColumns = [
+    // Plugin Migrations Table
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS plugin_migrations (
+      plugin_name TEXT PRIMARY KEY,
+      executed_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+  `);
+    // Universal Plugin Reports Table
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS plugin_reports (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      snapshot_id INTEGER NOT NULL,
+      plugin_name TEXT NOT NULL,
+      data TEXT NOT NULL,
+      total_score REAL,
+      score_count INTEGER,
+      score_weight_sum REAL,
+      score_calculated_at TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
+    );
+  `);
+    db.exec(`CREATE INDEX IF NOT EXISTS idx_plugin_reports_snapshot ON plugin_reports(snapshot_id);`);
+    db.exec(`CREATE INDEX IF NOT EXISTS idx_plugin_reports_composite ON plugin_reports(snapshot_id, plugin_name);`);
+    // Migrations for metrics
+    const metricsCols = [
         ['pagerank_score', 'REAL'],
+        ['hub_score', 'REAL'],
+        ['auth_score', 'REAL'],
         ['link_role', 'TEXT'],
-        ['duplicate_cluster_id', 'TEXT'],
-        ['duplicate_type', 'TEXT'],
-        ['is_cluster_primary', 'INTEGER DEFAULT 0'],
+        ['cluster_id', 'INTEGER'],
+        ['soft404_score', 'REAL'],
+        ['heading_score', 'REAL'],
+        ['orphan_type', 'TEXT'],
+        ['impact_level', 'TEXT'],
+        ['heading_data', 'TEXT'],
     ];
-    for (const [col, type] of metricsColumns) {
+    for (const [col, type] of metricsCols) {
         try {
             db.exec(`ALTER TABLE metrics ADD COLUMN ${col} ${type}`);
         }
-        catch { /* already exists */ }
+        catch { /* ignore */ }
+    }
+    // Final site column migrations
+    try {
+        db.exec('ALTER TABLE sites ADD COLUMN preferred_url TEXT');
+    }
+    catch { /* ignore */ }
+    try {
+        db.exec('ALTER TABLE sites ADD COLUMN ssl INTEGER');
     }
+    catch { /* ignore */ }
 }

package/dist/db/pluginRegistry.d.ts

@@ -0,0 +1,9 @@
+export declare class PluginRegistry {
+    private registeredPlugins;
+    private registeredTables;
+    registerPlugin(pluginName: string): void;
+    isPluginRegistered(pluginName: string): boolean;
+    registerTable(tableName: string, pluginName: string): void;
+    getPluginForTable(tableName: string): string | undefined;
+    isTableRegistered(tableName: string): boolean;
+}

package/dist/db/pluginRegistry.js

@@ -0,0 +1,19 @@
+export class PluginRegistry {
+    registeredPlugins = new Set();
+    registeredTables = new Map(); // tableName -> pluginName
+    registerPlugin(pluginName) {
+        this.registeredPlugins.add(pluginName);
+    }
+    isPluginRegistered(pluginName) {
+        return this.registeredPlugins.has(pluginName);
+    }
+    registerTable(tableName, pluginName) {
+        this.registeredTables.set(tableName, pluginName);
+    }
+    getPluginForTable(tableName) {
+        return this.registeredTables.get(tableName);
+    }
+    isTableRegistered(tableName) {
+        return this.registeredTables.has(tableName);
+    }
+}

package/dist/db/repositories/EdgeRepository.d.ts

@@ -19,6 +19,11 @@ export declare class EdgeRepository {
         weight: number;
         rel: string;
     }[]): void;
+    /**
+     * Remove all edges originating from a specific page within a snapshot.
+     * Used when re-crawling a page into a reused partial snapshot to avoid duplicates.
+     */
+    deleteEdgesForPage(snapshotId: number, sourcePageId: number): void;
     getEdgesBySnapshot(snapshotId: number): Edge[];
     getEdgesIteratorBySnapshot(snapshotId: number): IterableIterator<Edge>;
 }

package/dist/db/repositories/EdgeRepository.js

@@ -21,6 +21,13 @@ export class EdgeRepository {
         });
         tx(edges);
     }
+    /**
+     * Remove all edges originating from a specific page within a snapshot.
+     * Used when re-crawling a page into a reused partial snapshot to avoid duplicates.
+     */
+    deleteEdgesForPage(snapshotId, sourcePageId) {
+        this.db.prepare('DELETE FROM edges WHERE snapshot_id = ? AND source_page_id = ?').run(snapshotId, sourcePageId);
+    }
     getEdgesBySnapshot(snapshotId) {
         return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').all(snapshotId);
     }

package/dist/db/repositories/MetricsRepository.d.ts

@@ -2,19 +2,24 @@ import { Database } from 'better-sqlite3';
 export interface DbMetrics {
     snapshot_id: number;
     page_id: number;
-    authority_score: number | null;
-    hub_score: number | null;
-    pagerank: number | null;
-    pagerank_score: number | null;
-    link_role: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral' | null;
     crawl_status: string | null;
     word_count: number | null;
     thin_content_score: number | null;
     external_link_ratio: number | null;
-    orphan_score: number | null;
+    pagerank_score: number | null;
+    hub_score: number | null;
+    auth_score: number | null;
+    link_role: string | null;
     duplicate_cluster_id: string | null;
-    duplicate_type: 'exact' | 'near' | 'template_heavy' | 'none' | null;
-    is_cluster_primary: number;
+    duplicate_type: string | null;
+    cluster_id: number | null;
+    soft404_score: number | null;
+    heading_score: number | null;
+    orphan_score: number | null;
+    orphan_type: string | null;
+    impact_level: string | null;
+    heading_data: string | null;
+    is_cluster_primary: number | null;
 }
 export declare class MetricsRepository {
     private db;

package/dist/db/repositories/MetricsRepository.js

@@ -7,13 +7,21 @@ export class MetricsRepository {
         this.getByPageStmt = this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ? AND page_id = ?');
         this.insertStmt = this.db.prepare(`
       INSERT OR REPLACE INTO metrics (
-        snapshot_id, page_id, authority_score, hub_score, pagerank, pagerank_score,
-        link_role, crawl_status, word_count, thin_content_score, external_link_ratio,
-        orphan_score, duplicate_cluster_id, duplicate_type, is_cluster_primary
+        snapshot_id, page_id,
+        crawl_status, word_count, thin_content_score, external_link_ratio,
+        pagerank_score, hub_score, auth_score, link_role,
+        duplicate_cluster_id, duplicate_type, cluster_id,
+        soft404_score, heading_score,
+        orphan_score, orphan_type, impact_level,
+        heading_data, is_cluster_primary
       ) VALUES (
-        @snapshot_id, @page_id, @authority_score, @hub_score, @pagerank, @pagerank_score,
-        @link_role, @crawl_status, @word_count, @thin_content_score, @external_link_ratio,
-        @orphan_score, @duplicate_cluster_id, @duplicate_type, @is_cluster_primary
+        @snapshot_id, @page_id,
+        @crawl_status, @word_count, @thin_content_score, @external_link_ratio,
+        @pagerank_score, @hub_score, @auth_score, @link_role,
+        @duplicate_cluster_id, @duplicate_type, @cluster_id,
+        @soft404_score, @heading_score,
+        @orphan_score, @orphan_type, @impact_level,
+        @heading_data, @is_cluster_primary
       )
     `);
     }

package/dist/db/repositories/PageRepository.d.ts

@@ -12,14 +12,15 @@ export interface Page {
     etag: string | null;
     last_modified: string | null;
     html: string | null;
-    soft404_score: number | null;
     noindex: number;
     nofollow: number;
     security_error: string | null;
     retries: number;
     depth: number;
+    discovered_via_sitemap: number;
     redirect_chain: string | null;
     bytes_received: number | null;
+    is_internal: number;
     crawl_trap_flag: number;
     crawl_trap_risk: number | null;
     trap_type: string | null;
@@ -48,11 +49,12 @@ export declare class PageRepository {
         normalized_url: string;
         last_seen_snapshot_id: number;
     })[]): Map<string, number>;
-    getPagesBySnapshot(snapshotId: number): Page[];
+    getPagesBySnapshot(snapshotId: number, runType?: string): Page[];
     getPagesIdentityBySnapshot(snapshotId: number): {
         id: number;
         normalized_url: string;
     }[];
-    getPagesIteratorBySnapshot(snapshotId: number): IterableIterator<Page>;
+    getPagesIteratorBySnapshot(snapshotId: number, runType?: string): IterableIterator<Page>;
     getIdByUrl(siteId: number, url: string): number | undefined;
+    reconcileInternalUrls(siteId: number, siteOrigin: string): void;
 }

package/dist/db/repositories/PageRepository.js

@@ -8,17 +8,18 @@ export class PageRepository {
       INSERT INTO pages (
         site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id,
         http_status, canonical_url, content_hash, simhash, etag, last_modified, html,
-        soft404_score, noindex, nofollow, security_error, retries, depth,
-        redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type,
+        noindex, nofollow, security_error, retries, depth,
+        discovered_via_sitemap, redirect_chain, bytes_received, is_internal, crawl_trap_flag, crawl_trap_risk, trap_type,
         updated_at
       ) VALUES (
         @site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id,
         @http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html,
-        @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth,
-        @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
+        @noindex, @nofollow, @security_error, @retries, @depth,
+        @discovered_via_sitemap, @redirect_chain, @bytes_received, @is_internal, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
         datetime('now')
       )
       ON CONFLICT(site_id, normalized_url) DO UPDATE SET
+        first_seen_snapshot_id = COALESCE(pages.first_seen_snapshot_id, excluded.first_seen_snapshot_id),
         last_seen_snapshot_id = excluded.last_seen_snapshot_id,
         http_status = CASE WHEN excluded.http_status != 0 THEN excluded.http_status ELSE pages.http_status END,
         canonical_url = COALESCE(excluded.canonical_url, pages.canonical_url),
@@ -27,14 +28,15 @@ export class PageRepository {
         etag = COALESCE(excluded.etag, pages.etag),
         last_modified = COALESCE(excluded.last_modified, pages.last_modified),
         html = COALESCE(excluded.html, pages.html),
-        soft404_score = COALESCE(excluded.soft404_score, pages.soft404_score),
         noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END,
         nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END,
         security_error = COALESCE(excluded.security_error, pages.security_error),
         retries = MAX(pages.retries, excluded.retries),
         depth = MIN(pages.depth, excluded.depth),
+        discovered_via_sitemap = MAX(pages.discovered_via_sitemap, excluded.discovered_via_sitemap),
         redirect_chain = COALESCE(excluded.redirect_chain, pages.redirect_chain),
         bytes_received = COALESCE(excluded.bytes_received, pages.bytes_received),
+        is_internal = COALESCE(excluded.is_internal, pages.is_internal),
         crawl_trap_flag = MAX(pages.crawl_trap_flag, excluded.crawl_trap_flag),
         crawl_trap_risk = COALESCE(excluded.crawl_trap_risk, pages.crawl_trap_risk),
         trap_type = COALESCE(excluded.trap_type, pages.trap_type),
@@ -55,14 +57,15 @@ export class PageRepository {
             etag: page.etag ?? null,
             last_modified: page.last_modified ?? null,
             html: page.html ?? null,
-            soft404_score: page.soft404_score ?? null,
             noindex: page.noindex ?? 0,
             nofollow: page.nofollow ?? 0,
             security_error: page.security_error ?? null,
             retries: page.retries ?? 0,
             depth: page.depth ?? 0,
+            discovered_via_sitemap: page.discovered_via_sitemap ?? 0,
             redirect_chain: page.redirect_chain ?? null,
             bytes_received: page.bytes_received ?? null,
+            is_internal: page.is_internal ?? 1,
             crawl_trap_flag: page.crawl_trap_flag ?? 0,
             crawl_trap_risk: page.crawl_trap_risk ?? null,
             trap_type: page.trap_type ?? null,
@@ -103,17 +106,18 @@ export class PageRepository {
       INSERT INTO pages (
         site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id,
         http_status, canonical_url, content_hash, simhash, etag, last_modified, html,
-        soft404_score, noindex, nofollow, security_error, retries, depth,
-        redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type,
+        noindex, nofollow, security_error, retries, depth,
+        discovered_via_sitemap, redirect_chain, bytes_received, is_internal, crawl_trap_flag, crawl_trap_risk, trap_type,
         updated_at
       ) VALUES (
         @site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id,
         @http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html,
-        @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth,
-        @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
+        @noindex, @nofollow, @security_error, @retries, @depth,
+        @discovered_via_sitemap, @redirect_chain, @bytes_received, @is_internal, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
         datetime('now')
       )
       ON CONFLICT(site_id, normalized_url) DO UPDATE SET
+        first_seen_snapshot_id = COALESCE(pages.first_seen_snapshot_id, excluded.first_seen_snapshot_id),
         last_seen_snapshot_id = excluded.last_seen_snapshot_id,
         http_status = CASE WHEN excluded.http_status != 0 THEN excluded.http_status ELSE pages.http_status END,
         canonical_url = COALESCE(excluded.canonical_url, pages.canonical_url),
@@ -122,14 +126,15 @@ export class PageRepository {
         etag = COALESCE(excluded.etag, pages.etag),
         last_modified = COALESCE(excluded.last_modified, pages.last_modified),
         html = COALESCE(excluded.html, pages.html),
-        soft404_score = COALESCE(excluded.soft404_score, pages.soft404_score),
         noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END,
         nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END,
         security_error = COALESCE(excluded.security_error, pages.security_error),
         retries = MAX(pages.retries, excluded.retries),
         depth = MIN(pages.depth, excluded.depth),
+        discovered_via_sitemap = MAX(pages.discovered_via_sitemap, excluded.discovered_via_sitemap),
         redirect_chain = COALESCE(excluded.redirect_chain, pages.redirect_chain),
         bytes_received = COALESCE(excluded.bytes_received, pages.bytes_received),
+        is_internal = COALESCE(excluded.is_internal, pages.is_internal),
         crawl_trap_flag = MAX(pages.crawl_trap_flag, excluded.crawl_trap_flag),
         crawl_trap_risk = COALESCE(excluded.crawl_trap_risk, pages.crawl_trap_risk),
         trap_type = COALESCE(excluded.trap_type, pages.trap_type),
@@ -151,14 +156,15 @@ export class PageRepository {
                     etag: page.etag ?? null,
                     last_modified: page.last_modified ?? null,
                     html: page.html ?? null,
-                    soft404_score: page.soft404_score ?? null,
                     noindex: page.noindex ?? 0,
                     nofollow: page.nofollow ?? 0,
                     security_error: page.security_error ?? null,
                     retries: page.retries ?? 0,
                     depth: page.depth ?? 0,
+                    discovered_via_sitemap: page.discovered_via_sitemap ?? 0,
                     redirect_chain: page.redirect_chain ?? null,
                     bytes_received: page.bytes_received ?? null,
+                    is_internal: page.is_internal ?? 1,
                     crawl_trap_flag: page.crawl_trap_flag ?? 0,
                     crawl_trap_risk: page.crawl_trap_risk ?? null,
                     trap_type: page.trap_type ?? null,
@@ -170,17 +176,62 @@ export class PageRepository {
         tx(pages);
         return urlToId;
     }
-    getPagesBySnapshot(snapshotId) {
-        return this.db.prepare('SELECT p.* FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND p.first_seen_snapshot_id <= ?').all(snapshotId, snapshotId);
+    getPagesBySnapshot(snapshotId, runType = 'completed') {
+        if (runType === 'single') {
+            return this.db.prepare('SELECT p.* FROM pages p JOIN metrics m ON p.id = m.page_id WHERE m.snapshot_id = ?').all(snapshotId);
+        }
+        return this.db.prepare('SELECT p.* FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND COALESCE(p.first_seen_snapshot_id, p.last_seen_snapshot_id) <= ?').all(snapshotId, snapshotId);
     }
     getPagesIdentityBySnapshot(snapshotId) {
-        return this.db.prepare('SELECT p.id, p.normalized_url FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND p.first_seen_snapshot_id <= ?').all(snapshotId, snapshotId);
+        // For identities, always loading all up to this point is fine for the crawler to map URLs to IDs.
+        return this.db.prepare('SELECT p.id, p.normalized_url FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND COALESCE(p.first_seen_snapshot_id, p.last_seen_snapshot_id) <= ?').all(snapshotId, snapshotId);
     }
-    getPagesIteratorBySnapshot(snapshotId) {
-        return this.db.prepare('SELECT p.* FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND p.first_seen_snapshot_id <= ?').iterate(snapshotId, snapshotId);
+    getPagesIteratorBySnapshot(snapshotId, runType = 'completed') {
+        if (runType === 'single') {
+            return this.db.prepare('SELECT p.* FROM pages p JOIN metrics m ON p.id = m.page_id WHERE m.snapshot_id = ?').iterate(snapshotId);
+        }
+        return this.db.prepare('SELECT p.* FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND COALESCE(p.first_seen_snapshot_id, p.last_seen_snapshot_id) <= ?').iterate(snapshotId, snapshotId);
     }
     getIdByUrl(siteId, url) {
         const row = this.getIdStmt.get(siteId, url);
         return row?.id;
     }
+    reconcileInternalUrls(siteId, siteOrigin) {
+        const origin = siteOrigin.replace(/\/+$/, '');
+        const tx = this.db.transaction(() => {
+            const rows = this.db
+                .prepare("SELECT id, normalized_url FROM pages WHERE site_id = ? AND (normalized_url LIKE 'http://%' OR normalized_url LIKE 'https://%')")
+                .all(siteId);
+            for (const row of rows) {
+                let parsed;
+                try {
+                    parsed = new URL(row.normalized_url);
+                }
+                catch {
+                    continue;
+                }
+                if (parsed.origin !== origin) {
+                    continue;
+                }
+                const targetPath = `${parsed.pathname}${parsed.search}`;
+                if (targetPath === row.normalized_url) {
+                    continue;
+                }
+                const existing = this.db
+                    .prepare('SELECT id FROM pages WHERE site_id = ? AND normalized_url = ?')
+                    .get(siteId, targetPath);
+                if (existing && existing.id !== row.id) {
+                    this.db.prepare('UPDATE edges SET source_page_id = ? WHERE source_page_id = ?').run(existing.id, row.id);
+                    this.db.prepare('UPDATE edges SET target_page_id = ? WHERE target_page_id = ?').run(existing.id, row.id);
+                    this.db.prepare('UPDATE OR IGNORE metrics SET page_id = ? WHERE page_id = ?').run(existing.id, row.id);
+                    this.db.prepare('DELETE FROM metrics WHERE page_id = ?').run(row.id);
+                    this.db.prepare('DELETE FROM pages WHERE id = ?').run(row.id);
+                }
+                else {
+                    this.db.prepare('UPDATE pages SET normalized_url = ? WHERE id = ?').run(targetPath, row.id);
+                }
+            }
+        });
+        tx();
+    }
 }