@crawlith/core 0.1.0 → 0.1.2

package/dist/db/index.js

@@ -1,9 +1,12 @@
-import Database from 'better-sqlite3';
 import path from 'node:path';
 import fs from 'node:fs';
 import os from 'node:os';
-import { initSchema } from './schema.js';
+import { CrawlithDB } from './CrawlithDB.js';
 let dbInstance = null;
+let crawlithDbInstance = null;
+export * from './repositories/SiteRepository.js';
+export * from './repositories/SnapshotRepository.js';
+export * from './CrawlithDB.js';
 export function getDbPath() {
     if (process.env.NODE_ENV === 'test') {
         return ':memory:';
@@ -20,41 +23,50 @@ export function getDbPath() {
     }
     return path.join(crawlithDir, 'crawlith.db');
 }
-export function getDb() {
-    if (dbInstance) {
-        return dbInstance;
+/**
+ * Returns the higher-level CrawlithDB wrapper for plugins and new code.
+ */
+export function getCrawlithDB() {
+    if (crawlithDbInstance) {
+        return crawlithDbInstance;
     }
     const dbPath = getDbPath();
-    const db = new Database(dbPath);
-    // Hardening & Performance Configuration
-    db.pragma('journal_mode = WAL');
-    db.pragma('synchronous = NORMAL');
-    db.pragma('foreign_keys = ON');
-    db.pragma('temp_store = MEMORY');
-    db.pragma('mmap_size = 30000000000');
-    db.pragma('cache_size = -20000');
-    db.pragma('busy_timeout = 5000');
-    // Security controls
-    // Ensure file permissions are 600 (user read/write only)
+    crawlithDbInstance = new CrawlithDB(dbPath);
+    dbInstance = crawlithDbInstance.unsafeGetRawDb();
+    // Migrations for existing tables
     try {
-        fs.chmodSync(dbPath, 0o600);
+        dbInstance.exec(`ALTER TABLE pages ADD COLUMN discovered_via_sitemap INTEGER DEFAULT 0;`);
     }
-    catch (_e) {
-        // might fail on first creation if file doesn't exist yet, but better-sqlite3 creates it
-        // so we can try again or ignore if it's new
+    catch (_e) { /* ignore */ }
+    // Security controls: Ensure file permissions are 600 (user read/write only)
+    if (dbPath !== ':memory:') {
+        try {
+            fs.chmodSync(dbPath, 0o600);
+        }
+        catch (_e) {
+            // might fail if file doesn't exist yet but better-sqlite3 should have created it
+        }
     }
-    // Integrity check on startup
-    const integrity = db.pragma('integrity_check', { simple: true });
-    if (integrity !== 'ok') {
-        console.warn('Database integrity check failed:', integrity);
+    return crawlithDbInstance;
+}
+/**
+ * Returns the raw better-sqlite3 Database instance for legacy repositories.
+ */
+export function getDb() {
+    if (dbInstance) {
+        return dbInstance;
     }
-    // Initialize schema
-    initSchema(db);
-    dbInstance = db;
-    return db;
+    // Initializing via getCrawlithDB ensures consistent configuration
+    getCrawlithDB();
+    return dbInstance;
 }
 export function closeDb() {
-    if (dbInstance) {
+    if (crawlithDbInstance) {
+        crawlithDbInstance.close();
+        crawlithDbInstance = null;
+        dbInstance = null;
+    }
+    else if (dbInstance) {
         dbInstance.close();
         dbInstance = null;
     }

package/dist/db/migrations.d.ts

@@ -0,0 +1,2 @@
+import { Database } from 'better-sqlite3';
+export declare function runBaseMigrations(db: Database): void;

package/dist/db/{schema.js → migrations.js}

@@ -1,9 +1,11 @@
-export function initSchema(db) {
+export function runBaseMigrations(db) {
     // Sites Table
     db.exec(`
     CREATE TABLE IF NOT EXISTS sites (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       domain TEXT UNIQUE NOT NULL,
+      preferred_url TEXT,
+      ssl INTEGER,
       created_at TEXT DEFAULT (datetime('now')),
       settings_json TEXT,
       is_active INTEGER DEFAULT 1
@@ -14,18 +16,36 @@ export function initSchema(db) {
     CREATE TABLE IF NOT EXISTS snapshots (
       id INTEGER PRIMARY KEY AUTOINCREMENT,
       site_id INTEGER NOT NULL,
-      type TEXT CHECK(type IN ('full', 'partial', 'incremental')) NOT NULL,
+      run_type TEXT CHECK(run_type IN ('completed', 'incremental', 'single')) NOT NULL DEFAULT 'completed',
       created_at TEXT DEFAULT (datetime('now')),
       node_count INTEGER DEFAULT 0,
       edge_count INTEGER DEFAULT 0,
-      status TEXT CHECK(status IN ('running', 'completed', 'failed')) DEFAULT 'running',
+      status TEXT CHECK(status IN ('queued', 'running', 'completed', 'failed', 'cancelled')) DEFAULT 'running',
       limit_reached INTEGER DEFAULT 0,
       health_score REAL,
       orphan_count INTEGER,
       thin_content_count INTEGER,
+      total_score REAL,
+      score_count INTEGER,
+      score_weight_sum REAL,
+      score_calculated_at TEXT,
       FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE
     );
   `);
+    // Migration for snapshots: run_type and status
+    try {
+        db.exec(`ALTER TABLE snapshots ADD COLUMN run_type TEXT CHECK(run_type IN ('completed', 'incremental', 'single')) NOT NULL DEFAULT 'completed';`);
+    }
+    catch (_e) { /* ignore */ }
+    try {
+        // If type column exists, populate run_type from it
+        db.exec(`UPDATE snapshots SET run_type = CASE WHEN type = 'partial' THEN 'single' ELSE 'completed' END WHERE run_type IS NULL OR run_type = 'full' OR run_type = 'completed';`);
+    }
+    catch (_e) { /* ignore */ }
+    try {
+        db.exec(`ALTER TABLE snapshots DROP COLUMN type;`);
+    }
+    catch (_e) { /* ignore */ }
     // Pages Table
     db.exec(`
     CREATE TABLE IF NOT EXISTS pages (
@@ -41,14 +61,15 @@ export function initSchema(db) {
       etag TEXT,
       last_modified TEXT,
       html TEXT,
-      soft404_score REAL,
       noindex INTEGER DEFAULT 0,
       nofollow INTEGER DEFAULT 0,
       security_error TEXT,
       retries INTEGER DEFAULT 0,
       depth INTEGER DEFAULT 0,
+      discovered_via_sitemap INTEGER DEFAULT 0,
       redirect_chain TEXT,
       bytes_received INTEGER,
+      is_internal INTEGER DEFAULT 1,
       crawl_trap_flag INTEGER DEFAULT 0,
       crawl_trap_risk REAL,
       trap_type TEXT,
@@ -60,7 +81,15 @@ export function initSchema(db) {
       UNIQUE(site_id, normalized_url)
     );
   `);
-    // Index for Pages
+    // Migrations for existing tables
+    try {
+        db.exec(`ALTER TABLE pages ADD COLUMN is_internal INTEGER DEFAULT 1;`);
+    }
+    catch (_e) { /* ignore */ }
+    try {
+        db.exec(`ALTER TABLE pages ADD COLUMN discovered_via_sitemap INTEGER DEFAULT 0;`);
+    }
+    catch (_e) { /* ignore */ }
     db.exec(`CREATE INDEX IF NOT EXISTS idx_pages_site_last_seen ON pages(site_id, last_seen_snapshot_id);`);
     // Edges Table
     db.exec(`
@@ -76,26 +105,29 @@ export function initSchema(db) {
       FOREIGN KEY(target_page_id) REFERENCES pages(id) ON DELETE CASCADE
     );
   `);
-    // Index for Edges
-    db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot_source ON edges(snapshot_id, source_page_id);`);
     db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot ON edges(snapshot_id);`);
     // Metrics Table
     db.exec(`
     CREATE TABLE IF NOT EXISTS metrics (
       snapshot_id INTEGER NOT NULL,
       page_id INTEGER NOT NULL,
-      authority_score REAL,
-      hub_score REAL,
-      pagerank REAL,
-      pagerank_score REAL,
-      link_role TEXT CHECK(link_role IN ('hub', 'authority', 'power', 'balanced', 'peripheral')),
       crawl_status TEXT,
       word_count INTEGER,
       thin_content_score REAL,
       external_link_ratio REAL,
       orphan_score INTEGER,
+      pagerank_score REAL,
+      hub_score REAL,
+      auth_score REAL,
+      link_role TEXT,
       duplicate_cluster_id TEXT,
-      duplicate_type TEXT CHECK(duplicate_type IN ('exact', 'near', 'template_heavy', 'none')),
+      duplicate_type TEXT,
+      cluster_id INTEGER,
+      soft404_score REAL,
+      heading_score REAL,
+      orphan_type TEXT,
+      impact_level TEXT,
+      heading_data TEXT,
       is_cluster_primary INTEGER DEFAULT 0,
       PRIMARY KEY(snapshot_id, page_id),
       FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
@@ -129,41 +161,56 @@ export function initSchema(db) {
       FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
     );
   `);
-    // Migration: add columns to existing DBs that were created before this update
-    migrateSchema(db);
-}
-function migrateSchema(db) {
-    // Add missing columns to pages (safe: ALTER TABLE ADD COLUMN is idempotent-safe with try/catch)
-    const pageColumns = [
-        ['redirect_chain', 'TEXT'],
-        ['bytes_received', 'INTEGER'],
-        ['crawl_trap_flag', 'INTEGER DEFAULT 0'],
-        ['crawl_trap_risk', 'REAL'],
-        ['trap_type', 'TEXT'],
-    ];
-    for (const [col, type] of pageColumns) {
-        try {
-            db.exec(`ALTER TABLE pages ADD COLUMN ${col} ${type}`);
-        }
-        catch { /* already exists */ }
-    }
-    // Add missing columns to edges
-    try {
-        db.exec('ALTER TABLE edges ADD COLUMN weight REAL DEFAULT 1.0');
-    }
-    catch { /* already exists */ }
-    // Add missing columns to metrics
-    const metricsColumns = [
+    // Plugin Migrations Table
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS plugin_migrations (
+      plugin_name TEXT PRIMARY KEY,
+      executed_at TEXT NOT NULL DEFAULT (datetime('now'))
+    );
+  `);
+    // Universal Plugin Reports Table
+    db.exec(`
+    CREATE TABLE IF NOT EXISTS plugin_reports (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      snapshot_id INTEGER NOT NULL,
+      plugin_name TEXT NOT NULL,
+      data TEXT NOT NULL,
+      total_score REAL,
+      score_count INTEGER,
+      score_weight_sum REAL,
+      score_calculated_at TEXT,
+      created_at TEXT NOT NULL DEFAULT (datetime('now')),
+      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
+    );
+  `);
+    db.exec(`CREATE INDEX IF NOT EXISTS idx_plugin_reports_snapshot ON plugin_reports(snapshot_id);`);
+    db.exec(`CREATE INDEX IF NOT EXISTS idx_plugin_reports_composite ON plugin_reports(snapshot_id, plugin_name);`);
+    // Migrations for metrics
+    const metricsCols = [
         ['pagerank_score', 'REAL'],
+        ['hub_score', 'REAL'],
+        ['auth_score', 'REAL'],
         ['link_role', 'TEXT'],
-        ['duplicate_cluster_id', 'TEXT'],
-        ['duplicate_type', 'TEXT'],
-        ['is_cluster_primary', 'INTEGER DEFAULT 0'],
+        ['cluster_id', 'INTEGER'],
+        ['soft404_score', 'REAL'],
+        ['heading_score', 'REAL'],
+        ['orphan_type', 'TEXT'],
+        ['impact_level', 'TEXT'],
+        ['heading_data', 'TEXT'],
     ];
-    for (const [col, type] of metricsColumns) {
+    for (const [col, type] of metricsCols) {
         try {
             db.exec(`ALTER TABLE metrics ADD COLUMN ${col} ${type}`);
         }
-        catch { /* already exists */ }
+        catch { /* ignore */ }
+    }
+    // Final site column migrations
+    try {
+        db.exec('ALTER TABLE sites ADD COLUMN preferred_url TEXT');
+    }
+    catch { /* ignore */ }
+    try {
+        db.exec('ALTER TABLE sites ADD COLUMN ssl INTEGER');
     }
+    catch { /* ignore */ }
 }

package/dist/db/pluginRegistry.d.ts

@@ -0,0 +1,9 @@
+export declare class PluginRegistry {
+    private registeredPlugins;
+    private registeredTables;
+    registerPlugin(pluginName: string): void;
+    isPluginRegistered(pluginName: string): boolean;
+    registerTable(tableName: string, pluginName: string): void;
+    getPluginForTable(tableName: string): string | undefined;
+    isTableRegistered(tableName: string): boolean;
+}

package/dist/db/pluginRegistry.js

@@ -0,0 +1,19 @@
+export class PluginRegistry {
+    registeredPlugins = new Set();
+    registeredTables = new Map(); // tableName -> pluginName
+    registerPlugin(pluginName) {
+        this.registeredPlugins.add(pluginName);
+    }
+    isPluginRegistered(pluginName) {
+        return this.registeredPlugins.has(pluginName);
+    }
+    registerTable(tableName, pluginName) {
+        this.registeredTables.set(tableName, pluginName);
+    }
+    getPluginForTable(tableName) {
+        return this.registeredTables.get(tableName);
+    }
+    isTableRegistered(tableName) {
+        return this.registeredTables.has(tableName);
+    }
+}

package/dist/db/repositories/EdgeRepository.d.ts

@@ -12,5 +12,18 @@ export declare class EdgeRepository {
     private insertStmt;
     constructor(db: Database);
     insertEdge(snapshotId: number, sourcePageId: number, targetPageId: number, weight?: number, rel?: string): void;
+    insertEdges(edges: {
+        snapshot_id: number;
+        source_page_id: number;
+        target_page_id: number;
+        weight: number;
+        rel: string;
+    }[]): void;
+    /**
+     * Remove all edges originating from a specific page within a snapshot.
+     * Used when re-crawling a page into a reused partial snapshot to avoid duplicates.
+     */
+    deleteEdgesForPage(snapshotId: number, sourcePageId: number): void;
     getEdgesBySnapshot(snapshotId: number): Edge[];
+    getEdgesIteratorBySnapshot(snapshotId: number): IterableIterator<Edge>;
 }

package/dist/db/repositories/EdgeRepository.js

@@ -11,7 +11,27 @@ export class EdgeRepository {
     insertEdge(snapshotId, sourcePageId, targetPageId, weight = 1.0, rel = 'internal') {
         this.insertStmt.run(snapshotId, sourcePageId, targetPageId, weight, rel);
     }
+    insertEdges(edges) {
+        if (edges.length === 0)
+            return;
+        const tx = this.db.transaction((edgesBatch) => {
+            for (const edge of edgesBatch) {
+                this.insertStmt.run(edge.snapshot_id, edge.source_page_id, edge.target_page_id, edge.weight, edge.rel);
+            }
+        });
+        tx(edges);
+    }
+    /**
+     * Remove all edges originating from a specific page within a snapshot.
+     * Used when re-crawling a page into a reused partial snapshot to avoid duplicates.
+     */
+    deleteEdgesForPage(snapshotId, sourcePageId) {
+        this.db.prepare('DELETE FROM edges WHERE snapshot_id = ? AND source_page_id = ?').run(snapshotId, sourcePageId);
+    }
     getEdgesBySnapshot(snapshotId) {
         return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').all(snapshotId);
     }
+    getEdgesIteratorBySnapshot(snapshotId) {
+        return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').iterate(snapshotId);
+    }
 }

package/dist/db/repositories/MetricsRepository.d.ts

@@ -2,25 +2,33 @@ import { Database } from 'better-sqlite3';
 export interface DbMetrics {
     snapshot_id: number;
     page_id: number;
-    authority_score: number | null;
-    hub_score: number | null;
-    pagerank: number | null;
-    pagerank_score: number | null;
-    link_role: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral' | null;
     crawl_status: string | null;
     word_count: number | null;
     thin_content_score: number | null;
     external_link_ratio: number | null;
-    orphan_score: number | null;
+    pagerank_score: number | null;
+    hub_score: number | null;
+    auth_score: number | null;
+    link_role: string | null;
     duplicate_cluster_id: string | null;
-    duplicate_type: 'exact' | 'near' | 'template_heavy' | 'none' | null;
-    is_cluster_primary: number;
+    duplicate_type: string | null;
+    cluster_id: number | null;
+    soft404_score: number | null;
+    heading_score: number | null;
+    orphan_score: number | null;
+    orphan_type: string | null;
+    impact_level: string | null;
+    heading_data: string | null;
+    is_cluster_primary: number | null;
 }
 export declare class MetricsRepository {
     private db;
     private insertStmt;
+    private getByPageStmt;
     constructor(db: Database);
     insertMetrics(metrics: DbMetrics): void;
     getMetrics(snapshotId: number): DbMetrics[];
+    getMetricsIterator(snapshotId: number): IterableIterator<DbMetrics>;
     getMetricsForPage(snapshotId: number, pageId: number): DbMetrics | undefined;
+    insertMany(metricsList: DbMetrics[]): void;
 }

package/dist/db/repositories/MetricsRepository.js

@@ -1,17 +1,27 @@
 export class MetricsRepository {
     db;
     insertStmt;
+    getByPageStmt;
     constructor(db) {
         this.db = db;
+        this.getByPageStmt = this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ? AND page_id = ?');
         this.insertStmt = this.db.prepare(`
       INSERT OR REPLACE INTO metrics (
-        snapshot_id, page_id, authority_score, hub_score, pagerank, pagerank_score,
-        link_role, crawl_status, word_count, thin_content_score, external_link_ratio,
-        orphan_score, duplicate_cluster_id, duplicate_type, is_cluster_primary
+        snapshot_id, page_id,
+        crawl_status, word_count, thin_content_score, external_link_ratio,
+        pagerank_score, hub_score, auth_score, link_role,
+        duplicate_cluster_id, duplicate_type, cluster_id,
+        soft404_score, heading_score,
+        orphan_score, orphan_type, impact_level,
+        heading_data, is_cluster_primary
       ) VALUES (
-        @snapshot_id, @page_id, @authority_score, @hub_score, @pagerank, @pagerank_score,
-        @link_role, @crawl_status, @word_count, @thin_content_score, @external_link_ratio,
-        @orphan_score, @duplicate_cluster_id, @duplicate_type, @is_cluster_primary
+        @snapshot_id, @page_id,
+        @crawl_status, @word_count, @thin_content_score, @external_link_ratio,
+        @pagerank_score, @hub_score, @auth_score, @link_role,
+        @duplicate_cluster_id, @duplicate_type, @cluster_id,
+        @soft404_score, @heading_score,
+        @orphan_score, @orphan_type, @impact_level,
+        @heading_data, @is_cluster_primary
       )
     `);
     }
@@ -21,7 +31,18 @@ export class MetricsRepository {
     getMetrics(snapshotId) {
         return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ?').all(snapshotId);
     }
+    getMetricsIterator(snapshotId) {
+        return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ?').iterate(snapshotId);
+    }
     getMetricsForPage(snapshotId, pageId) {
-        return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ? AND page_id = ?').get(snapshotId, pageId);
+        return this.getByPageStmt.get(snapshotId, pageId);
+    }
+    insertMany(metricsList) {
+        const insert = this.insertStmt;
+        const tx = this.db.transaction((items) => {
+            for (const item of items)
+                insert.run(item);
+        });
+        tx(metricsList);
     }
 }

package/dist/db/repositories/PageRepository.d.ts

@@ -12,14 +12,15 @@ export interface Page {
     etag: string | null;
     last_modified: string | null;
     html: string | null;
-    soft404_score: number | null;
     noindex: number;
     nofollow: number;
     security_error: string | null;
     retries: number;
     depth: number;
+    discovered_via_sitemap: number;
     redirect_chain: string | null;
     bytes_received: number | null;
+    is_internal: number;
     crawl_trap_flag: number;
     crawl_trap_risk: number | null;
     trap_type: string | null;
@@ -42,6 +43,18 @@ export declare class PageRepository {
         last_seen_snapshot_id: number;
     }): number;
     getPage(siteId: number, url: string): Page | undefined;
-    getPagesBySnapshot(snapshotId: number): Page[];
+    getPagesByUrls(siteId: number, urls: string[]): Page[];
+    upsertMany(pages: (Partial<Page> & {
+        site_id: number;
+        normalized_url: string;
+        last_seen_snapshot_id: number;
+    })[]): Map<string, number>;
+    getPagesBySnapshot(snapshotId: number, runType?: string): Page[];
+    getPagesIdentityBySnapshot(snapshotId: number): {
+        id: number;
+        normalized_url: string;
+    }[];
+    getPagesIteratorBySnapshot(snapshotId: number, runType?: string): IterableIterator<Page>;
     getIdByUrl(siteId: number, url: string): number | undefined;
+    reconcileInternalUrls(siteId: number, siteOrigin: string): void;
 }