@crawlith/core 0.1.0 → 0.1.2

package/dist/db/CrawlithDB.js

@@ -0,0 +1,500 @@
+import Database from 'better-sqlite3';
+import { runBaseMigrations } from './migrations.js';
+import { Statements } from './statements.js';
+import { PluginRegistry } from './pluginRegistry.js';
+import { normalizeUrl } from '../crawler/normalize.js';
+export class CrawlithDB {
+    db;
+    statements;
+    registry;
+    /**
+     * @internal
+     * Dangerous: Returns the raw better-sqlite3 instance.
+     * Core only. Plugins must never use this.
+     */
+    unsafeGetRawDb() {
+        return this.db;
+    }
+    // Optional scoping properties
+    _pluginName;
+    _snapshotId;
+    /** Whether live fallback is allowed (from --live flag). Core-controlled. */
+    _live = false;
+    /** Whether this plugin makes network calls. Core-controlled via plugin.storage.fetchMode. */
+    _fetchMode = 'network';
+    constructor(dbPath) {
+        this.db = new Database(dbPath);
+        this.db.pragma('journal_mode = WAL');
+        this.db.pragma('synchronous = NORMAL');
+        this.db.pragma('foreign_keys = ON');
+        this.db.pragma('temp_store = MEMORY');
+        this.db.pragma('mmap_size = 30000000000');
+        this.db.pragma('cache_size = -20000');
+        this.db.pragma('busy_timeout = 5000');
+        // Integrity check on startup
+        const integrity = this.db.pragma('integrity_check', { simple: true });
+        if (integrity !== 'ok') {
+            console.warn('Database integrity check failed:', integrity);
+        }
+        this.registry = new PluginRegistry();
+        this.initialize();
+        this.statements = new Statements(this.db);
+    }
+    /**
+     * Schema API
+     */
+    get schema() {
+        return {
+            define: (columns) => this.registerPluginDataSchema(columns)
+        };
+    }
+    /**
+     * Fluent Data API (URL-scoped rows)
+     */
+    get data() {
+        return {
+            save: (input) => this.insertPluginRow(input),
+            find: (url, options) => this.getPluginRow(url, undefined, undefined, options),
+            all: () => this.getPluginRows(),
+            /**
+             * Cache-first with live fallback. Core-enforced pattern:
+             * 1. If cached data exists → return it (always, regardless of age)
+             * 2. If no cache + fetchMode='network' + live=false → return null (skip)
+             * 3. If no cache + (fetchMode='local' OR live=true) → call fetchFn, save, return
+             *
+             * Plugin authors NEVER touch ctx.live — the core injects it via scope().
+             */
+            getOrFetch: (url, fetchFn) => this._getOrFetch(url, fetchFn),
+        };
+    }
+    /**
+     * Report API (Global snapshot summary)
+     */
+    get report() {
+        return {
+            save: (summary, optionalScores) => this.insertPluginReport({ summary, ...optionalScores }),
+            find: () => this.getPluginReport()
+        };
+    }
+    initialize() {
+        runBaseMigrations(this.db);
+    }
+    /**
+     * Create a scoped instance for a specific plugin.
+     * Also bakes in live + fetchMode so getOrFetch() can enforce the protocol
+     * without exposing those controls to the plugin author.
+     */
+    scope(pluginName, snapshotId, options = {}) {
+        if (this._pluginName && this._pluginName !== pluginName) {
+            throw new Error(`Security Violation: Cannot re-scope a database instance already bound to "${this._pluginName}"`);
+        }
+        const scoped = Object.create(this);
+        scoped._pluginName = pluginName;
+        scoped._snapshotId = snapshotId;
+        scoped._live = options.live ?? false;
+        scoped._fetchMode = options.fetchMode ?? 'network';
+        return scoped;
+    }
+    registerPluginDataSchema(pluginNameOrColumns, extraColumns) {
+        let pluginName = this._pluginName;
+        let columns = pluginNameOrColumns;
+        if (typeof pluginNameOrColumns === 'string') {
+            pluginName = pluginNameOrColumns;
+            columns = extraColumns;
+        }
+        if (!pluginName)
+            throw new Error('Plugin name is required for registration (use unbound DB or scope() before calling)');
+        if (!columns)
+            throw new Error('Columns definition is required');
+        // Sanitize: hyphens are invalid in unquoted SQLite identifiers (e.g. "heading-health" → "heading_health_plugin")
+        const tableName = this._toTableName(pluginName);
+        // Validate columns
+        const reserved = ['id', 'snapshot_id', 'url_id', 'score', 'weight', 'created_at'];
+        for (const col of Object.keys(columns)) {
+            if (reserved.includes(col.toLowerCase())) {
+                throw new Error(`Plugin "${pluginName}" cannot define reserved column "${col}". Reserved: ${reserved.join(', ')}`);
+            }
+        }
+        if (this._isMigrationExecuted(pluginName)) {
+            // Even if executed, ensure the registry knows about the table name for this session
+            this.registry.registerTable(tableName, pluginName);
+            return;
+        }
+        const columnDefs = [
+            'id INTEGER PRIMARY KEY AUTOINCREMENT',
+            'snapshot_id INTEGER NOT NULL',
+            'url_id INTEGER NOT NULL',
+            ...Object.entries(columns).map(([col, type]) => `${col} ${type}`),
+            "score REAL",
+            "weight REAL DEFAULT 1.0",
+            "created_at TEXT DEFAULT (datetime('now'))",
+            'FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE',
+            'FOREIGN KEY(url_id) REFERENCES pages(id) ON DELETE CASCADE'
+        ];
+        const migrationSQL = `
+            CREATE TABLE IF NOT EXISTS ${tableName} (
+                ${columnDefs.join(',\n                ')}
+            );
+            CREATE INDEX IF NOT EXISTS idx_${tableName}_snapshot_url ON ${tableName}(snapshot_id, url_id);
+        `;
+        this.runInTransaction(() => {
+            this.registry.registerTable(tableName, pluginName);
+            this.db.exec(migrationSQL);
+            this.statements.insertMigration.run(pluginName);
+            this.registry.registerPlugin(pluginName);
+        });
+    }
+    /** @deprecated Use registerPluginDataSchema */
+    registerPluginMigration(pluginName, migrationSQL) {
+        this.runInTransaction(() => {
+            const tableMatches = migrationSQL.matchAll(/CREATE\s+TABLE\s+(?:IF\s+NOT\s+EXISTS\s+)?(\w+)/gi);
+            for (const match of tableMatches) {
+                this.registry.registerTable(match[1], pluginName);
+            }
+            this.db.exec(migrationSQL);
+            this.statements.insertMigration.run(pluginName);
+            this.registry.registerPlugin(pluginName);
+        });
+    }
+    getPageIdByUrl(snapshotId, url) {
+        const raw = url.trim();
+        if (!raw)
+            return null;
+        // Support both stored path format ("/foo") and absolute URL inputs.
+        const candidates = new Set();
+        const addCandidateWithSlashVariants = (value) => {
+            if (!value)
+                return;
+            candidates.add(value);
+            if (value === '/')
+                return;
+            if (value.endsWith('/')) {
+                candidates.add(value.slice(0, -1));
+            }
+            else {
+                candidates.add(`${value}/`);
+            }
+        };
+        if (raw.startsWith('/')) {
+            addCandidateWithSlashVariants(raw);
+        }
+        const normalizedPath = normalizeUrl(raw, '', { stripQuery: false, toPath: true });
+        if (normalizedPath) {
+            addCandidateWithSlashVariants(normalizedPath);
+        }
+        // Fallback for absolute URL parsing; normalizeUrl may return null for malformed/bare path input.
+        try {
+            const parsed = new URL(raw);
+            addCandidateWithSlashVariants(`${parsed.pathname}${parsed.search}`);
+        }
+        catch {
+            // ignore
+        }
+        for (const candidate of candidates) {
+            const row = this.statements.getPageIdByUrl.get(snapshotId, candidate);
+            if (row)
+                return row.id;
+        }
+        return null;
+    }
+    insertPluginReport(input) {
+        const snapshotId = input.snapshotId || this._snapshotId;
+        const pluginName = input.pluginName || this._pluginName;
+        if (!snapshotId)
+            throw new Error('snapshotId is required (not found in input or scope)');
+        if (!pluginName)
+            throw new Error('pluginName is required (not found in input or scope)');
+        this._assertSnapshotExists(snapshotId);
+        const data = JSON.stringify(input.summary);
+        this.statements.insertPluginReport.run(snapshotId, pluginName, data, input.totalScore ?? null, input.scoreCount ?? null, input.scoreWeightSum ?? null, input.scoreCalculatedAt ?? null);
+    }
+    insertPluginRow(input) {
+        const tableName = input.tableName || this._pluginName;
+        const snapshotId = input.snapshotId || this._snapshotId;
+        if (!tableName)
+            throw new Error('tableName/pluginName is required');
+        if (!snapshotId)
+            throw new Error('snapshotId is required');
+        const resolvedTable = this._resolveTableName(tableName);
+        this._assertSnapshotExists(snapshotId);
+        this._assertOwnership(resolvedTable);
+        this._assertTableRegistered(resolvedTable);
+        const urlId = this.getPageIdByUrl(snapshotId, input.url);
+        if (!urlId) {
+            throw new Error(`URL "${input.url}" not found in snapshot ${snapshotId}`);
+        }
+        const columns = Object.keys(input.data);
+        const placeholders = columns.map(() => '?').join(', ');
+        const fields = ['snapshot_id', 'url_id', ...columns].join(', ');
+        // We must use a dynamic but safe query here because T is unknown.
+        // However, since we validated tableName against the registry, it's safe.
+        // We still use parameters for all values.
+        const stmt = this.db.prepare(`
+      INSERT INTO ${resolvedTable} (${fields})
+      VALUES (?, ?, ${placeholders})
+    `);
+        const values = Object.values(input.data).map(v => typeof v === 'object' && v !== null ? JSON.stringify(v) : v);
+        stmt.run(snapshotId, urlId, ...values);
+    }
+    getPluginReport(snapshotId, pluginName) {
+        const sid = snapshotId || this._snapshotId;
+        const name = pluginName || this._pluginName;
+        if (!sid || !name)
+            throw new Error('snapshotId and pluginName are required');
+        const row = this.statements.getPluginReport.get(sid, name);
+        return row ? JSON.parse(row.data) : null;
+    }
+    getPluginRows(tableName, snapshotId) {
+        const targetTable = tableName || (this._pluginName ? this._toTableName(this._pluginName) : undefined);
+        const sid = snapshotId || this._snapshotId;
+        if (!targetTable || !sid)
+            throw new Error('Table name and snapshotId are required');
+        const resolvedTable = this._resolveTableName(targetTable);
+        this._assertTableRegistered(resolvedTable);
+        this._assertOwnership(resolvedTable);
+        const rows = this.db.prepare(`SELECT * FROM ${resolvedTable} WHERE snapshot_id = ?`).all(sid);
+        return rows.map(row => this._parseRow(row));
+    }
+    getPluginRow(tableNameOrUrl, snapshotId, url, options = {}) {
+        let targetTable = this._pluginName;
+        let sid = snapshotId || this._snapshotId;
+        let targetUrl = tableNameOrUrl;
+        // If called with 3 args or first arg is a registered table/plugin name
+        // Scoped instances MUST NOT allow overriding the target table to another plugin's table.
+        if (!this._pluginName && (url || this.registry.isTableRegistered(tableNameOrUrl) || this.registry.isTableRegistered(`${tableNameOrUrl}_plugin`))) {
+            targetTable = tableNameOrUrl;
+            sid = snapshotId || this._snapshotId;
+            targetUrl = url;
+        }
+        if (!targetTable || (!sid && !options.global) || !targetUrl) {
+            throw new Error(`Missing required arguments for getPluginRow: table=${targetTable}, snapshot=${sid}, url=${targetUrl}`);
+        }
+        const resolvedTable = this._resolveTableName(targetTable);
+        this._assertTableRegistered(resolvedTable);
+        this._assertOwnership(resolvedTable);
+        // We use normalized URL to get the ID, but for 'global' lookup we might need to be more careful.
+        // For now, we assume url_id maps to 'pages' which is snapshotted.
+        // Actually, if it's 'global', we should search by actual normalized URL across snapshots.
+        // Let's refine the query:
+        let query = `SELECT t.* FROM ${resolvedTable} t`;
+        const params = [];
+        if (options.global) {
+            // Join with pages to find the URL globally across all snapshots
+            query += ` JOIN pages p ON t.url_id = p.id WHERE p.normalized_url = ?`;
+            params.push(targetUrl);
+        }
+        else {
+            const urlId = this.getPageIdByUrl(sid, targetUrl);
+            if (!urlId)
+                return null;
+            query += ` WHERE t.snapshot_id = ? AND t.url_id = ?`;
+            params.push(sid, urlId);
+        }
+        if (options.maxAge) {
+            const seconds = typeof options.maxAge === 'number' ? options.maxAge : this._parseDuration(options.maxAge);
+            query += ` AND t.created_at >= datetime('now', '-${seconds} seconds')`;
+        }
+        query += ` ORDER BY t.id DESC LIMIT 1`;
+        const row = this.db.prepare(query).get(...params);
+        return row ? this._parseRow(row) : null;
+    }
+    _parseDuration(duration) {
+        const match = duration.match(/^(\d+)([hmds])$/);
+        if (!match)
+            throw new Error(`Invalid duration format: ${duration}. Use e.g. "24h", "1h", "600s"`);
+        const value = parseInt(match[1]);
+        const unit = match[2];
+        const multipliers = {
+            's': 1,
+            'm': 60,
+            'h': 3600,
+            'd': 86400
+        };
+        return value * multipliers[unit];
+    }
+    _parseRow(row) {
+        const result = { ...row };
+        for (const key in result) {
+            if (typeof result[key] === 'string' && (result[key].startsWith('{') || result[key].startsWith('['))) {
+                try {
+                    result[key] = JSON.parse(result[key]);
+                }
+                catch {
+                    // Not JSON or parse failed, keep as string
+                }
+            }
+        }
+        return result;
+    }
+    deleteSnapshotPlugins(snapshotId) {
+        this.runInTransaction(() => {
+            this.statements.deleteSnapshotPlugins.run(snapshotId);
+            // Also cleanup registered plugin tables
+            // We don't have a list of all rows in all tables, but we know the table names
+            // Registered in the registry.
+            // This implementation assumes plugins follow the convention of having a snapshot_id column.
+        });
+    }
+    async _getOrFetch(url, fetchFn) {
+        // 1. Check cache (global across snapshots)
+        const cached = this.getPluginRow(url, undefined, undefined, { global: true });
+        if (cached !== null) {
+            // Materialize a snapshot-local row even when cache hits globally,
+            // so per-snapshot score aggregation remains consistent.
+            if (this._pluginName && this._snapshotId) {
+                const existingForSnapshot = this.getPluginRow(url);
+                if (existingForSnapshot === null) {
+                    const cachedRow = cached;
+                    const { id: _id, snapshot_id: _snapshotId, url_id: _urlId, created_at: _createdAt, ...pluginData } = cachedRow;
+                    this.insertPluginRow({ url, data: pluginData });
+                }
+            }
+            return cached; // Always use cache when it exists
+        }
+        // 2. No cache. Can we fetch live?
+        const canFetchFresh = this._fetchMode === 'local' || (this._fetchMode === 'network' && this._live);
+        if (!canFetchFresh) {
+            return null; // Silent skip, no data to return
+        }
+        // 3. Compute/Fetch fresh data
+        const freshData = await fetchFn();
+        // 4. Save to DB
+        this.insertPluginRow({ url, data: freshData });
+        return freshData;
+    }
+    aggregateScoreProviders(snapshotId, plugins) {
+        this._assertSnapshotExists(snapshotId);
+        for (const plugin of plugins) {
+            if (!plugin.scoreProvider || !plugin.storage?.perPage?.columns)
+                continue;
+            const tableName = this._toTableName(plugin.name);
+            if (!this.registry.isTableRegistered(tableName))
+                continue;
+            try {
+                // Ensure the table schema strictly conforms by ignoring plugins that might have bypassed registry constraints
+                const aggregate = this.db.prepare(`
+                    SELECT 
+                        SUM(score * weight) as total_score,
+                        SUM(weight) as score_weight_sum,
+                        COUNT(score) as score_count
+                    FROM ${tableName}
+                    WHERE snapshot_id = ? AND score IS NOT NULL
+                `).get(snapshotId);
+                if (!aggregate || aggregate.score_count === 0)
+                    continue;
+                const { total_score, score_weight_sum, score_count } = aggregate;
+                const recentReport = this.db.prepare(`
+                    SELECT id FROM plugin_reports 
+                    WHERE snapshot_id = ? AND plugin_name = ?
+                    ORDER BY created_at DESC LIMIT 1
+                `).get(snapshotId, plugin.name);
+                if (recentReport) {
+                    this.db.prepare(`
+                        UPDATE plugin_reports 
+                        SET total_score = ?, score_weight_sum = ?, score_count = ?, score_calculated_at = datetime('now')
+                        WHERE id = ?
+                    `).run(total_score, score_weight_sum, score_count, recentReport.id);
+                }
+                else {
+                    this.db.prepare(`
+                        INSERT INTO plugin_reports 
+                        (snapshot_id, plugin_name, data, total_score, score_weight_sum, score_count, score_calculated_at)
+                        VALUES (?, ?, '{}', ?, ?, ?, datetime('now'))
+                    `).run(snapshotId, plugin.name, total_score, score_weight_sum, score_count);
+                }
+            }
+            catch (err) {
+                console.error(`[CrawlithDB.Aggregation] Failed to aggregate scores for plugin: ${plugin.name} - ${err.message}`);
+            }
+        }
+        try {
+            // After all plugins are aggregated, tally up the snapshot-level totals
+            const snapshotAggregate = this.db.prepare(`
+                SELECT 
+                    SUM(total_score) as overall_total_score,
+                    SUM(score_weight_sum) as overall_weight_sum,
+                    SUM(score_count) as overall_score_count
+                FROM plugin_reports
+                WHERE snapshot_id = ? AND total_score IS NOT NULL
+            `).get(snapshotId);
+            if (snapshotAggregate && snapshotAggregate.overall_score_count > 0) {
+                const pluginTotal = snapshotAggregate.overall_total_score || 0;
+                const pluginWeight = snapshotAggregate.overall_weight_sum || 0;
+                const pluginCount = snapshotAggregate.overall_score_count || 0;
+                // 1. Update the plugin-specific aggregate columns in the snapshots table
+                this.db.prepare(`
+                    UPDATE snapshots
+                    SET 
+                        total_score = ?, 
+                        score_weight_sum = ?, 
+                        score_count = ?, 
+                        score_calculated_at = datetime('now')
+                    WHERE id = ?
+                `).run(pluginTotal, pluginWeight, pluginCount, snapshotId);
+                // 2. Blend with the Core Health Score
+                // We fetch the current health_score (calculated by HealthService) and treat it 
+                // as a "Core" provider with a standard weight of 100.
+                const snapshot = this.db.prepare('SELECT health_score FROM snapshots WHERE id = ?').get(snapshotId);
+                const coreScore = snapshot?.health_score;
+                const CORE_WEIGHT = 100;
+                let blendedScore = null;
+                if (coreScore !== null && coreScore !== undefined) {
+                    // Weighted Average: (CoreScore * 100 + Sum(PluginScore * Weight)) / (100 + Sum(Weights))
+                    blendedScore = (coreScore * CORE_WEIGHT + pluginTotal) / (CORE_WEIGHT + pluginWeight);
+                }
+                else {
+                    // Fallback to pure plugin average if core health isn't computed
+                    blendedScore = pluginTotal / pluginWeight;
+                }
+                if (blendedScore !== null) {
+                    this.db.prepare('UPDATE snapshots SET health_score = ? WHERE id = ?').run(Number(blendedScore.toFixed(1)), snapshotId);
+                }
+            }
+        }
+        catch (err) {
+            console.error(`[CrawlithDB.Aggregation] Failed to aggregate snapshot-level scores: ${err.message}`);
+        }
+    }
+    runInTransaction(fn) {
+        const tx = this.db.transaction(fn);
+        tx();
+    }
+    _resolveTableName(name) {
+        if (this.registry.isTableRegistered(name))
+            return name;
+        const pluginTable = this._toTableName(name);
+        if (this.registry.isTableRegistered(pluginTable))
+            return pluginTable;
+        return name; // Will fail assertion later
+    }
+    /** Converts a plugin name to its canonical SQLite table name, sanitizing invalid characters. */
+    _toTableName(pluginName) {
+        return `${pluginName.replace(/-/g, '_')}_plugin`;
+    }
+    close() {
+        this.db.close();
+    }
+    _isMigrationExecuted(pluginName) {
+        const row = this.statements.getMigration.get(pluginName);
+        return !!row;
+    }
+    _assertSnapshotExists(snapshotId) {
+        const row = this.statements.getSnapshot.get(snapshotId);
+        if (!row) {
+            throw new Error(`Snapshot ID ${snapshotId} does not exist`);
+        }
+    }
+    _assertTableRegistered(tableName) {
+        if (!this.registry.isTableRegistered(tableName)) {
+            throw new Error(`Access Denied: Table "${tableName}" is not registered by any plugin migration.`);
+        }
+    }
+    _assertOwnership(tableName) {
+        if (!this._pluginName)
+            return; // Unbound instance has full access
+        const owner = this.registry.getPluginForTable(tableName);
+        if (owner !== this._pluginName) {
+            throw new Error(`Security Violation: Plugin "${this._pluginName}" attempted to access table "${tableName}" owned by "${owner}".`);
+        }
+    }
+}

package/dist/db/graphLoader.js

@@ -10,22 +10,38 @@ export function loadGraphFromSnapshot(snapshotId) {
     const edgeRepo = new EdgeRepository(db);
     const metricsRepo = new MetricsRepository(db);
     const snapshotRepo = new SnapshotRepository(db);
-    const pages = pageRepo.getPagesBySnapshot(snapshotId);
-    const metrics = metricsRepo.getMetrics(snapshotId);
     const snapshot = snapshotRepo.getSnapshot(snapshotId);
+    const pages = pageRepo.getPagesIteratorBySnapshot(snapshotId, snapshot?.run_type || 'completed');
+    const metrics = metricsRepo.getMetricsIterator(snapshotId);
     const metricsMap = new Map();
     for (const m of metrics) {
         metricsMap.set(m.page_id, m);
     }
     const graph = new Graph();
+    let pagesFetched = 0;
+    let pagesCached = 0;
+    let pagesSkipped = 0;
     if (snapshot) {
         graph.limitReached = !!snapshot.limit_reached;
     }
     const idMap = new Map();
     for (const p of pages) {
         idMap.set(p.id, p.normalized_url);
-        graph.addNode(p.normalized_url, p.depth, p.http_status || 0);
+        graph.addNode(p.normalized_url, p.depth, p.http_status || 0, !!p.is_internal);
         const m = metricsMap.get(p.id);
+        if (m) {
+            const isProcessed = m.crawl_status === 'fetched' ||
+                m.crawl_status === 'fetched_error' ||
+                m.crawl_status === 'network_error' ||
+                m.crawl_status === 'failed_after_retries' ||
+                m.crawl_status === 'blocked_by_robots';
+            if (isProcessed)
+                pagesFetched++;
+            else if (m.crawl_status === 'cached')
+                pagesCached++;
+            else if (m.crawl_status === 'skipped')
+                pagesSkipped++;
+        }
         let incrementalStatus;
         if (p.first_seen_snapshot_id === snapshotId) {
             incrementalStatus = 'new';
@@ -37,13 +53,14 @@ export function loadGraphFromSnapshot(snapshotId) {
             incrementalStatus = 'changed';
         }
         graph.updateNodeData(p.normalized_url, {
+            isInternal: !!p.is_internal,
             canonical: p.canonical_url || undefined,
+            discoveredViaSitemap: !!p.discovered_via_sitemap,
             contentHash: p.content_hash || undefined,
             simhash: p.simhash || undefined,
             etag: p.etag || undefined,
             lastModified: p.last_modified || undefined,
             html: p.html || undefined,
-            soft404Score: p.soft404_score || undefined,
             noindex: !!p.noindex,
             nofollow: !!p.nofollow,
             incrementalStatus,
@@ -55,18 +72,24 @@ export function loadGraphFromSnapshot(snapshotId) {
             crawlTrapRisk: p.crawl_trap_risk || undefined,
             trapType: p.trap_type || undefined,
             // Metrics
-            pageRank: m?.pagerank ?? undefined,
-            pageRankScore: m?.pagerank_score ?? m?.pagerank ?? undefined,
-            authorityScore: m?.authority_score ?? undefined,
+            // Additional metrics
+            crawlStatus: m?.crawl_status || undefined,
+            wordCount: m?.word_count != null ? m.word_count : undefined,
+            thinContentScore: m?.thin_content_score != null ? m.thin_content_score : undefined,
+            externalLinkRatio: m?.external_link_ratio != null ? m.external_link_ratio : undefined,
+            pagerankScore: m?.pagerank_score ?? undefined,
             hubScore: m?.hub_score ?? undefined,
+            authScore: m?.auth_score ?? undefined,
             linkRole: m?.link_role ?? undefined,
-            // Duplicate info
-            duplicateClusterId: m?.duplicate_cluster_id ?? undefined,
-            duplicateType: m?.duplicate_type ?? undefined,
-            isClusterPrimary: m?.is_cluster_primary ? true : undefined,
+            soft404Score: m?.soft404_score ?? undefined,
+            headingScore: m?.heading_score ?? undefined,
+            orphanScore: m?.orphan_score ?? undefined,
+            orphanType: m?.orphan_type ?? undefined,
+            impactLevel: m?.impact_level ?? undefined,
+            headingData: m?.heading_data ?? undefined,
         });
     }
-    const edges = edgeRepo.getEdgesBySnapshot(snapshotId);
+    const edges = edgeRepo.getEdgesIteratorBySnapshot(snapshotId);
     for (const e of edges) {
         const source = idMap.get(e.source_page_id);
         const target = idMap.get(e.target_page_id);
@@ -74,23 +97,12 @@ export function loadGraphFromSnapshot(snapshotId) {
             graph.addEdge(source, target, e.weight || 1.0);
         }
     }
-    // Load duplicate clusters
-    const dupClusters = db.prepare('SELECT * FROM duplicate_clusters WHERE snapshot_id = ?').all(snapshotId);
-    graph.duplicateClusters = dupClusters.map(c => ({
-        id: c.id,
-        type: c.type,
-        size: c.size,
-        representative: c.representative,
-        severity: c.severity
-    }));
-    // Load content clusters
-    const contentClusters = db.prepare('SELECT * FROM content_clusters WHERE snapshot_id = ?').all(snapshotId);
-    graph.contentClusters = contentClusters.map(c => ({
-        id: c.id,
-        count: c.count,
-        primaryUrl: c.primary_url,
-        risk: c.risk,
-        sharedPathPrefix: c.shared_path_prefix || undefined
-    }));
+    // Set session stats
+    graph.sessionStats = {
+        pagesFetched,
+        pagesCached,
+        pagesSkipped,
+        totalFound: idMap.size
+    };
     return graph;
 }

package/dist/db/index.d.ts

@@ -1,4 +1,15 @@
 import Database from 'better-sqlite3';
+import { CrawlithDB } from './CrawlithDB.js';
+export * from './repositories/SiteRepository.js';
+export * from './repositories/SnapshotRepository.js';
+export * from './CrawlithDB.js';
 export declare function getDbPath(): string;
+/**
+ * Returns the higher-level CrawlithDB wrapper for plugins and new code.
+ */
+export declare function getCrawlithDB(): CrawlithDB;
+/**
+ * Returns the raw better-sqlite3 Database instance for legacy repositories.
+ */
 export declare function getDb(): Database.Database;
 export declare function closeDb(): void;