@crawlith/core 0.1.1 → 0.1.2

package/src/db/repositories/PageRepository.ts

@@ -1,228 +0,0 @@
-import { Database } from 'better-sqlite3';
-
-export interface Page {
-  id: number;
-  site_id: number;
-  normalized_url: string;
-  first_seen_snapshot_id: number | null;
-  last_seen_snapshot_id: number | null;
-  http_status: number | null;
-  canonical_url: string | null;
-  content_hash: string | null;
-  simhash: string | null;
-  etag: string | null;
-  last_modified: string | null;
-  html: string | null;
-  soft404_score: number | null;
-  noindex: number;
-  nofollow: number;
-  security_error: string | null;
-  retries: number;
-  depth: number;
-  redirect_chain: string | null;
-  bytes_received: number | null;
-  crawl_trap_flag: number;
-  crawl_trap_risk: number | null;
-  trap_type: string | null;
-  created_at: string;
-  updated_at: string;
-}
-
-export class PageRepository {
-  private upsertStmt;
-  private getIdStmt;
-
-  constructor(private db: Database) {
-    this.upsertStmt = this.db.prepare(`
-      INSERT INTO pages (
-        site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id,
-        http_status, canonical_url, content_hash, simhash, etag, last_modified, html,
-        soft404_score, noindex, nofollow, security_error, retries, depth,
-        redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type,
-        updated_at
-      ) VALUES (
-        @site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id,
-        @http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html,
-        @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth,
-        @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
-        datetime('now')
-      )
-      ON CONFLICT(site_id, normalized_url) DO UPDATE SET
-        last_seen_snapshot_id = excluded.last_seen_snapshot_id,
-        http_status = CASE WHEN excluded.http_status != 0 THEN excluded.http_status ELSE pages.http_status END,
-        canonical_url = COALESCE(excluded.canonical_url, pages.canonical_url),
-        content_hash = COALESCE(excluded.content_hash, pages.content_hash),
-        simhash = COALESCE(excluded.simhash, pages.simhash),
-        etag = COALESCE(excluded.etag, pages.etag),
-        last_modified = COALESCE(excluded.last_modified, pages.last_modified),
-        html = COALESCE(excluded.html, pages.html),
-        soft404_score = COALESCE(excluded.soft404_score, pages.soft404_score),
-        noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END,
-        nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END,
-        security_error = COALESCE(excluded.security_error, pages.security_error),
-        retries = MAX(pages.retries, excluded.retries),
-        depth = MIN(pages.depth, excluded.depth),
-        redirect_chain = COALESCE(excluded.redirect_chain, pages.redirect_chain),
-        bytes_received = COALESCE(excluded.bytes_received, pages.bytes_received),
-        crawl_trap_flag = MAX(pages.crawl_trap_flag, excluded.crawl_trap_flag),
-        crawl_trap_risk = COALESCE(excluded.crawl_trap_risk, pages.crawl_trap_risk),
-        trap_type = COALESCE(excluded.trap_type, pages.trap_type),
-        updated_at = datetime('now')
-    `);
-
-    this.getIdStmt = this.db.prepare('SELECT id FROM pages WHERE site_id = ? AND normalized_url = ?');
-  }
-
-  upsertPage(page: Partial<Page> & { site_id: number; normalized_url: string; last_seen_snapshot_id: number }) {
-    const params = {
-      site_id: page.site_id,
-      normalized_url: page.normalized_url,
-      first_seen_snapshot_id: page.first_seen_snapshot_id ?? page.last_seen_snapshot_id,
-      last_seen_snapshot_id: page.last_seen_snapshot_id,
-      http_status: page.http_status ?? null,
-      canonical_url: page.canonical_url ?? null,
-      content_hash: page.content_hash ?? null,
-      simhash: page.simhash ?? null,
-      etag: page.etag ?? null,
-      last_modified: page.last_modified ?? null,
-      html: page.html ?? null,
-      soft404_score: page.soft404_score ?? null,
-      noindex: page.noindex ?? 0,
-      nofollow: page.nofollow ?? 0,
-      security_error: page.security_error ?? null,
-      retries: page.retries ?? 0,
-      depth: page.depth ?? 0,
-      redirect_chain: page.redirect_chain ?? null,
-      bytes_received: page.bytes_received ?? null,
-      crawl_trap_flag: page.crawl_trap_flag ?? 0,
-      crawl_trap_risk: page.crawl_trap_risk ?? null,
-      trap_type: page.trap_type ?? null,
-    };
-
-    const info = this.upsertStmt.run(params);
-    return info;
-  }
-
-  upsertAndGetId(page: Partial<Page> & { site_id: number; normalized_url: string; last_seen_snapshot_id: number }): number {
-    const tx = this.db.transaction(() => {
-      this.upsertPage(page);
-      const row = this.getIdStmt.get(page.site_id, page.normalized_url) as { id: number } | undefined;
-      if (!row) throw new Error(`Failed to retrieve ID for upserted page: ${page.normalized_url}`);
-      return row.id;
-    });
-    return tx();
-  }
-
-  getPage(siteId: number, url: string): Page | undefined {
-    return this.db.prepare('SELECT * FROM pages WHERE site_id = ? AND normalized_url = ?').get(siteId, url) as Page | undefined;
-  }
-
-  getPagesByUrls(siteId: number, urls: string[]): Page[] {
-    if (urls.length === 0) return [];
-    const chunkSize = 900;
-    const results: Page[] = [];
-
-    for (let i = 0; i < urls.length; i += chunkSize) {
-      const chunk = urls.slice(i, i + chunkSize);
-      const placeholders = chunk.map(() => '?').join(',');
-      const chunkResults = this.db.prepare(`SELECT * FROM pages WHERE site_id = ? AND normalized_url IN (${placeholders})`).all(siteId, ...chunk) as Page[];
-      results.push(...chunkResults);
-    }
-
-    return results;
-  }
-
-  upsertMany(pages: (Partial<Page> & { site_id: number; normalized_url: string; last_seen_snapshot_id: number })[]): Map<string, number> {
-    if (pages.length === 0) return new Map();
-
-    const upsertStmtWithReturn = this.db.prepare(`
-      INSERT INTO pages (
-        site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id,
-        http_status, canonical_url, content_hash, simhash, etag, last_modified, html,
-        soft404_score, noindex, nofollow, security_error, retries, depth,
-        redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type,
-        updated_at
-      ) VALUES (
-        @site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id,
-        @http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html,
-        @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth,
-        @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
-        datetime('now')
-      )
-      ON CONFLICT(site_id, normalized_url) DO UPDATE SET
-        last_seen_snapshot_id = excluded.last_seen_snapshot_id,
-        http_status = CASE WHEN excluded.http_status != 0 THEN excluded.http_status ELSE pages.http_status END,
-        canonical_url = COALESCE(excluded.canonical_url, pages.canonical_url),
-        content_hash = COALESCE(excluded.content_hash, pages.content_hash),
-        simhash = COALESCE(excluded.simhash, pages.simhash),
-        etag = COALESCE(excluded.etag, pages.etag),
-        last_modified = COALESCE(excluded.last_modified, pages.last_modified),
-        html = COALESCE(excluded.html, pages.html),
-        soft404_score = COALESCE(excluded.soft404_score, pages.soft404_score),
-        noindex = CASE WHEN excluded.http_status != 0 THEN excluded.noindex ELSE pages.noindex END,
-        nofollow = CASE WHEN excluded.http_status != 0 THEN excluded.nofollow ELSE pages.nofollow END,
-        security_error = COALESCE(excluded.security_error, pages.security_error),
-        retries = MAX(pages.retries, excluded.retries),
-        depth = MIN(pages.depth, excluded.depth),
-        redirect_chain = COALESCE(excluded.redirect_chain, pages.redirect_chain),
-        bytes_received = COALESCE(excluded.bytes_received, pages.bytes_received),
-        crawl_trap_flag = MAX(pages.crawl_trap_flag, excluded.crawl_trap_flag),
-        crawl_trap_risk = COALESCE(excluded.crawl_trap_risk, pages.crawl_trap_risk),
-        trap_type = COALESCE(excluded.trap_type, pages.trap_type),
-        updated_at = datetime('now')
-      RETURNING id
-    `);
-
-    const urlToId = new Map<string, number>();
-    const tx = this.db.transaction((pagesBatch) => {
-      for (const page of pagesBatch) {
-        const params = {
-          site_id: page.site_id,
-          normalized_url: page.normalized_url,
-          first_seen_snapshot_id: page.first_seen_snapshot_id ?? page.last_seen_snapshot_id,
-          last_seen_snapshot_id: page.last_seen_snapshot_id,
-          http_status: page.http_status ?? null,
-          canonical_url: page.canonical_url ?? null,
-          content_hash: page.content_hash ?? null,
-          simhash: page.simhash ?? null,
-          etag: page.etag ?? null,
-          last_modified: page.last_modified ?? null,
-          html: page.html ?? null,
-          soft404_score: page.soft404_score ?? null,
-          noindex: page.noindex ?? 0,
-          nofollow: page.nofollow ?? 0,
-          security_error: page.security_error ?? null,
-          retries: page.retries ?? 0,
-          depth: page.depth ?? 0,
-          redirect_chain: page.redirect_chain ?? null,
-          bytes_received: page.bytes_received ?? null,
-          crawl_trap_flag: page.crawl_trap_flag ?? 0,
-          crawl_trap_risk: page.crawl_trap_risk ?? null,
-          trap_type: page.trap_type ?? null,
-        };
-        const row = upsertStmtWithReturn.get(params) as { id: number };
-        urlToId.set(page.normalized_url, row.id);
-      }
-    });
-
-    tx(pages);
-    return urlToId;
-  }
-
-  getPagesBySnapshot(snapshotId: number): Page[] {
-    return this.db.prepare('SELECT p.* FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND p.first_seen_snapshot_id <= ?').all(snapshotId, snapshotId) as Page[];
-  }
-
-  getPagesIdentityBySnapshot(snapshotId: number): { id: number; normalized_url: string }[] {
-    return this.db.prepare('SELECT p.id, p.normalized_url FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND p.first_seen_snapshot_id <= ?').all(snapshotId, snapshotId) as { id: number; normalized_url: string }[];
-  }
-
-  getPagesIteratorBySnapshot(snapshotId: number): IterableIterator<Page> {
-    return this.db.prepare('SELECT p.* FROM pages p JOIN snapshots s ON p.site_id = s.site_id WHERE s.id = ? AND p.first_seen_snapshot_id <= ?').iterate(snapshotId, snapshotId) as IterableIterator<Page>;
-  }
-
-  getIdByUrl(siteId: number, url: string): number | undefined {
-    const row = this.getIdStmt.get(siteId, url) as { id: number } | undefined;
-    return row?.id;
-  }
-}

package/src/db/repositories/SiteRepository.ts

@@ -1,43 +0,0 @@
-import { Database } from 'better-sqlite3';
-
-export interface Site {
-  id: number;
-  domain: string;
-  created_at: string;
-  settings_json: string | null;
-  is_active: number;
-}
-
-export class SiteRepository {
-  constructor(private db: Database) { }
-
-  getSiteById(id: number): Site | undefined {
-    return this.db.prepare('SELECT * FROM sites WHERE id = ?').get(id) as Site | undefined;
-  }
-
-  getSite(domain: string): Site | undefined {
-    return this.db.prepare('SELECT * FROM sites WHERE domain = ?').get(domain) as Site | undefined;
-  }
-
-  getAllSites(): Site[] {
-    return this.db.prepare('SELECT * FROM sites ORDER BY domain ASC').all() as Site[];
-  }
-
-  createSite(domain: string): number {
-    const stmt = this.db.prepare('INSERT INTO sites (domain) VALUES (?)');
-    const info = stmt.run(domain);
-    return info.lastInsertRowid as number;
-  }
-
-  firstOrCreateSite(domain: string): Site {
-    let site = this.getSite(domain);
-    if (!site) {
-      this.createSite(domain);
-      site = this.getSite(domain);
-    }
-    return site!;
-  }
-  deleteSite(id: number): void {
-    this.db.prepare('DELETE FROM sites WHERE id = ?').run(id);
-  }
-}

package/src/db/repositories/SnapshotRepository.ts

@@ -1,99 +0,0 @@
-import { Database } from 'better-sqlite3';
-
-export interface Snapshot {
-  id: number;
-  site_id: number;
-  type: 'full' | 'partial' | 'incremental';
-  created_at: string;
-  node_count: number;
-  edge_count: number;
-  status: 'running' | 'completed' | 'failed';
-  limit_reached: number;
-  health_score: number | null;
-  orphan_count: number | null;
-  thin_content_count: number | null;
-}
-
-export class SnapshotRepository {
-  constructor(private db: Database) { }
-
-  createSnapshot(siteId: number, type: 'full' | 'partial' | 'incremental', status: 'running' | 'completed' | 'failed' = 'running'): number {
-    // Basic throttling or sleep if needed for tests, but generally SQLite is fast enough to have diff timestamps if not in same ms.
-    // However, if we run in memory, created_at is default current time.
-    // If two snapshots created in same second, ORDER BY created_at DESC is unstable or equal.
-    // We should rely on ID for stability if timestamps are equal, but the query uses created_at.
-    // Let's ensure we can also order by ID as tie-breaker.
-    const stmt = this.db.prepare('INSERT INTO snapshots (site_id, type, status) VALUES (?, ?, ?)');
-    const info = stmt.run(siteId, type, status);
-    return info.lastInsertRowid as number;
-  }
-
-  getLatestSnapshot(siteId: number, status?: 'completed' | 'running' | 'failed'): Snapshot | undefined {
-    let sql = 'SELECT * FROM snapshots WHERE site_id = ? AND type != \'partial\'';
-    const params: any[] = [siteId];
-    if (status) {
-      sql += ' AND status = ?';
-      params.push(status);
-    }
-    sql += ' ORDER BY created_at DESC, id DESC LIMIT 1';
-    return this.db.prepare(sql).get(...params) as Snapshot | undefined;
-  }
-
-  getSnapshotCount(siteId: number): number {
-    const result = this.db.prepare('SELECT COUNT(*) as count FROM snapshots WHERE site_id = ?').get(siteId) as { count: number };
-    return result.count;
-  }
-
-  updateSnapshotStatus(id: number, status: 'completed' | 'failed', stats: Partial<Snapshot> = {}) {
-    const sets: string[] = ['status = ?'];
-    const params: any[] = [status];
-
-    if (stats.node_count !== undefined) {
-      sets.push('node_count = ?');
-      params.push(stats.node_count);
-    }
-    if (stats.edge_count !== undefined) {
-      sets.push('edge_count = ?');
-      params.push(stats.edge_count);
-    }
-    if (stats.limit_reached !== undefined) {
-      sets.push('limit_reached = ?');
-      params.push(stats.limit_reached);
-    }
-    if (stats.health_score !== undefined) {
-      sets.push('health_score = ?');
-      params.push(stats.health_score);
-    }
-    if (stats.orphan_count !== undefined) {
-      sets.push('orphan_count = ?');
-      params.push(stats.orphan_count);
-    }
-    if (stats.thin_content_count !== undefined) {
-      sets.push('thin_content_count = ?');
-      params.push(stats.thin_content_count);
-    }
-
-    params.push(id);
-    const sql = `UPDATE snapshots SET ${sets.join(', ')} WHERE id = ?`;
-    this.db.prepare(sql).run(...params);
-  }
-
-  getSnapshot(id: number): Snapshot | undefined {
-    return this.db.prepare('SELECT * FROM snapshots WHERE id = ?').get(id) as Snapshot | undefined;
-  }
-
-  deleteSnapshot(id: number): void {
-    const tx = this.db.transaction(() => {
-      // Unlink pages from this snapshot to prevent FK constraint violations or data inconsistencies
-      this.db.prepare('UPDATE pages SET first_seen_snapshot_id = NULL WHERE first_seen_snapshot_id = ?').run(id);
-      this.db.prepare('UPDATE pages SET last_seen_snapshot_id = NULL WHERE last_seen_snapshot_id = ?').run(id);
-
-      // Cleanup: Delete pages that are no longer referenced by any snapshot
-      this.db.prepare('DELETE FROM pages WHERE first_seen_snapshot_id IS NULL AND last_seen_snapshot_id IS NULL').run();
-
-      // Delete the snapshot
-      this.db.prepare('DELETE FROM snapshots WHERE id = ?').run(id);
-    });
-    tx();
-  }
-}

package/src/db/schema.ts

@@ -1,177 +0,0 @@
-import { Database } from 'better-sqlite3';
-
-export function initSchema(db: Database) {
-  // Sites Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS sites (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      domain TEXT UNIQUE NOT NULL,
-      created_at TEXT DEFAULT (datetime('now')),
-      settings_json TEXT,
-      is_active INTEGER DEFAULT 1
-    );
-  `);
-
-  // Snapshots Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS snapshots (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      site_id INTEGER NOT NULL,
-      type TEXT CHECK(type IN ('full', 'partial', 'incremental')) NOT NULL,
-      created_at TEXT DEFAULT (datetime('now')),
-      node_count INTEGER DEFAULT 0,
-      edge_count INTEGER DEFAULT 0,
-      status TEXT CHECK(status IN ('running', 'completed', 'failed')) DEFAULT 'running',
-      limit_reached INTEGER DEFAULT 0,
-      health_score REAL,
-      orphan_count INTEGER,
-      thin_content_count INTEGER,
-      FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE
-    );
-  `);
-
-  // Pages Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS pages (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      site_id INTEGER NOT NULL,
-      normalized_url TEXT NOT NULL,
-      first_seen_snapshot_id INTEGER,
-      last_seen_snapshot_id INTEGER,
-      http_status INTEGER,
-      canonical_url TEXT,
-      content_hash TEXT,
-      simhash TEXT,
-      etag TEXT,
-      last_modified TEXT,
-      html TEXT,
-      soft404_score REAL,
-      noindex INTEGER DEFAULT 0,
-      nofollow INTEGER DEFAULT 0,
-      security_error TEXT,
-      retries INTEGER DEFAULT 0,
-      depth INTEGER DEFAULT 0,
-      redirect_chain TEXT,
-      bytes_received INTEGER,
-      crawl_trap_flag INTEGER DEFAULT 0,
-      crawl_trap_risk REAL,
-      trap_type TEXT,
-      created_at TEXT DEFAULT (datetime('now')),
-      updated_at TEXT DEFAULT (datetime('now')),
-      FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE,
-      FOREIGN KEY(first_seen_snapshot_id) REFERENCES snapshots(id),
-      FOREIGN KEY(last_seen_snapshot_id) REFERENCES snapshots(id),
-      UNIQUE(site_id, normalized_url)
-    );
-  `);
-
-  // Index for Pages
-  db.exec(`CREATE INDEX IF NOT EXISTS idx_pages_site_last_seen ON pages(site_id, last_seen_snapshot_id);`);
-
-  // Edges Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS edges (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      snapshot_id INTEGER NOT NULL,
-      source_page_id INTEGER NOT NULL,
-      target_page_id INTEGER NOT NULL,
-      weight REAL DEFAULT 1.0,
-      rel TEXT CHECK(rel IN ('nofollow', 'sponsored', 'ugc', 'internal', 'external', 'unknown')) DEFAULT 'internal',
-      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
-      FOREIGN KEY(source_page_id) REFERENCES pages(id) ON DELETE CASCADE,
-      FOREIGN KEY(target_page_id) REFERENCES pages(id) ON DELETE CASCADE
-    );
-  `);
-
-  // Index for Edges
-  db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot_source ON edges(snapshot_id, source_page_id);`);
-  db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot ON edges(snapshot_id);`);
-
-  // Metrics Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS metrics (
-      snapshot_id INTEGER NOT NULL,
-      page_id INTEGER NOT NULL,
-      authority_score REAL,
-      hub_score REAL,
-      pagerank REAL,
-      pagerank_score REAL,
-      link_role TEXT CHECK(link_role IN ('hub', 'authority', 'power', 'balanced', 'peripheral')),
-      crawl_status TEXT,
-      word_count INTEGER,
-      thin_content_score REAL,
-      external_link_ratio REAL,
-      orphan_score INTEGER,
-      duplicate_cluster_id TEXT,
-      duplicate_type TEXT CHECK(duplicate_type IN ('exact', 'near', 'template_heavy', 'none')),
-      is_cluster_primary INTEGER DEFAULT 0,
-      PRIMARY KEY(snapshot_id, page_id),
-      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
-      FOREIGN KEY(page_id) REFERENCES pages(id) ON DELETE CASCADE
-    );
-  `);
-
-  db.exec(`CREATE INDEX IF NOT EXISTS idx_metrics_snapshot ON metrics(snapshot_id);`);
-
-  // Duplicate Clusters Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS duplicate_clusters (
-      id TEXT NOT NULL,
-      snapshot_id INTEGER NOT NULL,
-      type TEXT CHECK(type IN ('exact', 'near', 'template_heavy')) NOT NULL,
-      size INTEGER NOT NULL,
-      representative TEXT NOT NULL,
-      severity TEXT CHECK(severity IN ('low', 'medium', 'high')) NOT NULL,
-      PRIMARY KEY(snapshot_id, id),
-      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
-    );
-  `);
-
-  // Content Clusters Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS content_clusters (
-      id INTEGER NOT NULL,
-      snapshot_id INTEGER NOT NULL,
-      count INTEGER NOT NULL,
-      primary_url TEXT NOT NULL,
-      risk TEXT CHECK(risk IN ('low', 'medium', 'high')) NOT NULL,
-      shared_path_prefix TEXT,
-      PRIMARY KEY(snapshot_id, id),
-      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
-    );
-  `);
-
-  // Migration: add columns to existing DBs that were created before this update
-  migrateSchema(db);
-}
-
-function migrateSchema(db: Database) {
-  // Add missing columns to pages (safe: ALTER TABLE ADD COLUMN is idempotent-safe with try/catch)
-  const pageColumns = [
-    ['redirect_chain', 'TEXT'],
-    ['bytes_received', 'INTEGER'],
-    ['crawl_trap_flag', 'INTEGER DEFAULT 0'],
-    ['crawl_trap_risk', 'REAL'],
-    ['trap_type', 'TEXT'],
-  ];
-
-  for (const [col, type] of pageColumns) {
-    try { db.exec(`ALTER TABLE pages ADD COLUMN ${col} ${type}`); } catch { /* already exists */ }
-  }
-
-  // Add missing columns to edges
-  try { db.exec('ALTER TABLE edges ADD COLUMN weight REAL DEFAULT 1.0'); } catch { /* already exists */ }
-
-  // Add missing columns to metrics
-  const metricsColumns = [
-    ['pagerank_score', 'REAL'],
-    ['link_role', 'TEXT'],
-    ['duplicate_cluster_id', 'TEXT'],
-    ['duplicate_type', 'TEXT'],
-    ['is_cluster_primary', 'INTEGER DEFAULT 0'],
-  ];
-
-  for (const [col, type] of metricsColumns) {
-    try { db.exec(`ALTER TABLE metrics ADD COLUMN ${col} ${type}`); } catch { /* already exists */ }
-  }
-}

package/src/diff/compare.ts

@@ -1,84 +0,0 @@
-import { Graph } from '../graph/graph.js';
-import { calculateMetrics } from '../graph/metrics.js';
-
-export interface DiffResult {
-  addedUrls: string[];
-  removedUrls: string[];
-  changedStatus: { url: string; oldStatus: number; newStatus: number }[];
-  changedCanonical: { url: string; oldCanonical: string | null; newCanonical: string | null }[];
-  changedDuplicateGroup: { url: string; oldGroup: string | null; newGroup: string | null }[];
-  metricDeltas: {
-    structuralEntropy: number;
-    orphanCount: number;
-    crawlEfficiency: number;
-  };
-}
-
-export function compareGraphs(oldGraph: Graph, newGraph: Graph): DiffResult {
-  const oldNodes = new Map(oldGraph.getNodes().map(n => [n.url, n]));
-  const newNodes = new Map(newGraph.getNodes().map(n => [n.url, n]));
-
-  const addedUrls: string[] = [];
-  const removedUrls: string[] = [];
-  const changedStatus: { url: string; oldStatus: number; newStatus: number }[] = [];
-  const changedCanonical: { url: string; oldCanonical: string | null; newCanonical: string | null }[] = [];
-  const changedDuplicateGroup: { url: string; oldGroup: string | null; newGroup: string | null }[] = [];
-
-  // Added & Changed
-  for (const [url, newNode] of newNodes) {
-    const oldNode = oldNodes.get(url);
-    if (!oldNode) {
-      addedUrls.push(url);
-    } else {
-      // Changed Status
-      if (oldNode.status !== newNode.status) {
-        changedStatus.push({ url, oldStatus: oldNode.status, newStatus: newNode.status });
-      }
-      // Changed Canonical
-      if (oldNode.canonical !== newNode.canonical) {
-        changedCanonical.push({
-          url,
-          oldCanonical: oldNode.canonical || null,
-          newCanonical: newNode.canonical || null
-        });
-      }
-      // Changed Duplicate Group
-      const oldGroup = oldNode.duplicateClusterId || null;
-      const newGroup = newNode.duplicateClusterId || null;
-      if (oldGroup !== newGroup) {
-        changedDuplicateGroup.push({
-          url,
-          oldGroup,
-          newGroup
-        });
-      }
-    }
-  }
-
-  // Removed
-  for (const url of oldNodes.keys()) {
-    if (!newNodes.has(url)) {
-      removedUrls.push(url);
-    }
-  }
-
-  // Metrics
-  // maxDepth is ignored by current calculateMetrics implementation but required by signature
-  const oldMetrics = calculateMetrics(oldGraph, 10);
-  const newMetrics = calculateMetrics(newGraph, 10);
-
-  const metricDeltas = {
-    structuralEntropy: newMetrics.structuralEntropy - oldMetrics.structuralEntropy,
-    orphanCount: newMetrics.orphanPages.length - oldMetrics.orphanPages.length,
-    crawlEfficiency: newMetrics.crawlEfficiencyScore - oldMetrics.crawlEfficiencyScore
-  };
-
-  return {
-    addedUrls,
-    removedUrls,
-    changedStatus,
-    changedCanonical,
-    changedDuplicateGroup,
-    metricDeltas
-  };
-}

package/src/events.ts

@@ -1,16 +0,0 @@
-export type CrawlEvent =
-  | { type: 'crawl:start'; url: string }
-  | { type: 'crawl:success'; url: string; status: number; durationMs: number; depth?: number }
-  | { type: 'crawl:error'; url: string; error: string; depth?: number }
-  | { type: 'crawl:limit-reached'; limit: number }
-  | { type: 'queue:enqueue'; url: string; depth: number }
-  | { type: 'metrics:start'; phase: string }
-  | { type: 'metrics:complete'; durationMs: number }
-  | { type: 'debug'; message: string; context?: unknown }
-  | { type: 'info'; message: string; context?: unknown }
-  | { type: 'warn'; message: string; context?: unknown }
-  | { type: 'error'; message: string; error?: unknown; context?: unknown };
-
-export interface EngineContext {
-  emit: (event: CrawlEvent) => void;
-}