@crawlith/core 0.1.0 → 0.1.2

package/src/db/index.ts

@@ -1,70 +0,0 @@
-import Database from 'better-sqlite3';
-import path from 'node:path';
-import fs from 'node:fs';
-import os from 'node:os';
-import { initSchema } from './schema.js';
-
-let dbInstance: Database.Database | null = null;
-
-export function getDbPath(): string {
-  if (process.env.NODE_ENV === 'test') {
-    return ':memory:';
-  }
-  if (process.env.CRAWLITH_DB_PATH) {
-    return process.env.CRAWLITH_DB_PATH;
-  }
-  const homeDir = os.homedir();
-  const crawlithDir = path.join(homeDir, '.crawlith');
-  if (!fs.existsSync(crawlithDir)) {
-    fs.mkdirSync(crawlithDir, { recursive: true });
-    // Set permissions to 700 (user only)
-    fs.chmodSync(crawlithDir, 0o700);
-  }
-  return path.join(crawlithDir, 'crawlith.db');
-}
-
-export function getDb(): Database.Database {
-  if (dbInstance) {
-    return dbInstance;
-  }
-
-  const dbPath = getDbPath();
-  const db = new Database(dbPath);
-
-  // Hardening & Performance Configuration
-  db.pragma('journal_mode = WAL');
-  db.pragma('synchronous = NORMAL');
-  db.pragma('foreign_keys = ON');
-  db.pragma('temp_store = MEMORY');
-  db.pragma('mmap_size = 30000000000');
-  db.pragma('cache_size = -20000');
-  db.pragma('busy_timeout = 5000');
-
-  // Security controls
-  // Ensure file permissions are 600 (user read/write only)
-  try {
-    fs.chmodSync(dbPath, 0o600);
-  } catch (_e) {
-    // might fail on first creation if file doesn't exist yet, but better-sqlite3 creates it
-    // so we can try again or ignore if it's new
-  }
-
-  // Integrity check on startup
-  const integrity = db.pragma('integrity_check', { simple: true });
-  if (integrity !== 'ok') {
-    console.warn('Database integrity check failed:', integrity);
-  }
-
-  // Initialize schema
-  initSchema(db);
-
-  dbInstance = db;
-  return db;
-}
-
-export function closeDb() {
-  if (dbInstance) {
-    dbInstance.close();
-    dbInstance = null;
-  }
-}

package/src/db/repositories/EdgeRepository.ts

@@ -1,29 +0,0 @@
-import { Database } from 'better-sqlite3';
-
-export interface Edge {
-  id: number;
-  snapshot_id: number;
-  source_page_id: number;
-  target_page_id: number;
-  weight: number;
-  rel: 'nofollow' | 'sponsored' | 'ugc' | 'internal' | 'external' | 'unknown';
-}
-
-export class EdgeRepository {
-  private insertStmt;
-
-  constructor(private db: Database) {
-    this.insertStmt = this.db.prepare(`
-      INSERT INTO edges (snapshot_id, source_page_id, target_page_id, weight, rel)
-      VALUES (?, ?, ?, ?, ?)
-    `);
-  }
-
-  insertEdge(snapshotId: number, sourcePageId: number, targetPageId: number, weight: number = 1.0, rel: string = 'internal') {
-    this.insertStmt.run(snapshotId, sourcePageId, targetPageId, weight, rel);
-  }
-
-  getEdgesBySnapshot(snapshotId: number): Edge[] {
-    return this.db.prepare('SELECT * FROM edges WHERE snapshot_id = ?').all(snapshotId) as Edge[];
-  }
-}

package/src/db/repositories/MetricsRepository.ts

@@ -1,49 +0,0 @@
-import { Database } from 'better-sqlite3';
-
-export interface DbMetrics {
-  snapshot_id: number;
-  page_id: number;
-  authority_score: number | null;
-  hub_score: number | null;
-  pagerank: number | null;
-  pagerank_score: number | null;
-  link_role: 'hub' | 'authority' | 'power' | 'balanced' | 'peripheral' | null;
-  crawl_status: string | null;
-  word_count: number | null;
-  thin_content_score: number | null;
-  external_link_ratio: number | null;
-  orphan_score: number | null;
-  duplicate_cluster_id: string | null;
-  duplicate_type: 'exact' | 'near' | 'template_heavy' | 'none' | null;
-  is_cluster_primary: number;
-}
-
-export class MetricsRepository {
-  private insertStmt;
-
-  constructor(private db: Database) {
-    this.insertStmt = this.db.prepare(`
-      INSERT OR REPLACE INTO metrics (
-        snapshot_id, page_id, authority_score, hub_score, pagerank, pagerank_score,
-        link_role, crawl_status, word_count, thin_content_score, external_link_ratio,
-        orphan_score, duplicate_cluster_id, duplicate_type, is_cluster_primary
-      ) VALUES (
-        @snapshot_id, @page_id, @authority_score, @hub_score, @pagerank, @pagerank_score,
-        @link_role, @crawl_status, @word_count, @thin_content_score, @external_link_ratio,
-        @orphan_score, @duplicate_cluster_id, @duplicate_type, @is_cluster_primary
-      )
-    `);
-  }
-
-  insertMetrics(metrics: DbMetrics) {
-    this.insertStmt.run(metrics);
-  }
-
-  getMetrics(snapshotId: number): DbMetrics[] {
-    return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ?').all(snapshotId) as DbMetrics[];
-  }
-
-  getMetricsForPage(snapshotId: number, pageId: number): DbMetrics | undefined {
-    return this.db.prepare('SELECT * FROM metrics WHERE snapshot_id = ? AND page_id = ?').get(snapshotId, pageId) as DbMetrics | undefined;
-  }
-}

package/src/db/repositories/PageRepository.ts

@@ -1,128 +0,0 @@
-import { Database } from 'better-sqlite3';
-
-export interface Page {
-  id: number;
-  site_id: number;
-  normalized_url: string;
-  first_seen_snapshot_id: number | null;
-  last_seen_snapshot_id: number | null;
-  http_status: number | null;
-  canonical_url: string | null;
-  content_hash: string | null;
-  simhash: string | null;
-  etag: string | null;
-  last_modified: string | null;
-  html: string | null;
-  soft404_score: number | null;
-  noindex: number;
-  nofollow: number;
-  security_error: string | null;
-  retries: number;
-  depth: number;
-  redirect_chain: string | null;
-  bytes_received: number | null;
-  crawl_trap_flag: number;
-  crawl_trap_risk: number | null;
-  trap_type: string | null;
-  created_at: string;
-  updated_at: string;
-}
-
-export class PageRepository {
-  private upsertStmt;
-  private getIdStmt;
-
-  constructor(private db: Database) {
-    this.upsertStmt = this.db.prepare(`
-      INSERT INTO pages (
-        site_id, normalized_url, first_seen_snapshot_id, last_seen_snapshot_id,
-        http_status, canonical_url, content_hash, simhash, etag, last_modified, html,
-        soft404_score, noindex, nofollow, security_error, retries, depth,
-        redirect_chain, bytes_received, crawl_trap_flag, crawl_trap_risk, trap_type,
-        updated_at
-      ) VALUES (
-        @site_id, @normalized_url, @first_seen_snapshot_id, @last_seen_snapshot_id,
-        @http_status, @canonical_url, @content_hash, @simhash, @etag, @last_modified, @html,
-        @soft404_score, @noindex, @nofollow, @security_error, @retries, @depth,
-        @redirect_chain, @bytes_received, @crawl_trap_flag, @crawl_trap_risk, @trap_type,
-        datetime('now')
-      )
-      ON CONFLICT(site_id, normalized_url) DO UPDATE SET
-        last_seen_snapshot_id = excluded.last_seen_snapshot_id,
-        http_status = excluded.http_status,
-        canonical_url = excluded.canonical_url,
-        content_hash = excluded.content_hash,
-        simhash = excluded.simhash,
-        etag = excluded.etag,
-        last_modified = excluded.last_modified,
-        html = excluded.html,
-        soft404_score = excluded.soft404_score,
-        noindex = excluded.noindex,
-        nofollow = excluded.nofollow,
-        security_error = excluded.security_error,
-        retries = excluded.retries,
-        depth = excluded.depth,
-        redirect_chain = excluded.redirect_chain,
-        bytes_received = excluded.bytes_received,
-        crawl_trap_flag = excluded.crawl_trap_flag,
-        crawl_trap_risk = excluded.crawl_trap_risk,
-        trap_type = excluded.trap_type,
-        updated_at = datetime('now')
-    `);
-
-    this.getIdStmt = this.db.prepare('SELECT id FROM pages WHERE site_id = ? AND normalized_url = ?');
-  }
-
-  upsertPage(page: Partial<Page> & { site_id: number; normalized_url: string; last_seen_snapshot_id: number }) {
-    const params = {
-      site_id: page.site_id,
-      normalized_url: page.normalized_url,
-      first_seen_snapshot_id: page.first_seen_snapshot_id ?? page.last_seen_snapshot_id,
-      last_seen_snapshot_id: page.last_seen_snapshot_id,
-      http_status: page.http_status ?? null,
-      canonical_url: page.canonical_url ?? null,
-      content_hash: page.content_hash ?? null,
-      simhash: page.simhash ?? null,
-      etag: page.etag ?? null,
-      last_modified: page.last_modified ?? null,
-      html: page.html ?? null,
-      soft404_score: page.soft404_score ?? null,
-      noindex: page.noindex ?? 0,
-      nofollow: page.nofollow ?? 0,
-      security_error: page.security_error ?? null,
-      retries: page.retries ?? 0,
-      depth: page.depth ?? 0,
-      redirect_chain: page.redirect_chain ?? null,
-      bytes_received: page.bytes_received ?? null,
-      crawl_trap_flag: page.crawl_trap_flag ?? 0,
-      crawl_trap_risk: page.crawl_trap_risk ?? null,
-      trap_type: page.trap_type ?? null,
-    };
-
-    const info = this.upsertStmt.run(params);
-    return info;
-  }
-
-  upsertAndGetId(page: Partial<Page> & { site_id: number; normalized_url: string; last_seen_snapshot_id: number }): number {
-    const tx = this.db.transaction(() => {
-      this.upsertPage(page);
-      const row = this.getIdStmt.get(page.site_id, page.normalized_url) as { id: number } | undefined;
-      if (!row) throw new Error(`Failed to retrieve ID for upserted page: ${page.normalized_url}`);
-      return row.id;
-    });
-    return tx();
-  }
-
-  getPage(siteId: number, url: string): Page | undefined {
-    return this.db.prepare('SELECT * FROM pages WHERE site_id = ? AND normalized_url = ?').get(siteId, url) as Page | undefined;
-  }
-
-  getPagesBySnapshot(snapshotId: number): Page[] {
-    return this.db.prepare('SELECT * FROM pages WHERE last_seen_snapshot_id = ?').all(snapshotId) as Page[];
-  }
-
-  getIdByUrl(siteId: number, url: string): number | undefined {
-    const row = this.getIdStmt.get(siteId, url) as { id: number } | undefined;
-    return row?.id;
-  }
-}

package/src/db/repositories/SiteRepository.ts

@@ -1,32 +0,0 @@
-import { Database } from 'better-sqlite3';
-
-export interface Site {
-  id: number;
-  domain: string;
-  created_at: string;
-  settings_json: string | null;
-  is_active: number;
-}
-
-export class SiteRepository {
-  constructor(private db: Database) { }
-
-  getSite(domain: string): Site | undefined {
-    return this.db.prepare('SELECT * FROM sites WHERE domain = ?').get(domain) as Site | undefined;
-  }
-
-  createSite(domain: string): number {
-    const stmt = this.db.prepare('INSERT INTO sites (domain) VALUES (?)');
-    const info = stmt.run(domain);
-    return info.lastInsertRowid as number;
-  }
-
-  firstOrCreateSite(domain: string): Site {
-    let site = this.getSite(domain);
-    if (!site) {
-      this.createSite(domain);
-      site = this.getSite(domain);
-    }
-    return site!;
-  }
-}

package/src/db/repositories/SnapshotRepository.ts

@@ -1,74 +0,0 @@
-import { Database } from 'better-sqlite3';
-
-export interface Snapshot {
-  id: number;
-  site_id: number;
-  type: 'full' | 'partial' | 'incremental';
-  created_at: string;
-  node_count: number;
-  edge_count: number;
-  status: 'running' | 'completed' | 'failed';
-  limit_reached: number;
-  health_score: number | null;
-  orphan_count: number | null;
-  thin_content_count: number | null;
-}
-
-export class SnapshotRepository {
-  constructor(private db: Database) {}
-
-  createSnapshot(siteId: number, type: 'full' | 'partial' | 'incremental', status: 'running' | 'completed' | 'failed' = 'running'): number {
-    const stmt = this.db.prepare('INSERT INTO snapshots (site_id, type, status) VALUES (?, ?, ?)');
-    const info = stmt.run(siteId, type, status);
-    return info.lastInsertRowid as number;
-  }
-
-  getLatestSnapshot(siteId: number, status?: 'completed' | 'running' | 'failed'): Snapshot | undefined {
-    let sql = 'SELECT * FROM snapshots WHERE site_id = ?';
-    const params: any[] = [siteId];
-    if (status) {
-      sql += ' AND status = ?';
-      params.push(status);
-    }
-    sql += ' ORDER BY created_at DESC LIMIT 1';
-    return this.db.prepare(sql).get(...params) as Snapshot | undefined;
-  }
-
-  updateSnapshotStatus(id: number, status: 'completed' | 'failed', stats: Partial<Snapshot> = {}) {
-    const sets: string[] = ['status = ?'];
-    const params: any[] = [status];
-
-    if (stats.node_count !== undefined) {
-      sets.push('node_count = ?');
-      params.push(stats.node_count);
-    }
-    if (stats.edge_count !== undefined) {
-      sets.push('edge_count = ?');
-      params.push(stats.edge_count);
-    }
-    if (stats.limit_reached !== undefined) {
-      sets.push('limit_reached = ?');
-      params.push(stats.limit_reached);
-    }
-    if (stats.health_score !== undefined) {
-      sets.push('health_score = ?');
-      params.push(stats.health_score);
-    }
-    if (stats.orphan_count !== undefined) {
-      sets.push('orphan_count = ?');
-      params.push(stats.orphan_count);
-    }
-    if (stats.thin_content_count !== undefined) {
-      sets.push('thin_content_count = ?');
-      params.push(stats.thin_content_count);
-    }
-
-    params.push(id);
-    const sql = `UPDATE snapshots SET ${sets.join(', ')} WHERE id = ?`;
-    this.db.prepare(sql).run(...params);
-  }
-
-  getSnapshot(id: number): Snapshot | undefined {
-    return this.db.prepare('SELECT * FROM snapshots WHERE id = ?').get(id) as Snapshot | undefined;
-  }
-}

package/src/db/schema.ts

@@ -1,177 +0,0 @@
-import { Database } from 'better-sqlite3';
-
-export function initSchema(db: Database) {
-  // Sites Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS sites (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      domain TEXT UNIQUE NOT NULL,
-      created_at TEXT DEFAULT (datetime('now')),
-      settings_json TEXT,
-      is_active INTEGER DEFAULT 1
-    );
-  `);
-
-  // Snapshots Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS snapshots (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      site_id INTEGER NOT NULL,
-      type TEXT CHECK(type IN ('full', 'partial', 'incremental')) NOT NULL,
-      created_at TEXT DEFAULT (datetime('now')),
-      node_count INTEGER DEFAULT 0,
-      edge_count INTEGER DEFAULT 0,
-      status TEXT CHECK(status IN ('running', 'completed', 'failed')) DEFAULT 'running',
-      limit_reached INTEGER DEFAULT 0,
-      health_score REAL,
-      orphan_count INTEGER,
-      thin_content_count INTEGER,
-      FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE
-    );
-  `);
-
-  // Pages Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS pages (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      site_id INTEGER NOT NULL,
-      normalized_url TEXT NOT NULL,
-      first_seen_snapshot_id INTEGER,
-      last_seen_snapshot_id INTEGER,
-      http_status INTEGER,
-      canonical_url TEXT,
-      content_hash TEXT,
-      simhash TEXT,
-      etag TEXT,
-      last_modified TEXT,
-      html TEXT,
-      soft404_score REAL,
-      noindex INTEGER DEFAULT 0,
-      nofollow INTEGER DEFAULT 0,
-      security_error TEXT,
-      retries INTEGER DEFAULT 0,
-      depth INTEGER DEFAULT 0,
-      redirect_chain TEXT,
-      bytes_received INTEGER,
-      crawl_trap_flag INTEGER DEFAULT 0,
-      crawl_trap_risk REAL,
-      trap_type TEXT,
-      created_at TEXT DEFAULT (datetime('now')),
-      updated_at TEXT DEFAULT (datetime('now')),
-      FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE,
-      FOREIGN KEY(first_seen_snapshot_id) REFERENCES snapshots(id),
-      FOREIGN KEY(last_seen_snapshot_id) REFERENCES snapshots(id),
-      UNIQUE(site_id, normalized_url)
-    );
-  `);
-
-  // Index for Pages
-  db.exec(`CREATE INDEX IF NOT EXISTS idx_pages_site_last_seen ON pages(site_id, last_seen_snapshot_id);`);
-
-  // Edges Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS edges (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      snapshot_id INTEGER NOT NULL,
-      source_page_id INTEGER NOT NULL,
-      target_page_id INTEGER NOT NULL,
-      weight REAL DEFAULT 1.0,
-      rel TEXT CHECK(rel IN ('nofollow', 'sponsored', 'ugc', 'internal', 'external', 'unknown')) DEFAULT 'internal',
-      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
-      FOREIGN KEY(source_page_id) REFERENCES pages(id) ON DELETE CASCADE,
-      FOREIGN KEY(target_page_id) REFERENCES pages(id) ON DELETE CASCADE
-    );
-  `);
-
-  // Index for Edges
-  db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot_source ON edges(snapshot_id, source_page_id);`);
-  db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot ON edges(snapshot_id);`);
-
-  // Metrics Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS metrics (
-      snapshot_id INTEGER NOT NULL,
-      page_id INTEGER NOT NULL,
-      authority_score REAL,
-      hub_score REAL,
-      pagerank REAL,
-      pagerank_score REAL,
-      link_role TEXT CHECK(link_role IN ('hub', 'authority', 'power', 'balanced', 'peripheral')),
-      crawl_status TEXT,
-      word_count INTEGER,
-      thin_content_score REAL,
-      external_link_ratio REAL,
-      orphan_score INTEGER,
-      duplicate_cluster_id TEXT,
-      duplicate_type TEXT CHECK(duplicate_type IN ('exact', 'near', 'template_heavy', 'none')),
-      is_cluster_primary INTEGER DEFAULT 0,
-      PRIMARY KEY(snapshot_id, page_id),
-      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
-      FOREIGN KEY(page_id) REFERENCES pages(id) ON DELETE CASCADE
-    );
-  `);
-
-  db.exec(`CREATE INDEX IF NOT EXISTS idx_metrics_snapshot ON metrics(snapshot_id);`);
-
-  // Duplicate Clusters Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS duplicate_clusters (
-      id TEXT NOT NULL,
-      snapshot_id INTEGER NOT NULL,
-      type TEXT CHECK(type IN ('exact', 'near', 'template_heavy')) NOT NULL,
-      size INTEGER NOT NULL,
-      representative TEXT NOT NULL,
-      severity TEXT CHECK(severity IN ('low', 'medium', 'high')) NOT NULL,
-      PRIMARY KEY(snapshot_id, id),
-      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
-    );
-  `);
-
-  // Content Clusters Table
-  db.exec(`
-    CREATE TABLE IF NOT EXISTS content_clusters (
-      id INTEGER NOT NULL,
-      snapshot_id INTEGER NOT NULL,
-      count INTEGER NOT NULL,
-      primary_url TEXT NOT NULL,
-      risk TEXT CHECK(risk IN ('low', 'medium', 'high')) NOT NULL,
-      shared_path_prefix TEXT,
-      PRIMARY KEY(snapshot_id, id),
-      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
-    );
-  `);
-
-  // Migration: add columns to existing DBs that were created before this update
-  migrateSchema(db);
-}
-
-function migrateSchema(db: Database) {
-  // Add missing columns to pages (safe: ALTER TABLE ADD COLUMN is idempotent-safe with try/catch)
-  const pageColumns = [
-    ['redirect_chain', 'TEXT'],
-    ['bytes_received', 'INTEGER'],
-    ['crawl_trap_flag', 'INTEGER DEFAULT 0'],
-    ['crawl_trap_risk', 'REAL'],
-    ['trap_type', 'TEXT'],
-  ];
-
-  for (const [col, type] of pageColumns) {
-    try { db.exec(`ALTER TABLE pages ADD COLUMN ${col} ${type}`); } catch { /* already exists */ }
-  }
-
-  // Add missing columns to edges
-  try { db.exec('ALTER TABLE edges ADD COLUMN weight REAL DEFAULT 1.0'); } catch { /* already exists */ }
-
-  // Add missing columns to metrics
-  const metricsColumns = [
-    ['pagerank_score', 'REAL'],
-    ['link_role', 'TEXT'],
-    ['duplicate_cluster_id', 'TEXT'],
-    ['duplicate_type', 'TEXT'],
-    ['is_cluster_primary', 'INTEGER DEFAULT 0'],
-  ];
-
-  for (const [col, type] of metricsColumns) {
-    try { db.exec(`ALTER TABLE metrics ADD COLUMN ${col} ${type}`); } catch { /* already exists */ }
-  }
-}

package/src/diff/compare.ts

@@ -1,84 +0,0 @@
-import { Graph } from '../graph/graph.js';
-import { calculateMetrics } from '../graph/metrics.js';
-
-export interface DiffResult {
-  addedUrls: string[];
-  removedUrls: string[];
-  changedStatus: { url: string; oldStatus: number; newStatus: number }[];
-  changedCanonical: { url: string; oldCanonical: string | null; newCanonical: string | null }[];
-  changedDuplicateGroup: { url: string; oldGroup: string | null; newGroup: string | null }[];
-  metricDeltas: {
-    structuralEntropy: number;
-    orphanCount: number;
-    crawlEfficiency: number;
-  };
-}
-
-export function compareGraphs(oldGraph: Graph, newGraph: Graph): DiffResult {
-  const oldNodes = new Map(oldGraph.getNodes().map(n => [n.url, n]));
-  const newNodes = new Map(newGraph.getNodes().map(n => [n.url, n]));
-
-  const addedUrls: string[] = [];
-  const removedUrls: string[] = [];
-  const changedStatus: { url: string; oldStatus: number; newStatus: number }[] = [];
-  const changedCanonical: { url: string; oldCanonical: string | null; newCanonical: string | null }[] = [];
-  const changedDuplicateGroup: { url: string; oldGroup: string | null; newGroup: string | null }[] = [];
-
-  // Added & Changed
-  for (const [url, newNode] of newNodes) {
-    const oldNode = oldNodes.get(url);
-    if (!oldNode) {
-      addedUrls.push(url);
-    } else {
-      // Changed Status
-      if (oldNode.status !== newNode.status) {
-        changedStatus.push({ url, oldStatus: oldNode.status, newStatus: newNode.status });
-      }
-      // Changed Canonical
-      if (oldNode.canonical !== newNode.canonical) {
-        changedCanonical.push({
-          url,
-          oldCanonical: oldNode.canonical || null,
-          newCanonical: newNode.canonical || null
-        });
-      }
-      // Changed Duplicate Group
-      const oldGroup = oldNode.duplicateClusterId || null;
-      const newGroup = newNode.duplicateClusterId || null;
-      if (oldGroup !== newGroup) {
-        changedDuplicateGroup.push({
-          url,
-          oldGroup,
-          newGroup
-        });
-      }
-    }
-  }
-
-  // Removed
-  for (const url of oldNodes.keys()) {
-    if (!newNodes.has(url)) {
-      removedUrls.push(url);
-    }
-  }
-
-  // Metrics
-  // maxDepth is ignored by current calculateMetrics implementation but required by signature
-  const oldMetrics = calculateMetrics(oldGraph, 10);
-  const newMetrics = calculateMetrics(newGraph, 10);
-
-  const metricDeltas = {
-    structuralEntropy: newMetrics.structuralEntropy - oldMetrics.structuralEntropy,
-    orphanCount: newMetrics.orphanPages.length - oldMetrics.orphanPages.length,
-    crawlEfficiency: newMetrics.crawlEfficiencyScore - oldMetrics.crawlEfficiencyScore
-  };
-
-  return {
-    addedUrls,
-    removedUrls,
-    changedStatus,
-    changedCanonical,
-    changedDuplicateGroup,
-    metricDeltas
-  };
-}