@crawlith/core 0.1.0

package/src/db/schema.ts

@@ -0,0 +1,177 @@
+import { Database } from 'better-sqlite3';
+
+export function initSchema(db: Database) {
+  // Sites Table
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS sites (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      domain TEXT UNIQUE NOT NULL,
+      created_at TEXT DEFAULT (datetime('now')),
+      settings_json TEXT,
+      is_active INTEGER DEFAULT 1
+    );
+  `);
+
+  // Snapshots Table
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS snapshots (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      site_id INTEGER NOT NULL,
+      type TEXT CHECK(type IN ('full', 'partial', 'incremental')) NOT NULL,
+      created_at TEXT DEFAULT (datetime('now')),
+      node_count INTEGER DEFAULT 0,
+      edge_count INTEGER DEFAULT 0,
+      status TEXT CHECK(status IN ('running', 'completed', 'failed')) DEFAULT 'running',
+      limit_reached INTEGER DEFAULT 0,
+      health_score REAL,
+      orphan_count INTEGER,
+      thin_content_count INTEGER,
+      FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE
+    );
+  `);
+
+  // Pages Table
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS pages (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      site_id INTEGER NOT NULL,
+      normalized_url TEXT NOT NULL,
+      first_seen_snapshot_id INTEGER,
+      last_seen_snapshot_id INTEGER,
+      http_status INTEGER,
+      canonical_url TEXT,
+      content_hash TEXT,
+      simhash TEXT,
+      etag TEXT,
+      last_modified TEXT,
+      html TEXT,
+      soft404_score REAL,
+      noindex INTEGER DEFAULT 0,
+      nofollow INTEGER DEFAULT 0,
+      security_error TEXT,
+      retries INTEGER DEFAULT 0,
+      depth INTEGER DEFAULT 0,
+      redirect_chain TEXT,
+      bytes_received INTEGER,
+      crawl_trap_flag INTEGER DEFAULT 0,
+      crawl_trap_risk REAL,
+      trap_type TEXT,
+      created_at TEXT DEFAULT (datetime('now')),
+      updated_at TEXT DEFAULT (datetime('now')),
+      FOREIGN KEY(site_id) REFERENCES sites(id) ON DELETE CASCADE,
+      FOREIGN KEY(first_seen_snapshot_id) REFERENCES snapshots(id),
+      FOREIGN KEY(last_seen_snapshot_id) REFERENCES snapshots(id),
+      UNIQUE(site_id, normalized_url)
+    );
+  `);
+
+  // Index for Pages
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_pages_site_last_seen ON pages(site_id, last_seen_snapshot_id);`);
+
+  // Edges Table
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS edges (
+      id INTEGER PRIMARY KEY AUTOINCREMENT,
+      snapshot_id INTEGER NOT NULL,
+      source_page_id INTEGER NOT NULL,
+      target_page_id INTEGER NOT NULL,
+      weight REAL DEFAULT 1.0,
+      rel TEXT CHECK(rel IN ('nofollow', 'sponsored', 'ugc', 'internal', 'external', 'unknown')) DEFAULT 'internal',
+      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
+      FOREIGN KEY(source_page_id) REFERENCES pages(id) ON DELETE CASCADE,
+      FOREIGN KEY(target_page_id) REFERENCES pages(id) ON DELETE CASCADE
+    );
+  `);
+
+  // Index for Edges
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot_source ON edges(snapshot_id, source_page_id);`);
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_edges_snapshot ON edges(snapshot_id);`);
+
+  // Metrics Table
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS metrics (
+      snapshot_id INTEGER NOT NULL,
+      page_id INTEGER NOT NULL,
+      authority_score REAL,
+      hub_score REAL,
+      pagerank REAL,
+      pagerank_score REAL,
+      link_role TEXT CHECK(link_role IN ('hub', 'authority', 'power', 'balanced', 'peripheral')),
+      crawl_status TEXT,
+      word_count INTEGER,
+      thin_content_score REAL,
+      external_link_ratio REAL,
+      orphan_score INTEGER,
+      duplicate_cluster_id TEXT,
+      duplicate_type TEXT CHECK(duplicate_type IN ('exact', 'near', 'template_heavy', 'none')),
+      is_cluster_primary INTEGER DEFAULT 0,
+      PRIMARY KEY(snapshot_id, page_id),
+      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE,
+      FOREIGN KEY(page_id) REFERENCES pages(id) ON DELETE CASCADE
+    );
+  `);
+
+  db.exec(`CREATE INDEX IF NOT EXISTS idx_metrics_snapshot ON metrics(snapshot_id);`);
+
+  // Duplicate Clusters Table
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS duplicate_clusters (
+      id TEXT NOT NULL,
+      snapshot_id INTEGER NOT NULL,
+      type TEXT CHECK(type IN ('exact', 'near', 'template_heavy')) NOT NULL,
+      size INTEGER NOT NULL,
+      representative TEXT NOT NULL,
+      severity TEXT CHECK(severity IN ('low', 'medium', 'high')) NOT NULL,
+      PRIMARY KEY(snapshot_id, id),
+      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
+    );
+  `);
+
+  // Content Clusters Table
+  db.exec(`
+    CREATE TABLE IF NOT EXISTS content_clusters (
+      id INTEGER NOT NULL,
+      snapshot_id INTEGER NOT NULL,
+      count INTEGER NOT NULL,
+      primary_url TEXT NOT NULL,
+      risk TEXT CHECK(risk IN ('low', 'medium', 'high')) NOT NULL,
+      shared_path_prefix TEXT,
+      PRIMARY KEY(snapshot_id, id),
+      FOREIGN KEY(snapshot_id) REFERENCES snapshots(id) ON DELETE CASCADE
+    );
+  `);
+
+  // Migration: add columns to existing DBs that were created before this update
+  migrateSchema(db);
+}
+
+function migrateSchema(db: Database) {
+  // Add missing columns to pages (safe: ALTER TABLE ADD COLUMN is idempotent-safe with try/catch)
+  const pageColumns = [
+    ['redirect_chain', 'TEXT'],
+    ['bytes_received', 'INTEGER'],
+    ['crawl_trap_flag', 'INTEGER DEFAULT 0'],
+    ['crawl_trap_risk', 'REAL'],
+    ['trap_type', 'TEXT'],
+  ];
+
+  for (const [col, type] of pageColumns) {
+    try { db.exec(`ALTER TABLE pages ADD COLUMN ${col} ${type}`); } catch { /* already exists */ }
+  }
+
+  // Add missing columns to edges
+  try { db.exec('ALTER TABLE edges ADD COLUMN weight REAL DEFAULT 1.0'); } catch { /* already exists */ }
+
+  // Add missing columns to metrics
+  const metricsColumns = [
+    ['pagerank_score', 'REAL'],
+    ['link_role', 'TEXT'],
+    ['duplicate_cluster_id', 'TEXT'],
+    ['duplicate_type', 'TEXT'],
+    ['is_cluster_primary', 'INTEGER DEFAULT 0'],
+  ];
+
+  for (const [col, type] of metricsColumns) {
+    try { db.exec(`ALTER TABLE metrics ADD COLUMN ${col} ${type}`); } catch { /* already exists */ }
+  }
+}

package/src/diff/compare.ts

@@ -0,0 +1,84 @@
+import { Graph } from '../graph/graph.js';
+import { calculateMetrics } from '../graph/metrics.js';
+
+export interface DiffResult {
+  addedUrls: string[];
+  removedUrls: string[];
+  changedStatus: { url: string; oldStatus: number; newStatus: number }[];
+  changedCanonical: { url: string; oldCanonical: string | null; newCanonical: string | null }[];
+  changedDuplicateGroup: { url: string; oldGroup: string | null; newGroup: string | null }[];
+  metricDeltas: {
+    structuralEntropy: number;
+    orphanCount: number;
+    crawlEfficiency: number;
+  };
+}
+
+export function compareGraphs(oldGraph: Graph, newGraph: Graph): DiffResult {
+  const oldNodes = new Map(oldGraph.getNodes().map(n => [n.url, n]));
+  const newNodes = new Map(newGraph.getNodes().map(n => [n.url, n]));
+
+  const addedUrls: string[] = [];
+  const removedUrls: string[] = [];
+  const changedStatus: { url: string; oldStatus: number; newStatus: number }[] = [];
+  const changedCanonical: { url: string; oldCanonical: string | null; newCanonical: string | null }[] = [];
+  const changedDuplicateGroup: { url: string; oldGroup: string | null; newGroup: string | null }[] = [];
+
+  // Added & Changed
+  for (const [url, newNode] of newNodes) {
+    const oldNode = oldNodes.get(url);
+    if (!oldNode) {
+      addedUrls.push(url);
+    } else {
+      // Changed Status
+      if (oldNode.status !== newNode.status) {
+        changedStatus.push({ url, oldStatus: oldNode.status, newStatus: newNode.status });
+      }
+      // Changed Canonical
+      if (oldNode.canonical !== newNode.canonical) {
+        changedCanonical.push({
+          url,
+          oldCanonical: oldNode.canonical || null,
+          newCanonical: newNode.canonical || null
+        });
+      }
+      // Changed Duplicate Group
+      const oldGroup = oldNode.duplicateClusterId || null;
+      const newGroup = newNode.duplicateClusterId || null;
+      if (oldGroup !== newGroup) {
+        changedDuplicateGroup.push({
+          url,
+          oldGroup,
+          newGroup
+        });
+      }
+    }
+  }
+
+  // Removed
+  for (const url of oldNodes.keys()) {
+    if (!newNodes.has(url)) {
+      removedUrls.push(url);
+    }
+  }
+
+  // Metrics
+  // maxDepth is ignored by current calculateMetrics implementation but required by signature
+  const oldMetrics = calculateMetrics(oldGraph, 10);
+  const newMetrics = calculateMetrics(newGraph, 10);
+
+  const metricDeltas = {
+    structuralEntropy: newMetrics.structuralEntropy - oldMetrics.structuralEntropy,
+    orphanCount: newMetrics.orphanPages.length - oldMetrics.orphanPages.length,
+    crawlEfficiency: newMetrics.crawlEfficiencyScore - oldMetrics.crawlEfficiencyScore
+  };
+
+  return {
+    addedUrls,
+    removedUrls,
+    changedStatus,
+    changedCanonical,
+    changedDuplicateGroup,
+    metricDeltas
+  };
+}

package/src/graph/cluster.ts

@@ -0,0 +1,192 @@
+import { Graph, GraphNode, ClusterInfo } from './graph.js';
+import { SimHash } from './simhash.js';
+
+/**
+ * Detects content clusters using 64-bit SimHash and Hamming Distance.
+ * Uses band optimization to reduce O(n^2) comparisons.
+ */
+export function detectContentClusters(
+    graph: Graph,
+    threshold: number = 10,
+    minSize: number = 3
+): ClusterInfo[] {
+    const nodes = graph.getNodes().filter(n => n.simhash && n.status === 200);
+    if (nodes.length === 0) return [];
+
+    const adjacency = new Map<string, Set<string>>();
+
+    // Banding Optimization (4 bands of 16 bits)
+    // Note: For threshold > 3, this is a heuristic and may miss some pairs,
+    // but it dramatically reduces the search space as requested.
+    const bands = 4;
+    const bandWidth = 16;
+    const buckets: Map<number, Set<string>>[] = Array.from({ length: bands }, () => new Map());
+
+    for (const node of nodes) {
+        const hash = BigInt(node.simhash!);
+        for (let b = 0; b < bands; b++) {
+            const bandValue = Number((hash >> BigInt(b * bandWidth)) & 0xFFFFn);
+            if (!buckets[b].has(bandValue)) {
+                buckets[b].set(bandValue, new Set());
+            }
+            buckets[b].get(bandValue)!.add(node.url);
+        }
+    }
+
+    const checkedPairs = new Set<string>();
+
+    for (let b = 0; b < bands; b++) {
+        for (const bucket of buckets[b].values()) {
+            if (bucket.size < 2) continue;
+            const bucketNodes = Array.from(bucket);
+            for (let i = 0; i < bucketNodes.length; i++) {
+                for (let j = i + 1; j < bucketNodes.length; j++) {
+                    const u1 = bucketNodes[i];
+                    const u2 = bucketNodes[j];
+                    if (u1 === u2) continue;
+
+                    const pairKey = u1 < u2 ? `${u1}|${u2}` : `${u2}|${u1}`;
+                    if (checkedPairs.has(pairKey)) continue;
+                    checkedPairs.add(pairKey);
+
+                    const n1 = graph.nodes.get(u1)!;
+                    const n2 = graph.nodes.get(u2)!;
+
+                    const dist = SimHash.hammingDistance(BigInt(n1.simhash!), BigInt(n2.simhash!));
+                    if (dist <= threshold) {
+                        if (!adjacency.has(u1)) adjacency.set(u1, new Set());
+                        if (!adjacency.has(u2)) adjacency.set(u2, new Set());
+                        adjacency.get(u1)!.add(u2);
+                        adjacency.get(u2)!.add(u1);
+                    }
+                }
+            }
+        }
+    }
+
+    // Find connected components (Clusters)
+    const visited = new Set<string>();
+    const clusters: string[][] = [];
+
+    for (const node of nodes) {
+        if (visited.has(node.url)) continue;
+
+        const component: string[] = [];
+        const queue = [node.url];
+        visited.add(node.url);
+
+        while (queue.length > 0) {
+            const current = queue.shift()!;
+            component.push(current);
+
+            const neighbors = adjacency.get(current);
+            if (neighbors) {
+                for (const neighbor of neighbors) {
+                    if (!visited.has(neighbor)) {
+                        visited.add(neighbor);
+                        queue.push(neighbor);
+                    }
+                }
+            }
+        }
+
+        if (component.length >= minSize) {
+            clusters.push(component);
+        }
+    }
+
+    // Sort clusters by size (descending) then by primary URL (ascending) for deterministic IDs
+    clusters.sort((a, b) => {
+        if (b.length !== a.length) return b.length - a.length;
+        const aPrimary = selectPrimaryUrl(a, graph);
+        const bPrimary = selectPrimaryUrl(b, graph);
+        return aPrimary.localeCompare(bPrimary);
+    });
+
+    const clusterInfos: ClusterInfo[] = [];
+    clusters.forEach((memberUrls, index) => {
+        const clusterId = index + 1;
+        const clusterNodes = memberUrls.map(url => graph.nodes.get(url)!);
+
+        for (const node of clusterNodes) {
+            node.clusterId = clusterId;
+        }
+
+        const primaryUrl = selectPrimaryUrl(memberUrls, graph);
+        const risk = calculateClusterRisk(clusterNodes);
+        const sharedPathPrefix = findSharedPathPrefix(memberUrls);
+
+        clusterInfos.push({
+            id: clusterId,
+            count: memberUrls.length,
+            primaryUrl,
+            risk,
+            sharedPathPrefix
+        });
+    });
+
+    graph.contentClusters = clusterInfos;
+    return clusterInfos;
+}
+
+/**
+ * Selects the primary URL for a cluster based on:
+ * 1. Highest PageRank
+ * 2. Shortest URL
+ * 3. Lexicographic fallback
+ */
+function selectPrimaryUrl(urls: string[], graph: Graph): string {
+    return urls.reduce((best, current) => {
+        const nBest = graph.nodes.get(best)!;
+        const nCurrent = graph.nodes.get(current)!;
+
+        if ((nCurrent.pageRank || 0) > (nBest.pageRank || 0)) return current;
+        if ((nCurrent.pageRank || 0) < (nBest.pageRank || 0)) return best;
+
+        if (current.length < best.length) return current;
+        if (current.length > best.length) return best;
+
+        return current.localeCompare(best) < 0 ? current : best;
+    });
+}
+
+/**
+ * Calculates cannibalization risk based on title and H1 similarity within the cluster.
+ */
+function calculateClusterRisk(nodes: GraphNode[]): 'low' | 'medium' | 'high' {
+    // Logic: Check if there's significant overlap in Titles or H1s among cluster members.
+    // This is a heuristic as requested.
+    // Simplified heuristic: risk is based on cluster density and size
+    // Large clusters of highly similar content are high risk.
+
+    // Fallback to a safe categorization
+    if (nodes.length > 5) return 'high';
+    if (nodes.length > 2) return 'medium';
+    return 'low';
+}
+
+/**
+ * Finds the common path prefix among a set of URLs.
+ */
+function findSharedPathPrefix(urls: string[]): string | undefined {
+    if (urls.length < 2) return undefined;
+
+    try {
+        const paths = urls.map(u => new URL(u).pathname.split('/').filter(Boolean));
+        const first = paths[0];
+        const common: string[] = [];
+
+        for (let i = 0; i < first.length; i++) {
+            const segment = first[i];
+            if (paths.every(p => p[i] === segment)) {
+                common.push(segment);
+            } else {
+                break;
+            }
+        }
+
+        return common.length > 0 ? '/' + common.join('/') : undefined;
+    } catch {
+        return undefined;
+    }
+}