messi-crawler 1.0.0

package/src/cli/renderer.ts

@@ -0,0 +1,83 @@
+/**
+ * Terminal rendering utilities — colours, symbols, layout helpers.
+ * Keeps all ANSI logic in one place so the wizard stays readable.
+ */
+
+export const c = {
+  reset:     "\x1b[0m",
+  bold:      "\x1b[1m",
+  dim:       "\x1b[2m",
+  italic:    "\x1b[3m",
+
+  // foreground
+  white:     "\x1b[97m",
+  gray:      "\x1b[90m",
+  cyan:      "\x1b[96m",
+  green:     "\x1b[92m",
+  yellow:    "\x1b[93m",
+  red:       "\x1b[91m",
+  blue:      "\x1b[94m",
+  magenta:   "\x1b[95m",
+  orange:    "\x1b[38;5;208m",
+};
+
+export const sym = {
+  dot:       "·",
+  bullet:    "•",
+  arrow:     "›",
+  check:     "✓",
+  cross:     "✗",
+  warn:      "⚠",
+  info:      "ℹ",
+  sparkle:   "◆",
+  bar:       "│",
+  corner:    "╰",
+  tee:       "├",
+  horiz:     "─",
+};
+
+/** Wraps text in an ANSI style sequence. */
+export function style(text: string, ...styles: string[]): string {
+  return styles.join("") + text + c.reset;
+}
+
+/** Prints a blank line. */
+export function blank(): void { console.log(); }
+
+/** dim separator line */
+export function divider(width = 52): void {
+  console.log(style(sym.horiz.repeat(width), c.dim, c.gray));
+}
+
+/** A styled section label, e.g.  ◆ Seeds */
+export function section(label: string): void {
+  console.log(style(`${sym.sparkle} ${label}`, c.bold, c.cyan));
+}
+
+/** A key/value summary row, e.g.   › depth    3 */
+export function row(key: string, value: string, valueColor = c.white): void {
+  const pad = 16;
+  const k = style(key.padEnd(pad), c.gray);
+  const v = style(value, valueColor);
+  console.log(`  ${style(sym.arrow, c.dim, c.gray)} ${k}${v}`);
+}
+
+/** Success line */
+export function ok(msg: string): void {
+  console.log(`  ${style(sym.check, c.green)} ${style(msg, c.white)}`);
+}
+
+/** Warning line */
+export function warn(msg: string): void {
+  console.log(`  ${style(sym.warn, c.yellow)} ${style(msg, c.yellow)}`);
+}
+
+/** Error line */
+export function err(msg: string): void {
+  console.log(`  ${style(sym.cross, c.red)} ${style(msg, c.red)}`);
+}
+
+/** Info line */
+export function info(msg: string): void {
+  console.log(`  ${style(sym.info, c.blue)} ${style(msg, c.gray)}`);
+}

package/src/config.ts

@@ -0,0 +1,22 @@
+import dotenv from "dotenv";
+dotenv.config();
+
+export const config = {
+  MAX_DEPTH: parseInt(process.env.MAX_DEPTH || "3", 10),
+  CRAWL_DELAY_MS: parseInt(process.env.CRAWL_DELAY_MS || "1000", 10),
+  WORKER_COUNT: parseInt(process.env.WORKER_COUNT || "10", 10),
+  REQUEST_TIMEOUT_MS: parseInt(process.env.REQUEST_TIMEOUT_MS || "10000", 10),
+  MAX_REDIRECTS: parseInt(process.env.MAX_REDIRECTS || "5", 10),
+  MAX_PAGES: parseInt(process.env.MAX_PAGES || "1000", 10),
+
+  /** Output destination: "database" | "pdf" */
+  OUTPUT_MODE: (process.env.OUTPUT_MODE || "database") as "database" | "pdf",
+
+  SEED_URLS: [
+    "https://www.akc.org/dog-breeds/",
+  ],
+
+  ALLOWED_DOMAINS: [
+    "www.akc.org",
+  ],
+};

package/src/db/clear.ts

@@ -0,0 +1,16 @@
+import { pool } from "./client.js";
+
+async function clearDatabase() {
+  console.log("Clearing database...");
+  try {
+    // Truncate all crawler tables and reset the auto-increment IDs
+    await pool.query("TRUNCATE TABLE links, crawled_pages, urls, domain_stats RESTART IDENTITY CASCADE;");
+    console.log("Database cleared successfully.");
+  } catch (error) {
+    console.error("Error clearing database:", error);
+  } finally {
+    await pool.end();
+  }
+}
+
+clearDatabase();

package/src/db/client.ts

@@ -0,0 +1,26 @@
+import pg from "pg";
+import dotenv from "dotenv";
+
+dotenv.config();
+
+const { Pool } = pg;
+
+const connectionString = process.env.DATABASE_URL;
+
+export const pool = connectionString
+  ? new Pool({ connectionString })
+  : new Pool({
+      host: process.env.PGHOST || "localhost",
+      port: parseInt(process.env.PGPORT || "5432", 10),
+      user: process.env.PGUSER || "postgres",
+      password: process.env.PGPASSWORD || "",
+      database: process.env.PGDATABASE || "web_crawler",
+    });
+
+export async function query(text: string, params?: any[]) {
+  return pool.query(text, params);
+}
+
+export async function closePool() {
+  await pool.end();
+}

package/src/db/queries.ts

@@ -0,0 +1,255 @@
+import { query, pool } from "./client.js";
+import type { ContentBlock, ExtractedImage } from "../worker/extractor.js";
+
+export interface URLRow {
+  id: number;
+  url: string;
+  domain: string;
+  status: string;
+  depth: number;
+}
+
+export interface CrawledPageContent {
+  title: string | null;
+  description: string | null;
+  canonicalUrl: string | null;
+  headings: { h1: string[]; h2: string[]; h3: string[] };
+  textContent: string | null;
+  blocks?: ContentBlock[];
+  images?: ExtractedImage[];
+}
+
+/**
+ * Claims the next PENDING URL for a given domain and marks it as FETCHING.
+ * Uses FOR UPDATE SKIP LOCKED to prevent multiple workers from claiming the same URL.
+ */
+export async function claimNextURL(domain: string): Promise<URLRow | null> {
+  const res = await query(
+    `UPDATE urls
+     SET status = 'FETCHING', fetched_at = NOW()
+     WHERE id = (
+       SELECT id FROM urls
+       WHERE status = 'PENDING' AND domain = $1
+       ORDER BY depth ASC, discovered_at ASC
+       LIMIT 1
+       FOR UPDATE SKIP LOCKED
+     )
+     RETURNING id, url, domain, status, depth`,
+    [domain]
+  );
+
+  if (res.rows.length === 0) {
+    return null;
+  }
+
+  return res.rows[0] as URLRow;
+}
+
+/**
+ * Atomically updates URL status to DONE and inserts the crawled page content.
+ */
+export async function markDone(urlId: number, content: CrawledPageContent): Promise<void> {
+  const client = await pool.connect();
+  try {
+    await client.query("BEGIN");
+
+    await client.query(
+      `INSERT INTO crawled_pages (url_id, title, description, canonical_url, headings, text_content)
+       VALUES ($1, $2, $3, $4, $5, $6)`,
+      [
+        urlId,
+        content.title,
+        content.description,
+        content.canonicalUrl,
+        JSON.stringify(content.headings),
+        content.textContent,
+      ]
+    );
+
+    await client.query(
+      `UPDATE urls
+       SET status = 'DONE', fetched_at = NOW()
+       WHERE id = $1`,
+      [urlId]
+    );
+
+    await client.query("COMMIT");
+  } catch (error) {
+    await client.query("ROLLBACK");
+    throw error;
+  } finally {
+    client.release();
+  }
+}
+
+/**
+ * Marks a URL status as FAILED and stores the error message.
+ */
+export async function markFailed(urlId: number, errorMessage: string): Promise<void> {
+  await query(
+    `UPDATE urls
+     SET status = 'FAILED', error_message = $2, fetched_at = NOW()
+     WHERE id = $1`,
+    [urlId, errorMessage]
+  );
+}
+
+/**
+ * Inserts a URL as PENDING if it doesn't already exist.
+ * Returns the ID of the URL (whether newly inserted or already existing).
+ */
+export async function insertURL(url: string, domain: string, depth: number): Promise<number> {
+  const res = await query(
+    `WITH ins AS (
+       INSERT INTO urls (url, domain, status, depth)
+       VALUES ($1, $2, 'PENDING', $3)
+       ON CONFLICT (url) DO NOTHING
+       RETURNING id
+     )
+     SELECT id FROM ins
+     UNION ALL
+     SELECT id FROM urls WHERE url = $1
+     LIMIT 1`,
+    [url, domain, depth]
+  );
+
+  return res.rows[0].id;
+}
+
+/**
+ * Inserts a link relationship between two URLs.
+ */
+export async function insertLink(fromUrlId: number, toUrlId: number): Promise<void> {
+  await query(
+    `INSERT INTO links (from_url_id, to_url_id)
+     VALUES ($1, $2)
+     ON CONFLICT (from_url_id, to_url_id) DO NOTHING`,
+    [fromUrlId, toUrlId]
+  );
+}
+
+/**
+ * Resets all URLs with FETCHING status back to PENDING.
+ * Used for crash recovery on startup to release stale locks.
+ */
+export async function resetStaleLocks(): Promise<void> {
+  await query(
+    `UPDATE urls
+     SET status = 'PENDING'
+     WHERE status = 'FETCHING'`
+  );
+}
+
+/**
+ * Deletes all PENDING URLs whose domain is not in the provided allowed list.
+ * This includes child links discovered during previous crawls, ensuring a new
+ * session scoped to different seeds starts with a clean queue.
+ */
+export async function clearPendingURLs(allowedDomains: string[]): Promise<void> {
+  if (allowedDomains.length === 0) return;
+
+  const result = await query(
+    `DELETE FROM urls
+     WHERE status = 'PENDING'
+       AND domain <> ALL($1::text[])`,
+    [allowedDomains]
+  );
+
+  const deleted = (result as any).rowCount ?? 0;
+  if (deleted > 0) {
+    console.log(`[setup] Cleared ${deleted} stale PENDING URL(s) outside allowed domains.`);
+  }
+}
+
+export interface GlobalStats {
+  pending: number;
+  fetching: number;
+  done: number;
+  failed: number;
+}
+
+export interface DomainStats {
+  domain: string;
+  pending_count: number;
+  fetching_count: number;
+  done_count: number;
+  failed_count: number;
+  last_crawled_at: Date | null;
+}
+
+/**
+ * Retrieves aggregate statistics across all URLs.
+ */
+export async function getGlobalStats(): Promise<GlobalStats> {
+  const res = await query(
+    `SELECT status, COUNT(*) as count
+     FROM urls
+     GROUP BY status`
+  );
+
+  const stats: GlobalStats = { pending: 0, fetching: 0, done: 0, failed: 0 };
+  for (const row of res.rows) {
+    const status = row.status.toLowerCase();
+    const count = parseInt(row.count, 10);
+    if (status === "pending") stats.pending = count;
+    else if (status === "fetching") stats.fetching = count;
+    else if (status === "done") stats.done = count;
+    else if (status === "failed") stats.failed = count;
+  }
+  return stats;
+}
+
+/**
+ * Recomputes and updates domain-level stats in the domain_stats table.
+ */
+export async function refreshDomainStats(): Promise<void> {
+  await query(`
+    CREATE TABLE IF NOT EXISTS domain_stats (
+      domain TEXT PRIMARY KEY,
+      pending_count INTEGER NOT NULL DEFAULT 0,
+      fetching_count INTEGER NOT NULL DEFAULT 0,
+      done_count INTEGER NOT NULL DEFAULT 0,
+      failed_count INTEGER NOT NULL DEFAULT 0,
+      last_crawled_at TIMESTAMPTZ
+    )
+  `);
+
+  await query(`
+    INSERT INTO domain_stats (domain, pending_count, fetching_count, done_count, failed_count, last_crawled_at)
+    SELECT
+      domain,
+      COUNT(*) FILTER (WHERE status = 'PENDING') as pending_count,
+      COUNT(*) FILTER (WHERE status = 'FETCHING') as fetching_count,
+      COUNT(*) FILTER (WHERE status = 'DONE') as done_count,
+      COUNT(*) FILTER (WHERE status = 'FAILED') as failed_count,
+      MAX(fetched_at) as last_crawled_at
+    FROM urls
+    GROUP BY domain
+    ON CONFLICT (domain) DO UPDATE SET
+      pending_count = EXCLUDED.pending_count,
+      fetching_count = EXCLUDED.fetching_count,
+      done_count = EXCLUDED.done_count,
+      failed_count = EXCLUDED.failed_count,
+      last_crawled_at = EXCLUDED.last_crawled_at
+  `);
+}
+
+/**
+ * Retrieves per-domain statistics.
+ */
+export async function getDomainStats(): Promise<DomainStats[]> {
+  const res = await query(
+    `SELECT domain, pending_count, fetching_count, done_count, failed_count, last_crawled_at
+     FROM domain_stats
+     ORDER BY domain ASC`
+  );
+  return res.rows.map((row) => ({
+    domain: row.domain,
+    pending_count: parseInt(row.pending_count, 10),
+    fetching_count: parseInt(row.fetching_count, 10),
+    done_count: parseInt(row.done_count, 10),
+    failed_count: parseInt(row.failed_count, 10),
+    last_crawled_at: row.last_crawled_at ? new Date(row.last_crawled_at) : null,
+  }));
+}
+

package/src/db/schema.sql

@@ -0,0 +1,43 @@
+-- All discovered URLs and their crawl state
+CREATE TABLE IF NOT EXISTS urls (
+  id            SERIAL PRIMARY KEY,
+  url           TEXT NOT NULL UNIQUE,
+  domain        TEXT NOT NULL,
+  status        TEXT NOT NULL DEFAULT 'PENDING',  -- PENDING | FETCHING | DONE | FAILED
+  depth         INTEGER NOT NULL DEFAULT 0,
+  error_message TEXT,
+  discovered_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+  fetched_at    TIMESTAMPTZ
+);
+
+CREATE INDEX IF NOT EXISTS idx_urls_status_domain ON urls (status, domain);
+
+-- Extracted page content
+CREATE TABLE IF NOT EXISTS crawled_pages (
+  id            SERIAL PRIMARY KEY,
+  url_id        INTEGER NOT NULL REFERENCES urls(id) ON DELETE CASCADE,
+  title         TEXT,
+  description   TEXT,
+  canonical_url TEXT,
+  headings      JSONB,     -- { h1: [...], h2: [...], h3: [...] }
+  text_content  TEXT,
+  crawled_at    TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
+-- Link graph
+CREATE TABLE IF NOT EXISTS links (
+  from_url_id   INTEGER NOT NULL REFERENCES urls(id) ON DELETE CASCADE,
+  to_url_id     INTEGER NOT NULL REFERENCES urls(id) ON DELETE CASCADE,
+  PRIMARY KEY (from_url_id, to_url_id)
+);
+
+-- Domain statistics for observability
+CREATE TABLE IF NOT EXISTS domain_stats (
+  domain TEXT PRIMARY KEY,
+  pending_count INTEGER NOT NULL DEFAULT 0,
+  fetching_count INTEGER NOT NULL DEFAULT 0,
+  done_count INTEGER NOT NULL DEFAULT 0,
+  failed_count INTEGER NOT NULL DEFAULT 0,
+  last_crawled_at TIMESTAMPTZ
+);
+

package/src/frontier/frontier.ts

@@ -0,0 +1,60 @@
+import { query } from "../db/client.js";
+import { config } from "../config.js";
+
+/**
+ * Retrieves the list of unique domains that have at least one pending URL,
+ * restricted to the domains allowed in the current crawl session.
+ */
+export async function getPendingDomains(): Promise<string[]> {
+  const allowed = config.ALLOWED_DOMAINS;
+
+  // If ALLOWED_DOMAINS is populated, filter at the DB level so the scheduler
+  // never even sees pending rows from outside the current session's scope.
+  if (allowed && allowed.length > 0) {
+    const res = await query(
+      `SELECT DISTINCT domain
+       FROM urls
+       WHERE status = 'PENDING'
+         AND domain = ANY($1::text[])`,
+      [allowed]
+    );
+    return res.rows.map((row) => row.domain);
+  }
+
+  const res = await query(
+    `SELECT DISTINCT domain
+     FROM urls
+     WHERE status = 'PENDING'`
+  );
+  return res.rows.map((row) => row.domain);
+}
+
+/**
+ * Retrieves the count of pending URLs bucketed by domain,
+ * restricted to the allowed domains in the current session.
+ */
+export async function getPendingCounts(): Promise<Record<string, number>> {
+  const allowed = config.ALLOWED_DOMAINS;
+
+  const res = allowed && allowed.length > 0
+    ? await query(
+        `SELECT domain, COUNT(*) as count
+         FROM urls
+         WHERE status = 'PENDING'
+           AND domain = ANY($1::text[])
+         GROUP BY domain`,
+        [allowed]
+      )
+    : await query(
+        `SELECT domain, COUNT(*) as count
+         FROM urls
+         WHERE status = 'PENDING'
+         GROUP BY domain`
+      );
+
+  const counts: Record<string, number> = {};
+  for (const row of res.rows) {
+    counts[row.domain] = parseInt(row.count, 10);
+  }
+  return counts;
+}

package/src/frontier/logger.ts

@@ -0,0 +1,75 @@
+import { getGlobalStats, refreshDomainStats, getDomainStats } from "../db/queries.js";
+
+let loggerInterval: any = null;
+let lastDoneAndFailedCount = 0;
+let startTime = 0;
+
+/**
+ * Starts a background interval to log crawler progress periodically.
+ */
+export async function startProgressLogger(intervalMs: number = 5000): Promise<void> {
+  if (loggerInterval) return;
+
+  startTime = Date.now();
+  try {
+    const initialStats = await getGlobalStats();
+    lastDoneAndFailedCount = initialStats.done + initialStats.failed;
+  } catch (err) {
+    lastDoneAndFailedCount = 0;
+  }
+
+  loggerInterval = setInterval(async () => {
+    try {
+      // 1. Sync statistics to domain_stats table
+      await refreshDomainStats();
+
+      // 2. Fetch global statistics
+      const globalStats = await getGlobalStats();
+
+      // 3. Fetch domain-level statistics
+      const domainStats = await getDomainStats();
+
+      // 4. Calculate crawl rates
+      const currentCompleted = globalStats.done + globalStats.failed;
+      const completedSinceStart = currentCompleted - lastDoneAndFailedCount;
+      const elapsedMinutes = (Date.now() - startTime) / 60000;
+      const crawlRate = elapsedMinutes > 0 ? (completedSinceStart / elapsedMinutes).toFixed(1) : "0.0";
+
+      // 5. Build and output the formatted log messages
+      console.log(`\n=== Crawler Progress Report ===`);
+      console.log(`Speed: ${crawlRate} pages/min`);
+      console.log(`Global Status Breakdown:`);
+      console.log(`  PENDING : ${globalStats.pending}`);
+      console.log(`  FETCHING: ${globalStats.fetching}`);
+      console.log(`  DONE    : ${globalStats.done}`);
+      console.log(`  FAILED  : ${globalStats.failed}`);
+
+      if (domainStats.length > 0) {
+        console.log(`Domain Breakdown:`);
+        for (const ds of domainStats) {
+          const lastCrawledStr = ds.last_crawled_at ? ds.last_crawled_at.toISOString() : "never";
+          console.log(
+            `  - ${ds.domain}: PENDING: ${ds.pending_count} | FETCHING: ${ds.fetching_count} | DONE: ${ds.done_count} | FAILED: ${ds.failed_count} (Last Crawled: ${lastCrawledStr})`
+          );
+        }
+      }
+      console.log(`================================\n`);
+    } catch (error) {
+      console.error("Error generating crawler progress logs:", error);
+    }
+  }, intervalMs);
+
+  if (loggerInterval && typeof loggerInterval.unref === "function") {
+    loggerInterval.unref();
+  }
+}
+
+/**
+ * Stops the progress logger interval.
+ */
+export function stopProgressLogger(): void {
+  if (loggerInterval) {
+    clearInterval(loggerInterval);
+    loggerInterval = null;
+  }
+}

package/src/frontier/robots.ts

@@ -0,0 +1,50 @@
+import { request } from "undici";
+// @ts-ignore
+import robotsParser from "robots-parser";
+import { getDomain } from "../normalizer.js";
+import { config } from "../config.js";
+
+const robotsCache = new Map<string, any>();
+const fetchFailures = new Set<string>();
+
+/**
+ * Checks if a URL is allowed to be crawled according to the domain's robots.txt rules.
+ * Caches robots.txt rules per domain to avoid duplicate requests.
+ */
+export async function isAllowedByRobots(urlStr: string): Promise<boolean> {
+  const domain = getDomain(urlStr);
+  if (!domain) return false;
+
+  if (fetchFailures.has(domain)) {
+    return true;
+  }
+
+  let parser = robotsCache.get(domain);
+
+  if (!parser) {
+    const robotsUrl = `https://${domain}/robots.txt`;
+    try {
+      const res = await request(robotsUrl, {
+        method: "GET",
+        headersTimeout: config.REQUEST_TIMEOUT_MS,
+        bodyTimeout: config.REQUEST_TIMEOUT_MS,
+      });
+
+      if (res.statusCode === 200) {
+        const content = await res.body.text();
+        const parserCreator = robotsParser as any;
+        parser = parserCreator(robotsUrl, content);
+        robotsCache.set(domain, parser);
+      } else {
+        fetchFailures.add(domain);
+        return true;
+      }
+    } catch (e) {
+      // On network/request errors, default to allowed but do not permanently cache failure
+      return true;
+    }
+  }
+
+  const isAllowed = parser.isAllowed(urlStr, "WebCrawler");
+  return isAllowed === undefined ? true : isAllowed;
+}