messi-crawler 1.0.0

package/src/output/pdf-strategy.ts

@@ -0,0 +1,388 @@
+import fs from "fs";
+import path from "path";
+import PDFDocumentCtor from "pdfkit";
+import type { OutputStrategy } from "./strategy.js";
+import { markDone, type CrawledPageContent } from "../db/queries.js";
+import { downloadImage } from "../worker/downloader.js";
+
+type PDFDocumentInstance = InstanceType<typeof PDFDocumentCtor>;
+
+const OUTPUT_DIR = "output";
+const BASE_NAME  = "documentation";
+
+function resolveOutputPath(): string {
+  const first = path.join(OUTPUT_DIR, `${BASE_NAME}.pdf`);
+  if (!fs.existsSync(first)) return first;
+  let n = 2;
+  while (fs.existsSync(path.join(OUTPUT_DIR, `${BASE_NAME}${n}.pdf`))) n++;
+  return path.join(OUTPUT_DIR, `${BASE_NAME}${n}.pdf`);
+}
+
+// ─── Layout constants ─────────────────────────────────────────────────────────
+
+const MARGIN          = 64;
+const FOOTER_HEIGHT   = 28;   // reserved space at bottom for footer
+const MAX_TEXT_CHARS  = 5000;
+
+// ─── Colour palette ───────────────────────────────────────────────────────────
+
+const C = {
+  title:     "#1a1a2e",
+  url:       "#4361ee",
+  desc:      "#444444",
+  section:   "#2d6a4f",
+  body:      "#222222",
+  truncated: "#aaaaaa",
+  rule:      "#dddddd",
+  coverBg:   "#1a1a2e",
+  coverFg:   "#ffffff",
+  coverSub:  "#a8dadc",
+  badge:     "#888888",
+  footer:    "#aaaaaa",
+};
+
+// ─── Helpers ──────────────────────────────────────────────────────────────────
+
+/**
+ * Draws a horizontal rule at the current cursor Y and advances by `gap` points.
+ * Uses an absolute move so it never inherits stale font metrics.
+ */
+function rule(doc: PDFDocumentInstance, color = C.rule, gap = 10): void {
+  const y = doc.y;
+  doc
+    .moveTo(MARGIN, y)
+    .lineTo(doc.page.width - MARGIN, y)
+    .strokeColor(color)
+    .lineWidth(0.5)
+    .stroke();
+  doc.y = y + gap;   // advance cursor by exact points, not line-height multiples
+}
+
+/**
+ * Normalises raw scraped text:
+ *  - Collapses runs of whitespace/newlines into a single space
+ *  - Trims leading/trailing whitespace
+ * This prevents the large blank gaps that `paragraphGap` creates when the
+ * extractor's output happens to contain stray newline characters.
+ */
+function normaliseText(raw: string): string {
+  return raw.replace(/\s+/g, " ").trim();
+}
+
+/**
+ * Returns the usable content height on the current page
+ * (page height minus top margin, bottom margin, and footer reservation).
+ */
+function contentBottom(doc: PDFDocumentInstance): number {
+  return doc.page.height - MARGIN - FOOTER_HEIGHT;
+}
+
+// ─── Strategy ────────────────────────────────────────────────────────────────
+
+export class PdfStrategy implements OutputStrategy {
+  private doc!: PDFDocumentInstance;
+  private stream!: fs.WriteStream;
+  private pageCount = 0;
+  private pdfPath!: string;
+
+  async init(): Promise<void> {
+    if (!fs.existsSync(OUTPUT_DIR)) fs.mkdirSync(OUTPUT_DIR, { recursive: true });
+
+    this.pdfPath = resolveOutputPath();
+
+    this.doc = new PDFDocumentCtor({
+      autoFirstPage: false,
+      bufferPages: true,
+      // Explicit margins so pdfkit never auto-paginates into blank pages
+      // due to cursor running past the bottom margin.
+      margins: { top: MARGIN, bottom: MARGIN + FOOTER_HEIGHT, left: MARGIN, right: MARGIN },
+      info: {
+        Title:   "Checkout the repo https://github.com/lightning4747/Web-crawler-cli",
+        Author:  "Web Crawler",
+        Subject: "Compiled documentation from crawled pages",
+      },
+    });
+
+    this.stream = fs.createWriteStream(this.pdfPath);
+    this.doc.pipe(this.stream);
+    this.renderCover();
+    console.log(`[PDF] Output file: ${this.pdfPath}`);
+  }
+
+  // ── Cover page ──────────────────────────────────────────────────────────────
+
+  private renderCover(): void {
+    const doc = this.doc;
+    doc.addPage();
+
+    doc.rect(0, 0, doc.page.width, doc.page.height).fill(C.coverBg);
+
+    const midY = doc.page.height / 2 - 60;
+
+    doc.fontSize(38).font("Helvetica-Bold").fillColor(C.coverFg)
+      .text("Web Crawler", MARGIN, midY, { align: "center" });
+
+    // Advance by exact points to avoid font-size-based gaps
+    doc.y += 8;
+
+    doc.fontSize(32).font("Helvetica-Bold").fillColor(C.coverSub)
+      .text("Documentation Book", { align: "center" });
+
+    doc.y += 28;
+
+    doc.fontSize(11).font("Helvetica").fillColor(C.coverFg)
+      .text("https://github.com/lightning4747/Web-crawler-cli", { align: "center", link: "https://github.com/lightning4747/Web-crawler-cli" });
+
+    doc.y += 8;
+
+    doc.fontSize(10).font("Helvetica").fillColor(C.coverSub)
+      .text(new Date().toUTCString(), { align: "center" });
+  }
+
+  // ── Chapter page ────────────────────────────────────────────────────────────
+
+  async save(urlId: number, url: string, content: CrawledPageContent): Promise<void> {
+    await markDone(urlId, content);
+
+    const doc   = this.doc;
+    this.pageCount++;
+    doc.addPage();
+
+    const W     = doc.page.width - MARGIN * 2;   // usable text width
+    const limit = contentBottom(doc);             // y-coordinate of content boundary
+
+    // ── Chapter badge (top-right, absolute position) ─────────────────────────
+    doc.fontSize(8).font("Helvetica").fillColor(C.badge)
+      .text(`CHAPTER ${this.pageCount}`, MARGIN, MARGIN, { width: W, align: "right" });
+
+    // Place cursor just below the badge — use exact points, not moveDown
+    doc.y = MARGIN + 14;
+
+    // ── Title ─────────────────────────────────────────────────────────────────
+    doc.fontSize(22).font("Helvetica-Bold").fillColor(C.title)
+      .text(content.title ?? url, { width: W, lineGap: 2 });
+
+    doc.y += 10;
+    rule(doc, "#4361ee", 10);
+
+    // ── Source URL ────────────────────────────────────────────────────────────
+    doc.fontSize(8.5).font("Helvetica-Oblique").fillColor(C.url)
+      .text(url, { width: W, link: url, underline: true, lineGap: 1 });
+
+    doc.y += 12;
+
+    // ── Description ───────────────────────────────────────────────────────────
+    if (content.description && doc.y < limit) {
+      doc.fontSize(11).font("Helvetica-Oblique").fillColor(C.desc)
+        .text(content.description.trim(), { width: W, lineGap: 2, align: "left" });
+
+      doc.y += 12;
+    }
+
+    // ── Headings summary ──────────────────────────────────────────────────────
+    const allHeadings = [
+      ...content.headings.h1,
+      ...content.headings.h2,
+      ...content.headings.h3,
+    ].slice(0, 12);
+
+    if (allHeadings.length > 0 && doc.y < limit) {
+      doc.fontSize(9).font("Helvetica-Bold").fillColor(C.section)
+        .text("CONTENTS OVERVIEW", { width: W, characterSpacing: 1 });
+
+      doc.y += 6;
+
+      for (const h of allHeadings) {
+        if (doc.y >= limit) break;
+
+        const bulletX = MARGIN;
+        const textX   = MARGIN + 14;
+        const y       = doc.y;
+
+        // Bullet dot — drawn absolutely, no cursor movement
+        doc.circle(bulletX + 3, y + 5, 2).fill(C.section);
+
+        doc.fontSize(10).font("Helvetica").fillColor(C.body)
+          .text(h, textX, y, { width: W - 14, lineGap: 2 });
+
+        // Advance by 4pt padding between bullet items
+        doc.y += 4;
+      }
+
+      doc.y += 8;
+
+      if (doc.y < limit) rule(doc, C.rule, 10);
+    }
+
+    // ── Page Content ──────────────────────────────────────────────────────────
+    if (content.blocks && content.blocks.length > 0) {
+      doc.fontSize(9).font("Helvetica-Bold").fillColor(C.section)
+        .text("PAGE CONTENT", { width: W, characterSpacing: 1 });
+
+      doc.y += 8;
+
+      for (const block of content.blocks) {
+        if (doc.y >= limit) {
+          doc.addPage();
+        }
+
+        if (block.type === "heading" && block.text) {
+          const headingSize = block.level === 1 ? 16 : block.level === 2 ? 14 : 12;
+          const headingHeight = doc.heightOfString(block.text, { width: W });
+          
+          if (doc.y + headingHeight + 40 > limit) {
+            doc.addPage();
+          }
+
+          doc.fontSize(headingSize).font("Helvetica-Bold").fillColor(C.title)
+            .text(block.text, { width: W, lineGap: 2 });
+          doc.y += 6;
+        } else if (block.type === "paragraph" && block.text) {
+          const text = block.text.trim();
+          if (!text) continue;
+
+          const textHeight = doc.heightOfString(text, { width: W });
+          if (doc.y + 20 > limit) {
+            doc.addPage();
+          }
+
+          doc.fontSize(10).font("Helvetica").fillColor(C.body)
+            .text(text, { width: W, lineGap: 3 });
+          doc.y += 8;
+        } else if (block.type === "list" && block.items && block.items.length > 0) {
+          if (doc.y + 20 > limit) {
+            doc.addPage();
+          }
+
+          for (const item of block.items) {
+            const itemText = item.trim();
+            if (!itemText) continue;
+
+            const bulletX = MARGIN + 10;
+            const textX = MARGIN + 22;
+            const itemHeight = doc.heightOfString(itemText, { width: W - 22 });
+
+            if (doc.y + itemHeight > limit) {
+              doc.addPage();
+            }
+
+            const y = doc.y;
+            doc.circle(bulletX + 3, y + 5, 2).fill(C.body);
+
+            doc.fontSize(9.5).font("Helvetica").fillColor(C.body)
+              .text(itemText, textX, y, { width: W - 22, lineGap: 2 });
+            doc.y += 4;
+          }
+          doc.y += 4;
+        } else if (block.type === "image" && block.src) {
+          try {
+            const imageBuffer = await downloadImage(block.src);
+            const maxImageHeight = 200;
+
+            if (doc.y + maxImageHeight + 20 > limit) {
+              doc.addPage();
+            }
+
+            doc.image(imageBuffer, {
+              fit: [W, maxImageHeight],
+              align: "center",
+            });
+            doc.y += maxImageHeight + 10;
+
+            if (block.alt) {
+              doc.fontSize(8.5).font("Helvetica-Oblique").fillColor(C.desc)
+                .text(block.alt, { width: W, align: "center" });
+              doc.y += 8;
+            }
+          } catch (err) {
+            const fallbackText = `[Image: ${block.alt || "No description available"} (${block.src})]`;
+            const boxHeight = 40;
+
+            if (doc.y + boxHeight > limit) {
+              doc.addPage();
+            }
+
+            const currentY = doc.y;
+            doc.rect(MARGIN, currentY, W, boxHeight)
+              .strokeColor(C.rule)
+              .lineWidth(0.5)
+              .stroke();
+
+            doc.fontSize(9).font("Helvetica-Oblique").fillColor(C.truncated)
+              .text(fallbackText, MARGIN + 10, currentY + 14, { width: W - 20, align: "center" });
+
+            doc.y = currentY + boxHeight + 10;
+          }
+        }
+      }
+    } else if (content.textContent && doc.y < limit) {
+      doc.fontSize(9).font("Helvetica-Bold").fillColor(C.section)
+        .text("PAGE CONTENT", { width: W, characterSpacing: 1 });
+
+      doc.y += 8;
+
+      const raw       = normaliseText(content.textContent);
+      const body      = raw.slice(0, MAX_TEXT_CHARS);
+      const truncated = raw.length > MAX_TEXT_CHARS;
+
+      doc.fontSize(10.5).font("Helvetica").fillColor(C.body)
+        .text(body, {
+          width: W,
+          lineGap: 3,
+          align: "left",
+        });
+
+      if (truncated && doc.y < limit) {
+        doc.y += 8;
+        doc.fontSize(8.5).font("Helvetica-Oblique").fillColor(C.truncated)
+          .text("[ content truncated for brevity ]", { width: W, align: "center" });
+      }
+    }
+  }
+
+  async finish(): Promise<void> {
+    const doc = this.doc;
+    const range = doc.bufferedPageRange();
+    const totalPages = range.count;
+
+    for (let i = 1; i < totalPages; i++) {
+      doc.switchToPage(i);
+
+      const W = doc.page.width - MARGIN * 2;
+      const footerY = doc.page.height - MARGIN - FOOTER_HEIGHT + 8;
+
+      // Draw running header
+      doc.fontSize(8).font("Helvetica").fillColor(C.badge)
+        .text("https://github.com/lightning4747/Web-crawler-cli", MARGIN, MARGIN - 24, { width: W, align: "left", link: "https://github.com/lightning4747/Web-crawler-cli" });
+
+      doc
+        .moveTo(MARGIN, MARGIN - 14)
+        .lineTo(doc.page.width - MARGIN, MARGIN - 14)
+        .strokeColor(C.rule)
+        .lineWidth(0.4)
+        .stroke();
+
+      // Draw running footer
+      doc
+        .moveTo(MARGIN, footerY)
+        .lineTo(doc.page.width - MARGIN, footerY)
+        .strokeColor(C.rule)
+        .lineWidth(0.4)
+        .stroke();
+
+      doc.fontSize(8).font("Helvetica").fillColor(C.footer)
+        .text(`Page ${i} of ${totalPages - 1}`, MARGIN, footerY + 5, {
+          width: W,
+          align: "center",
+          lineBreak: false,
+        });
+    }
+
+    await new Promise<void>((resolve, reject) => {
+      this.stream.on("finish", resolve);
+      this.stream.on("error", reject);
+      this.doc.end();
+    });
+    console.log(`[PDF] Done — ${this.pageCount} chapter(s) written to ${this.pdfPath}`);
+  }
+}

package/src/output/strategy.ts

@@ -0,0 +1,16 @@
+import type { CrawledPageContent } from "../db/queries.js";
+
+/**
+ * Strategy interface for output destinations.
+ * Implementations decide how to persist extracted page content.
+ */
+export interface OutputStrategy {
+  /** Called once before crawling begins. */
+  init(): Promise<void>;
+
+  /** Called for each successfully crawled page. */
+  save(urlId: number, url: string, content: CrawledPageContent): Promise<void>;
+
+  /** Called once after crawling finishes to flush/close any open resources. */
+  finish(): Promise<void>;
+}

package/src/security/ssrf.ts

@@ -0,0 +1,48 @@
+import { lookup } from "node:dns/promises";
+
+/**
+ * IP range patterns that must never be requested.
+ * Covers loopback, private, link-local, and cloud metadata addresses.
+ */
+const BLOCKED_IP_RANGES: RegExp[] = [
+  /^127\./,                           // loopback
+  /^10\./,                            // private Class A
+  /^172\.(1[6-9]|2\d|3[01])\./,      // private Class B
+  /^192\.168\./,                      // private Class C
+  /^169\.254\./,                      // link-local / cloud IMDS (AWS, GCP, Azure)
+  /^0\./,                             // current network
+  /^::1$/,                            // IPv6 loopback
+  /^fc00:/i,                          // IPv6 unique local
+  /^fe80:/i,                          // IPv6 link-local
+  /^100\.64\./,                       // shared address space (RFC 6598)
+];
+
+/**
+ * Hostnames that are blocked regardless of DNS resolution.
+ */
+const BLOCKED_HOSTNAMES = new Set([
+  "localhost",
+  "metadata.google.internal",
+  "169.254.169.254",
+]);
+
+/**
+ * Returns true if the hostname resolves to a private/internal address.
+ * Fails closed — if DNS lookup throws, the address is considered blocked.
+ */
+export async function isBlockedAddress(hostname: string): Promise<boolean> {
+  const lower = hostname.toLowerCase();
+
+  if (BLOCKED_HOSTNAMES.has(lower)) return true;
+
+  // Reject raw IP literals that match blocked ranges without a DNS lookup
+  if (BLOCKED_IP_RANGES.some((r) => r.test(hostname))) return true;
+
+  try {
+    const { address } = await lookup(hostname);
+    return BLOCKED_IP_RANGES.some((r) => r.test(address));
+  } catch {
+    // DNS resolution failed — fail closed
+    return true;
+  }
+}

package/src/security/validate-url.ts

@@ -0,0 +1,49 @@
+/**
+ * Validates that a string is a well-formed HTTP or HTTPS URL.
+ * Returns the parsed URL on success, or a descriptive error string on failure.
+ */
+export function validateSeedUrl(raw: string): { url: URL; error: null } | { url: null; error: string } {
+  let parsed: URL;
+
+  try {
+    parsed = new URL(raw);
+  } catch {
+    return { url: null, error: `"${raw}" is not a valid URL` };
+  }
+
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    return {
+      url: null,
+      error: `"${raw}" uses scheme "${parsed.protocol.replace(":", "")}" — only http and https are allowed`,
+    };
+  }
+
+  if (!parsed.hostname) {
+    return { url: null, error: `"${raw}" has no hostname` };
+  }
+
+  return { url: parsed, error: null };
+}
+
+/**
+ * Validates a list of URL strings.
+ * Returns valid URLs and a list of { input, reason } error objects.
+ */
+export function validateSeedUrls(raws: string[]): {
+  valid: string[];
+  invalid: Array<{ input: string; reason: string }>;
+} {
+  const valid: string[] = [];
+  const invalid: Array<{ input: string; reason: string }> = [];
+
+  for (const raw of raws) {
+    const result = validateSeedUrl(raw);
+    if (result.error) {
+      invalid.push({ input: raw, reason: result.error });
+    } else {
+      valid.push(raw);
+    }
+  }
+
+  return { valid, invalid };
+}

package/src/seed.ts

@@ -0,0 +1,18 @@
+import { config } from "./config.js";
+import { query } from "./db/client.js";
+import { getDomain } from "./normalizer.js";
+
+export async function seedDatabase() {
+  for (const url of config.SEED_URLS) {
+    const domain = getDomain(url);
+    if (!domain) {
+      continue;
+    }
+    await query(
+      `INSERT INTO urls (url, domain, status, depth)
+       VALUES ($1, $2, 'PENDING', 0)
+       ON CONFLICT (url) DO NOTHING`,
+      [url, domain]
+    );
+  }
+}

package/src/setup.ts

@@ -0,0 +1,170 @@
+/**
+ * Standalone CLI configuration wizard.
+ *
+ * Usage: npm run config
+ *
+ * Steps:
+ *  1. Reads seed URLs from seeds.txt
+ *  2. Prompts for crawler performance settings and output mode
+ *  3. Writes updated values to .env
+ *  4. Patches SEED_URLS and ALLOWED_DOMAINS in src/config.ts
+ */
+
+import { select, number, confirm } from "@inquirer/prompts";
+import fs from "fs";
+import path from "path";
+import { validateSeedUrls } from "./security/validate-url.js";
+
+const SEEDS_FILE = "seeds.txt";
+const ENV_FILE = ".env";
+const CONFIG_FILE = path.join("src", "config.ts");
+
+// ─── Helpers ─────────────────────────────────────────────────────────────────
+
+function readSeedsFile(): string[] {
+  if (!fs.existsSync(SEEDS_FILE)) {
+    console.warn(`[setup] ${SEEDS_FILE} not found. No seed URLs will be loaded.`);
+    return [];
+  }
+  const raw = fs
+    .readFileSync(SEEDS_FILE, "utf-8")
+    .split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l.length > 0 && !l.startsWith("#"));
+
+  const { valid, invalid } = validateSeedUrls(raw);
+
+  if (invalid.length > 0) {
+    console.warn(`[setup] Skipping ${invalid.length} invalid URL(s) in ${SEEDS_FILE}:`);
+    for (const e of invalid) {
+      console.warn(`         ✗ ${e.reason}`);
+    }
+  }
+
+  return valid;
+}
+
+function extractDomains(urls: string[]): string[] {
+  return urls.reduce<string[]>((acc, url) => {
+    try {
+      const { hostname } = new URL(url);
+      if (hostname && !acc.includes(hostname)) acc.push(hostname);
+    } catch {}
+    return acc;
+  }, []);
+}
+
+function writeEnvFile(values: Record<string, string>): void {
+  let existing: Record<string, string> = {};
+  if (fs.existsSync(ENV_FILE)) {
+    for (const line of fs.readFileSync(ENV_FILE, "utf-8").split("\n")) {
+      const trimmed = line.trim();
+      if (!trimmed || trimmed.startsWith("#")) continue;
+      const eqIdx = trimmed.indexOf("=");
+      if (eqIdx === -1) continue;
+      existing[trimmed.slice(0, eqIdx).trim()] = trimmed.slice(eqIdx + 1).trim();
+    }
+  }
+  const content =
+    Object.entries({ ...existing, ...values })
+      .map(([k, v]) => `${k}=${v}`)
+      .join("\n") + "\n";
+  fs.writeFileSync(ENV_FILE, content, "utf-8");
+  console.log(`[setup] .env updated.`);
+}
+
+function patchConfigTs(seedUrls: string[], allowedDomains: string[], outputMode: string): void {
+  if (!fs.existsSync(CONFIG_FILE)) {
+    console.warn(`[setup] ${CONFIG_FILE} not found — skipping patch.`);
+    return;
+  }
+  let src = fs.readFileSync(CONFIG_FILE, "utf-8");
+
+  const seedArray = "[\n" + seedUrls.map((u) => `    "${u}"`).join(",\n") + ",\n  ]";
+  src = src.replace(/SEED_URLS:\s*\[[\s\S]*?\]/, `SEED_URLS: ${seedArray}`);
+
+  const domainArray = "[\n" + allowedDomains.map((d) => `    "${d}"`).join(",\n") + ",\n  ]";
+  src = src.replace(/ALLOWED_DOMAINS:\s*\[[\s\S]*?\]/, `ALLOWED_DOMAINS: ${domainArray}`);
+
+  if (/OUTPUT_MODE:/.test(src)) {
+    src = src.replace(/OUTPUT_MODE:\s*["'][^"']*["']/, `OUTPUT_MODE: "${outputMode}"`);
+  } else {
+    src = src.replace(/(\n};)/, `\n  OUTPUT_MODE: "${outputMode}",\n};`);
+  }
+
+  fs.writeFileSync(CONFIG_FILE, src, "utf-8");
+  console.log(`[setup] src/config.ts patched.`);
+}
+
+// ─── Main wizard ──────────────────────────────────────────────────────────────
+
+async function main() {
+  console.log("\n╔══════════════════════════════════════════╗");
+  console.log("║   Web Crawler — Interactive Setup Wizard  ║");
+  console.log("╚══════════════════════════════════════════╝\n");
+
+  const seedUrls = readSeedsFile();
+  if (seedUrls.length === 0) {
+    console.warn(`[setup] No URLs found in ${SEEDS_FILE}. Add target URLs and re-run.\n`);
+  } else {
+    console.log(`[setup] Found ${seedUrls.length} seed URL(s):`);
+    seedUrls.forEach((u) => console.log(`         ${u}`));
+    console.log();
+  }
+
+  const outputMode = await select<string>({
+    message: "OUTPUT_MODE — where should crawled data be stored?",
+    choices: [
+      { name: "PostgreSQL database  (structured data)", value: "database" },
+      { name: "PDF eBook            (compiled document)", value: "pdf" },
+    ],
+    default: "database",
+  });
+
+  const maxDepth = await number({
+    message: "MAX_DEPTH — link hops from seed URLs:",
+    default: 3,
+    validate: (v) => (v !== undefined && v >= 0 ? true : "Must be 0 or greater"),
+  });
+
+  const crawlDelayMs = await number({
+    message: "CRAWL_DELAY_MS — politeness delay per domain (ms):",
+    default: 1000,
+    validate: (v) => (v !== undefined && v >= 0 ? true : "Must be 0 or greater"),
+  });
+
+  const workerCount = await number({
+    message: "WORKER_COUNT — concurrent workers:",
+    default: 5,
+    validate: (v) => (v !== undefined && v >= 1 ? true : "Must be at least 1"),
+  });
+
+  const maxPages = await number({
+    message: "MAX_PAGES — page limit (0 = unlimited):",
+    default: 1000,
+    validate: (v) => (v !== undefined && v >= 0 ? true : "Must be 0 or greater"),
+  });
+
+  const ok = await confirm({ message: "Save these settings?", default: true });
+  if (!ok) {
+    console.log("\nAborted.\n");
+    process.exit(0);
+  }
+
+  writeEnvFile({
+    MAX_DEPTH: String(maxDepth),
+    CRAWL_DELAY_MS: String(crawlDelayMs),
+    WORKER_COUNT: String(workerCount),
+    MAX_PAGES: String(maxPages),
+    OUTPUT_MODE: outputMode,
+  });
+
+  patchConfigTs(seedUrls, extractDomains(seedUrls), outputMode);
+
+  console.log("\n✓ Configuration saved. Run  npm run crawl  to start.\n");
+}
+
+main().catch((err) => {
+  console.error("[setup] Fatal error:", err);
+  process.exit(1);
+});