messi-crawler 1.0.0

package/dist/test/scheduler.test.js

@@ -0,0 +1,73 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+// Mock dependencies
+vi.mock("../config.js", () => {
+    return {
+        config: {
+            WORKER_COUNT: 2,
+            CRAWL_DELAY_MS: 1000,
+            MAX_PAGES: 100,
+        },
+    };
+});
+vi.mock("../db/queries.js", () => {
+    return {
+        claimNextURL: vi.fn(),
+        getGlobalStats: vi.fn().mockResolvedValue({ pending: 0, fetching: 0, done: 0, failed: 0 }),
+    };
+});
+vi.mock("../frontier/frontier.js", () => {
+    return {
+        getPendingDomains: vi.fn(),
+    };
+});
+vi.mock("../worker/worker.js", () => {
+    return {
+        processPage: vi.fn().mockResolvedValue(undefined),
+    };
+});
+import { claimNextURL } from "../db/queries.js";
+import { getPendingDomains } from "../frontier/frontier.js";
+import { startScheduler, stopScheduler, getCooldown } from "../frontier/scheduler.js";
+const mockClaimNextURL = vi.mocked(claimNextURL);
+const mockGetPendingDomains = vi.mocked(getPendingDomains);
+describe("Scheduler", () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+        vi.useFakeTimers();
+    });
+    afterEach(() => {
+        vi.useRealTimers();
+        stopScheduler();
+    });
+    it("should respect politeness delay (cooldowns) and round-robin domains", async () => {
+        // Two domains are pending
+        mockGetPendingDomains.mockResolvedValue(["react.dev", "typescriptlang.org"]);
+        // Mock claimNextURL responses
+        mockClaimNextURL
+            .mockResolvedValueOnce({ id: 1, url: "https://react.dev", domain: "react.dev", status: "FETCHING", depth: 0 })
+            .mockResolvedValueOnce({ id: 2, url: "https://typescriptlang.org", domain: "typescriptlang.org", status: "FETCHING", depth: 0 });
+        // Start the scheduler
+        const schedulerPromise = startScheduler();
+        // Allow the first loop iteration to execute
+        await vi.advanceTimersByTimeAsync(0);
+        // Verify it claimed react.dev first
+        expect(mockClaimNextURL).toHaveBeenNthCalledWith(1, "react.dev");
+        const cooldownReact = getCooldown("react.dev");
+        expect(cooldownReact).toBeGreaterThan(0);
+        // Advance time slightly (100ms, less than 1000ms cooldown)
+        await vi.advanceTimersByTimeAsync(100);
+        // It should check the next domain in round robin, which is typescriptlang.org
+        // Since typescriptlang.org has no cooldown, it should claim a URL for it
+        expect(mockClaimNextURL).toHaveBeenNthCalledWith(2, "typescriptlang.org");
+        const cooldownTS = getCooldown("typescriptlang.org");
+        expect(cooldownTS).toBeGreaterThan(0);
+        // Advance time slightly again
+        await vi.advanceTimersByTimeAsync(100);
+        // claimNextURL should not have been called a third time because both domains are on cooldown
+        expect(mockClaimNextURL).toHaveBeenCalledTimes(2);
+        // Stop scheduler to exit loop
+        stopScheduler();
+        await vi.advanceTimersByTimeAsync(100);
+        await schedulerPromise;
+    });
+});

package/dist/test/seed.test.js

@@ -0,0 +1,26 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+vi.mock("../db/client.js", () => {
+    return {
+        query: vi.fn().mockResolvedValue({ rows: [] }),
+    };
+});
+vi.mock("../config.js", () => {
+    return {
+        config: {
+            SEED_URLS: ["https://react.dev", "not-a-url"],
+        },
+    };
+});
+import { seedDatabase } from "../seed.js";
+import { query } from "../db/client.js";
+describe("Seeding Logic", () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+    });
+    it("should insert valid seed URLs and skip invalid ones", async () => {
+        await seedDatabase();
+        // query should only be called once, for "https://react.dev"
+        expect(query).toHaveBeenCalledTimes(1);
+        expect(query).toHaveBeenCalledWith(expect.stringContaining("INSERT INTO urls"), ["https://react.dev", "react.dev"]);
+    });
+});

package/dist/test/worker.test.js

@@ -0,0 +1,118 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+// Mock downloader
+vi.mock("../worker/downloader.js", () => {
+    return {
+        downloadPage: vi.fn(),
+    };
+});
+// Mock extractor
+vi.mock("../worker/extractor.js", () => {
+    return {
+        extractPageData: vi.fn(),
+    };
+});
+// Mock db queries
+vi.mock("../db/queries.js", () => {
+    return {
+        insertURL: vi.fn(),
+        insertLink: vi.fn(),
+        markDone: vi.fn(),
+        markFailed: vi.fn(),
+    };
+});
+vi.mock("../config.js", () => {
+    return {
+        config: {
+            ALLOWED_DOMAINS: ["react.dev"],
+            MAX_DEPTH: 2,
+        },
+    };
+});
+vi.mock("../frontier/robots.js", () => {
+    return {
+        isAllowedByRobots: vi.fn(),
+    };
+});
+import { downloadPage } from "../worker/downloader.js";
+import { extractPageData } from "../worker/extractor.js";
+import { insertURL, insertLink, markDone, markFailed } from "../db/queries.js";
+import { isAllowedByRobots } from "../frontier/robots.js";
+import { processPage } from "../worker/worker.js";
+const mockDownloadPage = vi.mocked(downloadPage);
+const mockExtractPageData = vi.mocked(extractPageData);
+const mockInsertURL = vi.mocked(insertURL);
+const mockInsertLink = vi.mocked(insertLink);
+const mockMarkDone = vi.mocked(markDone);
+const mockMarkFailed = vi.mocked(markFailed);
+const mockIsAllowedByRobots = vi.mocked(isAllowedByRobots);
+describe("Worker Pipeline", () => {
+    beforeEach(() => {
+        vi.clearAllMocks();
+        mockIsAllowedByRobots.mockResolvedValue(true);
+    });
+    it("should successfully process a page, extract content, and insert links", async () => {
+        mockDownloadPage.mockResolvedValue({
+            url: "https://react.dev/docs",
+            html: "<html>...</html>",
+            statusCode: 200,
+        });
+        mockExtractPageData.mockReturnValue({
+            title: "React Docs",
+            description: "Learn React",
+            canonicalUrl: "https://react.dev/docs",
+            headings: { h1: ["Docs"], h2: [], h3: [] },
+            textContent: "Learn React content",
+            links: ["/tutorial", "https://external.com", "https://react.dev/docs"],
+        });
+        mockInsertURL.mockResolvedValue(100);
+        await processPage({ id: 42, url: "https://react.dev/docs", depth: 1 });
+        expect(mockMarkDone).toHaveBeenCalledTimes(1);
+        expect(mockMarkDone).toHaveBeenCalledWith(42, {
+            title: "React Docs",
+            description: "Learn React",
+            canonicalUrl: "https://react.dev/docs",
+            headings: { h1: ["Docs"], h2: [], h3: [] },
+            textContent: "Learn React content",
+        });
+        expect(mockInsertURL).toHaveBeenCalledTimes(1);
+        expect(mockInsertURL).toHaveBeenCalledWith("https://react.dev/tutorial", "react.dev", 2);
+        expect(mockInsertLink).toHaveBeenCalledTimes(1);
+        expect(mockInsertLink).toHaveBeenCalledWith(42, 100);
+        expect(mockMarkFailed).not.toHaveBeenCalled();
+    });
+    it("should mark URL as FAILED if download fails", async () => {
+        mockDownloadPage.mockRejectedValue(new Error("Network Error"));
+        await expect(processPage({ id: 42, url: "https://react.dev/docs", depth: 1 })).rejects.toThrow("Network Error");
+        expect(mockMarkFailed).toHaveBeenCalledTimes(1);
+        expect(mockMarkFailed).toHaveBeenCalledWith(42, "Network Error");
+        expect(mockMarkDone).not.toHaveBeenCalled();
+    });
+    it("should discard links that exceed MAX_DEPTH", async () => {
+        mockDownloadPage.mockResolvedValue({
+            url: "https://react.dev/docs",
+            html: "<html>...</html>",
+            statusCode: 200,
+        });
+        mockExtractPageData.mockReturnValue({
+            title: "React Docs",
+            description: "Learn React",
+            canonicalUrl: "https://react.dev/docs",
+            headings: { h1: ["Docs"], h2: [], h3: [] },
+            textContent: "Learn React content",
+            links: ["/tutorial"],
+        });
+        // Run with current depth = 2, so nextDepth = 3 which exceeds MAX_DEPTH = 2
+        await processPage({ id: 42, url: "https://react.dev/docs", depth: 2 });
+        expect(mockMarkDone).toHaveBeenCalledTimes(1);
+        expect(mockInsertURL).not.toHaveBeenCalled();
+        expect(mockInsertLink).not.toHaveBeenCalled();
+    });
+    it("should abort crawl if URL is disallowed by robots.txt", async () => {
+        mockIsAllowedByRobots.mockResolvedValue(false);
+        await processPage({ id: 42, url: "https://react.dev/private", depth: 1 });
+        expect(mockMarkFailed).toHaveBeenCalledTimes(1);
+        expect(mockMarkFailed).toHaveBeenCalledWith(42, "Disallowed by robots.txt");
+        expect(mockDownloadPage).not.toHaveBeenCalled();
+        expect(mockMarkDone).not.toHaveBeenCalled();
+    });
+});

package/dist/worker/downloader.js

@@ -0,0 +1,114 @@
+import { request } from "undici";
+import { config } from "../config.js";
+import { isBlockedAddress } from "../security/ssrf.js";
+/**
+ * Guards against SSRF by resolving the hostname before the request is made.
+ * Throws if the address is private, loopback, or link-local.
+ */
+async function assertNotBlocked(url) {
+    const hostname = new URL(url).hostname;
+    if (await isBlockedAddress(hostname)) {
+        throw new Error(`SSRF blocked: "${hostname}" resolves to a private or internal address`);
+    }
+}
+/**
+ * Fetches the HTML content of a page, following redirects up to MAX_REDIRECTS.
+ * Tracks the final URL, enforces a request timeout, and blocks SSRF targets.
+ */
+export async function downloadPage(initialUrl) {
+    let currentUrl = initialUrl;
+    let redirectCount = 0;
+    // SSRF check on the initial URL before any network activity
+    await assertNotBlocked(currentUrl);
+    while (true) {
+        const res = await request(currentUrl, {
+            method: "GET",
+            headersTimeout: config.REQUEST_TIMEOUT_MS,
+            bodyTimeout: config.REQUEST_TIMEOUT_MS,
+        });
+        const statusCode = res.statusCode;
+        // Handle redirects (301, 302, 303, 307, 308)
+        if (statusCode >= 300 && statusCode < 400 && res.headers.location) {
+            if (redirectCount >= config.MAX_REDIRECTS) {
+                await res.body.text(); // consume body to release connection
+                throw new Error("Too many redirects");
+            }
+            const location = Array.isArray(res.headers.location)
+                ? res.headers.location[0]
+                : res.headers.location;
+            const nextUrl = new URL(location, currentUrl).href;
+            // SSRF check on every redirect target before following
+            await assertNotBlocked(nextUrl);
+            currentUrl = nextUrl;
+            redirectCount++;
+            await res.body.text(); // consume body
+            continue;
+        }
+        // Error on non-200 responses
+        if (statusCode !== 200) {
+            await res.body.text();
+            throw new Error(`HTTP status ${statusCode}`);
+        }
+        // Skip non-HTML content types
+        const contentTypeHeader = res.headers["content-type"];
+        const contentType = Array.isArray(contentTypeHeader)
+            ? contentTypeHeader[0]
+            : contentTypeHeader;
+        if (contentType && !contentType.includes("text/html")) {
+            await res.body.text();
+            throw new Error(`Non-HTML content type: ${contentType}`);
+        }
+        const html = await res.body.text();
+        return { url: currentUrl, html, statusCode };
+    }
+}
+/**
+ * Downloads image assets securely.
+ * Enforces the same SSRF/blocklist checks, redirection limits, and timeouts as pages.
+ */
+export async function downloadImage(initialUrl) {
+    let currentUrl = initialUrl;
+    let redirectCount = 0;
+    // SSRF check on initial URL
+    await assertNotBlocked(currentUrl);
+    while (true) {
+        const res = await request(currentUrl, {
+            method: "GET",
+            headersTimeout: config.REQUEST_TIMEOUT_MS,
+            bodyTimeout: config.REQUEST_TIMEOUT_MS,
+        });
+        const statusCode = res.statusCode;
+        // Handle redirects (301, 302, 303, 307, 308)
+        if (statusCode >= 300 && statusCode < 400 && res.headers.location) {
+            if (redirectCount >= config.MAX_REDIRECTS) {
+                await res.body.text();
+                throw new Error("Too many redirects fetching image");
+            }
+            const location = Array.isArray(res.headers.location)
+                ? res.headers.location[0]
+                : res.headers.location;
+            const nextUrl = new URL(location, currentUrl).href;
+            // SSRF check on target before redirecting
+            await assertNotBlocked(nextUrl);
+            currentUrl = nextUrl;
+            redirectCount++;
+            await res.body.text();
+            continue;
+        }
+        if (statusCode !== 200) {
+            await res.body.text();
+            throw new Error(`HTTP status ${statusCode} fetching image`);
+        }
+        // Validate that it's an image
+        const contentTypeHeader = res.headers["content-type"];
+        const contentType = Array.isArray(contentTypeHeader)
+            ? contentTypeHeader[0]
+            : contentTypeHeader;
+        if (contentType && !contentType.startsWith("image/")) {
+            await res.body.text();
+            throw new Error(`Non-image content type: ${contentType}`);
+        }
+        const arrayBuffer = await res.body.arrayBuffer();
+        return Buffer.from(arrayBuffer);
+    }
+}

package/dist/worker/extractor.js

@@ -0,0 +1,197 @@
+import * as cheerio from "cheerio";
+/**
+ * Extracts metadata, headings, structured text content blocks, images, and outgoing links from HTML.
+ * Strips site chrome and uses a text-density heuristic if no main content container is found.
+ */
+export function extractPageData(html, baseUrl) {
+    const $ = cheerio.load(html);
+    const title = $("title").text().trim() || null;
+    const description = $("meta[name=description]").attr("content")?.trim() || null;
+    const canonicalUrl = $("link[rel=canonical]").attr("href")?.trim() || null;
+    const h1 = [];
+    const h2 = [];
+    const h3 = [];
+    $("h1").each((_, el) => {
+        const text = $(el).text().trim();
+        if (text)
+            h1.push(text);
+    });
+    $("h2").each((_, el) => {
+        const text = $(el).text().trim();
+        if (text)
+            h2.push(text);
+    });
+    $("h3").each((_, el) => {
+        const text = $(el).text().trim();
+        if (text)
+            h3.push(text);
+    });
+    const links = [];
+    $("a[href]").each((_, el) => {
+        const href = $(el).attr("href")?.trim();
+        if (href) {
+            links.push(href);
+        }
+    });
+    // Determine resolution base URL for images
+    const resolutionBase = canonicalUrl || baseUrl || null;
+    // 1. Main-content heuristic selection
+    let mainNode = $("article").first();
+    if (mainNode.length === 0) {
+        mainNode = $("main").first();
+    }
+    if (mainNode.length === 0) {
+        mainNode = $("[role=main]").first();
+    }
+    // Fallback text-density heuristic
+    if (mainNode.length === 0) {
+        const totalBodyText = $("body").text().trim();
+        const minTextLength = Math.min(200, totalBodyText.length * 0.1);
+        let bestNode = $("body");
+        let maxScore = -1;
+        $("div, section").each((_, el) => {
+            const $el = $(el);
+            const text = $el.text().trim();
+            const textLength = text.length;
+            if (textLength < minTextLength)
+                return;
+            const tagCount = $el.find("*").length;
+            const score = textLength / (tagCount + 1);
+            if (score > maxScore) {
+                maxScore = score;
+                bestNode = $el;
+            }
+        });
+        mainNode = bestNode;
+    }
+    // 2. Clone and clean the chosen node
+    const cleanedNode = mainNode.clone();
+    cleanedNode.find("script, style, noscript, iframe, nav, footer, header").remove();
+    // 3. Extract in-order content blocks and overall images list
+    const blocks = [];
+    const images = [];
+    // Extract all images inside the cleaned main node
+    cleanedNode.find("img").each((_, img) => {
+        const src = $(img).attr("src")?.trim();
+        const alt = $(img).attr("alt")?.trim() || "";
+        if (src) {
+            let resolvedSrc = src;
+            if (resolutionBase) {
+                try {
+                    resolvedSrc = new URL(src, resolutionBase).href;
+                }
+                catch {
+                    // keep relative src if resolution fails
+                }
+            }
+            images.push({ src: resolvedSrc, alt });
+        }
+    });
+    // Track if we need to force a new paragraph on the next text node
+    let forceNewParagraph = true;
+    function walk(node) {
+        if (node.type === "text") {
+            const text = node.data.replace(/\s+/g, " ").trim();
+            if (text) {
+                const lastBlock = blocks[blocks.length - 1];
+                if (!forceNewParagraph && lastBlock && lastBlock.type === "paragraph") {
+                    lastBlock.text = (lastBlock.text + " " + text).replace(/\s+/g, " ").trim();
+                }
+                else {
+                    blocks.push({ type: "paragraph", text });
+                    forceNewParagraph = false;
+                }
+            }
+            return;
+        }
+        if (node.type !== "tag") {
+            return;
+        }
+        const el = node;
+        const tagName = el.tagName?.toLowerCase();
+        // Skip removed elements just in case
+        if (["script", "style", "noscript", "iframe", "nav", "footer", "header"].includes(tagName)) {
+            return;
+        }
+        if (/^h[1-6]$/.test(tagName)) {
+            const level = parseInt(tagName.substring(1), 10);
+            const text = $(el).text().replace(/\s+/g, " ").trim();
+            if (text) {
+                blocks.push({ type: "heading", level, text });
+            }
+            forceNewParagraph = true;
+        }
+        else if (tagName === "p") {
+            const text = $(el).text().replace(/\s+/g, " ").trim();
+            if (text) {
+                blocks.push({ type: "paragraph", text });
+            }
+            forceNewParagraph = true;
+        }
+        else if (tagName === "ul" || tagName === "ol") {
+            const items = [];
+            $(el).find("li").each((_, li) => {
+                const itemText = $(li).text().replace(/\s+/g, " ").trim();
+                if (itemText)
+                    items.push(itemText);
+            });
+            if (items.length > 0) {
+                blocks.push({ type: "list", items });
+            }
+            forceNewParagraph = true;
+        }
+        else if (tagName === "img") {
+            const src = $(el).attr("src")?.trim();
+            const alt = $(el).attr("alt")?.trim() || "";
+            if (src) {
+                let resolvedSrc = src;
+                if (resolutionBase) {
+                    try {
+                        resolvedSrc = new URL(src, resolutionBase).href;
+                    }
+                    catch {
+                        // keep as is
+                    }
+                }
+                blocks.push({ type: "image", src: resolvedSrc, alt });
+            }
+            forceNewParagraph = true;
+        }
+        else if (tagName === "br") {
+            forceNewParagraph = true;
+        }
+        else {
+            // For general container tags (div, span, etc.), walk contents recursively
+            $(el).contents().each((_, child) => {
+                walk(child);
+            });
+        }
+    }
+    cleanedNode.contents().each((_, child) => {
+        walk(child);
+    });
+    // Fallback textContent: concatenated paragraphs / lists for backwards compatibility
+    const textContentParts = [];
+    for (const block of blocks) {
+        if (block.type === "paragraph" && block.text) {
+            textContentParts.push(block.text);
+        }
+        else if (block.type === "heading" && block.text) {
+            textContentParts.push(block.text);
+        }
+        else if (block.type === "list" && block.items) {
+            textContentParts.push(block.items.join(" "));
+        }
+    }
+    const textContent = textContentParts.join(" ").replace(/\s+/g, " ").trim() || null;
+    return {
+        title,
+        description,
+        canonicalUrl,
+        headings: { h1, h2, h3 },
+        textContent,
+        links,
+        blocks,
+        images,
+    };
+}

package/dist/worker/worker.js

@@ -0,0 +1,87 @@
+import { downloadPage } from "./downloader.js";
+import { extractPageData } from "./extractor.js";
+import { normalizeURL, getDomain } from "../normalizer.js";
+import { insertURL, insertLink, markFailed } from "../db/queries.js";
+import { config } from "../config.js";
+import { isAllowedByRobots } from "../frontier/robots.js";
+import { getStrategy } from "../output/index.js";
+function isDomainAllowed(domain) {
+    if (!config.ALLOWED_DOMAINS || config.ALLOWED_DOMAINS.length === 0) {
+        return true;
+    }
+    return config.ALLOWED_DOMAINS.includes(domain);
+}
+/**
+ * Handles the complete crawling workflow for a single URL:
+ * 1. Downloads the page HTML (handling redirects & timeouts).
+ * 2. Extracts title, description, canonical, headings, text content, and outgoing links.
+ * 3. Delegates persistence to the active OutputStrategy (DB or PDF).
+ * 4. Filters, normalizes, and enqueues discovered links, establishing link graph relations.
+ */
+export async function processPage(urlRow) {
+    const urlId = urlRow.id;
+    const pageUrl = urlRow.url;
+    const currentDepth = urlRow.depth;
+    try {
+        // 0. Check robots.txt compliance
+        const allowed = await isAllowedByRobots(pageUrl);
+        if (!allowed) {
+            await markFailed(urlId, "Disallowed by robots.txt");
+            return;
+        }
+        // 1. Download page content
+        const downloadResult = await downloadPage(pageUrl);
+        // 2. Extract content & outgoing links
+        const extracted = extractPageData(downloadResult.html, downloadResult.url);
+        // Resolve final URL using canonical link if present
+        let finalUrl = downloadResult.url;
+        if (extracted.canonicalUrl) {
+            const normalizedCanonical = normalizeURL(extracted.canonicalUrl, finalUrl);
+            if (normalizedCanonical) {
+                finalUrl = normalizedCanonical;
+            }
+        }
+        // 3. Persist content via the active output strategy (DB or PDF)
+        const strategy = getStrategy();
+        await strategy.save(urlId, finalUrl, {
+            title: extracted.title,
+            description: extracted.description,
+            canonicalUrl: extracted.canonicalUrl,
+            headings: extracted.headings,
+            textContent: extracted.textContent,
+            blocks: extracted.blocks,
+            images: extracted.images,
+        });
+        // 4. Process outgoing links
+        const uniqueNormalizedLinks = new Set();
+        for (const link of extracted.links) {
+            const normalized = normalizeURL(link, finalUrl);
+            if (!normalized)
+                continue;
+            // Skip self-referential links
+            if (normalized === finalUrl || normalized === pageUrl)
+                continue;
+            const linkDomain = getDomain(normalized);
+            if (!linkDomain || !isDomainAllowed(linkDomain))
+                continue;
+            uniqueNormalizedLinks.add(normalized);
+        }
+        for (const normalizedLink of uniqueNormalizedLinks) {
+            const nextDepth = currentDepth + 1;
+            // Enforce MAX_DEPTH limit
+            if (nextDepth > config.MAX_DEPTH) {
+                continue;
+            }
+            const targetDomain = getDomain(normalizedLink);
+            // Insert target URL (ON CONFLICT DO NOTHING) and get its ID
+            const targetUrlId = await insertURL(normalizedLink, targetDomain, nextDepth);
+            // Establish link graph relation
+            await insertLink(urlId, targetUrlId);
+        }
+    }
+    catch (error) {
+        const errorMsg = error instanceof Error ? error.message : String(error);
+        await markFailed(urlId, errorMsg);
+        throw error;
+    }
+}

package/package.json

@@ -0,0 +1,48 @@
+{
+  "name": "messi-crawler",
+  "version": "1.0.0",
+  "description": "A web crawler built with Node.js and TypeScript for collecting programming-related documentation and web content.",
+  "main": "dist/index.js",
+  "bin": {
+    "messi-crawler": "./dist/index.js"
+  },
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "start": "node dist/index.js",
+    "dev": "tsx src/index.ts",
+    "test": "vitest",
+    "config": "tsx src/setup.ts",
+    "crawl": "npm run build && node dist/index.js",
+    "db:clear": "tsx src/db/clear.ts"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/lightning4747/Web-crawler-cli.git"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "bugs": {
+    "url": "https://github.com/lightning4747/Web-crawler-cli/issues"
+  },
+  "homepage": "https://github.com/lightning4747/Web-crawler-cli#readme",
+  "dependencies": {
+    "cheerio": "^1.2.0",
+    "dotenv": "^17.4.2",
+    "inquirer": "^14.0.2",
+    "pdfkit": "^0.19.1",
+    "pg": "^8.21.0",
+    "robots-parser": "^3.0.1",
+    "undici": "^8.3.0"
+  },
+  "devDependencies": {
+    "@types/inquirer": "^9.0.10",
+    "@types/node": "^25.9.1",
+    "@types/pdfkit": "^0.17.6",
+    "@types/pg": "^8.20.0",
+    "tsx": "^4.22.4",
+    "typescript": "^6.0.3",
+    "vitest": "^4.1.8"
+  }
+}

package/seeds.txt

@@ -0,0 +1,4 @@
+[https://www.akc.org/dog-breeds/](https://www.akc.org/dog-breeds/)
+[https://www.royalkennelclub.org/breeds-a-z/](https://www.google.com/search?q=https://www.royalkennelclub.org/breeds-a-z/)
+[https://dogapi.dog/api/v2/breeds](https://www.google.com/search?q=https://dogapi.dog/api/v2/breeds)
+[https://dog.ceo/api/breeds/list/all](https://www.google.com/search?q=https://dog.ceo/api/breeds/list/all)