messi-crawler 1.0.0

package/src/test/queries.test.ts

@@ -0,0 +1,247 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+// Mock the client module directly
+vi.mock("../db/client.js", () => {
+  return {
+    query: vi.fn(),
+    pool: {
+      connect: vi.fn(),
+    },
+  };
+});
+
+// Import the mocked query and pool
+import { query, pool } from "../db/client.js";
+import {
+  claimNextURL,
+  markDone,
+  markFailed,
+  insertURL,
+  insertLink,
+  resetStaleLocks,
+  getGlobalStats,
+  refreshDomainStats,
+  getDomainStats,
+} from "../db/queries.js";
+
+const mockedQuery = vi.mocked(query);
+const mockedPool = vi.mocked(pool);
+
+describe("Database Queries", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockedQuery.mockReset().mockResolvedValue({ rows: [] } as any);
+    mockedPool.connect.mockReset();
+  });
+
+  describe("claimNextURL", () => {
+    it("should return the row if a PENDING URL is found", async () => {
+      const mockRow = { id: 1, url: "https://react.dev", domain: "react.dev", status: "FETCHING", depth: 0 };
+      mockedQuery.mockResolvedValue({ rows: [mockRow] } as any);
+
+      const result = await claimNextURL("react.dev");
+
+      expect(mockedQuery).toHaveBeenCalledTimes(1);
+      expect(mockedQuery).toHaveBeenCalledWith(expect.stringContaining("UPDATE urls"), ["react.dev"]);
+      expect(result).toEqual(mockRow);
+    });
+
+    it("should return null if no PENDING URL is found", async () => {
+      mockedQuery.mockResolvedValue({ rows: [] } as any);
+
+      const result = await claimNextURL("react.dev");
+
+      expect(result).toBeNull();
+    });
+  });
+
+  describe("markDone", () => {
+    it("should execute queries in a transaction", async () => {
+      const mockClient = {
+        query: vi.fn().mockResolvedValue({}),
+        release: vi.fn(),
+      };
+      mockedPool.connect.mockResolvedValue(mockClient as any);
+
+      const content = {
+        title: "React",
+        description: "Library",
+        canonicalUrl: "https://react.dev",
+        headings: { h1: ["H1"], h2: [], h3: [] },
+        textContent: "Body content",
+      };
+
+      await markDone(1, content);
+
+      expect(mockedPool.connect).toHaveBeenCalledTimes(1);
+      expect(mockClient.query).toHaveBeenCalledWith("BEGIN");
+      expect(mockClient.query).toHaveBeenCalledWith(
+        expect.stringContaining("INSERT INTO crawled_pages"),
+        [1, "React", "Library", "https://react.dev", JSON.stringify(content.headings), "Body content"]
+      );
+      expect(mockClient.query).toHaveBeenCalledWith(
+        expect.stringContaining("UPDATE urls"),
+        [1]
+      );
+      expect(mockClient.query).toHaveBeenCalledWith("COMMIT");
+      expect(mockClient.release).toHaveBeenCalledTimes(1);
+    });
+
+    it("should rollback transaction on error", async () => {
+      const mockClient = {
+        query: vi.fn().mockImplementation((sql: string) => {
+          if (sql.includes("INSERT INTO crawled_pages")) {
+            throw new Error("DB Error");
+          }
+          return Promise.resolve({});
+        }),
+        release: vi.fn(),
+      };
+      mockedPool.connect.mockResolvedValue(mockClient as any);
+
+      const content = {
+        title: "React",
+        description: "Library",
+        canonicalUrl: "https://react.dev",
+        headings: { h1: ["H1"], h2: [], h3: [] },
+        textContent: "Body content",
+      };
+
+      await expect(markDone(1, content)).rejects.toThrow("DB Error");
+
+      expect(mockClient.query).toHaveBeenCalledWith("ROLLBACK");
+      expect(mockClient.release).toHaveBeenCalledTimes(1);
+    });
+  });
+
+  describe("markFailed", () => {
+    it("should update status to FAILED with error message", async () => {
+      mockedQuery.mockResolvedValue({ rows: [] } as any);
+      await markFailed(1, "Connection timeout");
+
+      expect(mockedQuery).toHaveBeenCalledTimes(1);
+      expect(mockedQuery).toHaveBeenCalledWith(
+        expect.stringContaining("UPDATE urls"),
+        [1, "Connection timeout"]
+      );
+    });
+  });
+
+  describe("insertURL", () => {
+    it("should return the ID of the URL", async () => {
+      mockedQuery.mockResolvedValue({ rows: [{ id: 42 }] } as any);
+
+      const id = await insertURL("https://react.dev", "react.dev", 1);
+
+      expect(id).toBe(42);
+      expect(mockedQuery).toHaveBeenCalledTimes(1);
+      expect(mockedQuery).toHaveBeenCalledWith(
+        expect.stringContaining("WITH ins AS"),
+        ["https://react.dev", "react.dev", 1]
+      );
+    });
+  });
+
+  describe("insertLink", () => {
+    it("should insert edge into links table", async () => {
+      mockedQuery.mockResolvedValue({ rows: [] } as any);
+      await insertLink(1, 2);
+
+      expect(mockedQuery).toHaveBeenCalledTimes(1);
+      expect(mockedQuery).toHaveBeenCalledWith(
+        expect.stringContaining("INSERT INTO links"),
+        [1, 2]
+      );
+    });
+  });
+
+  describe("resetStaleLocks", () => {
+    it("should reset FETCHING urls back to PENDING", async () => {
+      mockedQuery.mockResolvedValue({ rows: [] } as any);
+      await resetStaleLocks();
+
+      expect(mockedQuery).toHaveBeenCalledTimes(1);
+      expect(mockedQuery).toHaveBeenCalledWith(
+        expect.stringContaining("UPDATE urls")
+      );
+      expect(mockedQuery).toHaveBeenCalledWith(
+        expect.stringContaining("status = 'PENDING'")
+      );
+      expect(mockedQuery).toHaveBeenCalledWith(
+        expect.stringContaining("status = 'FETCHING'")
+      );
+    });
+  });
+
+  describe("getGlobalStats", () => {
+    it("should map query results to GlobalStats structure", async () => {
+      mockedQuery.mockResolvedValue({
+        rows: [
+          { status: "PENDING", count: "10" },
+          { status: "DONE", count: "5" },
+        ],
+      } as any);
+
+      const stats = await getGlobalStats();
+      expect(stats).toEqual({
+        pending: 10,
+        fetching: 0,
+        done: 5,
+        failed: 0,
+      });
+      expect(mockedQuery).toHaveBeenCalledWith(
+        expect.stringContaining("SELECT status, COUNT(*)")
+      );
+    });
+  });
+
+  describe("refreshDomainStats", () => {
+    it("should run CREATE TABLE and INSERT INTO domain_stats query", async () => {
+      mockedQuery.mockResolvedValue({ rows: [] } as any);
+      await refreshDomainStats();
+
+      expect(mockedQuery).toHaveBeenCalledTimes(2);
+      expect(mockedQuery).toHaveBeenNthCalledWith(
+        1,
+        expect.stringContaining("CREATE TABLE IF NOT EXISTS domain_stats")
+      );
+      expect(mockedQuery).toHaveBeenNthCalledWith(
+        2,
+        expect.stringContaining("INSERT INTO domain_stats")
+      );
+    });
+  });
+
+  describe("getDomainStats", () => {
+    it("should fetch and format domain stats", async () => {
+      const lastCrawled = new Date();
+      mockedQuery.mockResolvedValue({
+        rows: [
+          {
+            domain: "react.dev",
+            pending_count: "5",
+            fetching_count: "1",
+            done_count: "10",
+            failed_count: "2",
+            last_crawled_at: lastCrawled.toISOString(),
+          },
+        ],
+      } as any);
+
+      const stats = await getDomainStats();
+      expect(stats).toEqual([
+        {
+          domain: "react.dev",
+          pending_count: 5,
+          fetching_count: 1,
+          done_count: 10,
+          failed_count: 2,
+          last_crawled_at: lastCrawled,
+        },
+      ]);
+      expect(mockedQuery).toHaveBeenCalledWith(
+        expect.stringContaining("SELECT domain, pending_count")
+      );
+    });
+  });
+});

package/src/test/robots.test.ts

@@ -0,0 +1,56 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("undici", () => {
+  return {
+    request: vi.fn(),
+  };
+});
+
+vi.mock("../config.js", () => {
+  return {
+    config: {
+      REQUEST_TIMEOUT_MS: 1000,
+    },
+  };
+});
+
+import { request } from "undici";
+import { isAllowedByRobots } from "../frontier/robots.js";
+
+const mockedRequest = vi.mocked(request);
+
+describe("robots.txt compliance", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("should allow URLs if robots.txt allows it", async () => {
+    mockedRequest.mockResolvedValue({
+      statusCode: 200,
+      body: {
+        text: async () => `
+          User-agent: *
+          Disallow: /private/
+        `,
+      },
+    } as any);
+
+    const allowed = await isAllowedByRobots("https://react.dev/docs");
+    expect(allowed).toBe(true);
+
+    const disallowed = await isAllowedByRobots("https://react.dev/private/secret");
+    expect(disallowed).toBe(false);
+  });
+
+  it("should allow URLs if robots.txt returns 404", async () => {
+    mockedRequest.mockResolvedValue({
+      statusCode: 404,
+      body: {
+        text: async () => "Not Found",
+      },
+    } as any);
+
+    const allowed = await isAllowedByRobots("https://react.dev/docs");
+    expect(allowed).toBe(true);
+  });
+});

package/src/test/scheduler.test.ts

@@ -0,0 +1,90 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+
+// Mock dependencies
+vi.mock("../config.js", () => {
+  return {
+    config: {
+      WORKER_COUNT: 2,
+      CRAWL_DELAY_MS: 1000,
+      MAX_PAGES: 100,
+    },
+  };
+});
+
+vi.mock("../db/queries.js", () => {
+  return {
+    claimNextURL: vi.fn(),
+    getGlobalStats: vi.fn().mockResolvedValue({ pending: 0, fetching: 0, done: 0, failed: 0 }),
+  };
+});
+
+vi.mock("../frontier/frontier.js", () => {
+  return {
+    getPendingDomains: vi.fn(),
+  };
+});
+
+vi.mock("../worker/worker.js", () => {
+  return {
+    processPage: vi.fn().mockResolvedValue(undefined),
+  };
+});
+
+import { claimNextURL } from "../db/queries.js";
+import { getPendingDomains } from "../frontier/frontier.js";
+import { startScheduler, stopScheduler, getCooldown } from "../frontier/scheduler.js";
+
+const mockClaimNextURL = vi.mocked(claimNextURL);
+const mockGetPendingDomains = vi.mocked(getPendingDomains);
+
+describe("Scheduler", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+    stopScheduler();
+  });
+
+  it("should respect politeness delay (cooldowns) and round-robin domains", async () => {
+    // Two domains are pending
+    mockGetPendingDomains.mockResolvedValue(["react.dev", "typescriptlang.org"]);
+
+    // Mock claimNextURL responses
+    mockClaimNextURL
+      .mockResolvedValueOnce({ id: 1, url: "https://react.dev", domain: "react.dev", status: "FETCHING", depth: 0 })
+      .mockResolvedValueOnce({ id: 2, url: "https://typescriptlang.org", domain: "typescriptlang.org", status: "FETCHING", depth: 0 });
+
+    // Start the scheduler
+    const schedulerPromise = startScheduler();
+
+    // Allow the first loop iteration to execute
+    await vi.advanceTimersByTimeAsync(0);
+
+    // Verify it claimed react.dev first
+    expect(mockClaimNextURL).toHaveBeenNthCalledWith(1, "react.dev");
+    const cooldownReact = getCooldown("react.dev");
+    expect(cooldownReact).toBeGreaterThan(0);
+
+    // Advance time slightly (100ms, less than 1000ms cooldown)
+    await vi.advanceTimersByTimeAsync(100);
+
+    // It should check the next domain in round robin, which is typescriptlang.org
+    // Since typescriptlang.org has no cooldown, it should claim a URL for it
+    expect(mockClaimNextURL).toHaveBeenNthCalledWith(2, "typescriptlang.org");
+    const cooldownTS = getCooldown("typescriptlang.org");
+    expect(cooldownTS).toBeGreaterThan(0);
+
+    // Advance time slightly again
+    await vi.advanceTimersByTimeAsync(100);
+    // claimNextURL should not have been called a third time because both domains are on cooldown
+    expect(mockClaimNextURL).toHaveBeenCalledTimes(2);
+
+    // Stop scheduler to exit loop
+    stopScheduler();
+    await vi.advanceTimersByTimeAsync(100);
+    await schedulerPromise;
+  });
+});

package/src/test/seed.test.ts

@@ -0,0 +1,35 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+vi.mock("../db/client.js", () => {
+  return {
+    query: vi.fn().mockResolvedValue({ rows: [] }),
+  };
+});
+
+vi.mock("../config.js", () => {
+  return {
+    config: {
+      SEED_URLS: ["https://react.dev", "not-a-url"],
+    },
+  };
+});
+
+import { seedDatabase } from "../seed.js";
+import { query } from "../db/client.js";
+
+describe("Seeding Logic", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+  });
+
+  it("should insert valid seed URLs and skip invalid ones", async () => {
+    await seedDatabase();
+
+    // query should only be called once, for "https://react.dev"
+    expect(query).toHaveBeenCalledTimes(1);
+    expect(query).toHaveBeenCalledWith(
+      expect.stringContaining("INSERT INTO urls"),
+      ["https://react.dev", "react.dev"]
+    );
+  });
+});

package/src/test/worker.test.ts

@@ -0,0 +1,144 @@
+import { describe, it, expect, vi, beforeEach } from "vitest";
+
+// Mock downloader
+vi.mock("../worker/downloader.js", () => {
+  return {
+    downloadPage: vi.fn(),
+  };
+});
+
+// Mock extractor
+vi.mock("../worker/extractor.js", () => {
+  return {
+    extractPageData: vi.fn(),
+  };
+});
+
+// Mock db queries
+vi.mock("../db/queries.js", () => {
+  return {
+    insertURL: vi.fn(),
+    insertLink: vi.fn(),
+    markDone: vi.fn(),
+    markFailed: vi.fn(),
+  };
+});
+
+vi.mock("../config.js", () => {
+  return {
+    config: {
+      ALLOWED_DOMAINS: ["react.dev"],
+      MAX_DEPTH: 2,
+    },
+  };
+});
+
+vi.mock("../frontier/robots.js", () => {
+  return {
+    isAllowedByRobots: vi.fn(),
+  };
+});
+
+import { downloadPage } from "../worker/downloader.js";
+import { extractPageData } from "../worker/extractor.js";
+import { insertURL, insertLink, markDone, markFailed } from "../db/queries.js";
+import { isAllowedByRobots } from "../frontier/robots.js";
+import { processPage } from "../worker/worker.js";
+
+const mockDownloadPage = vi.mocked(downloadPage);
+const mockExtractPageData = vi.mocked(extractPageData);
+const mockInsertURL = vi.mocked(insertURL);
+const mockInsertLink = vi.mocked(insertLink);
+const mockMarkDone = vi.mocked(markDone);
+const mockMarkFailed = vi.mocked(markFailed);
+const mockIsAllowedByRobots = vi.mocked(isAllowedByRobots);
+
+describe("Worker Pipeline", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    mockIsAllowedByRobots.mockResolvedValue(true);
+  });
+
+  it("should successfully process a page, extract content, and insert links", async () => {
+    mockDownloadPage.mockResolvedValue({
+      url: "https://react.dev/docs",
+      html: "<html>...</html>",
+      statusCode: 200,
+    });
+
+    mockExtractPageData.mockReturnValue({
+      title: "React Docs",
+      description: "Learn React",
+      canonicalUrl: "https://react.dev/docs",
+      headings: { h1: ["Docs"], h2: [], h3: [] },
+      textContent: "Learn React content",
+      links: ["/tutorial", "https://external.com", "https://react.dev/docs"],
+    });
+
+    mockInsertURL.mockResolvedValue(100);
+
+    await processPage({ id: 42, url: "https://react.dev/docs", depth: 1 });
+
+    expect(mockMarkDone).toHaveBeenCalledTimes(1);
+    expect(mockMarkDone).toHaveBeenCalledWith(42, {
+      title: "React Docs",
+      description: "Learn React",
+      canonicalUrl: "https://react.dev/docs",
+      headings: { h1: ["Docs"], h2: [], h3: [] },
+      textContent: "Learn React content",
+    });
+
+    expect(mockInsertURL).toHaveBeenCalledTimes(1);
+    expect(mockInsertURL).toHaveBeenCalledWith("https://react.dev/tutorial", "react.dev", 2);
+
+    expect(mockInsertLink).toHaveBeenCalledTimes(1);
+    expect(mockInsertLink).toHaveBeenCalledWith(42, 100);
+
+    expect(mockMarkFailed).not.toHaveBeenCalled();
+  });
+
+  it("should mark URL as FAILED if download fails", async () => {
+    mockDownloadPage.mockRejectedValue(new Error("Network Error"));
+
+    await expect(processPage({ id: 42, url: "https://react.dev/docs", depth: 1 })).rejects.toThrow("Network Error");
+
+    expect(mockMarkFailed).toHaveBeenCalledTimes(1);
+    expect(mockMarkFailed).toHaveBeenCalledWith(42, "Network Error");
+    expect(mockMarkDone).not.toHaveBeenCalled();
+  });
+
+  it("should discard links that exceed MAX_DEPTH", async () => {
+    mockDownloadPage.mockResolvedValue({
+      url: "https://react.dev/docs",
+      html: "<html>...</html>",
+      statusCode: 200,
+    });
+
+    mockExtractPageData.mockReturnValue({
+      title: "React Docs",
+      description: "Learn React",
+      canonicalUrl: "https://react.dev/docs",
+      headings: { h1: ["Docs"], h2: [], h3: [] },
+      textContent: "Learn React content",
+      links: ["/tutorial"],
+    });
+
+    // Run with current depth = 2, so nextDepth = 3 which exceeds MAX_DEPTH = 2
+    await processPage({ id: 42, url: "https://react.dev/docs", depth: 2 });
+
+    expect(mockMarkDone).toHaveBeenCalledTimes(1);
+    expect(mockInsertURL).not.toHaveBeenCalled();
+    expect(mockInsertLink).not.toHaveBeenCalled();
+  });
+
+  it("should abort crawl if URL is disallowed by robots.txt", async () => {
+    mockIsAllowedByRobots.mockResolvedValue(false);
+
+    await processPage({ id: 42, url: "https://react.dev/private", depth: 1 });
+
+    expect(mockMarkFailed).toHaveBeenCalledTimes(1);
+    expect(mockMarkFailed).toHaveBeenCalledWith(42, "Disallowed by robots.txt");
+    expect(mockDownloadPage).not.toHaveBeenCalled();
+    expect(mockMarkDone).not.toHaveBeenCalled();
+  });
+});