extract-webpage 1.2.5

package/src/url-to-content/__tests__/url-to-html.test.ts

@@ -0,0 +1,301 @@
+/**
+ * @fileoverview Unit tests for URL scraping functionality
+ */
+import { scrapeURL, scrapeJINA } from "../url-to-html";
+import grab from "../utils/grab";
+
+// Mock grab-url
+jest.mock("grab-url");
+const mockGrab = grab as jest.MockedFunction<typeof grab>;
+
+describe("scrapeURL", () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+    // Reset environment variables
+    delete process.env.SCRAPER_URL;
+    delete process.env.SCRAPER_API_KEY;
+  });
+
+  it("should successfully scrape a URL", async () => {
+    const mockHtml = "<html><body><h1>Test Page</h1></body></html>";
+    mockGrab.mockResolvedValueOnce(mockHtml);
+
+    const result = await scrapeURL("https://example.com");
+
+    expect(result).toBe(mockHtml);
+    expect(mockGrab).toHaveBeenCalledWith(
+      "https://example.com",
+      expect.objectContaining({
+        responseType: "text",
+        "User-Agent": expect.any(String),
+        signal: expect.any(AbortSignal),
+      })
+    );
+  });
+
+  it("should use custom timeout", async () => {
+    const mockHtml = "<html><body>Content</body></html>";
+    mockGrab.mockResolvedValueOnce(mockHtml);
+
+    await scrapeURL("https://example.com", { timeout: 30 });
+
+    expect(mockGrab).toHaveBeenCalledWith(
+      "https://example.com",
+      expect.objectContaining({
+        signal: expect.any(AbortSignal),
+      })
+    );
+  });
+
+  it("should handle 403 Forbidden errors", async () => {
+    const error = new Error("Forbidden") as Error & { status?: number };
+    error.status = 403;
+    mockGrab.mockRejectedValueOnce(error);
+
+    // Mock Cloudflare scraper
+    global.fetch = jest.fn().mockRejectedValueOnce(new Error("Cloudflare failed"));
+
+    // Mock JINA fallback
+    mockGrab.mockResolvedValueOnce("Title: Test\nMarkdown Content:\nFallback content");
+
+    const result = await scrapeURL("https://blocked-site.com");
+
+    expect(typeof result).toBe("string");
+    expect(result).toContain("Fallback content");
+  });
+
+  it("should return error object when all methods fail", async () => {
+    const error = new Error("Network error") as Error & { status?: number };
+    error.status = 500;
+    mockGrab.mockRejectedValue(error);
+
+    global.fetch = jest.fn().mockRejectedValue(new Error("Cloudflare failed"));
+
+    const result = await scrapeURL("https://failing-site.com");
+
+    expect(typeof result).toBe("object");
+    expect((result as any).error).toBeDefined();
+  });
+
+  it("should detect bot protection and retry", async () => {
+    const botDetectionHtml =
+      "<html><body>Please verify you are a human</body></html>";
+    const successHtml = "<html><body>Actual content</body></html>";
+
+    // First attempt returns bot detection
+    mockGrab.mockResolvedValueOnce(botDetectionHtml);
+
+    // Mock successful Cloudflare bypass
+    global.fetch = jest.fn().mockResolvedValueOnce({
+      ok: true,
+      json: async () => ({
+        html: successHtml,
+        loadTime: 1000,
+        challengeBypassed: true,
+      }),
+    } as any);
+
+    const result = await scrapeURL("https://protected-site.com", {
+      checkBotDetection: true,
+    });
+
+    expect(result).toBe(successHtml);
+    expect(global.fetch).toHaveBeenCalled();
+  });
+
+  it("should handle bot detection with JINA fallback", async () => {
+    const botDetectionHtml = "<html><body>Access to this page has been denied</body></html>";
+
+    // First attempt returns bot detection
+    mockGrab.mockResolvedValueOnce(botDetectionHtml);
+
+    // Cloudflare fails
+    global.fetch = jest.fn().mockRejectedValueOnce(new Error("Cloudflare failed"));
+
+    // JINA succeeds
+    mockGrab.mockResolvedValueOnce(
+      "Title: Test Article\n===============\nMarkdown Content:\n# Article\nContent here"
+    );
+
+    const result = await scrapeURL("https://protected-site.com", {
+      checkBotDetection: true,
+    });
+
+    expect(typeof result).toBe("string");
+    expect(result).toContain("Article");
+  });
+
+  it("should return error when bot detection persists", async () => {
+    const botDetectionHtml = "<html><body>Cloudflare Ray ID found</body></html>";
+
+    // Initial request returns bot detection
+    mockGrab.mockResolvedValueOnce(botDetectionHtml);
+
+    // Cloudflare attempt also fails
+    global.fetch = jest.fn().mockResolvedValueOnce({
+      ok: true,
+      json: async () => ({
+        html: botDetectionHtml, // Still bot detection
+      }),
+    } as any);
+
+    const result = await scrapeURL("https://protected-site.com", {
+      checkBotDetection: true,
+    });
+
+    expect((result as any).error).toBe("Bot detected");
+  });
+
+  it("should prepend proxy to URL if provided", async () => {
+    const mockHtml = "<html><body>Content</body></html>";
+    mockGrab.mockResolvedValueOnce(mockHtml);
+
+    await scrapeURL("https://example.com", {
+      proxy: "https://proxy.example.com/",
+    });
+
+    expect(mockGrab).toHaveBeenCalledWith(
+      "https://proxy.example.com/https://example.com",
+      expect.any(Object)
+    );
+  });
+
+  it("should use different user agents", async () => {
+    const mockHtml = "<html><body>Content</body></html>";
+    mockGrab.mockResolvedValueOnce(mockHtml);
+
+    await scrapeURL("https://example.com", { userAgentIndex: 1 });
+
+    expect(mockGrab).toHaveBeenCalledWith(
+      "https://example.com",
+      expect.objectContaining({
+        "User-Agent": expect.stringContaining("Chrome/85"),
+      })
+    );
+  });
+
+  it("should add referer header when requested", async () => {
+    const mockHtml = "<html><body>Content</body></html>";
+    mockGrab.mockResolvedValueOnce(mockHtml);
+
+    await scrapeURL("https://example.com", { changeReferer: 1 });
+
+    expect(mockGrab).toHaveBeenCalledWith(
+      "https://example.com",
+      expect.objectContaining({
+        Referer: "https://www.google.com/",
+      })
+    );
+  });
+
+  it("should check robots.txt when requested", async () => {
+    mockGrab
+      .mockResolvedValueOnce("User-agent: *\nDisallow: /admin\n") // robots.txt
+      .mockResolvedValueOnce("<html><body>Content</body></html>"); // actual page
+
+    const result = await scrapeURL("https://example.com/page", {
+      checkRobotsAllowed: true,
+    });
+
+    expect(typeof result).toBe("string");
+    expect(result).toContain("Content");
+  });
+
+  it("should block scraping if robots.txt forbids", async () => {
+    mockGrab.mockResolvedValueOnce("User-agent: *\nDisallow: /\n"); // Block all
+
+    const result = await scrapeURL("https://example.com/page", {
+      checkRobotsAllowed: true,
+    });
+
+    expect((result as any).error).toBe("Robots.txt forbids to scrape there");
+  });
+});
+
+describe("scrapeJINA", () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  it("should extract content from JINA API", async () => {
+    const mockJinaResponse = `Title: Test Article
+URL Source: https://example.com
+Published Time: 2024-01-01
+Markdown Content:
+===============
+
+# Test Article
+
+This is the article content in markdown format.
+`;
+
+    mockGrab.mockResolvedValueOnce(mockJinaResponse);
+
+    const result = await scrapeJINA("https://example.com/article");
+
+    expect(mockGrab).toHaveBeenCalledWith(
+      "https://r.jina.ai/https://example.com/article",
+      {
+        responseType: "text",
+        timeout: 30,
+      }
+    );
+    expect(result).toContain("<title>Test Article</title>");
+    expect(result).toContain("article content");
+  });
+
+  it("should handle JINA with separator", async () => {
+    const mockJinaResponse = `Some header info
+===============
+Markdown Content:
+
+Content after separator
+`;
+
+    mockGrab.mockResolvedValueOnce(mockJinaResponse);
+
+    const result = await scrapeJINA("https://example.com");
+
+    expect(result).toContain("Content after separator");
+    expect(result).not.toContain("Some header info");
+  });
+
+  it("should throw error when JINA fetch fails", async () => {
+    mockGrab.mockRejectedValueOnce(new Error("JINA timeout"));
+
+    await expect(scrapeJINA("https://example.com")).rejects.toThrow(
+      "JINA scraping failed"
+    );
+  });
+
+  it("should throw error when JINA returns invalid data", async () => {
+    mockGrab.mockResolvedValueOnce(null as any);
+
+    await expect(scrapeJINA("https://example.com")).rejects.toThrow(
+      "JINA returned invalid or empty data"
+    );
+  });
+
+  it("should convert markdown to HTML", async () => {
+    const mockJinaResponse = `Title: Markdown Test
+===============
+Markdown Content:
+
+# Heading 1
+
+This is **bold** and *italic* text.
+
+- List item 1
+- List item 2
+`;
+
+    mockGrab.mockResolvedValueOnce(mockJinaResponse);
+
+    const result = await scrapeJINA("https://example.com");
+
+    expect(result).toContain("<h1>");
+    expect(result).toContain("<strong>");
+    expect(result).toContain("<em>");
+    expect(result).toContain("<li>");
+  });
+});