extract-webpage 1.2.187 → 1.2.188

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "extract-webpage",
-  "version": "1.2.187",
+  "version": "1.2.188",
   "module": "./dist/extract-webpage.es.js",
   "description": "Search, extract, cite, and outline the web for a topic with AI Research Agent.",
   "author": "vtempest <grokthiscontact@gmail.com>",
@@ -74,10 +74,10 @@
   "dependencies": {
     "@huggingface/transformers": "^3.8.1",
     "ai": "^5.0.0",
-    "chat-agent-toolkit": "^1.2.187",
+    "chat-agent-toolkit": "^1.2.188",
     "chrono-node": "^2.9.0",
     "drizzle-orm": "^0.45.1",
-    "extract-pdf": "^0.1.174",
+    "extract-pdf": "^0.1.175",
     "extract-youtube": "^1.0.103",
     "html-entities": "^2.6.0",
     "js-yaml": "^4.1.1",

package/src/url-to-content/__tests__/url-to-content.test.ts

@@ -1,50 +1,45 @@
 /**
  * @fileoverview Unit tests for URL content extraction
  */
-import { beforeEach, describe, expect, it, vi, type MockedFunction } from "vitest";
 import { extractContent } from "../url-to-content";
 import { scrapeURL } from "../url-to-html";
 import { extractContentAndCite } from "../../html-to-content/html-to-content";
-import { convertYoutubeToText, getURLYoutubeVideo } from "../youtube-helpers";
-import grab from "../../utils/grab";
+import { convertYoutubeToText } from "../youtube-helpers";
 import { convertPDFToHTML } from "extract-pdf";
 
 // Mock dependencies
-vi.mock("../url-to-html", () => ({
-  scrapeURL: vi.fn(),
+jest.mock("../url-to-html", () => ({
+  scrapeURL: jest.fn(),
 }));
 
-vi.mock("../../html-to-content/html-to-content", () => ({
-  extractContentAndCite: vi.fn(),
+jest.mock("../../html-to-content/html-to-content", () => ({
+  extractContentAndCite: jest.fn(),
 }));
 
-vi.mock("../youtube-helpers", () => ({
-  getURLYoutubeVideo: vi.fn(),
-  convertYoutubeToText: vi.fn(),
+jest.mock("../youtube-helpers", () => ({
+  getURLYoutubeVideo: jest.fn(),
+  convertYoutubeToText: jest.fn(),
 }));
 
-vi.mock("extract-pdf", () => ({
-  convertPDFToHTML: vi.fn(),
+jest.mock("extract-pdf", () => ({
+  convertPDFToHTML: jest.fn(),
 }));
 
-vi.mock("../../utils/grab");
+jest.mock("grab-url", () => jest.fn());
 
-const mockScrapeURL = scrapeURL as MockedFunction<typeof scrapeURL>;
-const mockExtractContentAndCite = extractContentAndCite as MockedFunction<
+const mockScrapeURL = scrapeURL as jest.MockedFunction<typeof scrapeURL>;
+const mockExtractContentAndCite = extractContentAndCite as jest.MockedFunction<
   typeof extractContentAndCite
 >;
 const mockConvertYoutubeToText =
-  convertYoutubeToText as MockedFunction<typeof convertYoutubeToText>;
-const mockConvertPDFToHTML = convertPDFToHTML as MockedFunction<
+  convertYoutubeToText as jest.MockedFunction<typeof convertYoutubeToText>;
+const mockConvertPDFToHTML = convertPDFToHTML as jest.MockedFunction<
   typeof convertPDFToHTML
 >;
-const mockGetURLYoutubeVideo =
-  getURLYoutubeVideo as MockedFunction<typeof getURLYoutubeVideo>;
-const mockGrab = grab as MockedFunction<typeof grab>;
 
 describe("extractContent", () => {
   beforeEach(() => {
-    vi.clearAllMocks();
+    jest.clearAllMocks();
   });
 
   describe("URL extraction", () => {
@@ -67,10 +62,17 @@ describe("extractContent", () => {
       expect(mockScrapeURL).toHaveBeenCalledWith("https://example.com/article", {
         proxy: null,
       });
-      expect(mockExtractContentAndCite).toHaveBeenCalledWith(
-        mockHtml,
-        expect.objectContaining({ url: "https://example.com/article" })
-      );
+      expect(mockExtractContentAndCite).toHaveBeenCalledWith(mockHtml, {
+        url: "https://example.com/article",
+        images: true,
+        links: true,
+        formatting: true,
+        absoluteURLs: true,
+        timeout: 10,
+        proxy: null,
+        citeFormatMonthFull: false,
+        citeFormatAuthorFull: true,
+      });
       expect(result.title).toBe("Test Article");
       expect(result.html).toBe("<p>Test content</p>");
     });
@@ -82,8 +84,7 @@ describe("extractContent", () => {
 
       const result = await extractContent("https://blocked-site.com/article");
 
-      // Any non-string scrapeURL result collapses to one generic message.
-      expect(result.error).toBe("Failed to fetch HTML content");
+      expect(result.error).toBe("HTTP error: 403 Forbidden");
       expect(mockExtractContentAndCite).not.toHaveBeenCalled();
     });
 
@@ -119,10 +120,9 @@ describe("extractContent", () => {
     it("should handle scrapeURL throwing an error", async () => {
       mockScrapeURL.mockRejectedValueOnce(new Error("Network timeout"));
 
-      const result = await extractContent("https://timeout.com/article");
-
-      // extractContent catches scrape failures and reports them in-band.
-      expect(result.error).toBe("Failed to scrape URL: Network timeout");
+      await expect(
+        extractContent("https://timeout.com/article")
+      ).rejects.toThrow("Network timeout");
     });
 
     it("should handle extraction returning no HTML", async () => {
@@ -192,7 +192,8 @@ describe("extractContent", () => {
 
   describe("YouTube extraction", () => {
     it("should extract YouTube video transcript", async () => {
-      mockGetURLYoutubeVideo.mockReturnValueOnce("dQw4w9WgXcQ");
+      const youtubeHelpers = require("../youtube-helpers");
+      youtubeHelpers.getURLYoutubeVideo.mockReturnValueOnce("dQw4w9WgXcQ");
 
       const mockTranscript = {
         title: "Test Video",
@@ -268,10 +269,6 @@ describe("extractContent", () => {
       };
 
       mockConvertPDFToHTML.mockResolvedValueOnce(mockPdfContent as any);
-      // isUrlPDF sniffs the leading "%PDF-" magic bytes via grab().
-      mockGrab.mockResolvedValueOnce(
-        new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d]).buffer as any
-      );
 
       await extractContent(
         "https://drive.google.com/file/d/ABC123/view"

package/src/url-to-content/__tests__/url-to-html.test.ts

@@ -1,17 +1,16 @@
 /**
  * @fileoverview Unit tests for URL scraping functionality
  */
-import { beforeEach, describe, expect, it, vi, type MockedFunction } from "vitest";
 import { scrapeURL, scrapeJINA } from "../url-to-html";
-import grab from "../../utils/grab";
+import grab from "../utils/grab";
 
-// The module under test imports the local grab wrapper, not grab-url.
-vi.mock("../../utils/grab");
-const mockGrab = grab as MockedFunction<typeof grab>;
+// Mock grab-url
+jest.mock("grab-url");
+const mockGrab = grab as jest.MockedFunction<typeof grab>;
 
 describe("scrapeURL", () => {
   beforeEach(() => {
-    vi.clearAllMocks();
+    jest.clearAllMocks();
     // Reset environment variables
     delete process.env.SCRAPER_URL;
     delete process.env.SCRAPER_API_KEY;
@@ -54,7 +53,7 @@ describe("scrapeURL", () => {
     mockGrab.mockRejectedValueOnce(error);
 
     // Mock Cloudflare scraper
-    global.fetch = vi.fn().mockRejectedValueOnce(new Error("Cloudflare failed"));
+    global.fetch = jest.fn().mockRejectedValueOnce(new Error("Cloudflare failed"));
 
     // Mock JINA fallback
     mockGrab.mockResolvedValueOnce("Title: Test\nMarkdown Content:\nFallback content");
@@ -70,12 +69,12 @@ describe("scrapeURL", () => {
     error.status = 500;
     mockGrab.mockRejectedValue(error);
 
-    global.fetch = vi.fn().mockRejectedValue(new Error("Cloudflare failed"));
+    global.fetch = jest.fn().mockRejectedValue(new Error("Cloudflare failed"));
 
-    // Exhausting every strategy throws rather than returning an error object.
-    await expect(scrapeURL("https://failing-site.com")).rejects.toThrow(
-      /All scraping methods failed/
-    );
+    const result = await scrapeURL("https://failing-site.com");
+
+    expect(typeof result).toBe("object");
+    expect((result as any).error).toBeDefined();
   });
 
   it("should detect bot protection and retry", async () => {
@@ -87,7 +86,7 @@ describe("scrapeURL", () => {
     mockGrab.mockResolvedValueOnce(botDetectionHtml);
 
     // Mock successful Cloudflare bypass
-    global.fetch = vi.fn().mockResolvedValueOnce({
+    global.fetch = jest.fn().mockResolvedValueOnce({
       ok: true,
       json: async () => ({
         html: successHtml,
@@ -111,7 +110,7 @@ describe("scrapeURL", () => {
     mockGrab.mockResolvedValueOnce(botDetectionHtml);
 
     // Cloudflare fails
-    global.fetch = vi.fn().mockRejectedValueOnce(new Error("Cloudflare failed"));
+    global.fetch = jest.fn().mockRejectedValueOnce(new Error("Cloudflare failed"));
 
     // JINA succeeds
     mockGrab.mockResolvedValueOnce(
@@ -127,26 +126,24 @@ describe("scrapeURL", () => {
   });
 
   it("should return error when bot detection persists", async () => {
-    // Note: the "Cloudflare Ray ID found " marker in checkHTMLForBotDetection
-    // carries a trailing space, so it does not match this fixture; use one of
-    // the markers that does.
-    const botDetectionHtml =
-      "<html><body>Please verify you are a human</body></html>";
+    const botDetectionHtml = "<html><body>Cloudflare Ray ID found</body></html>";
 
     // Initial request returns bot detection
     mockGrab.mockResolvedValueOnce(botDetectionHtml);
 
     // Cloudflare attempt also fails
-    global.fetch = vi.fn().mockResolvedValueOnce({
+    global.fetch = jest.fn().mockResolvedValueOnce({
       ok: true,
       json: async () => ({
         html: botDetectionHtml, // Still bot detection
       }),
     } as any);
 
-    await expect(
-      scrapeURL("https://protected-site.com", { checkBotDetection: true })
-    ).rejects.toThrow(/Bot detected/);
+    const result = await scrapeURL("https://protected-site.com", {
+      checkBotDetection: true,
+    });
+
+    expect((result as any).error).toBe("Bot detected");
   });
 
   it("should prepend proxy to URL if provided", async () => {
@@ -217,7 +214,7 @@ describe("scrapeURL", () => {
 
 describe("scrapeJINA", () => {
   beforeEach(() => {
-    vi.clearAllMocks();
+    jest.clearAllMocks();
   });
 
   it("should extract content from JINA API", async () => {
@@ -238,13 +235,10 @@ This is the article content in markdown format.
 
     expect(mockGrab).toHaveBeenCalledWith(
       "https://r.jina.ai/https://example.com/article",
-      expect.objectContaining({
+      {
         responseType: "text",
         timeout: 30,
-        // JINA is asked for HTML, and gets an Authorization header when
-        // JINA_API_KEY is set.
-        headers: expect.objectContaining({ Accept: "text/html" }),
-      })
+      }
     );
     expect(result).toContain("<title>Test Article</title>");
     expect(result).toContain("article content");

package/src/url-to-content/url-to-html.ts

@@ -62,16 +62,7 @@ export async function scrapeURL(url, options = {}) {
 
   if (checkRobotsAllowed) {
     const rules = await fetchScrapingRules(url);
-    // isAllowedToScrape matches rule paths with startsWith, so it needs the
-    // request path — passing the full URL made every rule (including
-    // `Disallow: /`) silently fail to match.
-    let requestPath: string;
-    try {
-      requestPath = new URL(url).pathname;
-    } catch {
-      requestPath = url;
-    }
-    if (!isAllowedToScrape(rules, requestPath)) {
+    if (!isAllowedToScrape(rules, url)) {
       return { error: "Robots.txt forbids to scrape there" };
     }
   }