npm - @purepageio/fetch-engines - Versions diffs - 0.2.9 → 0.2.11 - Mend

@purepageio/fetch-engines 0.2.9 → 0.2.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (60) hide show

package/dist/FetchEngine.d.ts +47 -0
package/dist/FetchEngine.d.ts.map +1 -0
package/dist/FetchEngine.js +114 -0
package/dist/FetchEngine.js.map +1 -0
package/dist/FetchEngine.test.d.ts +2 -0
package/dist/FetchEngine.test.d.ts.map +1 -0
package/dist/FetchEngine.test.js +44 -0
package/dist/FetchEngine.test.js.map +1 -0
package/dist/HybridEngine.d.ts +21 -0
package/dist/HybridEngine.d.ts.map +1 -0
package/dist/HybridEngine.js +62 -0
package/dist/HybridEngine.js.map +1 -0
package/dist/IEngine.d.ts +22 -0
package/dist/IEngine.d.ts.map +1 -0
package/dist/IEngine.js +2 -0
package/dist/IEngine.js.map +1 -0
package/dist/PlaywrightEngine.d.ts +90 -0
package/dist/PlaywrightEngine.d.ts.map +1 -0
package/dist/PlaywrightEngine.js +558 -0
package/dist/PlaywrightEngine.js.map +1 -0
package/dist/PlaywrightEngine.test.d.ts +2 -0
package/dist/PlaywrightEngine.test.d.ts.map +1 -0
package/dist/PlaywrightEngine.test.js +207 -0
package/dist/PlaywrightEngine.test.js.map +1 -0
package/dist/PuppeteerEngine.d.ts +21 -0
package/dist/PuppeteerEngine.d.ts.map +1 -0
package/dist/PuppeteerEngine.js +412 -0
package/dist/PuppeteerEngine.js.map +1 -0
package/dist/browser/BrowserPool.d.ts +29 -0
package/dist/browser/BrowserPool.d.ts.map +1 -0
package/dist/browser/BrowserPool.js +378 -0
package/dist/browser/BrowserPool.js.map +1 -0
package/dist/browser/PlaywrightBrowserPool.d.ts +48 -0
package/dist/browser/PlaywrightBrowserPool.d.ts.map +1 -0
package/dist/browser/PlaywrightBrowserPool.js +378 -0
package/dist/browser/PlaywrightBrowserPool.js.map +1 -0
package/dist/browser/PlaywrightBrowserPool.test.d.ts +2 -0
package/dist/browser/PlaywrightBrowserPool.test.d.ts.map +1 -0
package/dist/browser/PlaywrightBrowserPool.test.js +422 -0
package/dist/browser/PlaywrightBrowserPool.test.js.map +1 -0
package/dist/errors.d.ts +20 -0
package/dist/errors.d.ts.map +1 -0
package/dist/errors.js +30 -0
package/dist/errors.js.map +1 -0
package/dist/index.d.ts +8 -323
package/dist/index.d.ts.map +1 -0
package/dist/index.js +4 -1617
package/dist/index.js.map +1 -1
package/dist/types.d.ts +167 -0
package/dist/types.d.ts.map +1 -0
package/dist/types.js +2 -0
package/dist/types.js.map +1 -0
package/dist/utils/markdown-converter.d.ts +31 -0
package/dist/utils/markdown-converter.d.ts.map +1 -0
package/dist/utils/markdown-converter.js +796 -0
package/dist/utils/markdown-converter.js.map +1 -0
package/package.json +5 -14
package/dist/index.cjs +0 -1657
package/dist/index.cjs.map +0 -1
package/dist/index.d.cts +0 -323

package/dist/PlaywrightEngine.test.js ADDED Viewed

@@ -0,0 +1,207 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from "vitest";
+import { PlaywrightEngine } from "./PlaywrightEngine.js";
+import { PlaywrightBrowserPool } from "./browser/PlaywrightBrowserPool.js";
+import axios from "axios";
+import { EventEmitter } from "events";
+// Mock axios
+vi.mock("axios");
+// Mock the BrowserPool itself
+vi.mock("./browser/PlaywrightBrowserPool.js", () => {
+    // console.log("Mocking PlaywrightBrowserPool...");
+    const MockBrowserPool = vi.fn();
+    MockBrowserPool.prototype.initialize = vi.fn().mockResolvedValue(undefined);
+    MockBrowserPool.prototype.acquirePage = vi.fn(); // Mocked per test
+    MockBrowserPool.prototype.releasePage = vi.fn().mockResolvedValue(undefined);
+    MockBrowserPool.prototype.cleanup = vi.fn().mockResolvedValue(undefined);
+    MockBrowserPool.prototype.getMetrics = vi.fn().mockReturnValue([]); // Default to empty metrics
+    return { PlaywrightBrowserPool: MockBrowserPool };
+});
+// Helper function to create a mock Playwright Response
+const createMockResponse = () => {
+    return {
+        ok: vi.fn().mockReturnValue(true),
+        status: vi.fn().mockReturnValue(200),
+        headers: vi.fn().mockReturnValue({ "content-type": "text/html" }),
+        // Add other methods/properties if needed by the engine
+        // body: vi.fn().mockResolvedValue(Buffer.from("")),
+        // text: vi.fn().mockResolvedValue(""),
+        // json: vi.fn().mockResolvedValue({}),
+        // etc.
+    };
+};
+// Helper to create mock Page
+const createMockPage = () => {
+    const pageEmitter = new EventEmitter();
+    const mockPage = {
+        goto: vi.fn().mockResolvedValue(createMockResponse()), // Mock goto to return mock response
+        content: vi.fn().mockResolvedValue("<html>Mock HTML</html>"),
+        title: vi.fn().mockResolvedValue("Mock Title"),
+        url: vi.fn().mockReturnValue("http://mock.example.com"),
+        close: vi.fn().mockImplementation(() => {
+            pageEmitter.emit("close");
+            return Promise.resolve();
+        }),
+        isClosed: vi.fn().mockReturnValue(false),
+        on: pageEmitter.on.bind(pageEmitter),
+        once: pageEmitter.once.bind(pageEmitter),
+        off: pageEmitter.off.bind(pageEmitter),
+        evaluate: vi.fn(), // Mock evaluate if needed
+        mouse: { move: vi.fn().mockResolvedValue(undefined) }, // Mock mouse if behavior simulation tested
+        // Add other methods/properties as needed by the engine
+    };
+    return mockPage;
+};
+// --- Tests --- //
+// Increase timeout for tests involving retries and delays
+describe("PlaywrightEngine", { timeout: 20000 }, () => {
+    let engine;
+    let browserPoolMock;
+    beforeEach(() => {
+        vi.clearAllMocks(); // Clear mocks between tests
+        // --- Set up mocks BEFORE engine creation ---
+        // Get the mocked constructor instance from the module mock
+        browserPoolMock = new PlaywrightBrowserPool();
+        // Mock methods directly on the instance used by the engine
+        // Ensure acquirePage returns a fresh mock page for each test
+        vi.mocked(browserPoolMock.acquirePage).mockImplementation(async () => createMockPage());
+        vi.mocked(browserPoolMock.initialize).mockResolvedValue(undefined);
+        vi.mocked(browserPoolMock.cleanup).mockResolvedValue(undefined);
+        vi.mocked(browserPoolMock.getMetrics).mockReturnValue([]);
+        // Ensure the engine constructor uses THIS specific mocked instance
+        vi.mocked(PlaywrightBrowserPool).mockImplementation(() => browserPoolMock);
+        // Mock axios.get for HTTP fallback tests
+        const mockAxiosGet = vi.fn();
+        vi.mocked(axios.get).mockImplementation(mockAxiosGet);
+    });
+    afterEach(async () => {
+        if (engine) {
+            await engine.cleanup(); // Ensure engine resources are released
+        }
+        vi.restoreAllMocks();
+    });
+    it("should initialize the browser pool on first fetch and return content", async () => {
+        engine = new PlaywrightEngine({ useHttpFallback: false }); // Disable fallback for direct test
+        const url = "http://example.com/init";
+        const result = await engine.fetchHTML(url);
+        // Check calls on the INSTANCE mock
+        expect(browserPoolMock.initialize).toHaveBeenCalledTimes(1);
+        expect(browserPoolMock.acquirePage).toHaveBeenCalledTimes(1);
+        expect(result.html).toBe("<html>Mock HTML</html>");
+        expect(result.title).toBe("Mock Title");
+    });
+    it("should use HTTP fallback if enabled and successful", async () => {
+        engine = new PlaywrightEngine({ useHttpFallback: true });
+        const url = "http://example.com/fallback-success";
+        const mockHtml = "<html>Fallback OK</html>";
+        const mockAxiosGet = vi.mocked(axios.get).mockResolvedValue({
+            data: mockHtml,
+            status: 200,
+            headers: { "content-type": "text/html" },
+            config: { url: url }, // Mock config for URL extraction
+            request: { res: { responseUrl: url } }, // Mock responseUrl
+        });
+        const result = await engine.fetchHTML(url);
+        expect(mockAxiosGet).toHaveBeenCalledWith(url, expect.any(Object));
+        expect(browserPoolMock.acquirePage).not.toHaveBeenCalled(); // Browser pool should NOT be used
+        expect(result.html).toBe(mockHtml);
+        expect(result.title).toMatch(/Fallback OK/i); // Regex for simple title extraction
+    });
+    it("should retry browser fetch if initial attempt fails and fallback also fails", async () => {
+        engine = new PlaywrightEngine({
+            useHttpFallback: true,
+            maxRetries: 1,
+            retryDelay: 10,
+        });
+        const url = "http://example.com/retry";
+        // Mock HTTP fallback to fail
+        vi.mocked(axios.get).mockRejectedValue(new Error("Axios fallback failed"));
+        // Mock acquirePage -> goto to fail once, then succeed
+        const mockPage = createMockPage();
+        vi.mocked(browserPoolMock.acquirePage).mockResolvedValue(mockPage); // Use instance mock
+        vi.mocked(mockPage.goto)
+            .mockRejectedValueOnce(new Error("First goto failed"))
+            .mockResolvedValue(createMockResponse()); // Success on retry
+        const result = await engine.fetchHTML(url);
+        expect(axios.get).toHaveBeenCalledTimes(1);
+        expect(browserPoolMock.acquirePage).toHaveBeenCalledTimes(1); // Only acquired once
+        expect(mockPage.goto).toHaveBeenCalledTimes(2); // Initial attempt + 1 retry
+        expect(result.html).toBe("<html>Mock HTML</html>");
+    });
+    it("should call pool cleanup when engine cleanup is called", async () => {
+        engine = new PlaywrightEngine();
+        await engine.fetchHTML("http://example.com/cleanup-test");
+        await engine.cleanup();
+        expect(browserPoolMock.cleanup).toHaveBeenCalledTimes(1); // Check instance mock
+    });
+    it("should handle errors during HTTP fallback", async () => {
+        // Mock fetchHTMLWithHttpFallback to reject
+        vi.spyOn(PlaywrightEngine.prototype, "fetchHTMLWithHttpFallback").mockRejectedValue(new Error("HTTP Fallback Failed"));
+        engine = new PlaywrightEngine({ useHttpFallback: true, retryDelay: 10 });
+        // We expect it to fail the fallback, then proceed to playwright (which is mocked to succeed)
+        const result = await engine.fetchHTML("http://example.com/fail-fallback");
+        expect(result).toBeDefined(); // Should still succeed via Playwright mock
+        expect(browserPoolMock.acquirePage).toHaveBeenCalled(); // Check instance mock
+    });
+    it("should handle pool acquirePage errors", async () => {
+        // Mock acquirePage on the instance to reject
+        vi.mocked(browserPoolMock.acquirePage).mockRejectedValue(new Error("Pool Error"));
+        engine = new PlaywrightEngine({ useHttpFallback: false, retryDelay: 10 }); // Disable fallback
+        await expect(engine.fetchHTML("http://example.com/pool-error")).rejects.toThrow(/Fetch failed after/);
+        expect(browserPoolMock.acquirePage).toHaveBeenCalledTimes(1 + 3); // Check instance mock (Initial + 3 retries)
+    });
+    it("should handle page.goto errors", async () => {
+        // Mock page.goto to reject
+        const mockPage = createMockPage();
+        vi.mocked(browserPoolMock.acquirePage).mockResolvedValue(mockPage); // Use instance mock
+        vi.mocked(mockPage.goto).mockRejectedValue(new Error("Navigation Failed"));
+        engine = new PlaywrightEngine({ useHttpFallback: false, retryDelay: 10 });
+        await expect(engine.fetchHTML("http://example.com/goto-error")).rejects.toThrow(/Fetch failed after/);
+        expect(mockPage.goto).toHaveBeenCalledTimes(1 + 3); // Initial + 3 retries
+    });
+    // Example for testing retry logic
+    it("should exhaust retries and throw if browser fetch consistently fails", async () => {
+        const mockPage = createMockPage();
+        vi.mocked(browserPoolMock.acquirePage).mockResolvedValue(mockPage); // Use instance mock
+        vi.mocked(mockPage.goto).mockRejectedValue(new Error("Consistent Fail"));
+        engine = new PlaywrightEngine({
+            maxRetries: 2,
+            retryDelay: 10,
+            useHttpFallback: false,
+        }); // 2 retries
+        await expect(engine.fetchHTML("http://example.com/retry-fail")).rejects.toThrow(/Fetch failed after 2 retries: Playwright navigation failed: Consistent Fail/);
+        // acquirePage called once initially
+        // goto called initial + 2 retries = 3 times
+        expect(browserPoolMock.acquirePage).toHaveBeenCalledTimes(1); // Check instance mock
+        expect(mockPage.goto).toHaveBeenCalledTimes(3);
+    });
+    it("should switch to thorough mode on first fast mode failure", async () => {
+        const mockPage = createMockPage();
+        vi.mocked(browserPoolMock.acquirePage).mockResolvedValue(mockPage); // Use instance mock
+        // Fail once, then succeed
+        vi.mocked(mockPage.goto)
+            .mockRejectedValueOnce(new Error("Fast Mode Fail"))
+            .mockResolvedValue(createMockResponse()); // Succeed on second (thorough) try
+        vi.mocked(mockPage.content).mockResolvedValue("<html>Thorough Success</html>");
+        vi.mocked(mockPage.title).mockResolvedValue("Thorough Title");
+        engine = new PlaywrightEngine({
+            defaultFastMode: true,
+            maxRetries: 1,
+            retryDelay: 10,
+            useHttpFallback: false,
+        });
+        // Mock simulateHumanBehavior before calling fetchHTML
+        const simulateSpy = vi
+            .spyOn(engine, "simulateHumanBehavior")
+            .mockResolvedValue(undefined);
+        const result = await engine.fetchHTML("http://example.com/fast-fail");
+        expect(mockPage.goto).toHaveBeenCalledTimes(2);
+        expect(result.html).toBe("<html>Thorough Success</html>");
+        expect(result.title).toBe("Thorough Title");
+        // Check simulateHumanBehavior was called on the second (thorough) attempt
+        expect(simulateSpy).toHaveBeenCalledTimes(1);
+        simulateSpy.mockRestore(); // Clean up spy
+    });
+    // TODO: Test headed mode fallback logic
+    // TODO: Test caching logic
+});
+//# sourceMappingURL=PlaywrightEngine.test.js.map

package/dist/PlaywrightEngine.test.js.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"PlaywrightEngine.test.js","sourceRoot":"","sources":["../src/PlaywrightEngine.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAC;AACzE,OAAO,EAAE,gBAAgB,EAAE,MAAM,uBAAuB,CAAC;AACzD,OAAO,EAAE,qBAAqB,EAAE,MAAM,oCAAoC,CAAC;AAQ3E,OAAO,KAAK,MAAM,OAAO,CAAC;AAC1B,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AAEtC,aAAa;AACb,EAAE,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAEjB,8BAA8B;AAC9B,EAAE,CAAC,IAAI,CAAC,oCAAoC,EAAE,GAAG,EAAE;IACjD,mDAAmD;IACnD,MAAM,eAAe,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;IAChC,eAAe,CAAC,SAAS,CAAC,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAC5E,eAAe,CAAC,SAAS,CAAC,WAAW,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,kBAAkB;IACnE,eAAe,CAAC,SAAS,CAAC,WAAW,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;IAC7E,eAAe,CAAC,SAAS,CAAC,OAAO,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;IACzE,eAAe,CAAC,SAAS,CAAC,UAAU,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC,CAAC,2BAA2B;IAC/F,OAAO,EAAE,qBAAqB,EAAE,eAAe,EAAE,CAAC;AACpD,CAAC,CAAC,CAAC;AAEH,uDAAuD;AACvD,MAAM,kBAAkB,GAAG,GAAuB,EAAE;IAClD,OAAO;QACL,EAAE,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,IAAI,CAAC;QACjC,MAAM,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,GAAG,CAAC;QACpC,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,EAAE,cAAc,EAAE,WAAW,EAAE,CAAC;QACjE,uDAAuD;QACvD,oDAAoD;QACpD,uCAAuC;QACvC,uCAAuC;QACvC,OAAO;KACyB,CAAC;AACrC,CAAC,CAAC;AAEF,6BAA6B;AAC7B,MAAM,cAAc,GAAG,GAAS,EAAE;IAChC,MAAM,WAAW,GAAG,IAAI,YAAY,EAAE,CAAC;IACvC,MAAM,QAAQ,GAAG;QACf,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,kBAAkB,EAAE,CAAC,EAAE,oCAAoC;QAC3F,OAAO,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,wBAAwB,CAAC;QAC5D,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,YAAY,CAAC;QAC9C,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,yBAAyB,CAAC;QACvD,KAAK,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,kBAAkB,CAAC,GAAG,EAAE;YACrC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;YAC1B,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC,CAAC;QACF,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,eAAe,CAAC,KAAK,CAAC;QACxC,EAAE,EAAE,WAAW,CAAC,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC;QACpC,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,WAAW,CAAC;QACxC,GAAG,EAAE,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC;QACtC,QAAQ,EAAE,EAAE,CAAC,EAAE,EAAE,EAAE,0BAA0B;QAC7C,KAAK,EAAE,EAAE,IAAI,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC,iBAAiB,CAAC,SAAS,CAAC,EAAE,EAAE,2CAA2C;QAClG,uDAAuD;KACrC,CAAC;IACrB,OAAO,QAAQ,CAAC;AAClB,CAAC,CAAC;AAEF,mBAAmB;AAEnB,0DAA0D;AAC1D,QAAQ,CAAC,kBAAkB,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,EAAE,GAAG,EAAE;IACpD,IAAI,MAAwB,CAAC;IAC7B,IAAI,eAAsC,CAAC;IAE3C,UAAU,CAAC,GAAG,EAAE;QACd,EAAE,CAAC,aAAa,EAAE,CAAC,CAAC,4BAA4B;QAEhD,8CAA8C;QAC9C,2DAA2D;QAC3D,eAAe,GAAG,IAAI,qBAAqB,EAAE,CAAC;QAE9C,2DAA2D;QAC3D,6DAA6D;QAC7D,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,kBAAkB,CAAC,KAAK,IAAI,EAAE,CACnE,cAAc,EAAE,CACjB,CAAC;QACF,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QACnE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAChE,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC,eAAe,CAAC,EAAE,CAAC,CAAC;QAE1D,mEAAmE;QACnE,EAAE,CAAC,MAAM,CAAC,qBAAqB,CAAC,CAAC,kBAAkB,CAAC,GAAG,EAAE,CAAC,eAAe,CAAC,CAAC;QAE3E,yCAAyC;QACzC,MAAM,YAAY,GAAG,EAAE,CAAC,EAAE,EAAE,CAAC;QAC7B,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,kBAAkB,CAAC,YAAmB,CAAC,CAAC;IAC/D,CAAC,CAAC,CAAC;IAEH,SAAS,CAAC,KAAK,IAAI,EAAE;QACnB,IAAI,MAAM,EAAE,CAAC;YACX,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC,CAAC,uCAAuC;QACjE,CAAC;QACD,EAAE,CAAC,eAAe,EAAE,CAAC;IACvB,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sEAAsE,EAAE,KAAK,IAAI,EAAE;QACpF,MAAM,GAAG,IAAI,gBAAgB,CAAC,EAAE,eAAe,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,mCAAmC;QAC9F,MAAM,GAAG,GAAG,yBAAyB,CAAC;QACtC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAE3C,mCAAmC;QACnC,MAAM,CAAC,eAAe,CAAC,UAAU,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC5D,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC7D,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;QACnD,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IAC1C,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,oDAAoD,EAAE,KAAK,IAAI,EAAE;QAClE,MAAM,GAAG,IAAI,gBAAgB,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;QACzD,MAAM,GAAG,GAAG,qCAAqC,CAAC;QAClD,MAAM,QAAQ,GAAG,0BAA0B,CAAC;QAC5C,MAAM,YAAY,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAC;YAC1D,IAAI,EAAE,QAAQ;YACd,MAAM,EAAE,GAAG;YACX,OAAO,EAAE,EAAE,cAAc,EAAE,WAAW,EAAE;YACxC,MAAM,EAAE,EAAE,GAAG,EAAE,GAAG,EAAE,EAAE,iCAAiC;YACvD,OAAO,EAAE,EAAE,GAAG,EAAE,EAAE,WAAW,EAAE,GAAG,EAAE,EAAE,EAAE,mBAAmB;SACrD,CAAC,CAAC;QAEV,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAE3C,MAAM,CAAC,YAAY,CAAC,CAAC,oBAAoB,CAAC,GAAG,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACnE,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,GAAG,CAAC,gBAAgB,EAAE,CAAC,CAAC,kCAAkC;QAC9F,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,cAAc,CAAC,CAAC,CAAC,oCAAoC;IACpF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,6EAA6E,EAAE,KAAK,IAAI,EAAE;QAC3F,MAAM,GAAG,IAAI,gBAAgB,CAAC;YAC5B,eAAe,EAAE,IAAI;YACrB,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,EAAE;SACf,CAAC,CAAC;QACH,MAAM,GAAG,GAAG,0BAA0B,CAAC;QAEvC,6BAA6B;QAC7B,EAAE,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,uBAAuB,CAAC,CAAC,CAAC;QAE3E,sDAAsD;QACtD,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,oBAAoB;QACxF,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;aACrB,qBAAqB,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC;aACrD,iBAAiB,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,mBAAmB;QAE/D,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC;QAE3C,MAAM,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,qBAAqB;QACnF,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,4BAA4B;QAC5E,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,KAAK,IAAI,EAAE;QACtE,MAAM,GAAG,IAAI,gBAAgB,EAAE,CAAC;QAChC,MAAM,MAAM,CAAC,SAAS,CAAC,iCAAiC,CAAC,CAAC;QAC1D,MAAM,MAAM,CAAC,OAAO,EAAE,CAAC;QACvB,MAAM,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,sBAAsB;IAClF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2CAA2C,EAAE,KAAK,IAAI,EAAE;QACzD,2CAA2C;QAC3C,EAAE,CAAC,KAAK,CACN,gBAAgB,CAAC,SAAgB,EACjC,2BAA2B,CAC5B,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC,CAAC;QAEvD,MAAM,GAAG,IAAI,gBAAgB,CAAC,EAAE,eAAe,EAAE,IAAI,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;QACzE,6FAA6F;QAC7F,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,kCAAkC,CAAC,CAAC;QAC1E,MAAM,CAAC,MAAM,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,2CAA2C;QACzE,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,gBAAgB,EAAE,CAAC,CAAC,sBAAsB;IAChF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uCAAuC,EAAE,KAAK,IAAI,EAAE;QACrD,6CAA6C;QAC7C,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,iBAAiB,CACtD,IAAI,KAAK,CAAC,YAAY,CAAC,CACxB,CAAC;QACF,MAAM,GAAG,IAAI,gBAAgB,CAAC,EAAE,eAAe,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC,CAAC,mBAAmB;QAE9F,MAAM,MAAM,CACV,MAAM,CAAC,SAAS,CAAC,+BAA+B,CAAC,CAClD,CAAC,OAAO,CAAC,OAAO,CACf,oBAAoB,CACrB,CAAC;QACF,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,4CAA4C;IAChH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gCAAgC,EAAE,KAAK,IAAI,EAAE;QAC9C,2BAA2B;QAC3B,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,oBAAoB;QACxF,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,mBAAmB,CAAC,CAAC,CAAC;QAE3E,MAAM,GAAG,IAAI,gBAAgB,CAAC,EAAE,eAAe,EAAE,KAAK,EAAE,UAAU,EAAE,EAAE,EAAE,CAAC,CAAC;QAC1E,MAAM,MAAM,CACV,MAAM,CAAC,SAAS,CAAC,+BAA+B,CAAC,CAClD,CAAC,OAAO,CAAC,OAAO,CACf,oBAAoB,CACrB,CAAC;QACF,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,qBAAqB,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,sBAAsB;IAC5E,CAAC,CAAC,CAAC;IAEH,kCAAkC;IAClC,EAAE,CAAC,sEAAsE,EAAE,KAAK,IAAI,EAAE;QACpF,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,oBAAoB;QACxF,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,iBAAiB,CAAC,IAAI,KAAK,CAAC,iBAAiB,CAAC,CAAC,CAAC;QAEzE,MAAM,GAAG,IAAI,gBAAgB,CAAC;YAC5B,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,EAAE;YACd,eAAe,EAAE,KAAK;SACvB,CAAC,CAAC,CAAC,YAAY;QAEhB,MAAM,MAAM,CACV,MAAM,CAAC,SAAS,CAAC,+BAA+B,CAAC,CAClD,CAAC,OAAO,CAAC,OAAO,CACf,6EAA6E,CAC9E,CAAC;QAEF,oCAAoC;QACpC,4CAA4C;QAC5C,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,sBAAsB;QACpF,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;IACjD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,MAAM,QAAQ,GAAG,cAAc,EAAE,CAAC;QAClC,EAAE,CAAC,MAAM,CAAC,eAAe,CAAC,WAAW,CAAC,CAAC,iBAAiB,CAAC,QAAQ,CAAC,CAAC,CAAC,oBAAoB;QACxF,0BAA0B;QAC1B,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC;aACrB,qBAAqB,CAAC,IAAI,KAAK,CAAC,gBAAgB,CAAC,CAAC;aAClD,iBAAiB,CAAC,kBAAkB,EAAE,CAAC,CAAC,CAAC,mCAAmC;QAC/E,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,iBAAiB,CAC3C,+BAA+B,CAChC,CAAC;QACF,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,iBAAiB,CAAC,gBAAgB,CAAC,CAAC;QAE9D,MAAM,GAAG,IAAI,gBAAgB,CAAC;YAC5B,eAAe,EAAE,IAAI;YACrB,UAAU,EAAE,CAAC;YACb,UAAU,EAAE,EAAE;YACd,eAAe,EAAE,KAAK;SACvB,CAAC,CAAC;QAEH,sDAAsD;QACtD,MAAM,WAAW,GAAG,EAAE;aACnB,KAAK,CAAC,MAAa,EAAE,uBAAuB,CAAC;aAC7C,iBAAiB,CAAC,SAAS,CAAC,CAAC;QAEhC,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,SAAS,CAAC,8BAA8B,CAAC,CAAC;QAEtE,MAAM,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAC/C,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC1D,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;QAC5C,0EAA0E;QAC1E,MAAM,CAAC,WAAW,CAAC,CAAC,qBAAqB,CAAC,CAAC,CAAC,CAAC;QAE7C,WAAW,CAAC,WAAW,EAAE,CAAC,CAAC,eAAe;IAC5C,CAAC,CAAC,CAAC;IAEH,wCAAwC;IACxC,2BAA2B;AAC7B,CAAC,CAAC,CAAC"}

package/dist/PuppeteerEngine.d.ts ADDED Viewed

@@ -0,0 +1,21 @@
+import type { HTMLFetchResult, BrowserMetrics } from "./types";
+import type { IEngine } from "./IEngine";
+/**
+ * PuppeteerEngine - A headless browser engine that uses Puppeteer to render JavaScript-heavy pages
+ * Ideal for SPAs and sites with anti-scraping measures
+ */
+export declare class PuppeteerEngine implements IEngine {
+    private static browserPool;
+    private readonly queue;
+    private readonly cache;
+    private readonly cacheTTL;
+    private isInitializing;
+    constructor(concurrentPages?: number);
+    private initializeBrowserPool;
+    fetchHTML(url: string): Promise<HTMLFetchResult>;
+    private checkCache;
+    private cacheResult;
+    cleanup(): Promise<void>;
+    getMetrics(): BrowserMetrics[];
+}
+//# sourceMappingURL=PuppeteerEngine.d.ts.map

package/dist/PuppeteerEngine.d.ts.map ADDED Viewed

@@ -0,0 +1 @@

+ {"version":3,"file":"PuppeteerEngine.d.ts","sourceRoot":"","sources":["../src/PuppeteerEngine.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,eAAe,EAAE,cAAc,EAAE,MAAM,SAAS,CAAC;AAC/D,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAgBzC;;;GAGG;AACH,qBAAa,eAAgB,YAAW,OAAO;IAC7C,OAAO,CAAC,MAAM,CAAC,WAAW,CAA4B;IACtD,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAS;IAC/B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAsC;IAC5D,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA0B;IACnD,OAAO,CAAC,cAAc,CAAkB;gBAE5B,eAAe,SAAI;YAmBjB,qBAAqB;IA6B7B,SAAS,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAiUtD,OAAO,CAAC,UAAU;IAelB,OAAO,CAAC,WAAW;IAoBb,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAwB9B,UAAU,IAAI,cAAc,EAAE;CAG/B"}