@mendable/firecrawl-js 0.0.29-beta.3 → 0.0.29-beta.4

package/dist-cjs/__tests__/e2e_withAuth/index.test.js

@@ -0,0 +1,135 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const index_js_1 = require("../../index.js");
+const uuid_1 = require("uuid");
+const dotenv_1 = require("dotenv");
+dotenv_1.default.config();
+const TEST_API_KEY = process.env.TEST_API_KEY;
+const API_URL = "http://127.0.0.1:3002";
+describe('FirecrawlApp E2E Tests', () => {
+    test.concurrent('should throw error for no API key', () => {
+        expect(() => {
+            new index_js_1.default({ apiKey: null, apiUrl: API_URL });
+        }).toThrow("No API key provided");
+    });
+    test.concurrent('should throw error for invalid API key on scrape', async () => {
+        const invalidApp = new index_js_1.default({ apiKey: "invalid_api_key", apiUrl: API_URL });
+        await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
+    });
+    test.concurrent('should throw error for blocklisted URL on scrape', async () => {
+        const app = new index_js_1.default({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+        const blocklistedUrl = "https://facebook.com/fake-test";
+        await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow("Request failed with status code 403");
+    });
+    test.concurrent('should return successful response with valid preview token', async () => {
+        const app = new index_js_1.default({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL });
+        const response = await app.scrapeUrl('https://roastmywebsite.ai');
+        expect(response).not.toBeNull();
+        expect(response.data?.content).toContain("_Roast_");
+    }, 30000); // 30 seconds timeout
+    test.concurrent('should return successful response for valid scrape', async () => {
+        const app = new index_js_1.default({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+        const response = await app.scrapeUrl('https://roastmywebsite.ai');
+        expect(response).not.toBeNull();
+        expect(response.data?.content).toContain("_Roast_");
+        expect(response.data).toHaveProperty('markdown');
+        expect(response.data).toHaveProperty('metadata');
+        expect(response.data).not.toHaveProperty('html');
+    }, 30000); // 30 seconds timeout
+    test.concurrent('should return successful response with valid API key and include HTML', async () => {
+        const app = new index_js_1.default({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+        const response = await app.scrapeUrl('https://roastmywebsite.ai', { pageOptions: { includeHtml: true } });
+        expect(response).not.toBeNull();
+        expect(response.data?.content).toContain("_Roast_");
+        expect(response.data?.markdown).toContain("_Roast_");
+        expect(response.data?.html).toContain("<h1");
+    }, 30000); // 30 seconds timeout
+    test.concurrent('should return successful response for valid scrape with PDF file', async () => {
+        const app = new index_js_1.default({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+        const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf');
+        expect(response).not.toBeNull();
+        expect(response.data?.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
+    }, 30000); // 30 seconds timeout
+    test.concurrent('should return successful response for valid scrape with PDF file without explicit extension', async () => {
+        const app = new index_js_1.default({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+        const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001');
+        expect(response).not.toBeNull();
+        expect(response.data?.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
+    }, 30000); // 30 seconds timeout
+    test.concurrent('should throw error for invalid API key on crawl', async () => {
+        const invalidApp = new index_js_1.default({ apiKey: "invalid_api_key", apiUrl: API_URL });
+        await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
+    });
+    test.concurrent('should throw error for blocklisted URL on crawl', async () => {
+        const app = new index_js_1.default({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+        const blocklistedUrl = "https://twitter.com/fake-test";
+        await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow("Request failed with status code 403");
+    });
+    test.concurrent('should return successful response for crawl and wait for completion', async () => {
+        const app = new index_js_1.default({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+        const response = await app.crawlUrl('https://roastmywebsite.ai', { crawlerOptions: { excludes: ['blog/*'] } }, true, 30);
+        expect(response).not.toBeNull();
+        expect(response[0].content).toContain("_Roast_");
+    }, 60000); // 60 seconds timeout
+    test.concurrent('should handle idempotency key for crawl', async () => {
+        const app = new index_js_1.default({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+        const uniqueIdempotencyKey = (0, uuid_1.v4)();
+        const response = await app.crawlUrl('https://roastmywebsite.ai', { crawlerOptions: { excludes: ['blog/*'] } }, false, 2, uniqueIdempotencyKey);
+        expect(response).not.toBeNull();
+        expect(response.jobId).toBeDefined();
+        await expect(app.crawlUrl('https://roastmywebsite.ai', { crawlerOptions: { excludes: ['blog/*'] } }, true, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409");
+    });
+    test.concurrent('should check crawl status', async () => {
+        const app = new index_js_1.default({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+        const response = await app.crawlUrl('https://roastmywebsite.ai', { crawlerOptions: { excludes: ['blog/*'] } }, false);
+        expect(response).not.toBeNull();
+        expect(response.jobId).toBeDefined();
+        let statusResponse = await app.checkCrawlStatus(response.jobId);
+        const maxChecks = 15;
+        let checks = 0;
+        while (statusResponse.status === 'active' && checks < maxChecks) {
+            await new Promise(resolve => setTimeout(resolve, 1000));
+            expect(statusResponse.partial_data).not.toBeNull();
+            statusResponse = await app.checkCrawlStatus(response.jobId);
+            checks++;
+        }
+        expect(statusResponse).not.toBeNull();
+        expect(statusResponse.status).toBe('completed');
+        expect(statusResponse?.data?.length).toBeGreaterThan(0);
+    }, 35000); // 35 seconds timeout
+    test.concurrent('should return successful response for search', async () => {
+        const app = new index_js_1.default({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+        const response = await app.search("test query");
+        expect(response).not.toBeNull();
+        expect(response?.data?.[0]?.content).toBeDefined();
+        expect(response?.data?.length).toBeGreaterThan(2);
+    }, 30000); // 30 seconds timeout
+    test.concurrent('should throw error for invalid API key on search', async () => {
+        const invalidApp = new index_js_1.default({ apiKey: "invalid_api_key", apiUrl: API_URL });
+        await expect(invalidApp.search("test query")).rejects.toThrow("Request failed with status code 401");
+    });
+    test.concurrent('should perform LLM extraction', async () => {
+        const app = new index_js_1.default({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+        const response = await app.scrapeUrl("https://mendable.ai", {
+            extractorOptions: {
+                mode: 'llm-extraction',
+                extractionPrompt: "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
+                extractionSchema: {
+                    type: 'object',
+                    properties: {
+                        company_mission: { type: 'string' },
+                        supports_sso: { type: 'boolean' },
+                        is_open_source: { type: 'boolean' }
+                    },
+                    required: ['company_mission', 'supports_sso', 'is_open_source']
+                }
+            }
+        });
+        expect(response).not.toBeNull();
+        expect(response.data?.llm_extraction).toBeDefined();
+        const llmExtraction = response.data?.llm_extraction;
+        expect(llmExtraction?.company_mission).toBeDefined();
+        expect(typeof llmExtraction?.supports_sso).toBe('boolean');
+        expect(typeof llmExtraction?.is_open_source).toBe('boolean');
+    }, 30000); // 30 seconds timeout
+});

package/dist-cjs/__tests__/index.test.js

@@ -0,0 +1,38 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const globals_1 = require("@jest/globals");
+const axios_1 = require("axios");
+const index_js_1 = require("../index.js");
+const promises_1 = require("fs/promises");
+const path_1 = require("path");
+// Mock jest and set the type
+globals_1.jest.mock('axios');
+const mockedAxios = axios_1.default;
+// Get the fixure data from the JSON file in ./fixtures
+async function loadFixture(name) {
+    return await (0, promises_1.readFile)((0, path_1.join)(__dirname, 'fixtures', `${name}.json`), 'utf-8');
+}
+(0, globals_1.describe)('the firecrawl JS SDK', () => {
+    (0, globals_1.test)('Should require an API key to instantiate FirecrawlApp', async () => {
+        const fn = () => {
+            new index_js_1.default({ apiKey: undefined });
+        };
+        (0, globals_1.expect)(fn).toThrow('No API key provided');
+    });
+    (0, globals_1.test)('Should return scraped data from a /scrape API call', async () => {
+        const mockData = await loadFixture('scrape');
+        mockedAxios.post.mockResolvedValue({
+            status: 200,
+            data: JSON.parse(mockData),
+        });
+        const apiKey = 'YOUR_API_KEY';
+        const app = new index_js_1.default({ apiKey });
+        // Scrape a single URL
+        const url = 'https://mendable.ai';
+        const scrapedData = await app.scrapeUrl(url);
+        (0, globals_1.expect)(mockedAxios.post).toHaveBeenCalledTimes(1);
+        (0, globals_1.expect)(mockedAxios.post).toHaveBeenCalledWith(globals_1.expect.stringMatching(/^https:\/\/api.firecrawl.dev/), globals_1.expect.objectContaining({ url }), globals_1.expect.objectContaining({ headers: globals_1.expect.objectContaining({ 'Authorization': `Bearer ${apiKey}` }) }));
+        (0, globals_1.expect)(scrapedData.success).toBe(true);
+        (0, globals_1.expect)(scrapedData?.data?.metadata.title).toEqual('Mendable');
+    });
+});

package/dist-cjs/index.js

@@ -0,0 +1,263 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+const axios_1 = require("axios");
+const zod_1 = require("zod");
+const zod_to_json_schema_1 = require("zod-to-json-schema");
+/**
+ * Main class for interacting with the Firecrawl API.
+ */
+class FirecrawlApp {
+    /**
+     * Initializes a new instance of the FirecrawlApp class.
+     * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
+     */
+    constructor({ apiKey = null, apiUrl = null }) {
+        Object.defineProperty(this, "apiKey", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        Object.defineProperty(this, "apiUrl", {
+            enumerable: true,
+            configurable: true,
+            writable: true,
+            value: void 0
+        });
+        this.apiKey = apiKey || "";
+        this.apiUrl = apiUrl || "https://api.firecrawl.dev";
+        if (!this.apiKey) {
+            throw new Error("No API key provided");
+        }
+    }
+    /**
+     * Scrapes a URL using the Firecrawl API.
+     * @param {string} url - The URL to scrape.
+     * @param {Params | null} params - Additional parameters for the scrape request.
+     * @returns {Promise<ScrapeResponse>} The response from the scrape operation.
+     */
+    async scrapeUrl(url, params = null) {
+        const headers = {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiKey}`,
+        };
+        let jsonData = { url, ...params };
+        if (params?.extractorOptions?.extractionSchema) {
+            let schema = params.extractorOptions.extractionSchema;
+            // Check if schema is an instance of ZodSchema to correctly identify Zod schemas
+            if (schema instanceof zod_1.z.ZodSchema) {
+                schema = (0, zod_to_json_schema_1.zodToJsonSchema)(schema);
+            }
+            jsonData = {
+                ...jsonData,
+                extractorOptions: {
+                    ...params.extractorOptions,
+                    extractionSchema: schema,
+                    mode: params.extractorOptions.mode || "llm-extraction",
+                },
+            };
+        }
+        try {
+            const response = await axios_1.default.post(this.apiUrl + "/v0/scrape", jsonData, { headers });
+            if (response.status === 200) {
+                const responseData = response.data;
+                if (responseData.success) {
+                    return responseData;
+                }
+                else {
+                    throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
+                }
+            }
+            else {
+                this.handleError(response, "scrape URL");
+            }
+        }
+        catch (error) {
+            throw new Error(error.message);
+        }
+        return { success: false, error: "Internal server error." };
+    }
+    /**
+     * Searches for a query using the Firecrawl API.
+     * @param {string} query - The query to search for.
+     * @param {Params | null} params - Additional parameters for the search request.
+     * @returns {Promise<SearchResponse>} The response from the search operation.
+     */
+    async search(query, params = null) {
+        const headers = {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiKey}`,
+        };
+        let jsonData = { query };
+        if (params) {
+            jsonData = { ...jsonData, ...params };
+        }
+        try {
+            const response = await axios_1.default.post(this.apiUrl + "/v0/search", jsonData, { headers });
+            if (response.status === 200) {
+                const responseData = response.data;
+                if (responseData.success) {
+                    return responseData;
+                }
+                else {
+                    throw new Error(`Failed to search. Error: ${responseData.error}`);
+                }
+            }
+            else {
+                this.handleError(response, "search");
+            }
+        }
+        catch (error) {
+            throw new Error(error.message);
+        }
+        return { success: false, error: "Internal server error." };
+    }
+    /**
+     * Initiates a crawl job for a URL using the Firecrawl API.
+     * @param {string} url - The URL to crawl.
+     * @param {Params | null} params - Additional parameters for the crawl request.
+     * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
+     * @param {number} pollInterval - Time in seconds for job status checks.
+     * @param {string} idempotencyKey - Optional idempotency key for the request.
+     * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
+     */
+    async crawlUrl(url, params = null, waitUntilDone = true, pollInterval = 2, idempotencyKey) {
+        const headers = this.prepareHeaders(idempotencyKey);
+        let jsonData = { url };
+        if (params) {
+            jsonData = { ...jsonData, ...params };
+        }
+        try {
+            const response = await this.postRequest(this.apiUrl + "/v0/crawl", jsonData, headers);
+            if (response.status === 200) {
+                const jobId = response.data.jobId;
+                if (waitUntilDone) {
+                    return this.monitorJobStatus(jobId, headers, pollInterval);
+                }
+                else {
+                    return { success: true, jobId };
+                }
+            }
+            else {
+                this.handleError(response, "start crawl job");
+            }
+        }
+        catch (error) {
+            console.log(error);
+            throw new Error(error.message);
+        }
+        return { success: false, error: "Internal server error." };
+    }
+    /**
+     * Checks the status of a crawl job using the Firecrawl API.
+     * @param {string} jobId - The job ID of the crawl operation.
+     * @returns {Promise<JobStatusResponse>} The response containing the job status.
+     */
+    async checkCrawlStatus(jobId) {
+        const headers = this.prepareHeaders();
+        try {
+            const response = await this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
+            if (response.status === 200) {
+                return {
+                    success: true,
+                    status: response.data.status,
+                    data: response.data.data,
+                    partial_data: !response.data.data
+                        ? response.data.partial_data
+                        : undefined,
+                };
+            }
+            else {
+                this.handleError(response, "check crawl status");
+            }
+        }
+        catch (error) {
+            throw new Error(error.message);
+        }
+        return {
+            success: false,
+            status: "unknown",
+            error: "Internal server error.",
+        };
+    }
+    /**
+     * Prepares the headers for an API request.
+     * @returns {AxiosRequestHeaders} The prepared headers.
+     */
+    prepareHeaders(idempotencyKey) {
+        return {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiKey}`,
+            ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),
+        };
+    }
+    /**
+     * Sends a POST request to the specified URL.
+     * @param {string} url - The URL to send the request to.
+     * @param {Params} data - The data to send in the request.
+     * @param {AxiosRequestHeaders} headers - The headers for the request.
+     * @returns {Promise<AxiosResponse>} The response from the POST request.
+     */
+    postRequest(url, data, headers) {
+        return axios_1.default.post(url, data, { headers });
+    }
+    /**
+     * Sends a GET request to the specified URL.
+     * @param {string} url - The URL to send the request to.
+     * @param {AxiosRequestHeaders} headers - The headers for the request.
+     * @returns {Promise<AxiosResponse>} The response from the GET request.
+     */
+    getRequest(url, headers) {
+        return axios_1.default.get(url, { headers });
+    }
+    /**
+     * Monitors the status of a crawl job until completion or failure.
+     * @param {string} jobId - The job ID of the crawl operation.
+     * @param {AxiosRequestHeaders} headers - The headers for the request.
+     * @param {number} timeout - Timeout in seconds for job status checks.
+     * @returns {Promise<any>} The final job status or data.
+     */
+    async monitorJobStatus(jobId, headers, checkInterval) {
+        while (true) {
+            const statusResponse = await this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
+            if (statusResponse.status === 200) {
+                const statusData = statusResponse.data;
+                if (statusData.status === "completed") {
+                    if ("data" in statusData) {
+                        return statusData.data;
+                    }
+                    else {
+                        throw new Error("Crawl job completed but no data was returned");
+                    }
+                }
+                else if (["active", "paused", "pending", "queued"].includes(statusData.status)) {
+                    if (checkInterval < 2) {
+                        checkInterval = 2;
+                    }
+                    await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
+                }
+                else {
+                    throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`);
+                }
+            }
+            else {
+                this.handleError(statusResponse, "check crawl status");
+            }
+        }
+    }
+    /**
+     * Handles errors from API responses.
+     * @param {AxiosResponse} response - The response from the API.
+     * @param {string} action - The action being performed when the error occurred.
+     */
+    handleError(response, action) {
+        if ([402, 408, 409, 500].includes(response.status)) {
+            const errorMessage = response.data.error || "Unknown error occurred";
+            throw new Error(`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`);
+        }
+        else {
+            throw new Error(`Unexpected error occurred while trying to ${action}. Status code: ${response.status}`);
+        }
+    }
+}
+exports.default = FirecrawlApp;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "0.0.29-beta.3",
+  "version": "0.0.29-beta.4",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "build/index.js",
   "types": "types/index.d.ts",
@@ -8,7 +8,7 @@
   "scripts": {
     "build": "tsc",
     "build-and-publish": "npm run build && npm publish --access public",
-    "build-cjs": "tsc --project tsconfig.cjs.json",
+    "build-cjs": "tsc --outDir dist-cjs/ --project tsconfig.cjs.json",
     "publish-beta": "npm run build && npm publish --access public --tag beta",
     "test": "jest src/__tests__/**/*.test.ts"
   },

package/tsconfig.json

@@ -29,5 +29,9 @@
     "node_modules",
     "dist",
     "docs"
-  ]
+  ],
+  ".": {
+    "import": "./build/index.js",
+    "require": "./dist-cjs/index.js"
+  }
 }