@mendable/firecrawl-js 0.0.21 → 0.0.24

package/.env.example

@@ -0,0 +1,3 @@
+API_URL=http://localhost:3002
+TEST_API_KEY=fc-YOUR_API_KEY
+

package/build/index.js

@@ -19,6 +19,7 @@ export default class FirecrawlApp {
      * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
      */
     constructor({ apiKey = null }) {
+        this.apiUrl = "https://api.firecrawl.dev";
         this.apiKey = apiKey || "";
         if (!this.apiKey) {
             throw new Error("No API key provided");
@@ -47,7 +48,7 @@ export default class FirecrawlApp {
                 jsonData = Object.assign(Object.assign({}, jsonData), { extractorOptions: Object.assign(Object.assign({}, params.extractorOptions), { extractionSchema: schema, mode: params.extractorOptions.mode || "llm-extraction" }) });
             }
             try {
-                const response = yield axios.post("https://api.firecrawl.dev/v0/scrape", jsonData, { headers });
+                const response = yield axios.post(this.apiUrl + "/v0/scrape", jsonData, { headers });
                 if (response.status === 200) {
                     const responseData = response.data;
                     if (responseData.success) {
@@ -84,7 +85,7 @@ export default class FirecrawlApp {
                 jsonData = Object.assign(Object.assign({}, jsonData), params);
             }
             try {
-                const response = yield axios.post("https://api.firecrawl.dev/v0/search", jsonData, { headers });
+                const response = yield axios.post(this.apiUrl + "/v0/search", jsonData, { headers });
                 if (response.status === 200) {
                     const responseData = response.data;
                     if (responseData.success) {
@@ -109,22 +110,23 @@ export default class FirecrawlApp {
      * @param {string} url - The URL to crawl.
      * @param {Params | null} params - Additional parameters for the crawl request.
      * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
-     * @param {number} timeout - Timeout in seconds for job status checks.
+     * @param {number} pollInterval - Time in seconds for job status checks.
+     * @param {string} idempotencyKey - Optional idempotency key for the request.
      * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
      */
     crawlUrl(url_1) {
-        return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, timeout = 2) {
-            const headers = this.prepareHeaders();
+        return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, pollInterval = 2, idempotencyKey) {
+            const headers = this.prepareHeaders(idempotencyKey);
             let jsonData = { url };
             if (params) {
                 jsonData = Object.assign(Object.assign({}, jsonData), params);
             }
             try {
-                const response = yield this.postRequest("https://api.firecrawl.dev/v0/crawl", jsonData, headers);
+                const response = yield this.postRequest(this.apiUrl + "/v0/crawl", jsonData, headers);
                 if (response.status === 200) {
                     const jobId = response.data.jobId;
                     if (waitUntilDone) {
-                        return this.monitorJobStatus(jobId, headers, timeout);
+                        return this.monitorJobStatus(jobId, headers, pollInterval);
                     }
                     else {
                         return { success: true, jobId };
@@ -150,9 +152,14 @@ export default class FirecrawlApp {
         return __awaiter(this, void 0, void 0, function* () {
             const headers = this.prepareHeaders();
             try {
-                const response = yield this.getRequest(`https://api.firecrawl.dev/v0/crawl/status/${jobId}`, headers);
+                const response = yield this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
                 if (response.status === 200) {
-                    return response.data;
+                    return {
+                        success: true,
+                        status: response.data.status,
+                        data: response.data.data,
+                        partial_data: !response.data.data ? response.data.partial_data : undefined,
+                    };
                 }
                 else {
                     this.handleError(response, "check crawl status");
@@ -172,11 +179,8 @@ export default class FirecrawlApp {
      * Prepares the headers for an API request.
      * @returns {AxiosRequestHeaders} The prepared headers.
      */
-    prepareHeaders() {
-        return {
-            "Content-Type": "application/json",
-            Authorization: `Bearer ${this.apiKey}`,
-        };
+    prepareHeaders(idempotencyKey) {
+        return Object.assign({ 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.apiKey}` }, (idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}));
     }
     /**
      * Sends a POST request to the specified URL.
@@ -204,10 +208,10 @@ export default class FirecrawlApp {
      * @param {number} timeout - Timeout in seconds for job status checks.
      * @returns {Promise<any>} The final job status or data.
      */
-    monitorJobStatus(jobId, headers, timeout) {
+    monitorJobStatus(jobId, headers, checkInterval) {
         return __awaiter(this, void 0, void 0, function* () {
             while (true) {
-                const statusResponse = yield this.getRequest(`https://api.firecrawl.dev/v0/crawl/status/${jobId}`, headers);
+                const statusResponse = yield this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
                 if (statusResponse.status === 200) {
                     const statusData = statusResponse.data;
                     if (statusData.status === "completed") {
@@ -219,10 +223,10 @@ export default class FirecrawlApp {
                         }
                     }
                     else if (["active", "paused", "pending", "queued"].includes(statusData.status)) {
-                        if (timeout < 2) {
-                            timeout = 2;
+                        if (checkInterval < 2) {
+                            checkInterval = 2;
                         }
-                        yield new Promise((resolve) => setTimeout(resolve, timeout * 1000)); // Wait for the specified timeout before checking again
+                        yield new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
                     }
                     else {
                         throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "0.0.21",
+  "version": "0.0.24",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "build/index.js",
   "types": "types/index.d.ts",
@@ -9,7 +9,7 @@
     "build": "tsc",
     "publish": "npm run build && npm publish --access public",
     "publish-beta": "npm run build && npm publish --access public --tag beta",
-    "test": "jest src/**/*.test.ts"
+    "test": "jest src/__tests__/**/*.test.ts"
   },
   "repository": {
     "type": "git",
@@ -19,6 +19,8 @@
   "license": "MIT",
   "dependencies": {
     "axios": "^1.6.8",
+    "dotenv": "^16.4.5",
+    "uuid": "^9.0.1",
     "zod": "^3.23.8",
     "zod-to-json-schema": "^3.23.0"
   },
@@ -29,7 +31,10 @@
   "devDependencies": {
     "@jest/globals": "^29.7.0",
     "@types/axios": "^0.14.0",
-    "@types/node": "^20.12.7",
+    "@types/dotenv": "^8.2.0",
+    "@types/jest": "^29.5.12",
+    "@types/node": "^20.12.12",
+    "@types/uuid": "^9.0.8",
     "jest": "^29.7.0",
     "ts-jest": "^29.1.2",
     "typescript": "^5.4.5"

package/src/__tests__/e2e_withAuth/index.test.ts

@@ -0,0 +1,155 @@
+import FirecrawlApp from '../../index';
+import { v4 as uuidv4 } from 'uuid';
+import dotenv from 'dotenv';
+
+dotenv.config();
+
+const TEST_API_KEY = process.env.TEST_API_KEY;
+const API_URL = process.env.API_URL;
+
+describe('FirecrawlApp E2E Tests', () => {
+  test('should throw error for no API key', () => {
+    expect(() => {
+      new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
+    }).toThrow("No API key provided");
+  });
+
+  test('should throw error for invalid API key on scrape', async () => {
+    const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
+    await expect(invalidApp.scrapeUrl('https://firecrawl.dev')).rejects.toThrow("Request failed with status code 401");
+  });
+
+  test('should throw error for blocklisted URL on scrape', async () => {
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+    const blocklistedUrl = "https://facebook.com/fake-test";
+    await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow("Request failed with status code 403");
+  });
+
+  test('should return successful response with valid preview token', async () => {
+    const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL });
+    const response = await app.scrapeUrl('https://firecrawl.dev');
+    expect(response).not.toBeNull();
+    expect(response.data.content).toContain("🔥 Firecrawl");
+  }, 30000); // 30 seconds timeout
+
+  test('should return successful response for valid scrape', async () => {
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+    const response = await app.scrapeUrl('https://firecrawl.dev');
+    expect(response).not.toBeNull();
+    expect(response.data.content).toContain("🔥 Firecrawl");
+    expect(response.data).toHaveProperty('markdown');
+    expect(response.data).toHaveProperty('metadata');
+    expect(response.data).not.toHaveProperty('html');
+  }, 30000); // 30 seconds timeout
+
+  test('should return successful response with valid API key and include HTML', async () => {
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+    const response = await app.scrapeUrl('https://firecrawl.dev', { pageOptions: { includeHtml: true } });
+    expect(response).not.toBeNull();
+    expect(response.data.content).toContain("🔥 Firecrawl");
+    expect(response.data.markdown).toContain("🔥 Firecrawl");
+    expect(response.data.html).toContain("<h1");
+  }, 30000); // 30 seconds timeout
+
+  test('should return successful response for valid scrape with PDF file', async () => {
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+    const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf');
+    expect(response).not.toBeNull();
+    expect(response.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
+  }, 30000); // 30 seconds timeout
+
+  test('should return successful response for valid scrape with PDF file without explicit extension', async () => {
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+    const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001');
+    expect(response).not.toBeNull();
+    expect(response.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
+  }, 30000); // 30 seconds timeout
+
+  test('should throw error for invalid API key on crawl', async () => {
+    const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
+    await expect(invalidApp.crawlUrl('https://firecrawl.dev')).rejects.toThrow("Request failed with status code 401");
+  });
+
+  test('should throw error for blocklisted URL on crawl', async () => {
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+    const blocklistedUrl = "https://twitter.com/fake-test";
+    await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow("Request failed with status code 403");
+  });
+
+  test('should return successful response for crawl and wait for completion', async () => {
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+    const response = await app.crawlUrl('https://firecrawl.dev', { crawlerOptions: { excludes: ['blog/*'] } }, true, 30);
+    expect(response).not.toBeNull();
+    expect(response[0].content).toContain("🔥 Firecrawl");
+  }, 60000); // 60 seconds timeout
+
+  test('should handle idempotency key for crawl', async () => {
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+    const uniqueIdempotencyKey = uuidv4();
+    const response = await app.crawlUrl('https://firecrawl.dev', { crawlerOptions: { excludes: ['blog/*'] } }, false, 2, uniqueIdempotencyKey);
+    expect(response).not.toBeNull();
+    expect(response.jobId).toBeDefined();
+
+    await expect(app.crawlUrl('https://firecrawl.dev', { crawlerOptions: { excludes: ['blog/*'] } }, true, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409");
+  });
+
+  test('should check crawl status', async () => {
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+    const response = await app.crawlUrl('https://firecrawl.dev', { crawlerOptions: { excludes: ['blog/*'] } }, false);
+    expect(response).not.toBeNull();
+    expect(response.jobId).toBeDefined();
+
+    let statusResponse = await app.checkCrawlStatus(response.jobId);
+    const maxChecks = 15;
+    let checks = 0;
+
+    while (statusResponse.status === 'active' && checks < maxChecks) {
+      await new Promise(resolve => setTimeout(resolve, 1000));
+      expect(statusResponse.partial_data).not.toBeNull();
+      statusResponse = await app.checkCrawlStatus(response.jobId);
+      checks++;
+    }
+
+    expect(statusResponse).not.toBeNull();
+    expect(statusResponse.status).toBe('completed');
+    expect(statusResponse.data.length).toBeGreaterThan(0);
+  }, 35000); // 35 seconds timeout
+
+  test('should return successful response for search', async () => {
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+    const response = await app.search("test query");
+    expect(response).not.toBeNull();
+    expect(response.data[0].content).toBeDefined();
+    expect(response.data.length).toBeGreaterThan(2);
+  }, 30000); // 30 seconds timeout
+
+  test('should throw error for invalid API key on search', async () => {
+    const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
+    await expect(invalidApp.search("test query")).rejects.toThrow("Request failed with status code 401");
+  });
+
+  test('should perform LLM extraction', async () => {
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
+    const response = await app.scrapeUrl("https://mendable.ai", {
+      extractorOptions: {
+        mode: 'llm-extraction',
+        extractionPrompt: "Based on the information on the page, find what the company's mission is and whether it supports SSO, and whether it is open source",
+        extractionSchema: {
+          type: 'object',
+          properties: {
+            company_mission: { type: 'string' },
+            supports_sso: { type: 'boolean' },
+            is_open_source: { type: 'boolean' }
+          },
+          required: ['company_mission', 'supports_sso', 'is_open_source']
+        }
+      }
+    });
+    expect(response).not.toBeNull();
+    expect(response.data.llm_extraction).toBeDefined();
+    const llmExtraction = response.data.llm_extraction;
+    expect(llmExtraction.company_mission).toBeDefined();
+    expect(typeof llmExtraction.supports_sso).toBe('boolean');
+    expect(typeof llmExtraction.is_open_source).toBe('boolean');
+  }, 30000); // 30 seconds timeout
+});

package/src/index.ts

@@ -6,6 +6,7 @@ import { zodToJsonSchema } from "zod-to-json-schema";
  */
 export interface FirecrawlAppConfig {
   apiKey?: string | null;
+  apiUrl?: string | null;
 }
 
 /**
@@ -55,6 +56,7 @@ export interface JobStatusResponse {
   status: string;
   jobId?: string;
   data?: any;
+  partial_data?: any,
   error?: string;
 }
 
@@ -63,6 +65,7 @@ export interface JobStatusResponse {
  */
 export default class FirecrawlApp {
   private apiKey: string;
+  private apiUrl: string = "https://api.firecrawl.dev";
 
   /**
    * Initializes a new instance of the FirecrawlApp class.
@@ -107,7 +110,7 @@ export default class FirecrawlApp {
     }
     try {
       const response: AxiosResponse = await axios.post(
-        "https://api.firecrawl.dev/v0/scrape",
+        this.apiUrl + "/v0/scrape",
         jsonData,
         { headers },
       );
@@ -147,7 +150,7 @@ export default class FirecrawlApp {
     }
     try {
       const response: AxiosResponse = await axios.post(
-        "https://api.firecrawl.dev/v0/search",
+        this.apiUrl + "/v0/search",
         jsonData,
         { headers }
       );
@@ -172,30 +175,32 @@ export default class FirecrawlApp {
    * @param {string} url - The URL to crawl.
    * @param {Params | null} params - Additional parameters for the crawl request.
    * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
-   * @param {number} timeout - Timeout in seconds for job status checks.
+   * @param {number} pollInterval - Time in seconds for job status checks.
+   * @param {string} idempotencyKey - Optional idempotency key for the request.
    * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
    */
   async crawlUrl(
     url: string,
     params: Params | null = null,
     waitUntilDone: boolean = true,
-    timeout: number = 2
+    pollInterval: number = 2,
+    idempotencyKey?: string
   ): Promise<CrawlResponse | any> {
-    const headers = this.prepareHeaders();
+    const headers = this.prepareHeaders(idempotencyKey);
     let jsonData: Params = { url };
     if (params) {
       jsonData = { ...jsonData, ...params };
     }
     try {
       const response: AxiosResponse = await this.postRequest(
-        "https://api.firecrawl.dev/v0/crawl",
+        this.apiUrl + "/v0/crawl",
         jsonData,
         headers
       );
       if (response.status === 200) {
         const jobId: string = response.data.jobId;
         if (waitUntilDone) {
-          return this.monitorJobStatus(jobId, headers, timeout);
+          return this.monitorJobStatus(jobId, headers, pollInterval);
         } else {
           return { success: true, jobId };
         }
@@ -218,11 +223,16 @@ export default class FirecrawlApp {
     const headers: AxiosRequestHeaders = this.prepareHeaders();
     try {
       const response: AxiosResponse = await this.getRequest(
-        `https://api.firecrawl.dev/v0/crawl/status/${jobId}`,
+        this.apiUrl + `/v0/crawl/status/${jobId}`,
         headers
       );
       if (response.status === 200) {
-        return response.data;
+        return {
+          success: true,
+          status: response.data.status,
+          data: response.data.data,
+          partial_data: !response.data.data ? response.data.partial_data : undefined,
+        };
       } else {
         this.handleError(response, "check crawl status");
       }
@@ -240,11 +250,12 @@ export default class FirecrawlApp {
    * Prepares the headers for an API request.
    * @returns {AxiosRequestHeaders} The prepared headers.
    */
-  prepareHeaders(): AxiosRequestHeaders {
+  prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {
     return {
-      "Content-Type": "application/json",
-      Authorization: `Bearer ${this.apiKey}`,
-    } as AxiosRequestHeaders;
+      'Content-Type': 'application/json',
+      'Authorization': `Bearer ${this.apiKey}`,
+      ...(idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}),
+    } as AxiosRequestHeaders & { 'x-idempotency-key'?: string };
   }
 
   /**
@@ -285,11 +296,11 @@ export default class FirecrawlApp {
   async monitorJobStatus(
     jobId: string,
     headers: AxiosRequestHeaders,
-    timeout: number
+    checkInterval: number
   ): Promise<any> {
     while (true) {
       const statusResponse: AxiosResponse = await this.getRequest(
-        `https://api.firecrawl.dev/v0/crawl/status/${jobId}`,
+        this.apiUrl + `/v0/crawl/status/${jobId}`,
         headers
       );
       if (statusResponse.status === 200) {
@@ -303,10 +314,10 @@ export default class FirecrawlApp {
         } else if (
           ["active", "paused", "pending", "queued"].includes(statusData.status)
         ) {
-          if (timeout < 2) {
-            timeout = 2;
+          if (checkInterval < 2) {
+            checkInterval = 2;
           }
-          await new Promise((resolve) => setTimeout(resolve, timeout * 1000)); // Wait for the specified timeout before checking again
+          await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
         } else {
           throw new Error(
             `Crawl job failed or was stopped. Status: ${statusData.status}`

package/types/index.d.ts

@@ -5,6 +5,7 @@ import { z } from "zod";
  */
 export interface FirecrawlAppConfig {
     apiKey?: string | null;
+    apiUrl?: string | null;
 }
 /**
  * Generic parameter interface.
@@ -50,6 +51,7 @@ export interface JobStatusResponse {
     status: string;
     jobId?: string;
     data?: any;
+    partial_data?: any;
     error?: string;
 }
 /**
@@ -57,6 +59,7 @@ export interface JobStatusResponse {
  */
 export default class FirecrawlApp {
     private apiKey;
+    private apiUrl;
     /**
      * Initializes a new instance of the FirecrawlApp class.
      * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
@@ -81,10 +84,11 @@ export default class FirecrawlApp {
      * @param {string} url - The URL to crawl.
      * @param {Params | null} params - Additional parameters for the crawl request.
      * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
-     * @param {number} timeout - Timeout in seconds for job status checks.
+     * @param {number} pollInterval - Time in seconds for job status checks.
+     * @param {string} idempotencyKey - Optional idempotency key for the request.
      * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
      */
-    crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, timeout?: number): Promise<CrawlResponse | any>;
+    crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, pollInterval?: number, idempotencyKey?: string): Promise<CrawlResponse | any>;
     /**
      * Checks the status of a crawl job using the Firecrawl API.
      * @param {string} jobId - The job ID of the crawl operation.
@@ -95,7 +99,7 @@ export default class FirecrawlApp {
      * Prepares the headers for an API request.
      * @returns {AxiosRequestHeaders} The prepared headers.
      */
-    prepareHeaders(): AxiosRequestHeaders;
+    prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders;
     /**
      * Sends a POST request to the specified URL.
      * @param {string} url - The URL to send the request to.
@@ -118,7 +122,7 @@ export default class FirecrawlApp {
      * @param {number} timeout - Timeout in seconds for job status checks.
      * @returns {Promise<any>} The final job status or data.
      */
-    monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, timeout: number): Promise<any>;
+    monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<any>;
     /**
      * Handles errors from API responses.
      * @param {AxiosResponse} response - The response from the API.