@mendable/firecrawl-js 0.0.25 → 0.0.28

package/README.md

@@ -176,6 +176,11 @@ async function checkStatusExample(jobId) {
 checkStatusExample('your_job_id_here');
 ```
 
+## Running Locally
+To use the SDK when running Firecrawl locally, you can change the initial Firecrawl app instance to:
+```js
+const app = new FirecrawlApp({ apiKey: "YOUR_API_KEY", apiUrl: "http://localhost:3002" });
+```
 
 ## Error Handling
 

package/build/index.js

@@ -18,9 +18,9 @@ export default class FirecrawlApp {
      * Initializes a new instance of the FirecrawlApp class.
      * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
      */
-    constructor({ apiKey = null }) {
-        this.apiUrl = "https://api.firecrawl.dev";
+    constructor({ apiKey = null, apiUrl = null }) {
         this.apiKey = apiKey || "";
+        this.apiUrl = apiUrl || "https://api.firecrawl.dev";
         if (!this.apiKey) {
             throw new Error("No API key provided");
         }
@@ -158,7 +158,9 @@ export default class FirecrawlApp {
                         success: true,
                         status: response.data.status,
                         data: response.data.data,
-                        partial_data: !response.data.data ? response.data.partial_data : undefined,
+                        partial_data: !response.data.data
+                            ? response.data.partial_data
+                            : undefined,
                     };
                 }
                 else {
@@ -180,7 +182,7 @@ export default class FirecrawlApp {
      * @returns {AxiosRequestHeaders} The prepared headers.
      */
     prepareHeaders(idempotencyKey) {
-        return Object.assign({ 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.apiKey}` }, (idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}));
+        return Object.assign({ "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}` }, (idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}));
     }
     /**
      * Sends a POST request to the specified URL.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "0.0.25",
+  "version": "0.0.28",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "build/index.js",
   "types": "types/index.d.ts",
@@ -33,6 +33,7 @@
     "@types/axios": "^0.14.0",
     "@types/dotenv": "^8.2.0",
     "@types/jest": "^29.5.12",
+    "@types/mocha": "^10.0.6",
     "@types/node": "^20.12.12",
     "@types/uuid": "^9.0.8",
     "jest": "^29.7.0",

package/src/__tests__/e2e_withAuth/index.test.ts

@@ -2,100 +2,101 @@ import FirecrawlApp from '../../index';
 import { v4 as uuidv4 } from 'uuid';
 import dotenv from 'dotenv';
 
+
 dotenv.config();
 
 const TEST_API_KEY = process.env.TEST_API_KEY;
-const API_URL = process.env.API_URL;
+const API_URL = "http://127.0.0.1:3002";
 
 describe('FirecrawlApp E2E Tests', () => {
-  test('should throw error for no API key', () => {
+  test.concurrent('should throw error for no API key', () => {
     expect(() => {
       new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
     }).toThrow("No API key provided");
   });
 
-  test('should throw error for invalid API key on scrape', async () => {
+  test.concurrent('should throw error for invalid API key on scrape', async () => {
     const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-    await expect(invalidApp.scrapeUrl('https://firecrawl.dev')).rejects.toThrow("Request failed with status code 401");
+    await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
   });
 
-  test('should throw error for blocklisted URL on scrape', async () => {
+  test.concurrent('should throw error for blocklisted URL on scrape', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const blocklistedUrl = "https://facebook.com/fake-test";
     await expect(app.scrapeUrl(blocklistedUrl)).rejects.toThrow("Request failed with status code 403");
   });
 
-  test('should return successful response with valid preview token', async () => {
+  test.concurrent('should return successful response with valid preview token', async () => {
     const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL });
-    const response = await app.scrapeUrl('https://firecrawl.dev');
+    const response = await app.scrapeUrl('https://roastmywebsite.ai');
     expect(response).not.toBeNull();
-    expect(response.data.content).toContain("🔥 Firecrawl");
+    expect(response.data?.content).toContain("_Roast_");
   }, 30000); // 30 seconds timeout
 
-  test('should return successful response for valid scrape', async () => {
+  test.concurrent('should return successful response for valid scrape', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const response = await app.scrapeUrl('https://firecrawl.dev');
+    const response = await app.scrapeUrl('https://roastmywebsite.ai');
     expect(response).not.toBeNull();
-    expect(response.data.content).toContain("🔥 Firecrawl");
+    expect(response.data?.content).toContain("_Roast_");
     expect(response.data).toHaveProperty('markdown');
     expect(response.data).toHaveProperty('metadata');
     expect(response.data).not.toHaveProperty('html');
   }, 30000); // 30 seconds timeout
 
-  test('should return successful response with valid API key and include HTML', async () => {
+  test.concurrent('should return successful response with valid API key and include HTML', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const response = await app.scrapeUrl('https://firecrawl.dev', { pageOptions: { includeHtml: true } });
+    const response = await app.scrapeUrl('https://roastmywebsite.ai', { pageOptions: { includeHtml: true } });
     expect(response).not.toBeNull();
-    expect(response.data.content).toContain("🔥 Firecrawl");
-    expect(response.data.markdown).toContain("🔥 Firecrawl");
-    expect(response.data.html).toContain("<h1");
+    expect(response.data?.content).toContain("_Roast_");
+    expect(response.data?.markdown).toContain("_Roast_");
+    expect(response.data?.html).toContain("<h1");
   }, 30000); // 30 seconds timeout
 
-  test('should return successful response for valid scrape with PDF file', async () => {
+  test.concurrent('should return successful response for valid scrape with PDF file', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf');
     expect(response).not.toBeNull();
-    expect(response.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
+    expect(response.data?.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
   }, 30000); // 30 seconds timeout
 
-  test('should return successful response for valid scrape with PDF file without explicit extension', async () => {
+  test.concurrent('should return successful response for valid scrape with PDF file without explicit extension', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001');
     expect(response).not.toBeNull();
-    expect(response.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
+    expect(response.data?.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
   }, 30000); // 30 seconds timeout
 
-  test('should throw error for invalid API key on crawl', async () => {
+  test.concurrent('should throw error for invalid API key on crawl', async () => {
     const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-    await expect(invalidApp.crawlUrl('https://firecrawl.dev')).rejects.toThrow("Request failed with status code 401");
+    await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
   });
 
-  test('should throw error for blocklisted URL on crawl', async () => {
+  test.concurrent('should throw error for blocklisted URL on crawl', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const blocklistedUrl = "https://twitter.com/fake-test";
     await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow("Request failed with status code 403");
   });
 
-  test('should return successful response for crawl and wait for completion', async () => {
+  test.concurrent('should return successful response for crawl and wait for completion', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const response = await app.crawlUrl('https://firecrawl.dev', { crawlerOptions: { excludes: ['blog/*'] } }, true, 30);
+    const response = await app.crawlUrl('https://roastmywebsite.ai', { crawlerOptions: { excludes: ['blog/*'] } }, true, 30);
     expect(response).not.toBeNull();
-    expect(response[0].content).toContain("🔥 Firecrawl");
+    expect(response[0].content).toContain("_Roast_");
   }, 60000); // 60 seconds timeout
 
-  test('should handle idempotency key for crawl', async () => {
+  test.concurrent('should handle idempotency key for crawl', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const uniqueIdempotencyKey = uuidv4();
-    const response = await app.crawlUrl('https://firecrawl.dev', { crawlerOptions: { excludes: ['blog/*'] } }, false, 2, uniqueIdempotencyKey);
+    const response = await app.crawlUrl('https://roastmywebsite.ai', { crawlerOptions: { excludes: ['blog/*'] } }, false, 2, uniqueIdempotencyKey);
     expect(response).not.toBeNull();
     expect(response.jobId).toBeDefined();
 
-    await expect(app.crawlUrl('https://firecrawl.dev', { crawlerOptions: { excludes: ['blog/*'] } }, true, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409");
+    await expect(app.crawlUrl('https://roastmywebsite.ai', { crawlerOptions: { excludes: ['blog/*'] } }, true, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409");
   });
 
-  test('should check crawl status', async () => {
+  test.concurrent('should check crawl status', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const response = await app.crawlUrl('https://firecrawl.dev', { crawlerOptions: { excludes: ['blog/*'] } }, false);
+    const response = await app.crawlUrl('https://roastmywebsite.ai', { crawlerOptions: { excludes: ['blog/*'] } }, false);
     expect(response).not.toBeNull();
     expect(response.jobId).toBeDefined();
 
@@ -112,23 +113,23 @@ describe('FirecrawlApp E2E Tests', () => {
 
     expect(statusResponse).not.toBeNull();
     expect(statusResponse.status).toBe('completed');
-    expect(statusResponse.data.length).toBeGreaterThan(0);
+    expect(statusResponse?.data?.length).toBeGreaterThan(0);
   }, 35000); // 35 seconds timeout
 
-  test('should return successful response for search', async () => {
+  test.concurrent('should return successful response for search', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.search("test query");
     expect(response).not.toBeNull();
-    expect(response.data[0].content).toBeDefined();
-    expect(response.data.length).toBeGreaterThan(2);
+    expect(response?.data?.[0]?.content).toBeDefined();
+    expect(response?.data?.length).toBeGreaterThan(2);
   }, 30000); // 30 seconds timeout
 
-  test('should throw error for invalid API key on search', async () => {
+  test.concurrent('should throw error for invalid API key on search', async () => {
     const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
     await expect(invalidApp.search("test query")).rejects.toThrow("Request failed with status code 401");
   });
 
-  test('should perform LLM extraction', async () => {
+  test.concurrent('should perform LLM extraction', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.scrapeUrl("https://mendable.ai", {
       extractorOptions: {
@@ -146,10 +147,10 @@ describe('FirecrawlApp E2E Tests', () => {
       }
     });
     expect(response).not.toBeNull();
-    expect(response.data.llm_extraction).toBeDefined();
-    const llmExtraction = response.data.llm_extraction;
-    expect(llmExtraction.company_mission).toBeDefined();
-    expect(typeof llmExtraction.supports_sso).toBe('boolean');
-    expect(typeof llmExtraction.is_open_source).toBe('boolean');
+    expect(response.data?.llm_extraction).toBeDefined();
+    const llmExtraction = response.data?.llm_extraction;
+    expect(llmExtraction?.company_mission).toBeDefined();
+    expect(typeof llmExtraction?.supports_sso).toBe('boolean');
+    expect(typeof llmExtraction?.is_open_source).toBe('boolean');
   }, 30000); // 30 seconds timeout
 });

package/src/__tests__/index.test.ts

@@ -43,6 +43,6 @@ describe('the firecrawl JS SDK', () => {
       expect.objectContaining({ headers: expect.objectContaining({'Authorization': `Bearer ${apiKey}`}) }),
     )
     expect(scrapedData.success).toBe(true);
-    expect(scrapedData.data.metadata.title).toEqual('Mendable');
+    expect(scrapedData?.data?.metadata.title).toEqual('Mendable');
   });
 })

package/src/index.ts

@@ -10,15 +10,63 @@ export interface FirecrawlAppConfig {
 }
 
 /**
- * Generic parameter interface.
+ * Metadata for a Firecrawl document.
  */
-export interface Params {
+export interface FirecrawlDocumentMetadata {
+  title?: string;
+  description?: string;
+  language?: string;
+  keywords?: string;
+  robots?: string;
+  ogTitle?: string;
+  ogDescription?: string;
+  ogUrl?: string;
+  ogImage?: string;
+  ogAudio?: string;
+  ogDeterminer?: string;
+  ogLocale?: string;
+  ogLocaleAlternate?: string[];
+  ogSiteName?: string;
+  ogVideo?: string;
+  dctermsCreated?: string;
+  dcDateCreated?: string;
+  dcDate?: string;
+  dctermsType?: string;
+  dcType?: string;
+  dctermsAudience?: string;
+  dctermsSubject?: string;
+  dcSubject?: string;
+  dcDescription?: string;
+  dctermsKeywords?: string;
+  modifiedTime?: string;
+  publishedTime?: string;
+  articleTag?: string;
+  articleSection?: string;
+  sourceURL?: string;
+  pageStatusCode?: number;
+  pageError?: string;
   [key: string]: any;
-  extractorOptions?: {
-    extractionSchema: z.ZodSchema | any;
-    mode?: "llm-extraction";
-    extractionPrompt?: string;
-  };
+}
+
+/**
+ * Document interface for Firecrawl.
+ */
+export interface FirecrawlDocument {
+  id?: string;
+  url?: string;
+  content: string;
+  markdown?: string;
+  html?: string;
+  llm_extraction?: Record<string, any>;
+  createdAt?: Date;
+  updatedAt?: Date;
+  type?: string;
+  metadata: FirecrawlDocumentMetadata;
+  childrenLinks?: string[];
+  provider?: string;
+  warning?: string;
+
+  index?: number;
 }
 
 /**
@@ -26,16 +74,15 @@ export interface Params {
  */
 export interface ScrapeResponse {
   success: boolean;
-  data?: any;
+  data?: FirecrawlDocument;
   error?: string;
 }
-
 /**
  * Response interface for searching operations.
  */
 export interface SearchResponse {
   success: boolean;
-  data?: any;
+  data?: FirecrawlDocument[];
   error?: string;
 }
 /**
@@ -44,10 +91,9 @@ export interface SearchResponse {
 export interface CrawlResponse {
   success: boolean;
   jobId?: string;
-  data?: any;
+  data?: FirecrawlDocument[];
   error?: string;
 }
-
 /**
  * Response interface for job status checks.
  */
@@ -55,24 +101,35 @@ export interface JobStatusResponse {
   success: boolean;
   status: string;
   jobId?: string;
-  data?: any;
-  partial_data?: any,
+  data?: FirecrawlDocument[];
+  partial_data?: FirecrawlDocument[];
   error?: string;
 }
-
+/**
+ * Generic parameter interface.
+ */
+export interface Params {
+  [key: string]: any;
+  extractorOptions?: {
+    extractionSchema: z.ZodSchema | any;
+    mode?: "llm-extraction";
+    extractionPrompt?: string;
+  };
+}
 /**
  * Main class for interacting with the Firecrawl API.
  */
 export default class FirecrawlApp {
   private apiKey: string;
-  private apiUrl: string = "https://api.firecrawl.dev";
+  private apiUrl: string;
 
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
    */
-  constructor({ apiKey = null }: FirecrawlAppConfig) {
+  constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
     this.apiKey = apiKey || "";
+    this.apiUrl = apiUrl || "https://api.firecrawl.dev";
     if (!this.apiKey) {
       throw new Error("No API key provided");
     }
@@ -112,7 +169,7 @@ export default class FirecrawlApp {
       const response: AxiosResponse = await axios.post(
         this.apiUrl + "/v0/scrape",
         jsonData,
-        { headers },
+        { headers }
       );
       if (response.status === 200) {
         const responseData = response.data;
@@ -231,7 +288,9 @@ export default class FirecrawlApp {
           success: true,
           status: response.data.status,
           data: response.data.data,
-          partial_data: !response.data.data ? response.data.partial_data : undefined,
+          partial_data: !response.data.data
+            ? response.data.partial_data
+            : undefined,
         };
       } else {
         this.handleError(response, "check crawl status");
@@ -252,10 +311,10 @@ export default class FirecrawlApp {
    */
   prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {
     return {
-      'Content-Type': 'application/json',
-      'Authorization': `Bearer ${this.apiKey}`,
-      ...(idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}),
-    } as AxiosRequestHeaders & { 'x-idempotency-key'?: string };
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`,
+      ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),
+    } as AxiosRequestHeaders & { "x-idempotency-key"?: string };
   }
 
   /**
@@ -317,7 +376,9 @@ export default class FirecrawlApp {
           if (checkInterval < 2) {
             checkInterval = 2;
           }
-          await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
+          await new Promise((resolve) =>
+            setTimeout(resolve, checkInterval * 1000)
+          ); // Wait for the specified timeout before checking again
         } else {
           throw new Error(
             `Crawl job failed or was stopped. Status: ${statusData.status}`

package/types/index.d.ts

@@ -8,22 +8,68 @@ export interface FirecrawlAppConfig {
     apiUrl?: string | null;
 }
 /**
- * Generic parameter interface.
+ * Metadata for a Firecrawl document.
  */
-export interface Params {
+export interface FirecrawlDocumentMetadata {
+    title?: string;
+    description?: string;
+    language?: string;
+    keywords?: string;
+    robots?: string;
+    ogTitle?: string;
+    ogDescription?: string;
+    ogUrl?: string;
+    ogImage?: string;
+    ogAudio?: string;
+    ogDeterminer?: string;
+    ogLocale?: string;
+    ogLocaleAlternate?: string[];
+    ogSiteName?: string;
+    ogVideo?: string;
+    dctermsCreated?: string;
+    dcDateCreated?: string;
+    dcDate?: string;
+    dctermsType?: string;
+    dcType?: string;
+    dctermsAudience?: string;
+    dctermsSubject?: string;
+    dcSubject?: string;
+    dcDescription?: string;
+    dctermsKeywords?: string;
+    modifiedTime?: string;
+    publishedTime?: string;
+    articleTag?: string;
+    articleSection?: string;
+    sourceURL?: string;
+    pageStatusCode?: number;
+    pageError?: string;
     [key: string]: any;
-    extractorOptions?: {
-        extractionSchema: z.ZodSchema | any;
-        mode?: "llm-extraction";
-        extractionPrompt?: string;
-    };
+}
+/**
+ * Document interface for Firecrawl.
+ */
+export interface FirecrawlDocument {
+    id?: string;
+    url?: string;
+    content: string;
+    markdown?: string;
+    html?: string;
+    llm_extraction?: Record<string, any>;
+    createdAt?: Date;
+    updatedAt?: Date;
+    type?: string;
+    metadata: FirecrawlDocumentMetadata;
+    childrenLinks?: string[];
+    provider?: string;
+    warning?: string;
+    index?: number;
 }
 /**
  * Response interface for scraping operations.
  */
 export interface ScrapeResponse {
     success: boolean;
-    data?: any;
+    data?: FirecrawlDocument;
     error?: string;
 }
 /**
@@ -31,7 +77,7 @@ export interface ScrapeResponse {
  */
 export interface SearchResponse {
     success: boolean;
-    data?: any;
+    data?: FirecrawlDocument[];
     error?: string;
 }
 /**
@@ -40,7 +86,7 @@ export interface SearchResponse {
 export interface CrawlResponse {
     success: boolean;
     jobId?: string;
-    data?: any;
+    data?: FirecrawlDocument[];
     error?: string;
 }
 /**
@@ -50,10 +96,21 @@ export interface JobStatusResponse {
     success: boolean;
     status: string;
     jobId?: string;
-    data?: any;
-    partial_data?: any;
+    data?: FirecrawlDocument[];
+    partial_data?: FirecrawlDocument[];
     error?: string;
 }
+/**
+ * Generic parameter interface.
+ */
+export interface Params {
+    [key: string]: any;
+    extractorOptions?: {
+        extractionSchema: z.ZodSchema | any;
+        mode?: "llm-extraction";
+        extractionPrompt?: string;
+    };
+}
 /**
  * Main class for interacting with the Firecrawl API.
  */
@@ -64,7 +121,7 @@ export default class FirecrawlApp {
      * Initializes a new instance of the FirecrawlApp class.
      * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
      */
-    constructor({ apiKey }: FirecrawlAppConfig);
+    constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
     /**
      * Scrapes a URL using the Firecrawl API.
      * @param {string} url - The URL to scrape.