@mendable/firecrawl-js 0.0.26 → 0.0.29-beta.1

package/README.md

@@ -176,6 +176,11 @@ async function checkStatusExample(jobId) {
 checkStatusExample('your_job_id_here');
 ```
 
+## Running Locally
+To use the SDK when running Firecrawl locally, you can change the initial Firecrawl app instance to:
+```js
+const app = new FirecrawlApp({ apiKey: "YOUR_API_KEY", apiUrl: "http://localhost:3002" });
+```
 
 ## Error Handling
 

package/build/index.js

@@ -1,12 +1,3 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
 import axios from "axios";
 import { z } from "zod";
 import { zodToJsonSchema } from "zod-to-json-schema";
@@ -18,9 +9,9 @@ export default class FirecrawlApp {
      * Initializes a new instance of the FirecrawlApp class.
      * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
      */
-    constructor({ apiKey = null }) {
-        this.apiUrl = "https://api.firecrawl.dev";
+    constructor({ apiKey = null, apiUrl = null }) {
         this.apiKey = apiKey || "";
+        this.apiUrl = apiUrl || "https://api.firecrawl.dev";
         if (!this.apiKey) {
             throw new Error("No API key provided");
         }
@@ -31,42 +22,46 @@ export default class FirecrawlApp {
      * @param {Params | null} params - Additional parameters for the scrape request.
      * @returns {Promise<ScrapeResponse>} The response from the scrape operation.
      */
-    scrapeUrl(url_1) {
-        return __awaiter(this, arguments, void 0, function* (url, params = null) {
-            var _a;
-            const headers = {
-                "Content-Type": "application/json",
-                Authorization: `Bearer ${this.apiKey}`,
-            };
-            let jsonData = Object.assign({ url }, params);
-            if ((_a = params === null || params === void 0 ? void 0 : params.extractorOptions) === null || _a === void 0 ? void 0 : _a.extractionSchema) {
-                let schema = params.extractorOptions.extractionSchema;
-                // Check if schema is an instance of ZodSchema to correctly identify Zod schemas
-                if (schema instanceof z.ZodSchema) {
-                    schema = zodToJsonSchema(schema);
-                }
-                jsonData = Object.assign(Object.assign({}, jsonData), { extractorOptions: Object.assign(Object.assign({}, params.extractorOptions), { extractionSchema: schema, mode: params.extractorOptions.mode || "llm-extraction" }) });
+    async scrapeUrl(url, params = null) {
+        const headers = {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiKey}`,
+        };
+        let jsonData = { url, ...params };
+        if (params?.extractorOptions?.extractionSchema) {
+            let schema = params.extractorOptions.extractionSchema;
+            // Check if schema is an instance of ZodSchema to correctly identify Zod schemas
+            if (schema instanceof z.ZodSchema) {
+                schema = zodToJsonSchema(schema);
             }
-            try {
-                const response = yield axios.post(this.apiUrl + "/v0/scrape", jsonData, { headers });
-                if (response.status === 200) {
-                    const responseData = response.data;
-                    if (responseData.success) {
-                        return responseData;
-                    }
-                    else {
-                        throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
-                    }
+            jsonData = {
+                ...jsonData,
+                extractorOptions: {
+                    ...params.extractorOptions,
+                    extractionSchema: schema,
+                    mode: params.extractorOptions.mode || "llm-extraction",
+                },
+            };
+        }
+        try {
+            const response = await axios.post(this.apiUrl + "/v0/scrape", jsonData, { headers });
+            if (response.status === 200) {
+                const responseData = response.data;
+                if (responseData.success) {
+                    return responseData;
                 }
                 else {
-                    this.handleError(response, "scrape URL");
+                    throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
                 }
             }
-            catch (error) {
-                throw new Error(error.message);
+            else {
+                this.handleError(response, "scrape URL");
             }
-            return { success: false, error: "Internal server error." };
-        });
+        }
+        catch (error) {
+            throw new Error(error.message);
+        }
+        return { success: false, error: "Internal server error." };
     }
     /**
      * Searches for a query using the Firecrawl API.
@@ -74,36 +69,34 @@ export default class FirecrawlApp {
      * @param {Params | null} params - Additional parameters for the search request.
      * @returns {Promise<SearchResponse>} The response from the search operation.
      */
-    search(query_1) {
-        return __awaiter(this, arguments, void 0, function* (query, params = null) {
-            const headers = {
-                "Content-Type": "application/json",
-                Authorization: `Bearer ${this.apiKey}`,
-            };
-            let jsonData = { query };
-            if (params) {
-                jsonData = Object.assign(Object.assign({}, jsonData), params);
-            }
-            try {
-                const response = yield axios.post(this.apiUrl + "/v0/search", jsonData, { headers });
-                if (response.status === 200) {
-                    const responseData = response.data;
-                    if (responseData.success) {
-                        return responseData;
-                    }
-                    else {
-                        throw new Error(`Failed to search. Error: ${responseData.error}`);
-                    }
+    async search(query, params = null) {
+        const headers = {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiKey}`,
+        };
+        let jsonData = { query };
+        if (params) {
+            jsonData = { ...jsonData, ...params };
+        }
+        try {
+            const response = await axios.post(this.apiUrl + "/v0/search", jsonData, { headers });
+            if (response.status === 200) {
+                const responseData = response.data;
+                if (responseData.success) {
+                    return responseData;
                 }
                 else {
-                    this.handleError(response, "search");
+                    throw new Error(`Failed to search. Error: ${responseData.error}`);
                 }
             }
-            catch (error) {
-                throw new Error(error.message);
+            else {
+                this.handleError(response, "search");
             }
-            return { success: false, error: "Internal server error." };
-        });
+        }
+        catch (error) {
+            throw new Error(error.message);
+        }
+        return { success: false, error: "Internal server error." };
     }
     /**
      * Initiates a crawl job for a URL using the Firecrawl API.
@@ -114,73 +107,75 @@ export default class FirecrawlApp {
      * @param {string} idempotencyKey - Optional idempotency key for the request.
      * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
      */
-    crawlUrl(url_1) {
-        return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, pollInterval = 2, idempotencyKey) {
-            const headers = this.prepareHeaders(idempotencyKey);
-            let jsonData = { url };
-            if (params) {
-                jsonData = Object.assign(Object.assign({}, jsonData), params);
-            }
-            try {
-                const response = yield this.postRequest(this.apiUrl + "/v0/crawl", jsonData, headers);
-                if (response.status === 200) {
-                    const jobId = response.data.jobId;
-                    if (waitUntilDone) {
-                        return this.monitorJobStatus(jobId, headers, pollInterval);
-                    }
-                    else {
-                        return { success: true, jobId };
-                    }
+    async crawlUrl(url, params = null, waitUntilDone = true, pollInterval = 2, idempotencyKey) {
+        const headers = this.prepareHeaders(idempotencyKey);
+        let jsonData = { url };
+        if (params) {
+            jsonData = { ...jsonData, ...params };
+        }
+        try {
+            const response = await this.postRequest(this.apiUrl + "/v0/crawl", jsonData, headers);
+            if (response.status === 200) {
+                const jobId = response.data.jobId;
+                if (waitUntilDone) {
+                    return this.monitorJobStatus(jobId, headers, pollInterval);
                 }
                 else {
-                    this.handleError(response, "start crawl job");
+                    return { success: true, jobId };
                 }
             }
-            catch (error) {
-                console.log(error);
-                throw new Error(error.message);
+            else {
+                this.handleError(response, "start crawl job");
             }
-            return { success: false, error: "Internal server error." };
-        });
+        }
+        catch (error) {
+            console.log(error);
+            throw new Error(error.message);
+        }
+        return { success: false, error: "Internal server error." };
     }
     /**
      * Checks the status of a crawl job using the Firecrawl API.
      * @param {string} jobId - The job ID of the crawl operation.
      * @returns {Promise<JobStatusResponse>} The response containing the job status.
      */
-    checkCrawlStatus(jobId) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const headers = this.prepareHeaders();
-            try {
-                const response = yield this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
-                if (response.status === 200) {
-                    return {
-                        success: true,
-                        status: response.data.status,
-                        data: response.data.data,
-                        partial_data: !response.data.data ? response.data.partial_data : undefined,
-                    };
-                }
-                else {
-                    this.handleError(response, "check crawl status");
-                }
+    async checkCrawlStatus(jobId) {
+        const headers = this.prepareHeaders();
+        try {
+            const response = await this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
+            if (response.status === 200) {
+                return {
+                    success: true,
+                    status: response.data.status,
+                    data: response.data.data,
+                    partial_data: !response.data.data
+                        ? response.data.partial_data
+                        : undefined,
+                };
             }
-            catch (error) {
-                throw new Error(error.message);
+            else {
+                this.handleError(response, "check crawl status");
             }
-            return {
-                success: false,
-                status: "unknown",
-                error: "Internal server error.",
-            };
-        });
+        }
+        catch (error) {
+            throw new Error(error.message);
+        }
+        return {
+            success: false,
+            status: "unknown",
+            error: "Internal server error.",
+        };
     }
     /**
      * Prepares the headers for an API request.
      * @returns {AxiosRequestHeaders} The prepared headers.
      */
     prepareHeaders(idempotencyKey) {
-        return Object.assign({ 'Content-Type': 'application/json', 'Authorization': `Bearer ${this.apiKey}` }, (idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}));
+        return {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiKey}`,
+            ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),
+        };
     }
     /**
      * Sends a POST request to the specified URL.
@@ -208,35 +203,33 @@ export default class FirecrawlApp {
      * @param {number} timeout - Timeout in seconds for job status checks.
      * @returns {Promise<any>} The final job status or data.
      */
-    monitorJobStatus(jobId, headers, checkInterval) {
-        return __awaiter(this, void 0, void 0, function* () {
-            while (true) {
-                const statusResponse = yield this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
-                if (statusResponse.status === 200) {
-                    const statusData = statusResponse.data;
-                    if (statusData.status === "completed") {
-                        if ("data" in statusData) {
-                            return statusData.data;
-                        }
-                        else {
-                            throw new Error("Crawl job completed but no data was returned");
-                        }
-                    }
-                    else if (["active", "paused", "pending", "queued"].includes(statusData.status)) {
-                        if (checkInterval < 2) {
-                            checkInterval = 2;
-                        }
-                        yield new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
+    async monitorJobStatus(jobId, headers, checkInterval) {
+        while (true) {
+            const statusResponse = await this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
+            if (statusResponse.status === 200) {
+                const statusData = statusResponse.data;
+                if (statusData.status === "completed") {
+                    if ("data" in statusData) {
+                        return statusData.data;
                     }
                     else {
-                        throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`);
+                        throw new Error("Crawl job completed but no data was returned");
                     }
                 }
+                else if (["active", "paused", "pending", "queued"].includes(statusData.status)) {
+                    if (checkInterval < 2) {
+                        checkInterval = 2;
+                    }
+                    await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
+                }
                 else {
-                    this.handleError(statusResponse, "check crawl status");
+                    throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`);
                 }
             }
-        });
+            else {
+                this.handleError(statusResponse, "check crawl status");
+            }
+        }
     }
     /**
      * Handles errors from API responses.

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "0.0.26",
+  "version": "0.0.29-beta.1",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "build/index.js",
   "types": "types/index.d.ts",
@@ -33,6 +33,7 @@
     "@types/axios": "^0.14.0",
     "@types/dotenv": "^8.2.0",
     "@types/jest": "^29.5.12",
+    "@types/mocha": "^10.0.6",
     "@types/node": "^20.12.12",
     "@types/uuid": "^9.0.8",
     "jest": "^29.7.0",

package/src/__tests__/e2e_withAuth/index.test.ts

@@ -2,6 +2,7 @@ import FirecrawlApp from '../../index';
 import { v4 as uuidv4 } from 'uuid';
 import dotenv from 'dotenv';
 
+
 dotenv.config();
 
 const TEST_API_KEY = process.env.TEST_API_KEY;
@@ -29,14 +30,14 @@ describe('FirecrawlApp E2E Tests', () => {
     const app = new FirecrawlApp({ apiKey: "this_is_just_a_preview_token", apiUrl: API_URL });
     const response = await app.scrapeUrl('https://roastmywebsite.ai');
     expect(response).not.toBeNull();
-    expect(response.data.content).toContain("_Roast_");
+    expect(response.data?.content).toContain("_Roast_");
   }, 30000); // 30 seconds timeout
 
   test.concurrent('should return successful response for valid scrape', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.scrapeUrl('https://roastmywebsite.ai');
     expect(response).not.toBeNull();
-    expect(response.data.content).toContain("_Roast_");
+    expect(response.data?.content).toContain("_Roast_");
     expect(response.data).toHaveProperty('markdown');
     expect(response.data).toHaveProperty('metadata');
     expect(response.data).not.toHaveProperty('html');
@@ -46,23 +47,23 @@ describe('FirecrawlApp E2E Tests', () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.scrapeUrl('https://roastmywebsite.ai', { pageOptions: { includeHtml: true } });
     expect(response).not.toBeNull();
-    expect(response.data.content).toContain("_Roast_");
-    expect(response.data.markdown).toContain("_Roast_");
-    expect(response.data.html).toContain("<h1");
+    expect(response.data?.content).toContain("_Roast_");
+    expect(response.data?.markdown).toContain("_Roast_");
+    expect(response.data?.html).toContain("<h1");
   }, 30000); // 30 seconds timeout
 
   test.concurrent('should return successful response for valid scrape with PDF file', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001.pdf');
     expect(response).not.toBeNull();
-    expect(response.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
+    expect(response.data?.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
   }, 30000); // 30 seconds timeout
 
   test.concurrent('should return successful response for valid scrape with PDF file without explicit extension', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.scrapeUrl('https://arxiv.org/pdf/astro-ph/9301001');
     expect(response).not.toBeNull();
-    expect(response.data.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
+    expect(response.data?.content).toContain('We present spectrophotometric observations of the Broad Line Radio Galaxy');
   }, 30000); // 30 seconds timeout
 
   test.concurrent('should throw error for invalid API key on crawl', async () => {
@@ -112,15 +113,15 @@ describe('FirecrawlApp E2E Tests', () => {
 
     expect(statusResponse).not.toBeNull();
     expect(statusResponse.status).toBe('completed');
-    expect(statusResponse.data.length).toBeGreaterThan(0);
+    expect(statusResponse?.data?.length).toBeGreaterThan(0);
   }, 35000); // 35 seconds timeout
 
   test.concurrent('should return successful response for search', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.search("test query");
     expect(response).not.toBeNull();
-    expect(response.data[0].content).toBeDefined();
-    expect(response.data.length).toBeGreaterThan(2);
+    expect(response?.data?.[0]?.content).toBeDefined();
+    expect(response?.data?.length).toBeGreaterThan(2);
   }, 30000); // 30 seconds timeout
 
   test.concurrent('should throw error for invalid API key on search', async () => {
@@ -146,10 +147,10 @@ describe('FirecrawlApp E2E Tests', () => {
       }
     });
     expect(response).not.toBeNull();
-    expect(response.data.llm_extraction).toBeDefined();
-    const llmExtraction = response.data.llm_extraction;
-    expect(llmExtraction.company_mission).toBeDefined();
-    expect(typeof llmExtraction.supports_sso).toBe('boolean');
-    expect(typeof llmExtraction.is_open_source).toBe('boolean');
+    expect(response.data?.llm_extraction).toBeDefined();
+    const llmExtraction = response.data?.llm_extraction;
+    expect(llmExtraction?.company_mission).toBeDefined();
+    expect(typeof llmExtraction?.supports_sso).toBe('boolean');
+    expect(typeof llmExtraction?.is_open_source).toBe('boolean');
   }, 30000); // 30 seconds timeout
 });

package/src/__tests__/index.test.ts

@@ -43,6 +43,6 @@ describe('the firecrawl JS SDK', () => {
       expect.objectContaining({ headers: expect.objectContaining({'Authorization': `Bearer ${apiKey}`}) }),
     )
     expect(scrapedData.success).toBe(true);
-    expect(scrapedData.data.metadata.title).toEqual('Mendable');
+    expect(scrapedData?.data?.metadata.title).toEqual('Mendable');
   });
 })

package/src/index.ts

@@ -10,15 +10,63 @@ export interface FirecrawlAppConfig {
 }
 
 /**
- * Generic parameter interface.
+ * Metadata for a Firecrawl document.
  */
-export interface Params {
+export interface FirecrawlDocumentMetadata {
+  title?: string;
+  description?: string;
+  language?: string;
+  keywords?: string;
+  robots?: string;
+  ogTitle?: string;
+  ogDescription?: string;
+  ogUrl?: string;
+  ogImage?: string;
+  ogAudio?: string;
+  ogDeterminer?: string;
+  ogLocale?: string;
+  ogLocaleAlternate?: string[];
+  ogSiteName?: string;
+  ogVideo?: string;
+  dctermsCreated?: string;
+  dcDateCreated?: string;
+  dcDate?: string;
+  dctermsType?: string;
+  dcType?: string;
+  dctermsAudience?: string;
+  dctermsSubject?: string;
+  dcSubject?: string;
+  dcDescription?: string;
+  dctermsKeywords?: string;
+  modifiedTime?: string;
+  publishedTime?: string;
+  articleTag?: string;
+  articleSection?: string;
+  sourceURL?: string;
+  pageStatusCode?: number;
+  pageError?: string;
   [key: string]: any;
-  extractorOptions?: {
-    extractionSchema: z.ZodSchema | any;
-    mode?: "llm-extraction";
-    extractionPrompt?: string;
-  };
+}
+
+/**
+ * Document interface for Firecrawl.
+ */
+export interface FirecrawlDocument {
+  id?: string;
+  url?: string;
+  content: string;
+  markdown?: string;
+  html?: string;
+  llm_extraction?: Record<string, any>;
+  createdAt?: Date;
+  updatedAt?: Date;
+  type?: string;
+  metadata: FirecrawlDocumentMetadata;
+  childrenLinks?: string[];
+  provider?: string;
+  warning?: string;
+
+  index?: number;
 }
 
 /**
@@ -26,16 +74,15 @@ export interface Params {
  */
 export interface ScrapeResponse {
   success: boolean;
-  data?: any;
+  data?: FirecrawlDocument;
   error?: string;
 }
-
 /**
  * Response interface for searching operations.
  */
 export interface SearchResponse {
   success: boolean;
-  data?: any;
+  data?: FirecrawlDocument[];
   error?: string;
 }
 /**
@@ -44,10 +91,9 @@ export interface SearchResponse {
 export interface CrawlResponse {
   success: boolean;
   jobId?: string;
-  data?: any;
+  data?: FirecrawlDocument[];
   error?: string;
 }
-
 /**
  * Response interface for job status checks.
  */
@@ -55,24 +101,35 @@ export interface JobStatusResponse {
   success: boolean;
   status: string;
   jobId?: string;
-  data?: any;
-  partial_data?: any,
+  data?: FirecrawlDocument[];
+  partial_data?: FirecrawlDocument[];
   error?: string;
 }
-
+/**
+ * Generic parameter interface.
+ */
+export interface Params {
+  [key: string]: any;
+  extractorOptions?: {
+    extractionSchema: z.ZodSchema | any;
+    mode?: "llm-extraction";
+    extractionPrompt?: string;
+  };
+}
 /**
  * Main class for interacting with the Firecrawl API.
  */
 export default class FirecrawlApp {
   private apiKey: string;
-  private apiUrl: string = "https://api.firecrawl.dev";
+  private apiUrl: string;
 
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
    */
-  constructor({ apiKey = null }: FirecrawlAppConfig) {
+  constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
     this.apiKey = apiKey || "";
+    this.apiUrl = apiUrl || "https://api.firecrawl.dev";
     if (!this.apiKey) {
       throw new Error("No API key provided");
     }
@@ -112,7 +169,7 @@ export default class FirecrawlApp {
       const response: AxiosResponse = await axios.post(
         this.apiUrl + "/v0/scrape",
         jsonData,
-        { headers },
+        { headers }
       );
       if (response.status === 200) {
         const responseData = response.data;
@@ -231,7 +288,9 @@ export default class FirecrawlApp {
           success: true,
           status: response.data.status,
           data: response.data.data,
-          partial_data: !response.data.data ? response.data.partial_data : undefined,
+          partial_data: !response.data.data
+            ? response.data.partial_data
+            : undefined,
         };
       } else {
         this.handleError(response, "check crawl status");
@@ -252,10 +311,10 @@ export default class FirecrawlApp {
    */
   prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders {
     return {
-      'Content-Type': 'application/json',
-      'Authorization': `Bearer ${this.apiKey}`,
-      ...(idempotencyKey ? { 'x-idempotency-key': idempotencyKey } : {}),
-    } as AxiosRequestHeaders & { 'x-idempotency-key'?: string };
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`,
+      ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),
+    } as AxiosRequestHeaders & { "x-idempotency-key"?: string };
   }
 
   /**
@@ -317,7 +376,9 @@ export default class FirecrawlApp {
           if (checkInterval < 2) {
             checkInterval = 2;
           }
-          await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
+          await new Promise((resolve) =>
+            setTimeout(resolve, checkInterval * 1000)
+          ); // Wait for the specified timeout before checking again
         } else {
           throw new Error(
             `Crawl job failed or was stopped. Status: ${statusData.status}`

package/tsconfig.json

@@ -11,7 +11,6 @@
     // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
 
     /* Language and Environment */
-    "target": "es2016",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
     // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
     // "jsx": "preserve",                                /* Specify what JSX code is generated. */
     // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
@@ -25,9 +24,16 @@
     // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
 
     /* Modules */
-    "module": "NodeNext",                                /* Specify what module code is generated. */
     "rootDir": "./src",                                  /* Specify the root folder within your source files. */
-    "moduleResolution": "nodenext",                     /* Specify how TypeScript looks up a file from a given module specifier. */
+    
+    "target": "ES2021",
+    "lib": [
+      "ES2021",
+      "ES2022.Object",
+      "DOM"
+    ],
+    "module": "NodeNext",
+    "moduleResolution": "nodenext",/* Specify how TypeScript looks up a file from a given module specifier. */
     // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
     // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
     // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */

package/types/index.d.ts

@@ -8,22 +8,68 @@ export interface FirecrawlAppConfig {
     apiUrl?: string | null;
 }
 /**
- * Generic parameter interface.
+ * Metadata for a Firecrawl document.
  */
-export interface Params {
+export interface FirecrawlDocumentMetadata {
+    title?: string;
+    description?: string;
+    language?: string;
+    keywords?: string;
+    robots?: string;
+    ogTitle?: string;
+    ogDescription?: string;
+    ogUrl?: string;
+    ogImage?: string;
+    ogAudio?: string;
+    ogDeterminer?: string;
+    ogLocale?: string;
+    ogLocaleAlternate?: string[];
+    ogSiteName?: string;
+    ogVideo?: string;
+    dctermsCreated?: string;
+    dcDateCreated?: string;
+    dcDate?: string;
+    dctermsType?: string;
+    dcType?: string;
+    dctermsAudience?: string;
+    dctermsSubject?: string;
+    dcSubject?: string;
+    dcDescription?: string;
+    dctermsKeywords?: string;
+    modifiedTime?: string;
+    publishedTime?: string;
+    articleTag?: string;
+    articleSection?: string;
+    sourceURL?: string;
+    pageStatusCode?: number;
+    pageError?: string;
     [key: string]: any;
-    extractorOptions?: {
-        extractionSchema: z.ZodSchema | any;
-        mode?: "llm-extraction";
-        extractionPrompt?: string;
-    };
+}
+/**
+ * Document interface for Firecrawl.
+ */
+export interface FirecrawlDocument {
+    id?: string;
+    url?: string;
+    content: string;
+    markdown?: string;
+    html?: string;
+    llm_extraction?: Record<string, any>;
+    createdAt?: Date;
+    updatedAt?: Date;
+    type?: string;
+    metadata: FirecrawlDocumentMetadata;
+    childrenLinks?: string[];
+    provider?: string;
+    warning?: string;
+    index?: number;
 }
 /**
  * Response interface for scraping operations.
  */
 export interface ScrapeResponse {
     success: boolean;
-    data?: any;
+    data?: FirecrawlDocument;
     error?: string;
 }
 /**
@@ -31,7 +77,7 @@ export interface ScrapeResponse {
  */
 export interface SearchResponse {
     success: boolean;
-    data?: any;
+    data?: FirecrawlDocument[];
     error?: string;
 }
 /**
@@ -40,7 +86,7 @@ export interface SearchResponse {
 export interface CrawlResponse {
     success: boolean;
     jobId?: string;
-    data?: any;
+    data?: FirecrawlDocument[];
     error?: string;
 }
 /**
@@ -50,10 +96,21 @@ export interface JobStatusResponse {
     success: boolean;
     status: string;
     jobId?: string;
-    data?: any;
-    partial_data?: any;
+    data?: FirecrawlDocument[];
+    partial_data?: FirecrawlDocument[];
     error?: string;
 }
+/**
+ * Generic parameter interface.
+ */
+export interface Params {
+    [key: string]: any;
+    extractorOptions?: {
+        extractionSchema: z.ZodSchema | any;
+        mode?: "llm-extraction";
+        extractionPrompt?: string;
+    };
+}
 /**
  * Main class for interacting with the Firecrawl API.
  */
@@ -64,7 +121,7 @@ export default class FirecrawlApp {
      * Initializes a new instance of the FirecrawlApp class.
      * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
      */
-    constructor({ apiKey }: FirecrawlAppConfig);
+    constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
     /**
      * Scrapes a URL using the Firecrawl API.
      * @param {string} url - The URL to scrape.