@mendable/firecrawl-js 0.0.17-beta.4 → 0.0.17-beta.7

package/build/index.js

@@ -7,8 +7,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-import axios from 'axios';
-import { zodToJsonSchema } from 'zod-to-json-schema';
+import axios from "axios";
+import { zodToJsonSchema } from "zod-to-json-schema";
 /**
  * Main class for interacting with the Firecrawl API.
  */
@@ -18,9 +18,9 @@ export default class FirecrawlApp {
      * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
      */
     constructor({ apiKey = null }) {
-        this.apiKey = apiKey || '';
+        this.apiKey = apiKey || "";
         if (!this.apiKey) {
-            throw new Error('No API key provided');
+            throw new Error("No API key provided");
         }
     }
     /**
@@ -30,24 +30,19 @@ export default class FirecrawlApp {
      * @returns {Promise<ScrapeResponse>} The response from the scrape operation.
      */
     scrapeUrl(url_1) {
-        return __awaiter(this, arguments, void 0, function* (url, params = null, extractorOptions) {
+        return __awaiter(this, arguments, void 0, function* (url, params = null) {
+            var _a;
             const headers = {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`,
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${this.apiKey}`,
             };
-            let jsonData = { url };
-            if (params) {
-                jsonData = Object.assign(Object.assign({}, jsonData), params);
-            }
-            if (extractorOptions) {
-                jsonData.extractorOptions = Object.assign({}, extractorOptions);
-            }
-            if (extractorOptions === null || extractorOptions === void 0 ? void 0 : extractorOptions.extractionSchema) {
-                const schema = zodToJsonSchema(extractorOptions.extractionSchema);
-                jsonData = Object.assign(Object.assign({}, jsonData), { extractorOptions: Object.assign(Object.assign({}, extractorOptions), { extractionSchema: schema }) });
+            let jsonData = Object.assign({ url }, params);
+            if ((_a = params === null || params === void 0 ? void 0 : params.extractorOptions) === null || _a === void 0 ? void 0 : _a.extractionSchema) {
+                const schema = zodToJsonSchema(params.extractorOptions.extractionSchema);
+                jsonData = Object.assign(Object.assign({}, jsonData), { extractorOptions: Object.assign(Object.assign({}, params.extractorOptions), { extractionSchema: schema, mode: params.extractorOptions.mode || "llm-extraction" }) });
             }
             try {
-                const response = yield axios.post('https://api.firecrawl.dev/v0/scrape', jsonData, { headers });
+                const response = yield axios.post("https://api.firecrawl.dev/v0/scrape", jsonData, { headers });
                 if (response.status === 200) {
                     const responseData = response.data;
                     if (responseData.success) {
@@ -58,13 +53,13 @@ export default class FirecrawlApp {
                     }
                 }
                 else {
-                    this.handleError(response, 'scrape URL');
+                    this.handleError(response, "scrape URL");
                 }
             }
             catch (error) {
                 throw new Error(error.message);
             }
-            return { success: false, error: 'Internal server error.' };
+            return { success: false, error: "Internal server error." };
         });
     }
     /**
@@ -76,15 +71,15 @@ export default class FirecrawlApp {
     search(query_1) {
         return __awaiter(this, arguments, void 0, function* (query, params = null) {
             const headers = {
-                'Content-Type': 'application/json',
-                'Authorization': `Bearer ${this.apiKey}`,
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${this.apiKey}`,
             };
             let jsonData = { query };
             if (params) {
                 jsonData = Object.assign(Object.assign({}, jsonData), params);
             }
             try {
-                const response = yield axios.post('https://api.firecrawl.dev/v0/search', jsonData, { headers });
+                const response = yield axios.post("https://api.firecrawl.dev/v0/search", jsonData, { headers });
                 if (response.status === 200) {
                     const responseData = response.data;
                     if (responseData.success) {
@@ -95,13 +90,13 @@ export default class FirecrawlApp {
                     }
                 }
                 else {
-                    this.handleError(response, 'search');
+                    this.handleError(response, "search");
                 }
             }
             catch (error) {
                 throw new Error(error.message);
             }
-            return { success: false, error: 'Internal server error.' };
+            return { success: false, error: "Internal server error." };
         });
     }
     /**
@@ -120,7 +115,7 @@ export default class FirecrawlApp {
                 jsonData = Object.assign(Object.assign({}, jsonData), params);
             }
             try {
-                const response = yield this.postRequest('https://api.firecrawl.dev/v0/crawl', jsonData, headers);
+                const response = yield this.postRequest("https://api.firecrawl.dev/v0/crawl", jsonData, headers);
                 if (response.status === 200) {
                     const jobId = response.data.jobId;
                     if (waitUntilDone) {
@@ -131,14 +126,14 @@ export default class FirecrawlApp {
                     }
                 }
                 else {
-                    this.handleError(response, 'start crawl job');
+                    this.handleError(response, "start crawl job");
                 }
             }
             catch (error) {
                 console.log(error);
                 throw new Error(error.message);
             }
-            return { success: false, error: 'Internal server error.' };
+            return { success: false, error: "Internal server error." };
         });
     }
     /**
@@ -155,13 +150,17 @@ export default class FirecrawlApp {
                     return response.data;
                 }
                 else {
-                    this.handleError(response, 'check crawl status');
+                    this.handleError(response, "check crawl status");
                 }
             }
             catch (error) {
                 throw new Error(error.message);
             }
-            return { success: false, status: 'unknown', error: 'Internal server error.' };
+            return {
+                success: false,
+                status: "unknown",
+                error: "Internal server error.",
+            };
         });
     }
     /**
@@ -170,8 +169,8 @@ export default class FirecrawlApp {
      */
     prepareHeaders() {
         return {
-            'Content-Type': 'application/json',
-            'Authorization': `Bearer ${this.apiKey}`,
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiKey}`,
         };
     }
     /**
@@ -206,26 +205,26 @@ export default class FirecrawlApp {
                 const statusResponse = yield this.getRequest(`https://api.firecrawl.dev/v0/crawl/status/${jobId}`, headers);
                 if (statusResponse.status === 200) {
                     const statusData = statusResponse.data;
-                    if (statusData.status === 'completed') {
-                        if ('data' in statusData) {
+                    if (statusData.status === "completed") {
+                        if ("data" in statusData) {
                             return statusData.data;
                         }
                         else {
-                            throw new Error('Crawl job completed but no data was returned');
+                            throw new Error("Crawl job completed but no data was returned");
                         }
                     }
-                    else if (['active', 'paused', 'pending', 'queued'].includes(statusData.status)) {
+                    else if (["active", "paused", "pending", "queued"].includes(statusData.status)) {
                         if (timeout < 2) {
                             timeout = 2;
                         }
-                        yield new Promise(resolve => setTimeout(resolve, timeout * 1000)); // Wait for the specified timeout before checking again
+                        yield new Promise((resolve) => setTimeout(resolve, timeout * 1000)); // Wait for the specified timeout before checking again
                     }
                     else {
                         throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`);
                     }
                 }
                 else {
-                    this.handleError(statusResponse, 'check crawl status');
+                    this.handleError(statusResponse, "check crawl status");
                 }
             }
         });
@@ -237,7 +236,7 @@ export default class FirecrawlApp {
      */
     handleError(response, action) {
         if ([402, 409, 500].includes(response.status)) {
-            const errorMessage = response.data.error || 'Unknown error occurred';
+            const errorMessage = response.data.error || "Unknown error occurred";
             throw new Error(`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`);
         }
         else {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "0.0.17-beta.4",
+  "version": "0.0.17-beta.7",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "build/index.js",
   "types": "types/index.d.ts",

package/src/index.ts

@@ -1,6 +1,6 @@
-import axios, { AxiosResponse, AxiosRequestHeaders } from 'axios';
-import { z } from 'zod';
-import { zodToJsonSchema } from 'zod-to-json-schema'
+import axios, { AxiosResponse, AxiosRequestHeaders } from "axios";
+import { z } from "zod";
+import { zodToJsonSchema } from "zod-to-json-schema";
 /**
  * Configuration interface for FirecrawlApp.
  */
@@ -13,6 +13,11 @@ export interface FirecrawlAppConfig {
  */
 export interface Params {
   [key: string]: any;
+  extractorOptions?: {
+    extractionSchema: z.ZodSchema | any;
+    mode?: "llm-extraction";
+    extractionPrompt?: string;
+  };
 }
 
 /**
@@ -64,9 +69,9 @@ export default class FirecrawlApp {
    * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
    */
   constructor({ apiKey = null }: FirecrawlAppConfig) {
-    this.apiKey = apiKey || '';
+    this.apiKey = apiKey || "";
     if (!this.apiKey) {
-      throw new Error('No API key provided');
+      throw new Error("No API key provided");
     }
   }
 
@@ -76,38 +81,48 @@ export default class FirecrawlApp {
    * @param {Params | null} params - Additional parameters for the scrape request.
    * @returns {Promise<ScrapeResponse>} The response from the scrape operation.
    */
-  async scrapeUrl(url: string, params: Params | null = null, extractorOptions?: { mode: 'llm-extraction'; extractionPrompt?: string; extractionSchema: object }): Promise<ScrapeResponse> {
+  async scrapeUrl(
+    url: string,
+    params: Params | null = null
+  ): Promise<ScrapeResponse> {
     const headers: AxiosRequestHeaders = {
-      'Content-Type': 'application/json',
-      'Authorization': `Bearer ${this.apiKey}`,
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`,
     } as AxiosRequestHeaders;
-    let jsonData: Params & { extractorOptions?: { extractionSchema: z.ZodSchema | any, mode?: 'llm-extraction'; extractionPrompt?: string;  } } = { url };
-    if (params) {
-      jsonData = { ...jsonData, ...params };
-    }
-    if (extractorOptions) {
-      jsonData.extractorOptions = { ...extractorOptions };
-    }
-    if (extractorOptions?.extractionSchema) {
-      const schema = zodToJsonSchema(extractorOptions.extractionSchema as z.ZodSchema);
-      jsonData = { ...jsonData, extractorOptions: { ...extractorOptions, extractionSchema: schema } };
+    let jsonData: Params = { url, ...params };
+    if (params?.extractorOptions?.extractionSchema) {
+      const schema = zodToJsonSchema(
+        params.extractorOptions.extractionSchema as z.ZodSchema
+      );
+      jsonData = {
+        ...jsonData,
+        extractorOptions: {
+          ...params.extractorOptions,
+          extractionSchema: schema,
+          mode: params.extractorOptions.mode || "llm-extraction",
+        },
+      };
     }
     try {
-      const response: AxiosResponse = await axios.post('https://api.firecrawl.dev/v0/scrape', jsonData, { headers });
+      const response: AxiosResponse = await axios.post(
+        "https://api.firecrawl.dev/v0/scrape",
+        jsonData,
+        { headers }
+      );
       if (response.status === 200) {
         const responseData = response.data;
         if (responseData.success) {
-          return responseData; 
+          return responseData;
         } else {
           throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
         }
       } else {
-        this.handleError(response, 'scrape URL');
+        this.handleError(response, "scrape URL");
       }
     } catch (error: any) {
       throw new Error(error.message);
     }
-    return { success: false, error: 'Internal server error.' };
+    return { success: false, error: "Internal server error." };
   }
 
   /**
@@ -116,31 +131,38 @@ export default class FirecrawlApp {
    * @param {Params | null} params - Additional parameters for the search request.
    * @returns {Promise<SearchResponse>} The response from the search operation.
    */
-  async search(query: string, params: Params | null = null): Promise<SearchResponse> {
+  async search(
+    query: string,
+    params: Params | null = null
+  ): Promise<SearchResponse> {
     const headers: AxiosRequestHeaders = {
-      'Content-Type': 'application/json',
-      'Authorization': `Bearer ${this.apiKey}`,
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`,
     } as AxiosRequestHeaders;
     let jsonData: Params = { query };
     if (params) {
       jsonData = { ...jsonData, ...params };
     }
     try {
-      const response: AxiosResponse = await axios.post('https://api.firecrawl.dev/v0/search', jsonData, { headers });
+      const response: AxiosResponse = await axios.post(
+        "https://api.firecrawl.dev/v0/search",
+        jsonData,
+        { headers }
+      );
       if (response.status === 200) {
         const responseData = response.data;
         if (responseData.success) {
-          return responseData; 
+          return responseData;
         } else {
           throw new Error(`Failed to search. Error: ${responseData.error}`);
         }
       } else {
-        this.handleError(response, 'search');
+        this.handleError(response, "search");
       }
     } catch (error: any) {
       throw new Error(error.message);
     }
-    return { success: false, error: 'Internal server error.' };
+    return { success: false, error: "Internal server error." };
   }
 
   /**
@@ -151,14 +173,23 @@ export default class FirecrawlApp {
    * @param {number} timeout - Timeout in seconds for job status checks.
    * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
    */
-  async crawlUrl(url: string, params: Params | null = null, waitUntilDone: boolean = true, timeout: number = 2): Promise<CrawlResponse | any> {
+  async crawlUrl(
+    url: string,
+    params: Params | null = null,
+    waitUntilDone: boolean = true,
+    timeout: number = 2
+  ): Promise<CrawlResponse | any> {
     const headers = this.prepareHeaders();
     let jsonData: Params = { url };
     if (params) {
       jsonData = { ...jsonData, ...params };
     }
     try {
-      const response: AxiosResponse = await this.postRequest('https://api.firecrawl.dev/v0/crawl', jsonData, headers);
+      const response: AxiosResponse = await this.postRequest(
+        "https://api.firecrawl.dev/v0/crawl",
+        jsonData,
+        headers
+      );
       if (response.status === 200) {
         const jobId: string = response.data.jobId;
         if (waitUntilDone) {
@@ -167,13 +198,13 @@ export default class FirecrawlApp {
           return { success: true, jobId };
         }
       } else {
-        this.handleError(response, 'start crawl job');
+        this.handleError(response, "start crawl job");
       }
     } catch (error: any) {
-      console.log(error)
+      console.log(error);
       throw new Error(error.message);
     }
-    return { success: false, error: 'Internal server error.' };
+    return { success: false, error: "Internal server error." };
   }
 
   /**
@@ -184,16 +215,23 @@ export default class FirecrawlApp {
   async checkCrawlStatus(jobId: string): Promise<JobStatusResponse> {
     const headers: AxiosRequestHeaders = this.prepareHeaders();
     try {
-      const response: AxiosResponse = await this.getRequest(`https://api.firecrawl.dev/v0/crawl/status/${jobId}`, headers);
+      const response: AxiosResponse = await this.getRequest(
+        `https://api.firecrawl.dev/v0/crawl/status/${jobId}`,
+        headers
+      );
       if (response.status === 200) {
         return response.data;
       } else {
-        this.handleError(response, 'check crawl status');
+        this.handleError(response, "check crawl status");
       }
     } catch (error: any) {
       throw new Error(error.message);
     }
-    return { success: false, status: 'unknown', error: 'Internal server error.' };
+    return {
+      success: false,
+      status: "unknown",
+      error: "Internal server error.",
+    };
   }
 
   /**
@@ -202,8 +240,8 @@ export default class FirecrawlApp {
    */
   prepareHeaders(): AxiosRequestHeaders {
     return {
-      'Content-Type': 'application/json',
-      'Authorization': `Bearer ${this.apiKey}`,
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`,
     } as AxiosRequestHeaders;
   }
 
@@ -214,7 +252,11 @@ export default class FirecrawlApp {
    * @param {AxiosRequestHeaders} headers - The headers for the request.
    * @returns {Promise<AxiosResponse>} The response from the POST request.
    */
-  postRequest(url: string, data: Params, headers: AxiosRequestHeaders): Promise<AxiosResponse> {
+  postRequest(
+    url: string,
+    data: Params,
+    headers: AxiosRequestHeaders
+  ): Promise<AxiosResponse> {
     return axios.post(url, data, { headers });
   }
 
@@ -224,7 +266,10 @@ export default class FirecrawlApp {
    * @param {AxiosRequestHeaders} headers - The headers for the request.
    * @returns {Promise<AxiosResponse>} The response from the GET request.
    */
-  getRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse> {
+  getRequest(
+    url: string,
+    headers: AxiosRequestHeaders
+  ): Promise<AxiosResponse> {
     return axios.get(url, { headers });
   }
 
@@ -235,27 +280,38 @@ export default class FirecrawlApp {
    * @param {number} timeout - Timeout in seconds for job status checks.
    * @returns {Promise<any>} The final job status or data.
    */
-  async monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, timeout: number): Promise<any> {
+  async monitorJobStatus(
+    jobId: string,
+    headers: AxiosRequestHeaders,
+    timeout: number
+  ): Promise<any> {
     while (true) {
-      const statusResponse: AxiosResponse = await this.getRequest(`https://api.firecrawl.dev/v0/crawl/status/${jobId}`, headers);
+      const statusResponse: AxiosResponse = await this.getRequest(
+        `https://api.firecrawl.dev/v0/crawl/status/${jobId}`,
+        headers
+      );
       if (statusResponse.status === 200) {
         const statusData = statusResponse.data;
-        if (statusData.status === 'completed') {
-          if ('data' in statusData) {
+        if (statusData.status === "completed") {
+          if ("data" in statusData) {
             return statusData.data;
           } else {
-            throw new Error('Crawl job completed but no data was returned');
+            throw new Error("Crawl job completed but no data was returned");
           }
-        } else if (['active', 'paused', 'pending', 'queued'].includes(statusData.status)) {
+        } else if (
+          ["active", "paused", "pending", "queued"].includes(statusData.status)
+        ) {
           if (timeout < 2) {
             timeout = 2;
           }
-          await new Promise(resolve => setTimeout(resolve, timeout * 1000)); // Wait for the specified timeout before checking again
+          await new Promise((resolve) => setTimeout(resolve, timeout * 1000)); // Wait for the specified timeout before checking again
         } else {
-          throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`);
+          throw new Error(
+            `Crawl job failed or was stopped. Status: ${statusData.status}`
+          );
         }
       } else {
-        this.handleError(statusResponse, 'check crawl status');
+        this.handleError(statusResponse, "check crawl status");
       }
     }
   }
@@ -267,10 +323,15 @@ export default class FirecrawlApp {
    */
   handleError(response: AxiosResponse, action: string): void {
     if ([402, 409, 500].includes(response.status)) {
-      const errorMessage: string = response.data.error || 'Unknown error occurred';
-      throw new Error(`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`);
+      const errorMessage: string =
+        response.data.error || "Unknown error occurred";
+      throw new Error(
+        `Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`
+      );
     } else {
-      throw new Error(`Unexpected error occurred while trying to ${action}. Status code: ${response.status}`);
+      throw new Error(
+        `Unexpected error occurred while trying to ${action}. Status code: ${response.status}`
+      );
     }
   }
 }

package/types/index.d.ts

@@ -1,4 +1,5 @@
-import { AxiosResponse, AxiosRequestHeaders } from 'axios';
+import { AxiosResponse, AxiosRequestHeaders } from "axios";
+import { z } from "zod";
 /**
  * Configuration interface for FirecrawlApp.
  */
@@ -10,6 +11,11 @@ export interface FirecrawlAppConfig {
  */
 export interface Params {
     [key: string]: any;
+    extractorOptions?: {
+        extractionSchema: z.ZodSchema | any;
+        mode?: "llm-extraction";
+        extractionPrompt?: string;
+    };
 }
 /**
  * Response interface for scraping operations.
@@ -62,11 +68,7 @@ export default class FirecrawlApp {
      * @param {Params | null} params - Additional parameters for the scrape request.
      * @returns {Promise<ScrapeResponse>} The response from the scrape operation.
      */
-    scrapeUrl(url: string, params?: Params | null, extractorOptions?: {
-        mode: 'llm-extraction';
-        extractionPrompt?: string;
-        extractionSchema: object;
-    }): Promise<ScrapeResponse>;
+    scrapeUrl(url: string, params?: Params | null): Promise<ScrapeResponse>;
     /**
      * Searches for a query using the Firecrawl API.
      * @param {string} query - The query to search for.