@mendable/firecrawl-js 0.0.8 → 0.0.10

package/README.md

@@ -19,7 +19,7 @@ npm install @mendable/firecrawl-js
 Here's an example of how to use the SDK with error handling:
 
 ```js
-  import { FirecrawlApp } from '@mendable/firecrawl-js';
+  import FirecrawlApp from '@mendable/firecrawl-js';
 
   async function main() {
     try {
@@ -33,15 +33,18 @@ Here's an example of how to use the SDK with error handling:
       
       // Crawl a website
       const crawlUrl = 'https://mendable.ai';
-      const crawlParams = {
+      const params = {
       crawlerOptions: {
         excludes: ['blog/'],
         includes: [], // leave empty for all pages 
         limit: 1000,
+        },
+        pageOptions: {
+            onlyMainContent: true
         }
       };
 
-      const crawlResult = await app.crawlUrl(crawlUrl, crawlParams);
+      const crawlResult = await app.crawlUrl(crawlUrl, params);
       console.log(crawlResult);
 
     } catch (error) {
@@ -83,18 +86,21 @@ To crawl a website with error handling, use the `crawlUrl` method. It takes the
 async function crawlExample() {
   try {
     const crawlUrl = 'https://example.com';
-    const crawlParams = {
+    const params = {
       crawlerOptions: {
         excludes: ['blog/'],
         includes: [], // leave empty for all pages
         limit: 1000,
+      },
+      pageOptions: {
+        onlyMainContent: true
       }
     };
     const waitUntilDone = true;
     const timeout = 5;
     const crawlResult = await app.crawlUrl(
       crawlUrl,
-      crawlParams,
+      params,
       waitUntilDone,
       timeout
     );

package/build/index.js

@@ -10,13 +10,26 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 import axios from 'axios';
 import dotenv from 'dotenv';
 dotenv.config();
+/**
+ * Main class for interacting with the Firecrawl API.
+ */
 export default class FirecrawlApp {
+    /**
+     * Initializes a new instance of the FirecrawlApp class.
+     * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
+     */
     constructor({ apiKey = null }) {
         this.apiKey = apiKey || process.env.FIRECRAWL_API_KEY || '';
         if (!this.apiKey) {
             throw new Error('No API key provided');
         }
     }
+    /**
+     * Scrapes a URL using the Firecrawl API.
+     * @param {string} url - The URL to scrape.
+     * @param {Params | null} params - Additional parameters for the scrape request.
+     * @returns {Promise<ScrapeResponse>} The response from the scrape operation.
+     */
     scrapeUrl(url_1) {
         return __awaiter(this, arguments, void 0, function* (url, params = null) {
             const headers = {
@@ -32,7 +45,7 @@ export default class FirecrawlApp {
                 if (response.status === 200) {
                     const responseData = response.data;
                     if (responseData.success) {
-                        return responseData.data;
+                        return responseData;
                     }
                     else {
                         throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
@@ -45,8 +58,17 @@ export default class FirecrawlApp {
             catch (error) {
                 throw new Error(error.message);
             }
+            return { success: false, error: 'Internal server error.' };
         });
     }
+    /**
+     * Initiates a crawl job for a URL using the Firecrawl API.
+     * @param {string} url - The URL to crawl.
+     * @param {Params | null} params - Additional parameters for the crawl request.
+     * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
+     * @param {number} timeout - Timeout in seconds for job status checks.
+     * @returns {Promise<CrawlResponse>} The response from the crawl operation.
+     */
     crawlUrl(url_1) {
         return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, timeout = 2) {
             const headers = this.prepareHeaders();
@@ -62,7 +84,7 @@ export default class FirecrawlApp {
                         return this.monitorJobStatus(jobId, headers, timeout);
                     }
                     else {
-                        return { jobId };
+                        return { success: true, jobId };
                     }
                 }
                 else {
@@ -73,8 +95,14 @@ export default class FirecrawlApp {
                 console.log(error);
                 throw new Error(error.message);
             }
+            return { success: false, error: 'Internal server error.' };
         });
     }
+    /**
+     * Checks the status of a crawl job using the Firecrawl API.
+     * @param {string} jobId - The job ID of the crawl operation.
+     * @returns {Promise<JobStatusResponse>} The response containing the job status.
+     */
     checkCrawlStatus(jobId) {
         return __awaiter(this, void 0, void 0, function* () {
             const headers = this.prepareHeaders();
@@ -90,20 +118,45 @@ export default class FirecrawlApp {
             catch (error) {
                 throw new Error(error.message);
             }
+            return { success: false, status: 'unknown', error: 'Internal server error.' };
         });
     }
+    /**
+     * Prepares the headers for an API request.
+     * @returns {AxiosRequestHeaders} The prepared headers.
+     */
     prepareHeaders() {
         return {
             'Content-Type': 'application/json',
             'Authorization': `Bearer ${this.apiKey}`,
         };
     }
+    /**
+     * Sends a POST request to the specified URL.
+     * @param {string} url - The URL to send the request to.
+     * @param {Params} data - The data to send in the request.
+     * @param {AxiosRequestHeaders} headers - The headers for the request.
+     * @returns {Promise<AxiosResponse>} The response from the POST request.
+     */
     postRequest(url, data, headers) {
         return axios.post(url, data, { headers });
     }
+    /**
+     * Sends a GET request to the specified URL.
+     * @param {string} url - The URL to send the request to.
+     * @param {AxiosRequestHeaders} headers - The headers for the request.
+     * @returns {Promise<AxiosResponse>} The response from the GET request.
+     */
     getRequest(url, headers) {
         return axios.get(url, { headers });
     }
+    /**
+     * Monitors the status of a crawl job until completion or failure.
+     * @param {string} jobId - The job ID of the crawl operation.
+     * @param {AxiosRequestHeaders} headers - The headers for the request.
+     * @param {number} timeout - Timeout in seconds for job status checks.
+     * @returns {Promise<any>} The final job status or data.
+     */
     monitorJobStatus(jobId, headers, timeout) {
         return __awaiter(this, void 0, void 0, function* () {
             while (true) {
@@ -134,6 +187,11 @@ export default class FirecrawlApp {
             }
         });
     }
+    /**
+     * Handles errors from API responses.
+     * @param {AxiosResponse} response - The response from the API.
+     * @param {string} action - The action being performed when the error occurred.
+     */
     handleError(response, action) {
         if ([402, 409, 500].includes(response.status)) {
             const errorMessage = response.data.error || 'Unknown error occurred';

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "0.0.8",
+  "version": "0.0.10",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "build/index.js",
   "type": "module",
@@ -9,7 +9,7 @@
   },
   "repository": {
     "type": "git",
-    "url": "git+https://github.com/mendableai/firecrawl-js.git"
+    "url": "git+https://github.com/mendableai/firecrawl.git"
   },
   "author": "Mendable.ai",
   "license": "MIT",
@@ -18,9 +18,9 @@
     "dotenv": "^16.4.5"
   },
   "bugs": {
-    "url": "https://github.com/mendableai/firecrawl-js/issues"
+    "url": "https://github.com/mendableai/firecrawl/issues"
   },
-  "homepage": "https://github.com/mendableai/firecrawl-js#readme",
+  "homepage": "https://github.com/mendableai/firecrawl#readme",
   "devDependencies": {
     "@types/axios": "^0.14.0",
     "@types/dotenv": "^8.2.0",

package/src/index.ts

@@ -2,17 +2,60 @@ import axios, { AxiosResponse, AxiosRequestHeaders } from 'axios';
 import dotenv from 'dotenv';
 dotenv.config();
 
-interface FirecrawlAppConfig {
+/**
+ * Configuration interface for FirecrawlApp.
+ */
+export interface FirecrawlAppConfig {
   apiKey?: string | null;
 }
 
-interface Params {
+/**
+ * Generic parameter interface.
+ */
+export interface Params {
   [key: string]: any;
 }
 
+/**
+ * Response interface for scraping operations.
+ */
+export interface ScrapeResponse {
+  success: boolean;
+  data?: any;
+  error?: string;
+}
+
+/**
+ * Response interface for crawling operations.
+ */
+export interface CrawlResponse {
+  success: boolean;
+  jobId?: string;
+  data?: any;
+  error?: string;
+}
+
+/**
+ * Response interface for job status checks.
+ */
+export interface JobStatusResponse {
+  success: boolean;
+  status: string;
+  jobId?: string;
+  data?: any;
+  error?: string;
+}
+
+/**
+ * Main class for interacting with the Firecrawl API.
+ */
 export default class FirecrawlApp {
   private apiKey: string;
 
+  /**
+   * Initializes a new instance of the FirecrawlApp class.
+   * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
+   */
   constructor({ apiKey = null }: FirecrawlAppConfig) {
     this.apiKey = apiKey || process.env.FIRECRAWL_API_KEY || '';
     if (!this.apiKey) {
@@ -20,7 +63,13 @@ export default class FirecrawlApp {
     }
   }
 
-  async scrapeUrl(url: string, params: Params | null = null): Promise<any> {
+  /**
+   * Scrapes a URL using the Firecrawl API.
+   * @param {string} url - The URL to scrape.
+   * @param {Params | null} params - Additional parameters for the scrape request.
+   * @returns {Promise<ScrapeResponse>} The response from the scrape operation.
+   */
+  async scrapeUrl(url: string, params: Params | null = null): Promise<ScrapeResponse> {
     const headers: AxiosRequestHeaders = {
       'Content-Type': 'application/json',
       'Authorization': `Bearer ${this.apiKey}`,
@@ -34,7 +83,7 @@ export default class FirecrawlApp {
       if (response.status === 200) {
         const responseData = response.data;
         if (responseData.success) {
-          return responseData.data;
+          return responseData; 
         } else {
           throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
         }
@@ -44,9 +93,18 @@ export default class FirecrawlApp {
     } catch (error: any) {
       throw new Error(error.message);
     }
+    return { success: false, error: 'Internal server error.' };
   }
 
-  async crawlUrl(url: string, params: Params | null = null, waitUntilDone: boolean = true, timeout: number = 2): Promise<any> {
+  /**
+   * Initiates a crawl job for a URL using the Firecrawl API.
+   * @param {string} url - The URL to crawl.
+   * @param {Params | null} params - Additional parameters for the crawl request.
+   * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
+   * @param {number} timeout - Timeout in seconds for job status checks.
+   * @returns {Promise<CrawlResponse>} The response from the crawl operation.
+   */
+  async crawlUrl(url: string, params: Params | null = null, waitUntilDone: boolean = true, timeout: number = 2): Promise<CrawlResponse> {
     const headers = this.prepareHeaders();
     let jsonData: Params = { url };
     if (params) {
@@ -59,7 +117,7 @@ export default class FirecrawlApp {
         if (waitUntilDone) {
           return this.monitorJobStatus(jobId, headers, timeout);
         } else {
-          return { jobId };
+          return { success: true, jobId };
         }
       } else {
         this.handleError(response, 'start crawl job');
@@ -68,9 +126,15 @@ export default class FirecrawlApp {
       console.log(error)
       throw new Error(error.message);
     }
+    return { success: false, error: 'Internal server error.' };
   }
 
-  async checkCrawlStatus(jobId: string): Promise<any> {
+  /**
+   * Checks the status of a crawl job using the Firecrawl API.
+   * @param {string} jobId - The job ID of the crawl operation.
+   * @returns {Promise<JobStatusResponse>} The response containing the job status.
+   */
+  async checkCrawlStatus(jobId: string): Promise<JobStatusResponse> {
     const headers: AxiosRequestHeaders = this.prepareHeaders();
     try {
       const response: AxiosResponse = await this.getRequest(`https://api.firecrawl.dev/v0/crawl/status/${jobId}`, headers);
@@ -82,8 +146,13 @@ export default class FirecrawlApp {
     } catch (error: any) {
       throw new Error(error.message);
     }
+    return { success: false, status: 'unknown', error: 'Internal server error.' };
   }
 
+  /**
+   * Prepares the headers for an API request.
+   * @returns {AxiosRequestHeaders} The prepared headers.
+   */
   prepareHeaders(): AxiosRequestHeaders {
     return {
       'Content-Type': 'application/json',
@@ -91,14 +160,34 @@ export default class FirecrawlApp {
     } as AxiosRequestHeaders;
   }
 
+  /**
+   * Sends a POST request to the specified URL.
+   * @param {string} url - The URL to send the request to.
+   * @param {Params} data - The data to send in the request.
+   * @param {AxiosRequestHeaders} headers - The headers for the request.
+   * @returns {Promise<AxiosResponse>} The response from the POST request.
+   */
   postRequest(url: string, data: Params, headers: AxiosRequestHeaders): Promise<AxiosResponse> {
     return axios.post(url, data, { headers });
   }
 
+  /**
+   * Sends a GET request to the specified URL.
+   * @param {string} url - The URL to send the request to.
+   * @param {AxiosRequestHeaders} headers - The headers for the request.
+   * @returns {Promise<AxiosResponse>} The response from the GET request.
+   */
   getRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse> {
     return axios.get(url, { headers });
   }
 
+  /**
+   * Monitors the status of a crawl job until completion or failure.
+   * @param {string} jobId - The job ID of the crawl operation.
+   * @param {AxiosRequestHeaders} headers - The headers for the request.
+   * @param {number} timeout - Timeout in seconds for job status checks.
+   * @returns {Promise<any>} The final job status or data.
+   */
   async monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, timeout: number): Promise<any> {
     while (true) {
       const statusResponse: AxiosResponse = await this.getRequest(`https://api.firecrawl.dev/v0/crawl/status/${jobId}`, headers);
@@ -124,6 +213,11 @@ export default class FirecrawlApp {
     }
   }
 
+  /**
+   * Handles errors from API responses.
+   * @param {AxiosResponse} response - The response from the API.
+   * @param {string} action - The action being performed when the error occurred.
+   */
   handleError(response: AxiosResponse, action: string): void {
     if ([402, 409, 500].includes(response.status)) {
       const errorMessage: string = response.data.error || 'Unknown error occurred';