firecrawl 1.10.0 → 1.11.0

package/dist/index.cjs

@@ -117,13 +117,73 @@ var FirecrawlApp = class {
     return { success: false, error: "Internal server error." };
   }
   /**
-   * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+   * Searches using the Firecrawl API and optionally scrapes the results.
    * @param query - The search query string.
-   * @param params - Additional parameters for the search.
-   * @returns Throws an error advising to use version 0 of the API.
+   * @param params - Optional parameters for the search request.
+   * @returns The response from the search operation.
    */
   async search(query, params) {
-    throw new FirecrawlError("Search is not supported in v1, please downgrade Firecrawl to 0.0.36.", 400);
+    const headers = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`
+    };
+    let jsonData = {
+      query,
+      limit: params?.limit ?? 5,
+      tbs: params?.tbs,
+      filter: params?.filter,
+      lang: params?.lang ?? "en",
+      country: params?.country ?? "us",
+      location: params?.location,
+      origin: params?.origin ?? "api",
+      timeout: params?.timeout ?? 6e4,
+      scrapeOptions: params?.scrapeOptions ?? { formats: [] }
+    };
+    if (jsonData?.scrapeOptions?.extract?.schema) {
+      let schema = jsonData.scrapeOptions.extract.schema;
+      try {
+        schema = (0, import_zod_to_json_schema.zodToJsonSchema)(schema);
+      } catch (error) {
+      }
+      jsonData = {
+        ...jsonData,
+        scrapeOptions: {
+          ...jsonData.scrapeOptions,
+          extract: {
+            ...jsonData.scrapeOptions.extract,
+            schema
+          }
+        }
+      };
+    }
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/search`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        const responseData = response.data;
+        if (responseData.success) {
+          return {
+            success: true,
+            data: responseData.data,
+            warning: responseData.warning
+          };
+        } else {
+          throw new FirecrawlError(`Failed to search. Error: ${responseData.error}`, response.status);
+        }
+      } else {
+        this.handleError(response, "search");
+      }
+    } catch (error) {
+      if (error.response?.data?.error) {
+        throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
+      } else {
+        throw new FirecrawlError(error.message, 500);
+      }
+    }
+    return { success: false, error: "Internal server error.", data: [] };
   }
   /**
    * Initiates a crawl job for a URL using the Firecrawl API.
@@ -202,7 +262,7 @@ var FirecrawlApp = class {
           let statusData = response.data;
           if ("data" in statusData) {
             let data = statusData.data;
-            while ("next" in statusData) {
+            while (typeof statusData === "object" && "next" in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -397,7 +457,7 @@ var FirecrawlApp = class {
           let statusData = response.data;
           if ("data" in statusData) {
             let data = statusData.data;
-            while ("next" in statusData) {
+            while (typeof statusData === "object" && "next" in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -533,40 +593,44 @@ var FirecrawlApp = class {
    * @returns The final job status or data.
    */
   async monitorJobStatus(id, headers, checkInterval) {
-    while (true) {
-      let statusResponse = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
-        headers
-      );
-      if (statusResponse.status === 200) {
-        let statusData = statusResponse.data;
-        if (statusData.status === "completed") {
-          if ("data" in statusData) {
-            let data = statusData.data;
-            while ("next" in statusData) {
-              statusResponse = await this.getRequest(statusData.next, headers);
-              statusData = statusResponse.data;
-              data = data.concat(statusData.data);
+    try {
+      while (true) {
+        let statusResponse = await this.getRequest(
+          `${this.apiUrl}/v1/crawl/${id}`,
+          headers
+        );
+        if (statusResponse.status === 200) {
+          let statusData = statusResponse.data;
+          if (statusData.status === "completed") {
+            if ("data" in statusData) {
+              let data = statusData.data;
+              while (typeof statusData === "object" && "next" in statusData) {
+                statusResponse = await this.getRequest(statusData.next, headers);
+                statusData = statusResponse.data;
+                data = data.concat(statusData.data);
+              }
+              statusData.data = data;
+              return statusData;
+            } else {
+              throw new FirecrawlError("Crawl job completed but no data was returned", 500);
             }
-            statusData.data = data;
-            return statusData;
+          } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
+            checkInterval = Math.max(checkInterval, 2);
+            await new Promise(
+              (resolve) => setTimeout(resolve, checkInterval * 1e3)
+            );
           } else {
-            throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+            throw new FirecrawlError(
+              `Crawl job failed or was stopped. Status: ${statusData.status}`,
+              500
+            );
           }
-        } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
-          checkInterval = Math.max(checkInterval, 2);
-          await new Promise(
-            (resolve) => setTimeout(resolve, checkInterval * 1e3)
-          );
         } else {
-          throw new FirecrawlError(
-            `Crawl job failed or was stopped. Status: ${statusData.status}`,
-            500
-          );
+          this.handleError(statusResponse, "check crawl status");
         }
-      } else {
-        this.handleError(statusResponse, "check crawl status");
       }
+    } catch (error) {
+      throw new FirecrawlError(error, 500);
     }
   }
   /**

package/dist/index.d.cts

@@ -64,6 +64,8 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
     screenshot?: string;
     metadata?: FirecrawlDocumentMetadata;
     actions: ActionsSchema;
+    title?: string;
+    description?: string;
 }
 /**
  * Parameters for scraping operations.
@@ -258,6 +260,31 @@ declare class FirecrawlError extends Error {
     statusCode: number;
     constructor(message: string, statusCode: number);
 }
+/**
+ * Parameters for search operations.
+ * Defines options for searching and scraping search results.
+ */
+interface SearchParams {
+    limit?: number;
+    tbs?: string;
+    filter?: string;
+    lang?: string;
+    country?: string;
+    location?: string;
+    origin?: string;
+    timeout?: number;
+    scrapeOptions?: ScrapeParams;
+}
+/**
+ * Response interface for search operations.
+ * Defines the structure of the response received after a search operation.
+ */
+interface SearchResponse {
+    success: boolean;
+    data: FirecrawlDocument<undefined>[];
+    warning?: string;
+    error?: string;
+}
 /**
  * Main class for interacting with the Firecrawl API.
  * Provides methods for scraping, searching, crawling, and mapping web content.
@@ -279,12 +306,12 @@ declare class FirecrawlApp {
      */
     scrapeUrl<T extends zt.ZodSchema, ActionsSchema extends (Action[] | undefined) = undefined>(url: string, params?: ScrapeParams<T, ActionsSchema>): Promise<ScrapeResponse<zt.infer<T>, ActionsSchema extends Action[] ? ActionsResult : never> | ErrorResponse>;
     /**
-     * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+     * Searches using the Firecrawl API and optionally scrapes the results.
      * @param query - The search query string.
-     * @param params - Additional parameters for the search.
-     * @returns Throws an error advising to use version 0 of the API.
+     * @param params - Optional parameters for the search request.
+     * @returns The response from the search operation.
      */
-    search(query: string, params?: any): Promise<any>;
+    search(query: string, params?: SearchParams | Record<string, any>): Promise<SearchResponse>;
     /**
      * Initiates a crawl job for a URL using the Firecrawl API.
      * @param url - The URL to crawl.
@@ -422,4 +449,4 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     close(): void;
 }
 
-export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, FirecrawlApp as default };
+export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, type SearchParams, type SearchResponse, FirecrawlApp as default };

package/dist/index.d.ts

@@ -64,6 +64,8 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
     screenshot?: string;
     metadata?: FirecrawlDocumentMetadata;
     actions: ActionsSchema;
+    title?: string;
+    description?: string;
 }
 /**
  * Parameters for scraping operations.
@@ -258,6 +260,31 @@ declare class FirecrawlError extends Error {
     statusCode: number;
     constructor(message: string, statusCode: number);
 }
+/**
+ * Parameters for search operations.
+ * Defines options for searching and scraping search results.
+ */
+interface SearchParams {
+    limit?: number;
+    tbs?: string;
+    filter?: string;
+    lang?: string;
+    country?: string;
+    location?: string;
+    origin?: string;
+    timeout?: number;
+    scrapeOptions?: ScrapeParams;
+}
+/**
+ * Response interface for search operations.
+ * Defines the structure of the response received after a search operation.
+ */
+interface SearchResponse {
+    success: boolean;
+    data: FirecrawlDocument<undefined>[];
+    warning?: string;
+    error?: string;
+}
 /**
  * Main class for interacting with the Firecrawl API.
  * Provides methods for scraping, searching, crawling, and mapping web content.
@@ -279,12 +306,12 @@ declare class FirecrawlApp {
      */
     scrapeUrl<T extends zt.ZodSchema, ActionsSchema extends (Action[] | undefined) = undefined>(url: string, params?: ScrapeParams<T, ActionsSchema>): Promise<ScrapeResponse<zt.infer<T>, ActionsSchema extends Action[] ? ActionsResult : never> | ErrorResponse>;
     /**
-     * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+     * Searches using the Firecrawl API and optionally scrapes the results.
      * @param query - The search query string.
-     * @param params - Additional parameters for the search.
-     * @returns Throws an error advising to use version 0 of the API.
+     * @param params - Optional parameters for the search request.
+     * @returns The response from the search operation.
      */
-    search(query: string, params?: any): Promise<any>;
+    search(query: string, params?: SearchParams | Record<string, any>): Promise<SearchResponse>;
     /**
      * Initiates a crawl job for a URL using the Firecrawl API.
      * @param url - The URL to crawl.
@@ -422,4 +449,4 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     close(): void;
 }
 
-export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, FirecrawlApp as default };
+export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, type SearchParams, type SearchResponse, FirecrawlApp as default };

package/dist/index.js

@@ -81,13 +81,73 @@ var FirecrawlApp = class {
     return { success: false, error: "Internal server error." };
   }
   /**
-   * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+   * Searches using the Firecrawl API and optionally scrapes the results.
    * @param query - The search query string.
-   * @param params - Additional parameters for the search.
-   * @returns Throws an error advising to use version 0 of the API.
+   * @param params - Optional parameters for the search request.
+   * @returns The response from the search operation.
    */
   async search(query, params) {
-    throw new FirecrawlError("Search is not supported in v1, please downgrade Firecrawl to 0.0.36.", 400);
+    const headers = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`
+    };
+    let jsonData = {
+      query,
+      limit: params?.limit ?? 5,
+      tbs: params?.tbs,
+      filter: params?.filter,
+      lang: params?.lang ?? "en",
+      country: params?.country ?? "us",
+      location: params?.location,
+      origin: params?.origin ?? "api",
+      timeout: params?.timeout ?? 6e4,
+      scrapeOptions: params?.scrapeOptions ?? { formats: [] }
+    };
+    if (jsonData?.scrapeOptions?.extract?.schema) {
+      let schema = jsonData.scrapeOptions.extract.schema;
+      try {
+        schema = zodToJsonSchema(schema);
+      } catch (error) {
+      }
+      jsonData = {
+        ...jsonData,
+        scrapeOptions: {
+          ...jsonData.scrapeOptions,
+          extract: {
+            ...jsonData.scrapeOptions.extract,
+            schema
+          }
+        }
+      };
+    }
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/search`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        const responseData = response.data;
+        if (responseData.success) {
+          return {
+            success: true,
+            data: responseData.data,
+            warning: responseData.warning
+          };
+        } else {
+          throw new FirecrawlError(`Failed to search. Error: ${responseData.error}`, response.status);
+        }
+      } else {
+        this.handleError(response, "search");
+      }
+    } catch (error) {
+      if (error.response?.data?.error) {
+        throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
+      } else {
+        throw new FirecrawlError(error.message, 500);
+      }
+    }
+    return { success: false, error: "Internal server error.", data: [] };
   }
   /**
    * Initiates a crawl job for a URL using the Firecrawl API.
@@ -166,7 +226,7 @@ var FirecrawlApp = class {
           let statusData = response.data;
           if ("data" in statusData) {
             let data = statusData.data;
-            while ("next" in statusData) {
+            while (typeof statusData === "object" && "next" in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -361,7 +421,7 @@ var FirecrawlApp = class {
           let statusData = response.data;
           if ("data" in statusData) {
             let data = statusData.data;
-            while ("next" in statusData) {
+            while (typeof statusData === "object" && "next" in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -497,40 +557,44 @@ var FirecrawlApp = class {
    * @returns The final job status or data.
    */
   async monitorJobStatus(id, headers, checkInterval) {
-    while (true) {
-      let statusResponse = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
-        headers
-      );
-      if (statusResponse.status === 200) {
-        let statusData = statusResponse.data;
-        if (statusData.status === "completed") {
-          if ("data" in statusData) {
-            let data = statusData.data;
-            while ("next" in statusData) {
-              statusResponse = await this.getRequest(statusData.next, headers);
-              statusData = statusResponse.data;
-              data = data.concat(statusData.data);
+    try {
+      while (true) {
+        let statusResponse = await this.getRequest(
+          `${this.apiUrl}/v1/crawl/${id}`,
+          headers
+        );
+        if (statusResponse.status === 200) {
+          let statusData = statusResponse.data;
+          if (statusData.status === "completed") {
+            if ("data" in statusData) {
+              let data = statusData.data;
+              while (typeof statusData === "object" && "next" in statusData) {
+                statusResponse = await this.getRequest(statusData.next, headers);
+                statusData = statusResponse.data;
+                data = data.concat(statusData.data);
+              }
+              statusData.data = data;
+              return statusData;
+            } else {
+              throw new FirecrawlError("Crawl job completed but no data was returned", 500);
             }
-            statusData.data = data;
-            return statusData;
+          } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
+            checkInterval = Math.max(checkInterval, 2);
+            await new Promise(
+              (resolve) => setTimeout(resolve, checkInterval * 1e3)
+            );
           } else {
-            throw new FirecrawlError("Crawl job completed but no data was returned", 500);
+            throw new FirecrawlError(
+              `Crawl job failed or was stopped. Status: ${statusData.status}`,
+              500
+            );
           }
-        } else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
-          checkInterval = Math.max(checkInterval, 2);
-          await new Promise(
-            (resolve) => setTimeout(resolve, checkInterval * 1e3)
-          );
         } else {
-          throw new FirecrawlError(
-            `Crawl job failed or was stopped. Status: ${statusData.status}`,
-            500
-          );
+          this.handleError(statusResponse, "check crawl status");
         }
-      } else {
-        this.handleError(statusResponse, "check crawl status");
       }
+    } catch (error) {
+      throw new FirecrawlError(error, 500);
     }
   }
   /**

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "1.10.0",
+  "version": "1.11.0",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/__tests__/v1/e2e_withAuth/index.test.ts

@@ -381,8 +381,45 @@ describe('FirecrawlApp E2E Tests', () => {
     expect(filteredLinks?.length).toBeGreaterThan(0);
   }, 30000); // 30 seconds timeout
 
-  test('should throw NotImplementedError for search on v1', async () => {
+  
+
+  test('should search with string query', async () => {
     const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: TEST_API_KEY });
-    await expect(app.search("test query")).rejects.toThrow("Search is not supported in v1");
+    const response = await app.search("firecrawl");
+    expect(response.success).toBe(true);
+    console.log(response.data);
+    expect(response.data?.length).toBeGreaterThan(0);
+    expect(response.data?.[0]?.markdown).toBeDefined();
+    expect(response.data?.[0]?.metadata).toBeDefined();
+    expect(response.data?.[0]?.metadata?.title).toBeDefined();
+    expect(response.data?.[0]?.metadata?.description).toBeDefined();
+  });
+
+  test('should search with params object', async () => {
+    const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: TEST_API_KEY });
+    const response = await app.search("firecrawl", {
+      limit: 3,
+      lang: 'en',
+      country: 'us',
+      scrapeOptions: {
+        formats: ['markdown', 'html', 'links'],
+        onlyMainContent: true
+      }
+    });
+    expect(response.success).toBe(true);
+    expect(response.data.length).toBeLessThanOrEqual(3);
+    for (const doc of response.data) {
+      expect(doc.markdown).toBeDefined();
+      expect(doc.html).toBeDefined();
+      expect(doc.links).toBeDefined();
+      expect(doc.metadata).toBeDefined();
+      expect(doc.metadata?.title).toBeDefined();
+      expect(doc.metadata?.description).toBeDefined();
+    }
+  });
+
+  test('should handle invalid API key for search', async () => {
+    const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: "invalid_api_key" });
+    await expect(app.search("test query")).rejects.toThrow("Request failed with status code 404");
   });
 });

package/src/index.ts

@@ -68,6 +68,9 @@ export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult
   screenshot?: string;
   metadata?: FirecrawlDocumentMetadata;
   actions: ActionsSchema;
+  // v1 search only
+  title?: string;
+  description?: string;
 }
 
 /**
@@ -282,6 +285,33 @@ export class FirecrawlError extends Error {
   }
 }
 
+/**
+ * Parameters for search operations.
+ * Defines options for searching and scraping search results.
+ */
+export interface SearchParams {
+  limit?: number;
+  tbs?: string;
+  filter?: string;
+  lang?: string;
+  country?: string;
+  location?: string;
+  origin?: string;
+  timeout?: number;
+  scrapeOptions?: ScrapeParams;
+}
+
+/**
+ * Response interface for search operations.
+ * Defines the structure of the response received after a search operation.
+ */
+export interface SearchResponse {
+  success: boolean;
+  data: FirecrawlDocument<undefined>[];
+  warning?: string;
+  error?: string;
+}
+
 /**
  * Main class for interacting with the Firecrawl API.
  * Provides methods for scraping, searching, crawling, and mapping web content.
@@ -369,16 +399,80 @@ export default class FirecrawlApp {
   }
 
   /**
-   * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+   * Searches using the Firecrawl API and optionally scrapes the results.
    * @param query - The search query string.
-   * @param params - Additional parameters for the search.
-   * @returns Throws an error advising to use version 0 of the API.
+   * @param params - Optional parameters for the search request.
+   * @returns The response from the search operation.
    */
-  async search(
-    query: string,
-    params?: any
-  ): Promise<any> {
-    throw new FirecrawlError("Search is not supported in v1, please downgrade Firecrawl to 0.0.36.", 400);
+  async search(query: string, params?: SearchParams | Record<string, any>): Promise<SearchResponse> {
+    const headers: AxiosRequestHeaders = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`,
+    } as AxiosRequestHeaders;
+
+    let jsonData: any = {
+      query,
+      limit: params?.limit ?? 5,
+      tbs: params?.tbs,
+      filter: params?.filter,
+      lang: params?.lang ?? "en",
+      country: params?.country ?? "us",
+      location: params?.location,
+      origin: params?.origin ?? "api",
+      timeout: params?.timeout ?? 60000,
+      scrapeOptions: params?.scrapeOptions ?? { formats: [] },
+    };
+
+    if (jsonData?.scrapeOptions?.extract?.schema) {
+      let schema = jsonData.scrapeOptions.extract.schema;
+
+      // Try parsing the schema as a Zod schema
+      try {
+        schema = zodToJsonSchema(schema);
+      } catch (error) {
+        
+      }
+      jsonData = {
+        ...jsonData,
+        scrapeOptions: {
+          ...jsonData.scrapeOptions,
+          extract: {
+            ...jsonData.scrapeOptions.extract,
+            schema: schema,
+          },
+        },
+      };
+    }
+
+    try {
+      const response: AxiosResponse = await this.postRequest(
+        this.apiUrl + `/v1/search`,
+        jsonData,
+        headers
+      );
+
+      if (response.status === 200) {
+        const responseData = response.data;
+        if (responseData.success) {
+          return {
+            success: true,
+            data: responseData.data as FirecrawlDocument<any>[],
+            warning: responseData.warning,
+          };
+        } else {
+          throw new FirecrawlError(`Failed to search. Error: ${responseData.error}`, response.status);
+        }
+      } else {
+        this.handleError(response, "search");
+      }
+    } catch (error: any) {
+      if (error.response?.data?.error) {
+        throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+      } else {
+        throw new FirecrawlError(error.message, 500);
+      }
+    }
+    return { success: false, error: "Internal server error.", data: [] };
   }
 
   /**
@@ -470,7 +564,7 @@ export default class FirecrawlApp {
           let statusData = response.data
           if ("data" in statusData) {
             let data = statusData.data;
-            while ('next' in statusData) {
+            while (typeof statusData === 'object' && 'next' in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -704,7 +798,7 @@ export default class FirecrawlApp {
           let statusData = response.data
           if ("data" in statusData) {
             let data = statusData.data;
-            while ('next' in statusData) {
+            while (typeof statusData === 'object' && 'next' in statusData) {
               statusData = (await this.getRequest(statusData.next, headers)).data;
               data = data.concat(statusData.data);
             }
@@ -863,42 +957,46 @@ export default class FirecrawlApp {
     headers: AxiosRequestHeaders,
     checkInterval: number
   ): Promise<CrawlStatusResponse | ErrorResponse> {
-    while (true) {
-      let statusResponse: AxiosResponse = await this.getRequest(
-        `${this.apiUrl}/v1/crawl/${id}`,
-        headers
-      );
-      if (statusResponse.status === 200) {
-        let statusData = statusResponse.data;
-          if (statusData.status === "completed") {
-            if ("data" in statusData) {
-              let data = statusData.data;
-              while ('next' in statusData) {
-                statusResponse = await this.getRequest(statusData.next, headers);
-                statusData = statusResponse.data;
-                data = data.concat(statusData.data);
+    try {
+      while (true) {
+        let statusResponse: AxiosResponse = await this.getRequest(
+          `${this.apiUrl}/v1/crawl/${id}`,
+          headers
+        );
+        if (statusResponse.status === 200) {
+          let statusData = statusResponse.data;
+            if (statusData.status === "completed") {
+              if ("data" in statusData) {
+                let data = statusData.data;
+                while (typeof statusData === 'object' && 'next' in statusData) {
+                  statusResponse = await this.getRequest(statusData.next, headers);
+                  statusData = statusResponse.data;
+                  data = data.concat(statusData.data);
+                }
+                statusData.data = data;
+                return statusData;
+              } else {
+                throw new FirecrawlError("Crawl job completed but no data was returned", 500);
               }
-              statusData.data = data;
-              return statusData;
-            } else {
-              throw new FirecrawlError("Crawl job completed but no data was returned", 500);
-            }
-          } else if (
-          ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
-        ) {
-          checkInterval = Math.max(checkInterval, 2);
-          await new Promise((resolve) =>
-            setTimeout(resolve, checkInterval * 1000)
-          );
+            } else if (
+            ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
+          ) {
+            checkInterval = Math.max(checkInterval, 2);
+            await new Promise((resolve) =>
+              setTimeout(resolve, checkInterval * 1000)
+            );
+          } else {
+            throw new FirecrawlError(
+              `Crawl job failed or was stopped. Status: ${statusData.status}`,
+              500
+            );
+          }
         } else {
-          throw new FirecrawlError(
-            `Crawl job failed or was stopped. Status: ${statusData.status}`,
-            500
-          );
+          this.handleError(statusResponse, "check crawl status");
         }
-      } else {
-        this.handleError(statusResponse, "check crawl status");
       }
+    } catch (error: any) {
+      throw new FirecrawlError(error, 500);
     }
   }