firecrawl 1.10.1 → 1.11.0

package/dist/index.cjs

@@ -49,16 +49,20 @@ var FirecrawlError = class extends Error {
 var FirecrawlApp = class {
   apiKey;
   apiUrl;
+  isCloudService(url) {
+    return url.includes("api.firecrawl.dev");
+  }
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
    */
   constructor({ apiKey = null, apiUrl = null }) {
-    if (typeof apiKey !== "string") {
+    const baseUrl = apiUrl || "https://api.firecrawl.dev";
+    if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
       throw new FirecrawlError("No API key provided", 401);
     }
-    this.apiKey = apiKey;
-    this.apiUrl = apiUrl || "https://api.firecrawl.dev";
+    this.apiKey = apiKey || "";
+    this.apiUrl = baseUrl;
   }
   /**
    * Scrapes a URL using the Firecrawl API.
@@ -113,13 +117,73 @@ var FirecrawlApp = class {
     return { success: false, error: "Internal server error." };
   }
   /**
-   * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+   * Searches using the Firecrawl API and optionally scrapes the results.
    * @param query - The search query string.
-   * @param params - Additional parameters for the search.
-   * @returns Throws an error advising to use version 0 of the API.
+   * @param params - Optional parameters for the search request.
+   * @returns The response from the search operation.
    */
   async search(query, params) {
-    throw new FirecrawlError("Search is not supported in v1, please downgrade Firecrawl to 0.0.36.", 400);
+    const headers = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`
+    };
+    let jsonData = {
+      query,
+      limit: params?.limit ?? 5,
+      tbs: params?.tbs,
+      filter: params?.filter,
+      lang: params?.lang ?? "en",
+      country: params?.country ?? "us",
+      location: params?.location,
+      origin: params?.origin ?? "api",
+      timeout: params?.timeout ?? 6e4,
+      scrapeOptions: params?.scrapeOptions ?? { formats: [] }
+    };
+    if (jsonData?.scrapeOptions?.extract?.schema) {
+      let schema = jsonData.scrapeOptions.extract.schema;
+      try {
+        schema = (0, import_zod_to_json_schema.zodToJsonSchema)(schema);
+      } catch (error) {
+      }
+      jsonData = {
+        ...jsonData,
+        scrapeOptions: {
+          ...jsonData.scrapeOptions,
+          extract: {
+            ...jsonData.scrapeOptions.extract,
+            schema
+          }
+        }
+      };
+    }
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/search`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        const responseData = response.data;
+        if (responseData.success) {
+          return {
+            success: true,
+            data: responseData.data,
+            warning: responseData.warning
+          };
+        } else {
+          throw new FirecrawlError(`Failed to search. Error: ${responseData.error}`, response.status);
+        }
+      } else {
+        this.handleError(response, "search");
+      }
+    } catch (error) {
+      if (error.response?.data?.error) {
+        throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
+      } else {
+        throw new FirecrawlError(error.message, 500);
+      }
+    }
+    return { success: false, error: "Internal server error.", data: [] };
   }
   /**
    * Initiates a crawl job for a URL using the Firecrawl API.
@@ -295,9 +359,9 @@ var FirecrawlApp = class {
    * @param webhook - Optional webhook for the batch scrape.
    * @returns The response from the crawl operation.
    */
-  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook) {
+  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, ...params };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -333,9 +397,9 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
-  async asyncBatchScrapeUrls(urls, params, idempotencyKey) {
+  async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, ...params ?? {} };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params ?? {} };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -363,8 +427,8 @@ var FirecrawlApp = class {
    * @param idempotencyKey - Optional idempotency key for the request.
    * @returns A CrawlWatcher instance to monitor the crawl job.
    */
-  async batchScrapeUrlsAndWatch(urls, params, idempotencyKey) {
-    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey);
+  async batchScrapeUrlsAndWatch(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
+    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);
     if (crawl.success && crawl.id) {
       const id = crawl.id;
       return new CrawlWatcher(id, this);
@@ -593,8 +657,10 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
   ws;
   data;
   status;
+  id;
   constructor(id, app) {
     super();
+    this.id = id;
     this.ws = new import_isows.WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
     this.status = "scraping";
     this.data = [];
@@ -604,7 +670,8 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
         this.dispatchTypedEvent("done", new CustomEvent("done", {
           detail: {
             status: this.status,
-            data: this.data
+            data: this.data,
+            id: this.id
           }
         }));
       } else if (msg.type === "error") {
@@ -613,7 +680,8 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
           detail: {
             status: this.status,
             data: this.data,
-            error: msg.error
+            error: msg.error,
+            id: this.id
           }
         }));
       } else if (msg.type === "catchup") {
@@ -621,12 +689,18 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
         this.data.push(...msg.data.data ?? []);
         for (const doc of this.data) {
           this.dispatchTypedEvent("document", new CustomEvent("document", {
-            detail: doc
+            detail: {
+              ...doc,
+              id: this.id
+            }
           }));
         }
       } else if (msg.type === "document") {
         this.dispatchTypedEvent("document", new CustomEvent("document", {
-          detail: msg.data
+          detail: {
+            ...msg.data,
+            id: this.id
+          }
         }));
       }
     };
@@ -635,12 +709,20 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
         this.ws.close();
         return;
       }
-      const msg = JSON.parse(ev.data);
-      messageHandler(msg);
+      try {
+        const msg = JSON.parse(ev.data);
+        messageHandler(msg);
+      } catch (error) {
+        console.error("Error on message", error);
+      }
     }).bind(this);
     this.ws.onclose = ((ev) => {
-      const msg = JSON.parse(ev.reason);
-      messageHandler(msg);
+      try {
+        const msg = JSON.parse(ev.reason);
+        messageHandler(msg);
+      } catch (error) {
+        console.error("Error on close", error);
+      }
     }).bind(this);
     this.ws.onerror = ((_) => {
       this.status = "failed";
@@ -648,7 +730,8 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
         detail: {
           status: this.status,
           data: this.data,
-          error: "WebSocket error"
+          error: "WebSocket error",
+          id: this.id
         }
       }));
     }).bind(this);

package/dist/index.d.cts

@@ -64,6 +64,8 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
     screenshot?: string;
     metadata?: FirecrawlDocumentMetadata;
     actions: ActionsSchema;
+    title?: string;
+    description?: string;
 }
 /**
  * Parameters for scraping operations.
@@ -171,6 +173,7 @@ interface BatchScrapeResponse {
     url?: string;
     success: true;
     error?: string;
+    invalidURLs?: string[];
 }
 /**
  * Response interface for job status checks.
@@ -225,10 +228,11 @@ interface MapResponse {
  * Defines options for extracting information from URLs.
  */
 interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
-    prompt: string;
+    prompt?: string;
     schema?: LLMSchema;
     systemPrompt?: string;
     allowExternalLinks?: boolean;
+    includeSubdomains?: boolean;
 }
 /**
  * Response interface for extracting information from URLs.
@@ -256,6 +260,31 @@ declare class FirecrawlError extends Error {
     statusCode: number;
     constructor(message: string, statusCode: number);
 }
+/**
+ * Parameters for search operations.
+ * Defines options for searching and scraping search results.
+ */
+interface SearchParams {
+    limit?: number;
+    tbs?: string;
+    filter?: string;
+    lang?: string;
+    country?: string;
+    location?: string;
+    origin?: string;
+    timeout?: number;
+    scrapeOptions?: ScrapeParams;
+}
+/**
+ * Response interface for search operations.
+ * Defines the structure of the response received after a search operation.
+ */
+interface SearchResponse {
+    success: boolean;
+    data: FirecrawlDocument<undefined>[];
+    warning?: string;
+    error?: string;
+}
 /**
  * Main class for interacting with the Firecrawl API.
  * Provides methods for scraping, searching, crawling, and mapping web content.
@@ -263,6 +292,7 @@ declare class FirecrawlError extends Error {
 declare class FirecrawlApp {
     apiKey: string;
     apiUrl: string;
+    private isCloudService;
     /**
      * Initializes a new instance of the FirecrawlApp class.
      * @param config - Configuration options for the FirecrawlApp instance.
@@ -276,12 +306,12 @@ declare class FirecrawlApp {
      */
     scrapeUrl<T extends zt.ZodSchema, ActionsSchema extends (Action[] | undefined) = undefined>(url: string, params?: ScrapeParams<T, ActionsSchema>): Promise<ScrapeResponse<zt.infer<T>, ActionsSchema extends Action[] ? ActionsResult : never> | ErrorResponse>;
     /**
-     * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+     * Searches using the Firecrawl API and optionally scrapes the results.
      * @param query - The search query string.
-     * @param params - Additional parameters for the search.
-     * @returns Throws an error advising to use version 0 of the API.
+     * @param params - Optional parameters for the search request.
+     * @returns The response from the search operation.
      */
-    search(query: string, params?: any): Promise<any>;
+    search(query: string, params?: SearchParams | Record<string, any>): Promise<SearchResponse>;
     /**
      * Initiates a crawl job for a URL using the Firecrawl API.
      * @param url - The URL to crawl.
@@ -329,8 +359,8 @@ declare class FirecrawlApp {
      * @param webhook - Optional webhook for the batch scrape.
      * @returns The response from the crawl operation.
      */
-    batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"]): Promise<BatchScrapeStatusResponse | ErrorResponse>;
-    asyncBatchScrapeUrls(urls: string[], params?: ScrapeParams, idempotencyKey?: string): Promise<BatchScrapeResponse | ErrorResponse>;
+    batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;
+    asyncBatchScrapeUrls(urls: string[], params?: ScrapeParams, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeResponse | ErrorResponse>;
     /**
      * Initiates a batch scrape job and returns a CrawlWatcher to monitor the job via WebSocket.
      * @param urls - The URL to scrape.
@@ -338,7 +368,7 @@ declare class FirecrawlApp {
      * @param idempotencyKey - Optional idempotency key for the request.
      * @returns A CrawlWatcher instance to monitor the crawl job.
      */
-    batchScrapeUrlsAndWatch(urls: string[], params?: ScrapeParams, idempotencyKey?: string): Promise<CrawlWatcher>;
+    batchScrapeUrlsAndWatch(urls: string[], params?: ScrapeParams, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<CrawlWatcher>;
     /**
      * Checks the status of a batch scrape job using the Firecrawl API.
      * @param id - The ID of the batch scrape operation.
@@ -414,8 +444,9 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     private ws;
     data: FirecrawlDocument<undefined>[];
     status: CrawlStatusResponse["status"];
+    id: string;
     constructor(id: string, app: FirecrawlApp);
     close(): void;
 }
 
-export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, FirecrawlApp as default };
+export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, type SearchParams, type SearchResponse, FirecrawlApp as default };

package/dist/index.d.ts

@@ -64,6 +64,8 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
     screenshot?: string;
     metadata?: FirecrawlDocumentMetadata;
     actions: ActionsSchema;
+    title?: string;
+    description?: string;
 }
 /**
  * Parameters for scraping operations.
@@ -171,6 +173,7 @@ interface BatchScrapeResponse {
     url?: string;
     success: true;
     error?: string;
+    invalidURLs?: string[];
 }
 /**
  * Response interface for job status checks.
@@ -225,10 +228,11 @@ interface MapResponse {
  * Defines options for extracting information from URLs.
  */
 interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
-    prompt: string;
+    prompt?: string;
     schema?: LLMSchema;
     systemPrompt?: string;
     allowExternalLinks?: boolean;
+    includeSubdomains?: boolean;
 }
 /**
  * Response interface for extracting information from URLs.
@@ -256,6 +260,31 @@ declare class FirecrawlError extends Error {
     statusCode: number;
     constructor(message: string, statusCode: number);
 }
+/**
+ * Parameters for search operations.
+ * Defines options for searching and scraping search results.
+ */
+interface SearchParams {
+    limit?: number;
+    tbs?: string;
+    filter?: string;
+    lang?: string;
+    country?: string;
+    location?: string;
+    origin?: string;
+    timeout?: number;
+    scrapeOptions?: ScrapeParams;
+}
+/**
+ * Response interface for search operations.
+ * Defines the structure of the response received after a search operation.
+ */
+interface SearchResponse {
+    success: boolean;
+    data: FirecrawlDocument<undefined>[];
+    warning?: string;
+    error?: string;
+}
 /**
  * Main class for interacting with the Firecrawl API.
  * Provides methods for scraping, searching, crawling, and mapping web content.
@@ -263,6 +292,7 @@ declare class FirecrawlError extends Error {
 declare class FirecrawlApp {
     apiKey: string;
     apiUrl: string;
+    private isCloudService;
     /**
      * Initializes a new instance of the FirecrawlApp class.
      * @param config - Configuration options for the FirecrawlApp instance.
@@ -276,12 +306,12 @@ declare class FirecrawlApp {
      */
     scrapeUrl<T extends zt.ZodSchema, ActionsSchema extends (Action[] | undefined) = undefined>(url: string, params?: ScrapeParams<T, ActionsSchema>): Promise<ScrapeResponse<zt.infer<T>, ActionsSchema extends Action[] ? ActionsResult : never> | ErrorResponse>;
     /**
-     * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+     * Searches using the Firecrawl API and optionally scrapes the results.
      * @param query - The search query string.
-     * @param params - Additional parameters for the search.
-     * @returns Throws an error advising to use version 0 of the API.
+     * @param params - Optional parameters for the search request.
+     * @returns The response from the search operation.
      */
-    search(query: string, params?: any): Promise<any>;
+    search(query: string, params?: SearchParams | Record<string, any>): Promise<SearchResponse>;
     /**
      * Initiates a crawl job for a URL using the Firecrawl API.
      * @param url - The URL to crawl.
@@ -329,8 +359,8 @@ declare class FirecrawlApp {
      * @param webhook - Optional webhook for the batch scrape.
      * @returns The response from the crawl operation.
      */
-    batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"]): Promise<BatchScrapeStatusResponse | ErrorResponse>;
-    asyncBatchScrapeUrls(urls: string[], params?: ScrapeParams, idempotencyKey?: string): Promise<BatchScrapeResponse | ErrorResponse>;
+    batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;
+    asyncBatchScrapeUrls(urls: string[], params?: ScrapeParams, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeResponse | ErrorResponse>;
     /**
      * Initiates a batch scrape job and returns a CrawlWatcher to monitor the job via WebSocket.
      * @param urls - The URL to scrape.
@@ -338,7 +368,7 @@ declare class FirecrawlApp {
      * @param idempotencyKey - Optional idempotency key for the request.
      * @returns A CrawlWatcher instance to monitor the crawl job.
      */
-    batchScrapeUrlsAndWatch(urls: string[], params?: ScrapeParams, idempotencyKey?: string): Promise<CrawlWatcher>;
+    batchScrapeUrlsAndWatch(urls: string[], params?: ScrapeParams, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<CrawlWatcher>;
     /**
      * Checks the status of a batch scrape job using the Firecrawl API.
      * @param id - The ID of the batch scrape operation.
@@ -414,8 +444,9 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     private ws;
     data: FirecrawlDocument<undefined>[];
     status: CrawlStatusResponse["status"];
+    id: string;
     constructor(id: string, app: FirecrawlApp);
     close(): void;
 }
 
-export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, FirecrawlApp as default };
+export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, type SearchParams, type SearchResponse, FirecrawlApp as default };

package/dist/index.js

@@ -13,16 +13,20 @@ var FirecrawlError = class extends Error {
 var FirecrawlApp = class {
   apiKey;
   apiUrl;
+  isCloudService(url) {
+    return url.includes("api.firecrawl.dev");
+  }
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
    */
   constructor({ apiKey = null, apiUrl = null }) {
-    if (typeof apiKey !== "string") {
+    const baseUrl = apiUrl || "https://api.firecrawl.dev";
+    if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
       throw new FirecrawlError("No API key provided", 401);
     }
-    this.apiKey = apiKey;
-    this.apiUrl = apiUrl || "https://api.firecrawl.dev";
+    this.apiKey = apiKey || "";
+    this.apiUrl = baseUrl;
   }
   /**
    * Scrapes a URL using the Firecrawl API.
@@ -77,13 +81,73 @@ var FirecrawlApp = class {
     return { success: false, error: "Internal server error." };
   }
   /**
-   * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+   * Searches using the Firecrawl API and optionally scrapes the results.
    * @param query - The search query string.
-   * @param params - Additional parameters for the search.
-   * @returns Throws an error advising to use version 0 of the API.
+   * @param params - Optional parameters for the search request.
+   * @returns The response from the search operation.
    */
   async search(query, params) {
-    throw new FirecrawlError("Search is not supported in v1, please downgrade Firecrawl to 0.0.36.", 400);
+    const headers = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`
+    };
+    let jsonData = {
+      query,
+      limit: params?.limit ?? 5,
+      tbs: params?.tbs,
+      filter: params?.filter,
+      lang: params?.lang ?? "en",
+      country: params?.country ?? "us",
+      location: params?.location,
+      origin: params?.origin ?? "api",
+      timeout: params?.timeout ?? 6e4,
+      scrapeOptions: params?.scrapeOptions ?? { formats: [] }
+    };
+    if (jsonData?.scrapeOptions?.extract?.schema) {
+      let schema = jsonData.scrapeOptions.extract.schema;
+      try {
+        schema = zodToJsonSchema(schema);
+      } catch (error) {
+      }
+      jsonData = {
+        ...jsonData,
+        scrapeOptions: {
+          ...jsonData.scrapeOptions,
+          extract: {
+            ...jsonData.scrapeOptions.extract,
+            schema
+          }
+        }
+      };
+    }
+    try {
+      const response = await this.postRequest(
+        this.apiUrl + `/v1/search`,
+        jsonData,
+        headers
+      );
+      if (response.status === 200) {
+        const responseData = response.data;
+        if (responseData.success) {
+          return {
+            success: true,
+            data: responseData.data,
+            warning: responseData.warning
+          };
+        } else {
+          throw new FirecrawlError(`Failed to search. Error: ${responseData.error}`, response.status);
+        }
+      } else {
+        this.handleError(response, "search");
+      }
+    } catch (error) {
+      if (error.response?.data?.error) {
+        throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
+      } else {
+        throw new FirecrawlError(error.message, 500);
+      }
+    }
+    return { success: false, error: "Internal server error.", data: [] };
   }
   /**
    * Initiates a crawl job for a URL using the Firecrawl API.
@@ -259,9 +323,9 @@ var FirecrawlApp = class {
    * @param webhook - Optional webhook for the batch scrape.
    * @returns The response from the crawl operation.
    */
-  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook) {
+  async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, ...params };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
       try {
@@ -297,9 +361,9 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
-  async asyncBatchScrapeUrls(urls, params, idempotencyKey) {
+  async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, ...params ?? {} };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params ?? {} };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -327,8 +391,8 @@ var FirecrawlApp = class {
    * @param idempotencyKey - Optional idempotency key for the request.
    * @returns A CrawlWatcher instance to monitor the crawl job.
    */
-  async batchScrapeUrlsAndWatch(urls, params, idempotencyKey) {
-    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey);
+  async batchScrapeUrlsAndWatch(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
+    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);
     if (crawl.success && crawl.id) {
       const id = crawl.id;
       return new CrawlWatcher(id, this);
@@ -557,8 +621,10 @@ var CrawlWatcher = class extends TypedEventTarget {
   ws;
   data;
   status;
+  id;
   constructor(id, app) {
     super();
+    this.id = id;
     this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
     this.status = "scraping";
     this.data = [];
@@ -568,7 +634,8 @@ var CrawlWatcher = class extends TypedEventTarget {
         this.dispatchTypedEvent("done", new CustomEvent("done", {
           detail: {
             status: this.status,
-            data: this.data
+            data: this.data,
+            id: this.id
           }
         }));
       } else if (msg.type === "error") {
@@ -577,7 +644,8 @@ var CrawlWatcher = class extends TypedEventTarget {
           detail: {
             status: this.status,
             data: this.data,
-            error: msg.error
+            error: msg.error,
+            id: this.id
           }
         }));
       } else if (msg.type === "catchup") {
@@ -585,12 +653,18 @@ var CrawlWatcher = class extends TypedEventTarget {
         this.data.push(...msg.data.data ?? []);
         for (const doc of this.data) {
           this.dispatchTypedEvent("document", new CustomEvent("document", {
-            detail: doc
+            detail: {
+              ...doc,
+              id: this.id
+            }
           }));
         }
       } else if (msg.type === "document") {
         this.dispatchTypedEvent("document", new CustomEvent("document", {
-          detail: msg.data
+          detail: {
+            ...msg.data,
+            id: this.id
+          }
         }));
       }
     };
@@ -599,12 +673,20 @@ var CrawlWatcher = class extends TypedEventTarget {
         this.ws.close();
         return;
       }
-      const msg = JSON.parse(ev.data);
-      messageHandler(msg);
+      try {
+        const msg = JSON.parse(ev.data);
+        messageHandler(msg);
+      } catch (error) {
+        console.error("Error on message", error);
+      }
     }).bind(this);
     this.ws.onclose = ((ev) => {
-      const msg = JSON.parse(ev.reason);
-      messageHandler(msg);
+      try {
+        const msg = JSON.parse(ev.reason);
+        messageHandler(msg);
+      } catch (error) {
+        console.error("Error on close", error);
+      }
     }).bind(this);
     this.ws.onerror = ((_) => {
       this.status = "failed";
@@ -612,7 +694,8 @@ var CrawlWatcher = class extends TypedEventTarget {
         detail: {
           status: this.status,
           data: this.data,
-          error: "WebSocket error"
+          error: "WebSocket error",
+          id: this.id
         }
       }));
     }).bind(this);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "1.10.1",
+  "version": "1.11.0",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/__tests__/index.test.ts

@@ -1,9 +1,9 @@
-import { describe, test, expect, jest } from '@jest/globals';
-import axios from 'axios';
-import FirecrawlApp from '../index';
+import { describe, expect, jest, test } from '@jest/globals';
 
-import { readFile } from 'fs/promises';
+import FirecrawlApp from '../index';
+import axios from 'axios';
 import { join } from 'path';
+import { readFile } from 'fs/promises';
 
 // Mock jest and set the type
 jest.mock('axios');
@@ -14,13 +14,22 @@ async function loadFixture(name: string): Promise<string> {
   return await readFile(join(__dirname, 'fixtures', `${name}.json`), 'utf-8')
 }
 
+const API_URL = process.env.API_URL ?? "https://api.firecrawl.dev";
+
 describe('the firecrawl JS SDK', () => {
 
-  test('Should require an API key to instantiate FirecrawlApp', async () => {
-    const fn = () => {
-      new FirecrawlApp({ apiKey: undefined });
-    };
-    expect(fn).toThrow('No API key provided');
+  test('Should require an API key only for cloud service', async () => {
+    if (API_URL.includes('api.firecrawl.dev')) {
+      // Should throw for cloud service
+      expect(() => {
+        new FirecrawlApp({ apiKey: undefined, apiUrl: API_URL });
+      }).toThrow('No API key provided');
+    } else {
+      // Should not throw for self-hosted
+      expect(() => {
+        new FirecrawlApp({ apiKey: undefined, apiUrl: API_URL });
+      }).not.toThrow();
+    }
   });
 
   test('Should return scraped data from a /scrape API call', async () => {

package/src/__tests__/v1/e2e_withAuth/index.test.ts

@@ -9,15 +9,28 @@ const TEST_API_KEY = process.env.TEST_API_KEY;
 const API_URL = process.env.API_URL ?? "https://api.firecrawl.dev";
 
 describe('FirecrawlApp E2E Tests', () => {
-  test.concurrent('should throw error for no API key', async () => {
-    expect(() => {
-      new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
-    }).toThrow("No API key provided");
+  test.concurrent('should throw error for no API key only for cloud service', async () => {
+    if (API_URL.includes('api.firecrawl.dev')) {
+      // Should throw for cloud service
+      expect(() => {
+        new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
+      }).toThrow("No API key provided");
+    } else {
+      // Should not throw for self-hosted
+      expect(() => {
+        new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
+      }).not.toThrow();
+    }
   });
 
   test.concurrent('should throw error for invalid API key on scrape', async () => {
-    const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-    await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
+    if (API_URL.includes('api.firecrawl.dev')) {
+      const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
+      await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 404");
+    } else {
+      const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
+      await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
+    }
   });
 
   test.concurrent('should throw error for blocklisted URL on scrape', async () => {
@@ -155,14 +168,13 @@ describe('FirecrawlApp E2E Tests', () => {
   }, 30000); // 30 seconds timeout
 
   test.concurrent('should throw error for invalid API key on crawl', async () => {
-    const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-    await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
-  });
-
-  test.concurrent('should throw error for blocklisted URL on crawl', async () => {
-    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const blocklistedUrl = "https://twitter.com/fake-test";
-    await expect(app.crawlUrl(blocklistedUrl)).rejects.toThrow("URL is blocked. Firecrawl currently does not support social media scraping due to policy restrictions.");
+    if (API_URL.includes('api.firecrawl.dev')) {
+      const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
+      await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 404");
+    } else {
+      const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
+      await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
+    }
   });
 
   test.concurrent('should return successful response for crawl and wait for completion', async () => {
@@ -337,8 +349,13 @@ describe('FirecrawlApp E2E Tests', () => {
   }, 60000); // 60 seconds timeout
 
   test.concurrent('should throw error for invalid API key on map', async () => {
-    const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-    await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
+    if (API_URL.includes('api.firecrawl.dev')) {
+      const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
+      await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 404");
+    } else {
+      const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
+      await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
+    }
   });
 
   test.concurrent('should throw error for blocklisted URL on map', async () => {
@@ -355,8 +372,7 @@ describe('FirecrawlApp E2E Tests', () => {
   }, 30000); // 30 seconds timeout
 
   test.concurrent('should return successful response for valid map', async () => {
-    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });    const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;
     expect(response).not.toBeNull();
     
     expect(response.links?.length).toBeGreaterThan(0);
@@ -365,8 +381,45 @@ describe('FirecrawlApp E2E Tests', () => {
     expect(filteredLinks?.length).toBeGreaterThan(0);
   }, 30000); // 30 seconds timeout
 
-  test('should throw NotImplementedError for search on v1', async () => {
+  
+
+  test('should search with string query', async () => {
     const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: TEST_API_KEY });
-    await expect(app.search("test query")).rejects.toThrow("Search is not supported in v1");
+    const response = await app.search("firecrawl");
+    expect(response.success).toBe(true);
+    console.log(response.data);
+    expect(response.data?.length).toBeGreaterThan(0);
+    expect(response.data?.[0]?.markdown).toBeDefined();
+    expect(response.data?.[0]?.metadata).toBeDefined();
+    expect(response.data?.[0]?.metadata?.title).toBeDefined();
+    expect(response.data?.[0]?.metadata?.description).toBeDefined();
+  });
+
+  test('should search with params object', async () => {
+    const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: TEST_API_KEY });
+    const response = await app.search("firecrawl", {
+      limit: 3,
+      lang: 'en',
+      country: 'us',
+      scrapeOptions: {
+        formats: ['markdown', 'html', 'links'],
+        onlyMainContent: true
+      }
+    });
+    expect(response.success).toBe(true);
+    expect(response.data.length).toBeLessThanOrEqual(3);
+    for (const doc of response.data) {
+      expect(doc.markdown).toBeDefined();
+      expect(doc.html).toBeDefined();
+      expect(doc.links).toBeDefined();
+      expect(doc.metadata).toBeDefined();
+      expect(doc.metadata?.title).toBeDefined();
+      expect(doc.metadata?.description).toBeDefined();
+    }
+  });
+
+  test('should handle invalid API key for search', async () => {
+    const app = new FirecrawlApp({ apiUrl: API_URL, apiKey: "invalid_api_key" });
+    await expect(app.search("test query")).rejects.toThrow("Request failed with status code 404");
   });
 });

package/src/index.ts

@@ -68,6 +68,9 @@ export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult
   screenshot?: string;
   metadata?: FirecrawlDocumentMetadata;
   actions: ActionsSchema;
+  // v1 search only
+  title?: string;
+  description?: string;
 }
 
 /**
@@ -183,6 +186,7 @@ export interface BatchScrapeResponse {
   url?: string;
   success: true;
   error?: string;
+  invalidURLs?: string[];
 }
 
 /**
@@ -242,10 +246,11 @@ export interface MapResponse {
  * Defines options for extracting information from URLs.
  */
 export interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
-  prompt: string;
+  prompt?: string;
   schema?: LLMSchema;
   systemPrompt?: string;
   allowExternalLinks?: boolean;
+  includeSubdomains?: boolean;
 }
 
 /**
@@ -280,6 +285,33 @@ export class FirecrawlError extends Error {
   }
 }
 
+/**
+ * Parameters for search operations.
+ * Defines options for searching and scraping search results.
+ */
+export interface SearchParams {
+  limit?: number;
+  tbs?: string;
+  filter?: string;
+  lang?: string;
+  country?: string;
+  location?: string;
+  origin?: string;
+  timeout?: number;
+  scrapeOptions?: ScrapeParams;
+}
+
+/**
+ * Response interface for search operations.
+ * Defines the structure of the response received after a search operation.
+ */
+export interface SearchResponse {
+  success: boolean;
+  data: FirecrawlDocument<undefined>[];
+  warning?: string;
+  error?: string;
+}
+
 /**
  * Main class for interacting with the Firecrawl API.
  * Provides methods for scraping, searching, crawling, and mapping web content.
@@ -288,17 +320,23 @@ export default class FirecrawlApp {
   public apiKey: string;
   public apiUrl: string;
 
+  private isCloudService(url: string): boolean {
+    return url.includes('api.firecrawl.dev');
+  }
+
   /**
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
    */
   constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
-    if (typeof apiKey !== "string") {
+    const baseUrl = apiUrl || "https://api.firecrawl.dev";
+    
+    if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
       throw new FirecrawlError("No API key provided", 401);
     }
 
-    this.apiKey = apiKey;
-    this.apiUrl = apiUrl || "https://api.firecrawl.dev";
+    this.apiKey = apiKey || '';
+    this.apiUrl = baseUrl;
   }
 
   /**
@@ -361,16 +399,80 @@ export default class FirecrawlApp {
   }
 
   /**
-   * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+   * Searches using the Firecrawl API and optionally scrapes the results.
    * @param query - The search query string.
-   * @param params - Additional parameters for the search.
-   * @returns Throws an error advising to use version 0 of the API.
+   * @param params - Optional parameters for the search request.
+   * @returns The response from the search operation.
    */
-  async search(
-    query: string,
-    params?: any
-  ): Promise<any> {
-    throw new FirecrawlError("Search is not supported in v1, please downgrade Firecrawl to 0.0.36.", 400);
+  async search(query: string, params?: SearchParams | Record<string, any>): Promise<SearchResponse> {
+    const headers: AxiosRequestHeaders = {
+      "Content-Type": "application/json",
+      Authorization: `Bearer ${this.apiKey}`,
+    } as AxiosRequestHeaders;
+
+    let jsonData: any = {
+      query,
+      limit: params?.limit ?? 5,
+      tbs: params?.tbs,
+      filter: params?.filter,
+      lang: params?.lang ?? "en",
+      country: params?.country ?? "us",
+      location: params?.location,
+      origin: params?.origin ?? "api",
+      timeout: params?.timeout ?? 60000,
+      scrapeOptions: params?.scrapeOptions ?? { formats: [] },
+    };
+
+    if (jsonData?.scrapeOptions?.extract?.schema) {
+      let schema = jsonData.scrapeOptions.extract.schema;
+
+      // Try parsing the schema as a Zod schema
+      try {
+        schema = zodToJsonSchema(schema);
+      } catch (error) {
+        
+      }
+      jsonData = {
+        ...jsonData,
+        scrapeOptions: {
+          ...jsonData.scrapeOptions,
+          extract: {
+            ...jsonData.scrapeOptions.extract,
+            schema: schema,
+          },
+        },
+      };
+    }
+
+    try {
+      const response: AxiosResponse = await this.postRequest(
+        this.apiUrl + `/v1/search`,
+        jsonData,
+        headers
+      );
+
+      if (response.status === 200) {
+        const responseData = response.data;
+        if (responseData.success) {
+          return {
+            success: true,
+            data: responseData.data as FirecrawlDocument<any>[],
+            warning: responseData.warning,
+          };
+        } else {
+          throw new FirecrawlError(`Failed to search. Error: ${responseData.error}`, response.status);
+        }
+      } else {
+        this.handleError(response, "search");
+      }
+    } catch (error: any) {
+      if (error.response?.data?.error) {
+        throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+      } else {
+        throw new FirecrawlError(error.message, 500);
+      }
+    }
+    return { success: false, error: "Internal server error.", data: [] };
   }
 
   /**
@@ -576,9 +678,10 @@ export default class FirecrawlApp {
     pollInterval: number = 2,
     idempotencyKey?: string,
     webhook?: CrawlParams["webhook"],
+    ignoreInvalidURLs?: boolean,
   ): Promise<BatchScrapeStatusResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { urls, ...params };
+    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...params };
     if (jsonData?.extract?.schema) {
       let schema = jsonData.extract.schema;
 
@@ -621,10 +724,12 @@ export default class FirecrawlApp {
   async asyncBatchScrapeUrls(
     urls: string[],
     params?: ScrapeParams,
-    idempotencyKey?: string
+    idempotencyKey?: string,
+    webhook?: CrawlParams["webhook"],
+    ignoreInvalidURLs?: boolean,
   ): Promise<BatchScrapeResponse | ErrorResponse> {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData: any = { urls, ...(params ?? {}) };
+    let jsonData: any = { urls, webhook, ignoreInvalidURLs, ...(params ?? {}) };
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -657,8 +762,10 @@ export default class FirecrawlApp {
     urls: string[],
     params?: ScrapeParams,
     idempotencyKey?: string,
+    webhook?: CrawlParams["webhook"],
+    ignoreInvalidURLs?: boolean,
   ) {
-    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey);
+    const crawl = await this.asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs);
 
     if (crawl.success && crawl.id) {
       const id = crawl.id;
@@ -932,9 +1039,11 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
   private ws: WebSocket;
   public data: FirecrawlDocument<undefined>[];
   public status: CrawlStatusResponse["status"];
+  public id: string;
 
   constructor(id: string, app: FirecrawlApp) {
     super();
+    this.id = id;
     this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
     this.status = "scraping";
     this.data = [];
@@ -965,6 +1074,7 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
           detail: {
             status: this.status,
             data: this.data,
+            id: this.id,
           },
         }));
       } else if (msg.type === "error") {
@@ -974,6 +1084,7 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
             status: this.status,
             data: this.data,
             error: msg.error,
+            id: this.id,
           },
         }));
       } else if (msg.type === "catchup") {
@@ -981,12 +1092,18 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
         this.data.push(...(msg.data.data ?? []));
         for (const doc of this.data) {
           this.dispatchTypedEvent("document", new CustomEvent("document", {
-            detail: doc,
+            detail: {
+              ...doc,
+              id: this.id,
+            },
           }));
         }
       } else if (msg.type === "document") {
         this.dispatchTypedEvent("document", new CustomEvent("document", {
-          detail: msg.data,
+          detail: {
+            ...msg.data,
+            id: this.id,
+          },
         }));
       }
     }
@@ -996,14 +1113,21 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
         this.ws.close();
         return;
       }
-
-      const msg = JSON.parse(ev.data) as Message;
-      messageHandler(msg);
+      try {
+        const msg = JSON.parse(ev.data) as Message;
+        messageHandler(msg);
+      } catch (error) {
+        console.error("Error on message", error);
+      }
     }).bind(this);
 
     this.ws.onclose = ((ev: CloseEvent) => {
-      const msg = JSON.parse(ev.reason) as Message;
-      messageHandler(msg);
+      try {
+        const msg = JSON.parse(ev.reason) as Message;
+        messageHandler(msg);
+      } catch (error) {
+        console.error("Error on close", error);
+      }
     }).bind(this);
 
     this.ws.onerror = ((_: Event) => {
@@ -1013,6 +1137,7 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
           status: this.status,
           data: this.data,
           error: "WebSocket error",
+          id: this.id,
         },
       }));
     }).bind(this);