firecrawl 1.14.0 → 1.15.0

package/dist/index.cjs

@@ -42,9 +42,11 @@ var import_isows = require("isows");
 var import_typescript_event_target = require("typescript-event-target");
 var FirecrawlError = class extends Error {
   statusCode;
-  constructor(message, statusCode) {
+  details;
+  constructor(message, statusCode, details) {
     super(message);
     this.statusCode = statusCode;
+    this.details = details;
   }
 };
 var FirecrawlApp = class {
@@ -91,6 +93,20 @@ var FirecrawlApp = class {
         }
       };
     }
+    if (jsonData?.jsonOptions?.schema) {
+      let schema = jsonData.jsonOptions.schema;
+      try {
+        schema = (0, import_zod_to_json_schema.zodToJsonSchema)(schema);
+      } catch (error) {
+      }
+      jsonData = {
+        ...jsonData,
+        jsonOptions: {
+          ...jsonData.jsonOptions,
+          schema
+        }
+      };
+    }
     try {
       const response = await import_axios.default.post(
         this.apiUrl + `/v1/scrape`,
@@ -312,6 +328,28 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
+  /**
+   * Returns information about crawl errors.
+   * @param id - The ID of the crawl operation.
+   * @returns Information about crawl errors.
+   */
+  async checkCrawlErrors(id) {
+    const headers = this.prepareHeaders();
+    try {
+      const response = await this.deleteRequest(
+        `${this.apiUrl}/v1/crawl/${id}/errors`,
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "check crawl errors");
+      }
+    } catch (error) {
+      throw new FirecrawlError(error.message, 500);
+    }
+    return { success: false, error: "Internal server error." };
+  }
   /**
    * Cancels a crawl job using the Firecrawl API.
    * @param id - The ID of the crawl operation.
@@ -400,6 +438,20 @@ var FirecrawlApp = class {
         }
       };
     }
+    if (jsonData?.jsonOptions?.schema) {
+      let schema = jsonData.jsonOptions.schema;
+      try {
+        schema = (0, import_zod_to_json_schema.zodToJsonSchema)(schema);
+      } catch (error) {
+      }
+      jsonData = {
+        ...jsonData,
+        jsonOptions: {
+          ...jsonData.jsonOptions,
+          schema
+        }
+      };
+    }
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -530,6 +582,28 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
+  /**
+   * Returns information about batch scrape errors.
+   * @param id - The ID of the batch scrape operation.
+   * @returns Information about batch scrape errors.
+   */
+  async checkBatchScrapeErrors(id) {
+    const headers = this.prepareHeaders();
+    try {
+      const response = await this.deleteRequest(
+        `${this.apiUrl}/v1/batch/scrape/${id}/errors`,
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "check batch scrape errors");
+      }
+    } catch (error) {
+      throw new FirecrawlError(error.message, 500);
+    }
+    return { success: false, error: "Internal server error." };
+  }
   /**
    * Extracts information from URLs using the Firecrawl API.
    * Currently in Beta. Expect breaking changes on future minor versions.
@@ -587,7 +661,7 @@ var FirecrawlApp = class {
         this.handleError(response, "extract");
       }
     } catch (error) {
-      throw new FirecrawlError(error.message, 500);
+      throw new FirecrawlError(error.message, 500, error.response?.data?.details);
     }
     return { success: false, error: "Internal server error." };
   }
@@ -623,7 +697,7 @@ var FirecrawlApp = class {
         this.handleError(response, "start extract job");
       }
     } catch (error) {
-      throw new FirecrawlError(error.message, 500);
+      throw new FirecrawlError(error.message, 500, error.response?.data?.details);
     }
     return { success: false, error: "Internal server error." };
   }

package/dist/index.d.cts

@@ -72,7 +72,7 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
  * Defines the options and configurations available for scraping web content.
  */
 interface CrawlScrapeOptions {
-    formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract")[];
+    formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json")[];
     headers?: Record<string, string>;
     includeTags?: string[];
     excludeTags?: string[];
@@ -119,6 +119,11 @@ interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema exten
         schema?: LLMSchema;
         systemPrompt?: string;
     };
+    json?: {
+        prompt?: string;
+        schema?: LLMSchema;
+        systemPrompt?: string;
+    };
     actions?: ActionsSchema;
 }
 interface ActionsResult {
@@ -258,7 +263,8 @@ interface ErrorResponse {
  */
 declare class FirecrawlError extends Error {
     statusCode: number;
-    constructor(message: string, statusCode: number);
+    details?: any;
+    constructor(message: string, statusCode: number, details?: any);
 }
 /**
  * Parameters for search operations.
@@ -285,6 +291,24 @@ interface SearchResponse {
     warning?: string;
     error?: string;
 }
+/**
+ * Response interface for crawl/batch scrape error monitoring.
+ */
+interface CrawlErrorsResponse {
+    /**
+     * Scrapes that errored out + error details
+     */
+    errors: {
+        id: string;
+        timestamp?: string;
+        url: string;
+        error: string;
+    }[];
+    /**
+     * URLs blocked by robots.txt
+     */
+    robotsBlocked: string[];
+}
 /**
  * Main class for interacting with the Firecrawl API.
  * Provides methods for scraping, searching, crawling, and mapping web content.
@@ -332,6 +356,12 @@ declare class FirecrawlApp {
      * @returns The response containing the job status.
      */
     checkCrawlStatus(id?: string, getAllData?: boolean, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse>;
+    /**
+     * Returns information about crawl errors.
+     * @param id - The ID of the crawl operation.
+     * @returns Information about crawl errors.
+     */
+    checkCrawlErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse>;
     /**
      * Cancels a crawl job using the Firecrawl API.
      * @param id - The ID of the crawl operation.
@@ -382,6 +412,12 @@ declare class FirecrawlApp {
      * @returns The response containing the job status.
      */
     checkBatchScrapeStatus(id?: string, getAllData?: boolean, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse>;
+    /**
+     * Returns information about batch scrape errors.
+     * @param id - The ID of the batch scrape operation.
+     * @returns Information about batch scrape errors.
+     */
+    checkBatchScrapeErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse>;
     /**
      * Extracts information from URLs using the Firecrawl API.
      * Currently in Beta. Expect breaking changes on future minor versions.
@@ -469,4 +505,4 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     close(): void;
 }
 
-export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, type SearchParams, type SearchResponse, FirecrawlApp as default };
+export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlErrorsResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, type SearchParams, type SearchResponse, FirecrawlApp as default };

package/dist/index.d.ts

@@ -72,7 +72,7 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
  * Defines the options and configurations available for scraping web content.
  */
 interface CrawlScrapeOptions {
-    formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract")[];
+    formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json")[];
     headers?: Record<string, string>;
     includeTags?: string[];
     excludeTags?: string[];
@@ -119,6 +119,11 @@ interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema exten
         schema?: LLMSchema;
         systemPrompt?: string;
     };
+    json?: {
+        prompt?: string;
+        schema?: LLMSchema;
+        systemPrompt?: string;
+    };
     actions?: ActionsSchema;
 }
 interface ActionsResult {
@@ -258,7 +263,8 @@ interface ErrorResponse {
  */
 declare class FirecrawlError extends Error {
     statusCode: number;
-    constructor(message: string, statusCode: number);
+    details?: any;
+    constructor(message: string, statusCode: number, details?: any);
 }
 /**
  * Parameters for search operations.
@@ -285,6 +291,24 @@ interface SearchResponse {
     warning?: string;
     error?: string;
 }
+/**
+ * Response interface for crawl/batch scrape error monitoring.
+ */
+interface CrawlErrorsResponse {
+    /**
+     * Scrapes that errored out + error details
+     */
+    errors: {
+        id: string;
+        timestamp?: string;
+        url: string;
+        error: string;
+    }[];
+    /**
+     * URLs blocked by robots.txt
+     */
+    robotsBlocked: string[];
+}
 /**
  * Main class for interacting with the Firecrawl API.
  * Provides methods for scraping, searching, crawling, and mapping web content.
@@ -332,6 +356,12 @@ declare class FirecrawlApp {
      * @returns The response containing the job status.
      */
     checkCrawlStatus(id?: string, getAllData?: boolean, nextURL?: string, skip?: number, limit?: number): Promise<CrawlStatusResponse | ErrorResponse>;
+    /**
+     * Returns information about crawl errors.
+     * @param id - The ID of the crawl operation.
+     * @returns Information about crawl errors.
+     */
+    checkCrawlErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse>;
     /**
      * Cancels a crawl job using the Firecrawl API.
      * @param id - The ID of the crawl operation.
@@ -382,6 +412,12 @@ declare class FirecrawlApp {
      * @returns The response containing the job status.
      */
     checkBatchScrapeStatus(id?: string, getAllData?: boolean, nextURL?: string, skip?: number, limit?: number): Promise<BatchScrapeStatusResponse | ErrorResponse>;
+    /**
+     * Returns information about batch scrape errors.
+     * @param id - The ID of the batch scrape operation.
+     * @returns Information about batch scrape errors.
+     */
+    checkBatchScrapeErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse>;
     /**
      * Extracts information from URLs using the Firecrawl API.
      * Currently in Beta. Expect breaking changes on future minor versions.
@@ -469,4 +505,4 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     close(): void;
 }
 
-export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, type SearchParams, type SearchResponse, FirecrawlApp as default };
+export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlErrorsResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, type SearchParams, type SearchResponse, FirecrawlApp as default };

package/dist/index.js

@@ -6,9 +6,11 @@ import { WebSocket } from "isows";
 import { TypedEventTarget } from "typescript-event-target";
 var FirecrawlError = class extends Error {
   statusCode;
-  constructor(message, statusCode) {
+  details;
+  constructor(message, statusCode, details) {
     super(message);
     this.statusCode = statusCode;
+    this.details = details;
   }
 };
 var FirecrawlApp = class {
@@ -55,6 +57,20 @@ var FirecrawlApp = class {
         }
       };
     }
+    if (jsonData?.jsonOptions?.schema) {
+      let schema = jsonData.jsonOptions.schema;
+      try {
+        schema = zodToJsonSchema(schema);
+      } catch (error) {
+      }
+      jsonData = {
+        ...jsonData,
+        jsonOptions: {
+          ...jsonData.jsonOptions,
+          schema
+        }
+      };
+    }
     try {
       const response = await axios.post(
         this.apiUrl + `/v1/scrape`,
@@ -276,6 +292,28 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
+  /**
+   * Returns information about crawl errors.
+   * @param id - The ID of the crawl operation.
+   * @returns Information about crawl errors.
+   */
+  async checkCrawlErrors(id) {
+    const headers = this.prepareHeaders();
+    try {
+      const response = await this.deleteRequest(
+        `${this.apiUrl}/v1/crawl/${id}/errors`,
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "check crawl errors");
+      }
+    } catch (error) {
+      throw new FirecrawlError(error.message, 500);
+    }
+    return { success: false, error: "Internal server error." };
+  }
   /**
    * Cancels a crawl job using the Firecrawl API.
    * @param id - The ID of the crawl operation.
@@ -364,6 +402,20 @@ var FirecrawlApp = class {
         }
       };
     }
+    if (jsonData?.jsonOptions?.schema) {
+      let schema = jsonData.jsonOptions.schema;
+      try {
+        schema = zodToJsonSchema(schema);
+      } catch (error) {
+      }
+      jsonData = {
+        ...jsonData,
+        jsonOptions: {
+          ...jsonData.jsonOptions,
+          schema
+        }
+      };
+    }
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -494,6 +546,28 @@ var FirecrawlApp = class {
     }
     return { success: false, error: "Internal server error." };
   }
+  /**
+   * Returns information about batch scrape errors.
+   * @param id - The ID of the batch scrape operation.
+   * @returns Information about batch scrape errors.
+   */
+  async checkBatchScrapeErrors(id) {
+    const headers = this.prepareHeaders();
+    try {
+      const response = await this.deleteRequest(
+        `${this.apiUrl}/v1/batch/scrape/${id}/errors`,
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "check batch scrape errors");
+      }
+    } catch (error) {
+      throw new FirecrawlError(error.message, 500);
+    }
+    return { success: false, error: "Internal server error." };
+  }
   /**
    * Extracts information from URLs using the Firecrawl API.
    * Currently in Beta. Expect breaking changes on future minor versions.
@@ -551,7 +625,7 @@ var FirecrawlApp = class {
         this.handleError(response, "extract");
       }
     } catch (error) {
-      throw new FirecrawlError(error.message, 500);
+      throw new FirecrawlError(error.message, 500, error.response?.data?.details);
     }
     return { success: false, error: "Internal server error." };
   }
@@ -587,7 +661,7 @@ var FirecrawlApp = class {
         this.handleError(response, "start extract job");
       }
     } catch (error) {
-      throw new FirecrawlError(error.message, 500);
+      throw new FirecrawlError(error.message, 500, error.response?.data?.details);
     }
     return { success: false, error: "Internal server error." };
   }

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "1.14.0",
+  "version": "1.15.0",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/index.ts

@@ -78,7 +78,7 @@ export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult
  * Defines the options and configurations available for scraping web content.
  */
 export interface CrawlScrapeOptions {
-  formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract")[];
+  formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json")[];
   headers?: Record<string, string>;
   includeTags?: string[];
   excludeTags?: string[];
@@ -127,6 +127,11 @@ export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchem
     schema?: LLMSchema;
     systemPrompt?: string;
   };
+  json?:{
+    prompt?: string;
+    schema?: LLMSchema;
+    systemPrompt?: string;
+  }
   actions?: ActionsSchema;
 }
 
@@ -279,9 +284,11 @@ export interface ErrorResponse {
  */
 export class FirecrawlError extends Error {
   statusCode: number;
-  constructor(message: string, statusCode: number) {
+  details?: any;
+  constructor(message: string, statusCode: number, details?: any) {
     super(message);
     this.statusCode = statusCode;
+    this.details = details;
   }
 }
 
@@ -312,6 +319,26 @@ export interface SearchResponse {
   error?: string;
 }
 
+/**
+ * Response interface for crawl/batch scrape error monitoring.
+ */
+export interface CrawlErrorsResponse {
+  /**
+   * Scrapes that errored out + error details
+   */
+  errors: {
+    id: string,
+    timestamp?: string,
+    url: string,
+    error: string,
+  }[];
+
+  /**
+   * URLs blocked by robots.txt
+   */
+  robotsBlocked: string[];
+};
+
 /**
  * Main class for interacting with the Firecrawl API.
  * Provides methods for scraping, searching, crawling, and mapping web content.
@@ -371,6 +398,23 @@ export default class FirecrawlApp {
         },
       };
     }
+
+    if (jsonData?.jsonOptions?.schema) {
+      let schema = jsonData.jsonOptions.schema;
+      // Try parsing the schema as a Zod schema
+      try {
+        schema = zodToJsonSchema(schema);
+      } catch (error) {
+        
+      }
+      jsonData = {
+        ...jsonData,
+        jsonOptions: {
+          ...jsonData.jsonOptions,
+          schema: schema,
+        },
+      };
+    }
     try {
       const response: AxiosResponse = await axios.post(
         this.apiUrl + `/v1/scrape`,
@@ -619,6 +663,29 @@ export default class FirecrawlApp {
     return { success: false, error: "Internal server error." };
   }
 
+  /**
+   * Returns information about crawl errors.
+   * @param id - The ID of the crawl operation.
+   * @returns Information about crawl errors.
+   */
+  async checkCrawlErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {
+    const headers = this.prepareHeaders();
+    try {
+      const response: AxiosResponse = await this.deleteRequest(
+        `${this.apiUrl}/v1/crawl/${id}/errors`,
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "check crawl errors");
+      }
+    } catch (error: any) {
+      throw new FirecrawlError(error.message, 500);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+
   /**
    * Cancels a crawl job using the Firecrawl API.
    * @param id - The ID of the crawl operation.
@@ -727,6 +794,23 @@ export default class FirecrawlApp {
         },
       };
     }
+    if (jsonData?.jsonOptions?.schema) {
+      let schema = jsonData.jsonOptions.schema;
+
+      // Try parsing the schema as a Zod schema
+      try {
+        schema = zodToJsonSchema(schema);
+      } catch (error) {
+        
+      }
+      jsonData = {
+        ...jsonData,
+        jsonOptions: {
+          ...jsonData.jsonOptions,
+          schema: schema,
+        },
+      };
+    }
     try {
       const response: AxiosResponse = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -881,6 +965,29 @@ export default class FirecrawlApp {
     return { success: false, error: "Internal server error." };
   }
 
+  /**
+   * Returns information about batch scrape errors.
+   * @param id - The ID of the batch scrape operation.
+   * @returns Information about batch scrape errors.
+   */
+  async checkBatchScrapeErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse> {
+    const headers = this.prepareHeaders();
+    try {
+      const response: AxiosResponse = await this.deleteRequest(
+        `${this.apiUrl}/v1/batch/scrape/${id}/errors`,
+        headers
+      );
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "check batch scrape errors");
+      }
+    } catch (error: any) {
+      throw new FirecrawlError(error.message, 500);
+    }
+    return { success: false, error: "Internal server error." };
+  }
+
   /**
    * Extracts information from URLs using the Firecrawl API.
    * Currently in Beta. Expect breaking changes on future minor versions.
@@ -941,9 +1048,9 @@ export default class FirecrawlApp {
         this.handleError(response, "extract");
       }
     } catch (error: any) {
-      throw new FirecrawlError(error.message, 500);
+      throw new FirecrawlError(error.message, 500, error.response?.data?.details);
     }
-    return { success: false, error: "Internal server error." };
+    return { success: false, error: "Internal server error."};
   }
 
   /**
@@ -985,7 +1092,7 @@ export default class FirecrawlApp {
         this.handleError(response, "start extract job");
       }
     } catch (error: any) {
-      throw new FirecrawlError(error.message, 500);
+      throw new FirecrawlError(error.message, 500, error.response?.data?.details);
     }
     return { success: false, error: "Internal server error." };
   }