@mendable/firecrawl 1.19.2 → 1.21.1

package/dist/index.cjs

@@ -419,6 +419,7 @@ var FirecrawlApp = class {
    * @param pollInterval - Time in seconds for job status checks.
    * @param idempotencyKey - Optional idempotency key for the request.
    * @param webhook - Optional webhook for the batch scrape.
+   * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.
    * @returns The response from the crawl operation.
    */
   async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
@@ -607,7 +608,7 @@ var FirecrawlApp = class {
   /**
    * Extracts information from URLs using the Firecrawl API.
    * Currently in Beta. Expect breaking changes on future minor versions.
-   * @param url - The URL to extract information from.
+   * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
    * @param params - Additional parameters for the extract request.
    * @returns The response from the extract operation.
    */
@@ -841,7 +842,7 @@ var FirecrawlApp = class {
    * @param {string} action - The action being performed when the error occurred.
    */
   handleError(response, action) {
-    if ([400, 402, 408, 409, 500].includes(response.status)) {
+    if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
       const errorMessage = response.data.error || "Unknown error occurred";
       const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : "";
       throw new FirecrawlError(
@@ -922,10 +923,25 @@ var FirecrawlApp = class {
    */
   async asyncDeepResearch(query, params) {
     const headers = this.prepareHeaders();
+    let jsonData = { query, ...params };
+    if (jsonData?.jsonOptions?.schema) {
+      let schema = jsonData.jsonOptions.schema;
+      try {
+        schema = (0, import_zod_to_json_schema.zodToJsonSchema)(schema);
+      } catch (error) {
+      }
+      jsonData = {
+        ...jsonData,
+        jsonOptions: {
+          ...jsonData.jsonOptions,
+          schema
+        }
+      };
+    }
     try {
       const response = await this.postRequest(
         `${this.apiUrl}/v1/deep-research`,
-        { query, ...params },
+        jsonData,
         headers
       );
       if (response.status === 200) {

package/dist/index.d.cts

@@ -65,6 +65,11 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
     screenshot?: string;
     metadata?: FirecrawlDocumentMetadata;
     actions: ActionsSchema;
+    compare?: {
+        previousScrapeAt: string | null;
+        changeStatus: "new" | "same" | "changed" | "removed";
+        visibility: "visible" | "hidden";
+    };
     title?: string;
     description?: string;
 }
@@ -73,7 +78,7 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
  * Defines the options and configurations available for scraping web content.
  */
 interface CrawlScrapeOptions {
-    formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json")[];
+    formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json" | "compare")[];
     headers?: Record<string, string>;
     includeTags?: string[];
     excludeTags?: string[];
@@ -97,6 +102,7 @@ type Action = {
 } | {
     type: "click";
     selector: string;
+    all?: boolean;
 } | {
     type: "screenshot";
     fullPage?: boolean;
@@ -131,6 +137,14 @@ interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema exten
 }
 interface ActionsResult {
     screenshots: string[];
+    scrapes: ({
+        url: string;
+        html: string;
+    })[];
+    javascriptReturns: {
+        type: string;
+        value: unknown;
+    }[];
 }
 /**
  * Response interface for scraping operations.
@@ -325,7 +339,7 @@ interface CrawlErrorsResponse {
  * Parameters for deep research operations.
  * Defines options for conducting deep research on a query.
  */
-interface DeepResearchParams {
+interface DeepResearchParams<LLMSchema extends zt.ZodSchema = any> {
     /**
      * Maximum depth of research iterations (1-10)
      * @default 7
@@ -342,9 +356,25 @@ interface DeepResearchParams {
      */
     maxUrls?: number;
     /**
-     * Experimental flag for streaming steps
+     * The prompt to use for the final analysis
+     */
+    analysisPrompt?: string;
+    /**
+     * The system prompt to use for the research agent
+     */
+    systemPrompt?: string;
+    /**
+     * The formats to use for the final analysis
+     */
+    formats?: ("markdown" | "json")[];
+    /**
+     * The JSON options to use for the final analysis
      */
-    __experimental_streamSteps?: boolean;
+    jsonOptions?: {
+        prompt?: string;
+        schema?: LLMSchema;
+        systemPrompt?: string;
+    };
 }
 /**
  * Response interface for deep research operations.
@@ -512,6 +542,7 @@ declare class FirecrawlApp {
      * @param pollInterval - Time in seconds for job status checks.
      * @param idempotencyKey - Optional idempotency key for the request.
      * @param webhook - Optional webhook for the batch scrape.
+     * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.
      * @returns The response from the crawl operation.
      */
     batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;
@@ -543,11 +574,11 @@ declare class FirecrawlApp {
     /**
      * Extracts information from URLs using the Firecrawl API.
      * Currently in Beta. Expect breaking changes on future minor versions.
-     * @param url - The URL to extract information from.
+     * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
      * @param params - Additional parameters for the extract request.
      * @returns The response from the extract operation.
      */
-    extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
+    extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
     /**
      * Initiates an asynchronous extract job for a URL using the Firecrawl API.
      * @param url - The URL to extract data from.
@@ -613,7 +644,7 @@ declare class FirecrawlApp {
      * @param onSource - Optional callback to receive source updates in real-time.
      * @returns The final research results.
      */
-    deepResearch(query: string, params: DeepResearchParams, onActivity?: (activity: {
+    deepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>, onActivity?: (activity: {
         type: string;
         status: string;
         message: string;
@@ -630,7 +661,7 @@ declare class FirecrawlApp {
      * @param params - Parameters for the deep research operation.
      * @returns The response containing the research job ID.
      */
-    asyncDeepResearch(query: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse>;
+    asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse>;
     /**
      * Checks the status of a deep research operation.
      * @param id - The ID of the deep research operation.

package/dist/index.d.ts

@@ -65,6 +65,11 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
     screenshot?: string;
     metadata?: FirecrawlDocumentMetadata;
     actions: ActionsSchema;
+    compare?: {
+        previousScrapeAt: string | null;
+        changeStatus: "new" | "same" | "changed" | "removed";
+        visibility: "visible" | "hidden";
+    };
     title?: string;
     description?: string;
 }
@@ -73,7 +78,7 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
  * Defines the options and configurations available for scraping web content.
  */
 interface CrawlScrapeOptions {
-    formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json")[];
+    formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json" | "compare")[];
     headers?: Record<string, string>;
     includeTags?: string[];
     excludeTags?: string[];
@@ -97,6 +102,7 @@ type Action = {
 } | {
     type: "click";
     selector: string;
+    all?: boolean;
 } | {
     type: "screenshot";
     fullPage?: boolean;
@@ -131,6 +137,14 @@ interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema exten
 }
 interface ActionsResult {
     screenshots: string[];
+    scrapes: ({
+        url: string;
+        html: string;
+    })[];
+    javascriptReturns: {
+        type: string;
+        value: unknown;
+    }[];
 }
 /**
  * Response interface for scraping operations.
@@ -325,7 +339,7 @@ interface CrawlErrorsResponse {
  * Parameters for deep research operations.
  * Defines options for conducting deep research on a query.
  */
-interface DeepResearchParams {
+interface DeepResearchParams<LLMSchema extends zt.ZodSchema = any> {
     /**
      * Maximum depth of research iterations (1-10)
      * @default 7
@@ -342,9 +356,25 @@ interface DeepResearchParams {
      */
     maxUrls?: number;
     /**
-     * Experimental flag for streaming steps
+     * The prompt to use for the final analysis
+     */
+    analysisPrompt?: string;
+    /**
+     * The system prompt to use for the research agent
+     */
+    systemPrompt?: string;
+    /**
+     * The formats to use for the final analysis
+     */
+    formats?: ("markdown" | "json")[];
+    /**
+     * The JSON options to use for the final analysis
      */
-    __experimental_streamSteps?: boolean;
+    jsonOptions?: {
+        prompt?: string;
+        schema?: LLMSchema;
+        systemPrompt?: string;
+    };
 }
 /**
  * Response interface for deep research operations.
@@ -512,6 +542,7 @@ declare class FirecrawlApp {
      * @param pollInterval - Time in seconds for job status checks.
      * @param idempotencyKey - Optional idempotency key for the request.
      * @param webhook - Optional webhook for the batch scrape.
+     * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.
      * @returns The response from the crawl operation.
      */
     batchScrapeUrls(urls: string[], params?: ScrapeParams, pollInterval?: number, idempotencyKey?: string, webhook?: CrawlParams["webhook"], ignoreInvalidURLs?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;
@@ -543,11 +574,11 @@ declare class FirecrawlApp {
     /**
      * Extracts information from URLs using the Firecrawl API.
      * Currently in Beta. Expect breaking changes on future minor versions.
-     * @param url - The URL to extract information from.
+     * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
      * @param params - Additional parameters for the extract request.
      * @returns The response from the extract operation.
      */
-    extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
+    extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
     /**
      * Initiates an asynchronous extract job for a URL using the Firecrawl API.
      * @param url - The URL to extract data from.
@@ -613,7 +644,7 @@ declare class FirecrawlApp {
      * @param onSource - Optional callback to receive source updates in real-time.
      * @returns The final research results.
      */
-    deepResearch(query: string, params: DeepResearchParams, onActivity?: (activity: {
+    deepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>, onActivity?: (activity: {
         type: string;
         status: string;
         message: string;
@@ -630,7 +661,7 @@ declare class FirecrawlApp {
      * @param params - Parameters for the deep research operation.
      * @returns The response containing the research job ID.
      */
-    asyncDeepResearch(query: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse>;
+    asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse>;
     /**
      * Checks the status of a deep research operation.
      * @param id - The ID of the deep research operation.

package/dist/index.js

@@ -383,6 +383,7 @@ var FirecrawlApp = class {
    * @param pollInterval - Time in seconds for job status checks.
    * @param idempotencyKey - Optional idempotency key for the request.
    * @param webhook - Optional webhook for the batch scrape.
+   * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.
    * @returns The response from the crawl operation.
    */
   async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs) {
@@ -571,7 +572,7 @@ var FirecrawlApp = class {
   /**
    * Extracts information from URLs using the Firecrawl API.
    * Currently in Beta. Expect breaking changes on future minor versions.
-   * @param url - The URL to extract information from.
+   * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
    * @param params - Additional parameters for the extract request.
    * @returns The response from the extract operation.
    */
@@ -805,7 +806,7 @@ var FirecrawlApp = class {
    * @param {string} action - The action being performed when the error occurred.
    */
   handleError(response, action) {
-    if ([400, 402, 408, 409, 500].includes(response.status)) {
+    if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
       const errorMessage = response.data.error || "Unknown error occurred";
       const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : "";
       throw new FirecrawlError(
@@ -886,10 +887,25 @@ var FirecrawlApp = class {
    */
   async asyncDeepResearch(query, params) {
     const headers = this.prepareHeaders();
+    let jsonData = { query, ...params };
+    if (jsonData?.jsonOptions?.schema) {
+      let schema = jsonData.jsonOptions.schema;
+      try {
+        schema = zodToJsonSchema(schema);
+      } catch (error) {
+      }
+      jsonData = {
+        ...jsonData,
+        jsonOptions: {
+          ...jsonData.jsonOptions,
+          schema
+        }
+      };
+    }
     try {
       const response = await this.postRequest(
         `${this.apiUrl}/v1/deep-research`,
-        { query, ...params },
+        jsonData,
         headers
       );
       if (response.status === 200) {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl",
-  "version": "1.19.2",
+  "version": "1.21.1",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/index.ts

@@ -69,6 +69,11 @@ export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult
   screenshot?: string;
   metadata?: FirecrawlDocumentMetadata;
   actions: ActionsSchema;
+  compare?: {
+    previousScrapeAt: string | null;
+    changeStatus: "new" | "same" | "changed" | "removed";
+    visibility: "visible" | "hidden";
+  };
   // v1 search only
   title?: string;
   description?: string;
@@ -79,7 +84,7 @@ export interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult
  * Defines the options and configurations available for scraping web content.
  */
 export interface CrawlScrapeOptions {
-  formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json")[];
+  formats?: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract" | "json" | "compare")[];
   headers?: Record<string, string>;
   includeTags?: string[];
   excludeTags?: string[];
@@ -104,6 +109,7 @@ export type Action = {
 } | {
   type: "click",
   selector: string,
+  all?: boolean,
 } | {
   type: "screenshot",
   fullPage?: boolean,
@@ -140,6 +146,14 @@ export interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchem
 
 export interface ActionsResult {
   screenshots: string[];
+  scrapes: ({
+    url: string;
+    html: string;
+  })[];
+  javascriptReturns: {
+    type: string;
+    value: unknown
+  }[];
 }
 
 /**
@@ -355,7 +369,7 @@ export interface CrawlErrorsResponse {
  * Parameters for deep research operations.
  * Defines options for conducting deep research on a query.
  */
-export interface DeepResearchParams {
+export interface DeepResearchParams<LLMSchema extends zt.ZodSchema = any>  {
   /**
    * Maximum depth of research iterations (1-10)
    * @default 7
@@ -372,9 +386,29 @@ export interface DeepResearchParams {
    */
   maxUrls?: number;
   /**
+   * The prompt to use for the final analysis
+   */
+  analysisPrompt?: string;
+  /**
+   * The system prompt to use for the research agent
+   */
+  systemPrompt?: string;
+  /**
+   * The formats to use for the final analysis
+   */
+  formats?: ("markdown" | "json")[];
+  /**
+   * The JSON options to use for the final analysis
+   */
+  jsonOptions?:{
+    prompt?: string;
+    schema?: LLMSchema;
+    systemPrompt?: string;
+  };
+  /** 
    * Experimental flag for streaming steps
    */
-  __experimental_streamSteps?: boolean;
+  // __experimental_streamSteps?: boolean;
 }
 
 /**
@@ -893,6 +927,7 @@ export default class FirecrawlApp {
    * @param pollInterval - Time in seconds for job status checks.
    * @param idempotencyKey - Optional idempotency key for the request.
    * @param webhook - Optional webhook for the batch scrape.
+   * @param ignoreInvalidURLs - Optional flag to ignore invalid URLs.
    * @returns The response from the crawl operation.
    */
   async batchScrapeUrls(
@@ -1119,14 +1154,14 @@ export default class FirecrawlApp {
   /**
    * Extracts information from URLs using the Firecrawl API.
    * Currently in Beta. Expect breaking changes on future minor versions.
-   * @param url - The URL to extract information from.
+   * @param urls - The URLs to extract information from. Optional if using other methods for data extraction.
    * @param params - Additional parameters for the extract request.
    * @returns The response from the extract operation.
    */
-  async extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {
+  async extract<T extends zt.ZodSchema = any>(urls?: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse> {
     const headers = this.prepareHeaders();
 
-    let jsonData: { urls: string[] } & ExtractParams<T> = { urls,  ...params };
+    let jsonData: { urls?: string[] } & ExtractParams<T> = { urls: urls,  ...params };
     let jsonSchema: any;
     try {
       if (!params?.schema) {
@@ -1388,7 +1423,7 @@ export default class FirecrawlApp {
    * @param {string} action - The action being performed when the error occurred.
    */
   handleError(response: AxiosResponse, action: string): void {
-    if ([400, 402, 408, 409, 500].includes(response.status)) {
+    if ([400, 402, 403, 408, 409, 500].includes(response.status)) {
       const errorMessage: string =
         response.data.error || "Unknown error occurred";
       const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : '';
@@ -1415,7 +1450,7 @@ export default class FirecrawlApp {
    */
   async deepResearch(
     query: string, 
-    params: DeepResearchParams,
+    params: DeepResearchParams<zt.ZodSchema>,
     onActivity?: (activity: {
       type: string;
       status: string;
@@ -1500,12 +1535,31 @@ export default class FirecrawlApp {
    * @param params - Parameters for the deep research operation.
    * @returns The response containing the research job ID.
    */
-  async asyncDeepResearch(query: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {
+  async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
     const headers = this.prepareHeaders();
+    let jsonData: any = { query, ...params };
+
+    if (jsonData?.jsonOptions?.schema) {
+      let schema = jsonData.jsonOptions.schema;
+      // Try parsing the schema as a Zod schema
+      try {
+        schema = zodToJsonSchema(schema);
+      } catch (error) {
+        
+      }
+      jsonData = {
+        ...jsonData,
+        jsonOptions: {
+          ...jsonData.jsonOptions,
+          schema: schema,
+        },
+      };
+    }
+
     try {
       const response: AxiosResponse = await this.postRequest(
         `${this.apiUrl}/v1/deep-research`,
-        { query, ...params },
+        jsonData,
         headers
       );