firecrawl 1.18.1 → 1.18.4

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "1.18.1",
+  "version": "1.18.4",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/src/index.ts

@@ -364,6 +364,11 @@ export interface DeepResearchParams {
    * @default 270
    */
   timeLimit?: number;
+  /**
+   * Maximum number of URLs to analyze (1-1000)
+   * @default 20
+   */
+  maxUrls?: number;
   /**
    * Experimental flag for streaming steps
    */
@@ -413,6 +418,48 @@ export interface DeepResearchStatusResponse {
   summaries: string[];
 }
 
+/**
+ * Parameters for LLMs.txt generation operations.
+ */
+export interface GenerateLLMsTextParams {
+  /**
+   * Maximum number of URLs to process (1-100)
+   * @default 10
+   */
+  maxUrls?: number;
+  /**
+   * Whether to show the full LLMs-full.txt in the response
+   * @default false
+   */
+  showFullText?: boolean;
+  /**
+   * Experimental flag for streaming
+   */
+  __experimental_stream?: boolean;
+}
+
+/**
+ * Response interface for LLMs.txt generation operations.
+ */
+export interface GenerateLLMsTextResponse {
+  success: boolean;
+  id: string;
+}
+
+/**
+ * Status response interface for LLMs.txt generation operations.
+ */
+export interface GenerateLLMsTextStatusResponse {
+  success: boolean;
+  data: {
+    llmstxt: string;
+    llmsfulltxt?: string;
+  };
+  status: "processing" | "completed" | "failed";
+  error?: string;
+  expiresAt: string;
+}
+
 /**
  * Main class for interacting with the Firecrawl API.
  * Provides methods for scraping, searching, crawling, and mapping web content.
@@ -493,7 +540,7 @@ export default class FirecrawlApp {
       const response: AxiosResponse = await axios.post(
         this.apiUrl + `/v1/scrape`,
         jsonData,
-        { headers }
+        { headers, timeout: params?.timeout !== undefined ? (params.timeout + 5000) : undefined },
       );
       if (response.status === 200) {
         const responseData = response.data;
@@ -1220,7 +1267,7 @@ export default class FirecrawlApp {
     data: any,
     headers: AxiosRequestHeaders
   ): Promise<AxiosResponse> {
-    return axios.post(url, data, { headers });
+    return axios.post(url, data, { headers, timeout: (data?.timeout ? (data.timeout + 5000) : undefined) });
   }
 
   /**
@@ -1349,10 +1396,22 @@ export default class FirecrawlApp {
 
   /**
    * Initiates a deep research operation on a given topic and polls until completion.
+   * @param topic - The topic to research.
    * @param params - Parameters for the deep research operation.
+   * @param onActivity - Optional callback to receive activity updates in real-time.
    * @returns The final research results.
    */
-  async __deepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchStatusResponse | ErrorResponse> {
+  async __deepResearch(
+    topic: string, 
+    params: DeepResearchParams,
+    onActivity?: (activity: {
+      type: string;
+      status: string;
+      message: string;
+      timestamp: string;
+      depth: number;
+    }) => void
+  ): Promise<DeepResearchStatusResponse | ErrorResponse> {
     try {
       const response = await this.__asyncDeepResearch(topic, params);
       
@@ -1366,16 +1425,24 @@ export default class FirecrawlApp {
 
       const jobId = response.id;
       let researchStatus;
+      let lastActivityCount = 0;
 
       while (true) {
-        // console.log("Checking research status...");
         researchStatus = await this.__checkDeepResearchStatus(jobId);
-        // console.log("Research status:", researchStatus);
         
         if ('error' in researchStatus && !researchStatus.success) {
           return researchStatus;
         }
 
+        // Stream new activities through the callback if provided
+        if (onActivity && researchStatus.activities) {
+          const newActivities = researchStatus.activities.slice(lastActivityCount);
+          for (const activity of newActivities) {
+            onActivity(activity);
+          }
+          lastActivityCount = researchStatus.activities.length;
+        }
+
         if (researchStatus.status === "completed") {
           return researchStatus;
         }
@@ -1393,7 +1460,6 @@ export default class FirecrawlApp {
 
         await new Promise(resolve => setTimeout(resolve, 2000));
       }
-      // console.log("Research status finished:", researchStatus);
 
       return { success: false, error: "Research job terminated unexpectedly" };
     } catch (error: any) {
@@ -1459,6 +1525,118 @@ export default class FirecrawlApp {
     }
     return { success: false, error: "Internal server error." };
   }
+
+  /**
+   * Generates LLMs.txt for a given URL and polls until completion.
+   * @param url - The URL to generate LLMs.txt from.
+   * @param params - Parameters for the LLMs.txt generation operation.
+   * @returns The final generation results.
+   */
+  async generateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {
+    try {
+      const response = await this.asyncGenerateLLMsText(url, params);
+      
+      if (!response.success || 'error' in response) {
+        return { success: false, error: 'error' in response ? response.error : 'Unknown error' };
+      }
+
+      if (!response.id) {
+        throw new FirecrawlError(`Failed to start LLMs.txt generation. No job ID returned.`, 500);
+      }
+
+      const jobId = response.id;
+      let generationStatus;
+
+      while (true) {
+        generationStatus = await this.checkGenerateLLMsTextStatus(jobId);
+        
+        if ('error' in generationStatus && !generationStatus.success) {
+          return generationStatus;
+        }
+
+        if (generationStatus.status === "completed") {
+          return generationStatus;
+        }
+
+        if (generationStatus.status === "failed") {
+          throw new FirecrawlError(
+            `LLMs.txt generation job ${generationStatus.status}. Error: ${generationStatus.error}`, 
+            500
+          );
+        }
+
+        if (generationStatus.status !== "processing") {
+          break;
+        }
+
+        await new Promise(resolve => setTimeout(resolve, 2000));
+      }
+
+      return { success: false, error: "LLMs.txt generation job terminated unexpectedly" };
+    } catch (error: any) {
+      throw new FirecrawlError(error.message, 500, error.response?.data?.details);
+    }
+  }
+
+  /**
+   * Initiates a LLMs.txt generation operation without polling.
+   * @param url - The URL to generate LLMs.txt from.
+   * @param params - Parameters for the LLMs.txt generation operation.
+   * @returns The response containing the generation job ID.
+   */
+  async asyncGenerateLLMsText(url: string, params?: GenerateLLMsTextParams): Promise<GenerateLLMsTextResponse | ErrorResponse> {
+    const headers = this.prepareHeaders();
+    try {
+      const response: AxiosResponse = await this.postRequest(
+        `${this.apiUrl}/v1/llmstxt`,
+        { url, ...params },
+        headers
+      );
+
+      if (response.status === 200) {
+        return response.data;
+      } else {
+        this.handleError(response, "start LLMs.txt generation");
+      }
+    } catch (error: any) {
+      if (error.response?.data?.error) {
+        throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+      } else {
+        throw new FirecrawlError(error.message, 500);
+      }
+    }
+    return { success: false, error: "Internal server error." };
+  }
+
+  /**
+   * Checks the status of a LLMs.txt generation operation.
+   * @param id - The ID of the LLMs.txt generation operation.
+   * @returns The current status and results of the generation operation.
+   */
+  async checkGenerateLLMsTextStatus(id: string): Promise<GenerateLLMsTextStatusResponse | ErrorResponse> {
+    const headers = this.prepareHeaders();
+    try {
+      const response: AxiosResponse = await this.getRequest(
+        `${this.apiUrl}/v1/llmstxt/${id}`,
+        headers
+      );
+
+      if (response.status === 200) {
+        return response.data;
+      } else if (response.status === 404) {
+        throw new FirecrawlError("LLMs.txt generation job not found", 404);
+      } else {
+        this.handleError(response, "check LLMs.txt generation status");
+      }
+    } catch (error: any) {
+      if (error.response?.data?.error) {
+        throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+      } else {
+        throw new FirecrawlError(error.message, 500);
+      }
+    }
+    return { success: false, error: "Internal server error." };
+  }
 }
 
 interface CrawlWatcherEvents {