firecrawl 1.2.0 → 1.4.2

package/src/__tests__/v1/e2e_withAuth/index.test.ts

@@ -1,4 +1,4 @@
-import FirecrawlApp, { CrawlParams, CrawlResponse, CrawlStatusResponse, MapResponse, ScrapeParams, ScrapeResponse } from '../../../index';
+import FirecrawlApp, { type CrawlParams, type CrawlResponse, type CrawlStatusResponse, type MapResponse, type ScrapeResponse } from '../../../index';
 import { v4 as uuidv4 } from 'uuid';
 import dotenv from 'dotenv';
 import { describe, test, expect } from '@jest/globals';
@@ -6,7 +6,7 @@ import { describe, test, expect } from '@jest/globals';
 dotenv.config();
 
 const TEST_API_KEY = process.env.TEST_API_KEY;
-const API_URL = "http://127.0.0.1:3002";
+const API_URL = process.env.API_URL ?? "https://api.firecrawl.dev";
 
 describe('FirecrawlApp E2E Tests', () => {
   test.concurrent('should throw error for no API key', async () => {
@@ -71,6 +71,7 @@ describe('FirecrawlApp E2E Tests', () => {
     expect(response.links?.length).toBeGreaterThan(0);
     expect(response.links?.[0]).toContain("https://");
     expect(response.metadata).not.toBeNull();
+    expect(response.metadata).not.toBeUndefined();
     expect(response.metadata).toHaveProperty("title");
     expect(response.metadata).toHaveProperty("description");
     expect(response.metadata).toHaveProperty("keywords");
@@ -85,19 +86,21 @@ describe('FirecrawlApp E2E Tests', () => {
     expect(response.metadata).not.toHaveProperty("pageStatusCode");
     expect(response.metadata).toHaveProperty("statusCode");
     expect(response.metadata).not.toHaveProperty("pageError");
-    expect(response.metadata.error).toBeUndefined();
-    expect(response.metadata.title).toBe("Roast My Website");
-    expect(response.metadata.description).toBe("Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️");
-    expect(response.metadata.keywords).toBe("Roast My Website,Roast,Website,GitHub,Firecrawl");
-    expect(response.metadata.robots).toBe("follow, index");
-    expect(response.metadata.ogTitle).toBe("Roast My Website");
-    expect(response.metadata.ogDescription).toBe("Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️");
-    expect(response.metadata.ogUrl).toBe("https://www.roastmywebsite.ai");
-    expect(response.metadata.ogImage).toBe("https://www.roastmywebsite.ai/og.png");
-    expect(response.metadata.ogLocaleAlternate).toStrictEqual([]);
-    expect(response.metadata.ogSiteName).toBe("Roast My Website");
-    expect(response.metadata.sourceURL).toBe("https://roastmywebsite.ai");
-    expect(response.metadata.statusCode).toBe(200);
+    if (response.metadata !== undefined) {
+      expect(response.metadata.error).toBeUndefined();
+      expect(response.metadata.title).toBe("Roast My Website");
+      expect(response.metadata.description).toBe("Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️");
+      expect(response.metadata.keywords).toBe("Roast My Website,Roast,Website,GitHub,Firecrawl");
+      expect(response.metadata.robots).toBe("follow, index");
+      expect(response.metadata.ogTitle).toBe("Roast My Website");
+      expect(response.metadata.ogDescription).toBe("Welcome to Roast My Website, the ultimate tool for putting your website through the wringer! This repository harnesses the power of Firecrawl to scrape and capture screenshots of websites, and then unleashes the latest LLM vision models to mercilessly roast them. 🌶️");
+      expect(response.metadata.ogUrl).toBe("https://www.roastmywebsite.ai");
+      expect(response.metadata.ogImage).toBe("https://www.roastmywebsite.ai/og.png");
+      expect(response.metadata.ogLocaleAlternate).toStrictEqual([]);
+      expect(response.metadata.ogSiteName).toBe("Roast My Website");
+      expect(response.metadata.sourceURL).toBe("https://roastmywebsite.ai");
+      expect(response.metadata.statusCode).toBe(200);
+    }
   }, 30000); // 30 seconds timeout
 
   test.concurrent('should return successful response for valid scrape with PDF file', async () => {
@@ -127,7 +130,7 @@ describe('FirecrawlApp E2E Tests', () => {
 
   test.concurrent('should return successful response for crawl and wait for completion', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const response = await app.crawlUrl('https://roastmywebsite.ai', {}, true, 30) as CrawlStatusResponse;
+    const response = await app.crawlUrl('https://roastmywebsite.ai', {}, 30) as CrawlStatusResponse;
     expect(response).not.toBeNull();
     expect(response).toHaveProperty("total");
     expect(response.total).toBeGreaterThan(0);
@@ -138,21 +141,25 @@ describe('FirecrawlApp E2E Tests', () => {
     expect(response).toHaveProperty("status");
     expect(response.status).toBe("completed");
     expect(response).not.toHaveProperty("next"); // wait until done
-    expect(response.data?.length).toBeGreaterThan(0);
-    expect(response.data?.[0]).toHaveProperty("markdown");
-    expect(response.data?.[0].markdown).toContain("_Roast_");
-    expect(response.data?.[0]).not.toHaveProperty('content'); // v0
-    expect(response.data?.[0]).not.toHaveProperty("html");
-    expect(response.data?.[0]).not.toHaveProperty("rawHtml");
-    expect(response.data?.[0]).not.toHaveProperty("screenshot");
-    expect(response.data?.[0]).not.toHaveProperty("links");
-    expect(response.data?.[0]).toHaveProperty("metadata");
-    expect(response.data?.[0].metadata).toHaveProperty("title");
-    expect(response.data?.[0].metadata).toHaveProperty("description");
-    expect(response.data?.[0].metadata).toHaveProperty("language");
-    expect(response.data?.[0].metadata).toHaveProperty("sourceURL");
-    expect(response.data?.[0].metadata).toHaveProperty("statusCode");
-    expect(response.data?.[0].metadata).not.toHaveProperty("error");
+    expect(response.data.length).toBeGreaterThan(0);
+    expect(response.data[0]).not.toBeNull();
+    expect(response.data[0]).not.toBeUndefined();
+    if (response.data[0]) {
+      expect(response.data[0]).toHaveProperty("markdown");
+      expect(response.data[0].markdown).toContain("_Roast_");
+      expect(response.data[0]).not.toHaveProperty('content'); // v0
+      expect(response.data[0]).not.toHaveProperty("html");
+      expect(response.data[0]).not.toHaveProperty("rawHtml");
+      expect(response.data[0]).not.toHaveProperty("screenshot");
+      expect(response.data[0]).not.toHaveProperty("links");
+      expect(response.data[0]).toHaveProperty("metadata");
+      expect(response.data[0].metadata).toHaveProperty("title");
+      expect(response.data[0].metadata).toHaveProperty("description");
+      expect(response.data[0].metadata).toHaveProperty("language");
+      expect(response.data[0].metadata).toHaveProperty("sourceURL");
+      expect(response.data[0].metadata).toHaveProperty("statusCode");
+      expect(response.data[0].metadata).not.toHaveProperty("error");
+    }
   }, 60000); // 60 seconds timeout
 
   test.concurrent('should return successful response for crawl with options and wait for completion', async () => {    
@@ -173,7 +180,7 @@ describe('FirecrawlApp E2E Tests', () => {
         onlyMainContent: true,
         waitFor: 1000
       }
-    } as CrawlParams, true, 30) as CrawlStatusResponse;
+    } as CrawlParams, 30) as CrawlStatusResponse;
     expect(response).not.toBeNull();
     expect(response).toHaveProperty("total");
     expect(response.total).toBeGreaterThan(0);
@@ -184,41 +191,45 @@ describe('FirecrawlApp E2E Tests', () => {
     expect(response).toHaveProperty("status");
     expect(response.status).toBe("completed");
     expect(response).not.toHaveProperty("next");
-    expect(response.data?.length).toBeGreaterThan(0);
-    expect(response.data?.[0]).toHaveProperty("markdown");
-    expect(response.data?.[0].markdown).toContain("_Roast_");
-    expect(response.data?.[0]).not.toHaveProperty('content'); // v0
-    expect(response.data?.[0]).toHaveProperty("html");
-    expect(response.data?.[0].html).toContain("<h1");
-    expect(response.data?.[0]).toHaveProperty("rawHtml");
-    expect(response.data?.[0].rawHtml).toContain("<h1");
-    expect(response.data?.[0]).toHaveProperty("screenshot");
-    expect(response.data?.[0].screenshot).toContain("https://");
-    expect(response.data?.[0]).toHaveProperty("links");
-    expect(response.data?.[0].links).not.toBeNull();
-    expect(response.data?.[0].links?.length).toBeGreaterThan(0);
-    expect(response.data?.[0]).toHaveProperty("metadata");
-    expect(response.data?.[0].metadata).toHaveProperty("title");
-    expect(response.data?.[0].metadata).toHaveProperty("description");
-    expect(response.data?.[0].metadata).toHaveProperty("language");
-    expect(response.data?.[0].metadata).toHaveProperty("sourceURL");
-    expect(response.data?.[0].metadata).toHaveProperty("statusCode");
-    expect(response.data?.[0].metadata).not.toHaveProperty("error");
+    expect(response.data.length).toBeGreaterThan(0);
+    expect(response.data[0]).not.toBeNull();
+    expect(response.data[0]).not.toBeUndefined();
+    if (response.data[0]) {
+      expect(response.data[0]).toHaveProperty("markdown");
+      expect(response.data[0].markdown).toContain("_Roast_");
+      expect(response.data[0]).not.toHaveProperty('content'); // v0
+      expect(response.data[0]).toHaveProperty("html");
+      expect(response.data[0].html).toContain("<h1");
+      expect(response.data[0]).toHaveProperty("rawHtml");
+      expect(response.data[0].rawHtml).toContain("<h1");
+      expect(response.data[0]).toHaveProperty("screenshot");
+      expect(response.data[0].screenshot).toContain("https://");
+      expect(response.data[0]).toHaveProperty("links");
+      expect(response.data[0].links).not.toBeNull();
+      expect(response.data[0].links?.length).toBeGreaterThan(0);
+      expect(response.data[0]).toHaveProperty("metadata");
+      expect(response.data[0].metadata).toHaveProperty("title");
+      expect(response.data[0].metadata).toHaveProperty("description");
+      expect(response.data[0].metadata).toHaveProperty("language");
+      expect(response.data[0].metadata).toHaveProperty("sourceURL");
+      expect(response.data[0].metadata).toHaveProperty("statusCode");
+      expect(response.data[0].metadata).not.toHaveProperty("error");
+    }
   }, 60000); // 60 seconds timeout
 
   test.concurrent('should handle idempotency key for crawl', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const uniqueIdempotencyKey = uuidv4();
-    const response = await app.crawlUrl('https://roastmywebsite.ai', {}, false, 2, uniqueIdempotencyKey) as CrawlResponse;
+    const response = await app.asyncCrawlUrl('https://roastmywebsite.ai', {}, uniqueIdempotencyKey) as CrawlResponse;
     expect(response).not.toBeNull();
     expect(response.id).toBeDefined();
 
-    await expect(app.crawlUrl('https://roastmywebsite.ai', {}, true, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409");
+    await expect(app.crawlUrl('https://roastmywebsite.ai', {}, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409");
   });
 
   test.concurrent('should check crawl status', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const response = await app.crawlUrl('https://firecrawl.dev', { scrapeOptions: { formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links']}} as CrawlParams, false) as CrawlResponse;
+    const response = await app.asyncCrawlUrl('https://firecrawl.dev', { scrapeOptions: { formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links']}} as CrawlParams) as CrawlResponse;
     expect(response).not.toBeNull();
     expect(response.id).toBeDefined();
 
@@ -226,7 +237,8 @@ describe('FirecrawlApp E2E Tests', () => {
     const maxChecks = 15;
     let checks = 0;
 
-    while (statusResponse.status === 'scraping' && checks < maxChecks) {
+    expect(statusResponse.success).toBe(true);
+    while ((statusResponse as any).status === 'scraping' && checks < maxChecks) {
       await new Promise(resolve => setTimeout(resolve, 5000));
       expect(statusResponse).not.toHaveProperty("partial_data"); // v0
       expect(statusResponse).not.toHaveProperty("current"); // v0
@@ -236,44 +248,55 @@ describe('FirecrawlApp E2E Tests', () => {
       expect(statusResponse).toHaveProperty("expiresAt");
       expect(statusResponse).toHaveProperty("status");
       expect(statusResponse).toHaveProperty("next");
-      expect(statusResponse.total).toBeGreaterThan(0);
-      expect(statusResponse.creditsUsed).toBeGreaterThan(0);
-      expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now());
-      expect(statusResponse.status).toBe("scraping");
-      expect(statusResponse.next).toContain("/v1/crawl/");
+      expect(statusResponse.success).toBe(true);
+      if (statusResponse.success === true) {
+        expect(statusResponse.total).toBeGreaterThan(0);
+        expect(statusResponse.creditsUsed).toBeGreaterThan(0);
+        expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now());
+        expect(statusResponse.status).toBe("scraping");
+        expect(statusResponse.next).toContain("/v1/crawl/");
+      }
       statusResponse = await app.checkCrawlStatus(response.id) as CrawlStatusResponse;
+      expect(statusResponse.success).toBe(true);
       checks++;
     }
 
     expect(statusResponse).not.toBeNull();
     expect(statusResponse).toHaveProperty("total");
-    expect(statusResponse.total).toBeGreaterThan(0);
-    expect(statusResponse).toHaveProperty("creditsUsed");
-    expect(statusResponse.creditsUsed).toBeGreaterThan(0);
-    expect(statusResponse).toHaveProperty("expiresAt");
-    expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now());
-    expect(statusResponse).toHaveProperty("status");
-    expect(statusResponse.status).toBe("completed");
-    expect(statusResponse.data?.length).toBeGreaterThan(0);
-    expect(statusResponse.data?.[0]).toHaveProperty("markdown");
-    expect(statusResponse.data?.[0].markdown?.length).toBeGreaterThan(10);
-    expect(statusResponse.data?.[0]).not.toHaveProperty('content'); // v0
-    expect(statusResponse.data?.[0]).toHaveProperty("html");
-    expect(statusResponse.data?.[0].html).toContain("<div");
-    expect(statusResponse.data?.[0]).toHaveProperty("rawHtml");
-    expect(statusResponse.data?.[0].rawHtml).toContain("<div");
-    expect(statusResponse.data?.[0]).toHaveProperty("screenshot");
-    expect(statusResponse.data?.[0].screenshot).toContain("https://");
-    expect(statusResponse.data?.[0]).toHaveProperty("links");
-    expect(statusResponse.data?.[0].links).not.toBeNull();
-    expect(statusResponse.data?.[0].links?.length).toBeGreaterThan(0);
-    expect(statusResponse.data?.[0]).toHaveProperty("metadata");
-    expect(statusResponse.data?.[0].metadata).toHaveProperty("title");
-    expect(statusResponse.data?.[0].metadata).toHaveProperty("description");
-    expect(statusResponse.data?.[0].metadata).toHaveProperty("language");
-    expect(statusResponse.data?.[0].metadata).toHaveProperty("sourceURL");
-    expect(statusResponse.data?.[0].metadata).toHaveProperty("statusCode");
-    expect(statusResponse.data?.[0].metadata).not.toHaveProperty("error");
+    expect(statusResponse.success).toBe(true);
+    if (statusResponse.success === true) {
+      expect(statusResponse.total).toBeGreaterThan(0);
+      expect(statusResponse).toHaveProperty("creditsUsed");
+      expect(statusResponse.creditsUsed).toBeGreaterThan(0);
+      expect(statusResponse).toHaveProperty("expiresAt");
+      expect(statusResponse.expiresAt.getTime()).toBeGreaterThan(Date.now());
+      expect(statusResponse).toHaveProperty("status");
+      expect(statusResponse.status).toBe("completed");
+      expect(statusResponse.data.length).toBeGreaterThan(0);
+      expect(statusResponse.data[0]).not.toBeNull();
+      expect(statusResponse.data[0]).not.toBeUndefined();
+      if (statusResponse.data[0]) {
+        expect(statusResponse.data[0]).toHaveProperty("markdown");
+        expect(statusResponse.data[0].markdown?.length).toBeGreaterThan(10);
+        expect(statusResponse.data[0]).not.toHaveProperty('content'); // v0
+        expect(statusResponse.data[0]).toHaveProperty("html");
+        expect(statusResponse.data[0].html).toContain("<div");
+        expect(statusResponse.data[0]).toHaveProperty("rawHtml");
+        expect(statusResponse.data[0].rawHtml).toContain("<div");
+        expect(statusResponse.data[0]).toHaveProperty("screenshot");
+        expect(statusResponse.data[0].screenshot).toContain("https://");
+        expect(statusResponse.data[0]).toHaveProperty("links");
+        expect(statusResponse.data[0].links).not.toBeNull();
+        expect(statusResponse.data[0].links?.length).toBeGreaterThan(0);
+        expect(statusResponse.data[0]).toHaveProperty("metadata");
+        expect(statusResponse.data[0].metadata).toHaveProperty("title");
+        expect(statusResponse.data[0].metadata).toHaveProperty("description");
+        expect(statusResponse.data[0].metadata).toHaveProperty("language");
+        expect(statusResponse.data[0].metadata).toHaveProperty("sourceURL");
+        expect(statusResponse.data[0].metadata).toHaveProperty("statusCode");
+        expect(statusResponse.data[0].metadata).not.toHaveProperty("error");
+      }
+    }
   }, 60000); // 60 seconds timeout
 
   test.concurrent('should throw error for invalid API key on map', async () => {

package/src/index.ts

@@ -1,5 +1,5 @@
-import axios, { AxiosResponse, AxiosRequestHeaders } from "axios";
-import { z } from "zod";
+import axios, { type AxiosResponse, type AxiosRequestHeaders } from "axios";
+import type * as zt from "zod";
 import { zodToJsonSchema } from "zod-to-json-schema";
 import { WebSocket } from "isows";
 import { TypedEventTarget } from "typescript-event-target";
@@ -58,13 +58,13 @@ export interface FirecrawlDocumentMetadata {
  * Document interface for Firecrawl.
  * Represents a document retrieved or processed by Firecrawl.
  */
-export interface FirecrawlDocument {
+export interface FirecrawlDocument<T> {
   url?: string;
   markdown?: string;
   html?: string;
   rawHtml?: string;
   links?: string[];
-  extract?: Record<any, any>;
+  extract?: T;
   screenshot?: string;
   metadata?: FirecrawlDocumentMetadata;
 }
@@ -73,26 +73,29 @@ export interface FirecrawlDocument {
  * Parameters for scraping operations.
  * Defines the options and configurations available for scraping web content.
  */
-export interface ScrapeParams {
+export interface CrawlScrapeOptions {
   formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "extract" | "full@scrennshot")[];
   headers?: Record<string, string>;
   includeTags?: string[];
   excludeTags?: string[];
   onlyMainContent?: boolean;
+  waitFor?: number;
+  timeout?: number;
+}
+
+export interface ScrapeParams<LLMSchema extends zt.ZodSchema> extends CrawlScrapeOptions {
   extract?: {
     prompt?: string;
-    schema?: z.ZodSchema | any;
+    schema?: LLMSchema;
     systemPrompt?: string;
   };
-  waitFor?: number;
-  timeout?: number;
 }
 
 /**
  * Response interface for scraping operations.
  * Defines the structure of the response received after a scraping operation.
  */
-export interface ScrapeResponse extends FirecrawlDocument {
+export interface ScrapeResponse<LLMResult> extends FirecrawlDocument<LLMResult> {
   success: true;
   warning?: string;
   error?: string;
@@ -110,7 +113,8 @@ export interface CrawlParams {
   allowBackwardLinks?: boolean;
   allowExternalLinks?: boolean;
   ignoreSitemap?: boolean;
-  scrapeOptions?: ScrapeParams;
+  scrapeOptions?: CrawlScrapeOptions;
+  webhook?: string;
 }
 
 /**
@@ -130,15 +134,14 @@ export interface CrawlResponse {
  */
 export interface CrawlStatusResponse {
   success: true;
-  total: number;
+  status: "scraping" | "completed" | "failed" | "cancelled";
   completed: number;
+  total: number;
   creditsUsed: number;
   expiresAt: Date;
-  status: "scraping" | "completed" | "failed";
-  next: string;
-  data?: FirecrawlDocument[];
-  error?: string;
-}
+  next?: string;
+  data: FirecrawlDocument<undefined>[];
+};
 
 /**
  * Parameters for mapping operations.
@@ -183,7 +186,11 @@ export default class FirecrawlApp {
    * @param config - Configuration options for the FirecrawlApp instance.
    */
   constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
-    this.apiKey = apiKey || "";
+    if (typeof apiKey !== "string") {
+      throw new Error("No API key provided");
+    }
+
+    this.apiKey = apiKey;
     this.apiUrl = apiUrl || "https://api.firecrawl.dev";
   }
 
@@ -193,10 +200,10 @@ export default class FirecrawlApp {
    * @param params - Additional parameters for the scrape request.
    * @returns The response from the scrape operation.
    */
-  async scrapeUrl(
+  async scrapeUrl<T extends zt.ZodSchema>(
     url: string,
-    params?: ScrapeParams
-  ): Promise<ScrapeResponse | ErrorResponse> {
+    params?: ScrapeParams<T>
+  ): Promise<ScrapeResponse<zt.infer<T>> | ErrorResponse> {
     const headers: AxiosRequestHeaders = {
       "Content-Type": "application/json",
       Authorization: `Bearer ${this.apiKey}`,
@@ -328,9 +335,10 @@ export default class FirecrawlApp {
   /**
    * Checks the status of a crawl job using the Firecrawl API.
    * @param id - The ID of the crawl operation.
+   * @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
    * @returns The response containing the job status.
    */
-  async checkCrawlStatus(id?: string): Promise<CrawlStatusResponse | ErrorResponse> {
+  async checkCrawlStatus(id?: string, getAllData = false): Promise<CrawlStatusResponse | ErrorResponse> {
     if (!id) {
       throw new Error("No crawl ID provided");
     }
@@ -342,16 +350,28 @@ export default class FirecrawlApp {
         headers
       );
       if (response.status === 200) {
+        let allData = response.data.data;
+        if (getAllData && response.data.status === "completed") {
+          let statusData = response.data
+          if ("data" in statusData) {
+            let data = statusData.data;
+            while ('next' in statusData) {
+              statusData = (await this.getRequest(statusData.next, headers)).data;
+              data = data.concat(statusData.data);
+            }
+            allData = data;
+          }
+        }
         return ({
-          success: true,
+          success: response.data.success,
           status: response.data.status,
           total: response.data.total,
           completed: response.data.completed,
           creditsUsed: response.data.creditsUsed,
           expiresAt: new Date(response.data.expiresAt),
           next: response.data.next,
-          data: response.data.data,
-          error: response.data.error
+          data: allData,
+          error: response.data.error,
         })
       } else {
         this.handleError(response, "check crawl status");
@@ -451,22 +471,29 @@ export default class FirecrawlApp {
     id: string,
     headers: AxiosRequestHeaders,
     checkInterval: number
-  ): Promise<CrawlStatusResponse> {
+  ): Promise<CrawlStatusResponse | ErrorResponse> {
     while (true) {
-      const statusResponse: AxiosResponse = await this.getRequest(
+      let statusResponse: AxiosResponse = await this.getRequest(
         `${this.apiUrl}/v1/crawl/${id}`,
         headers
       );
       if (statusResponse.status === 200) {
-        const statusData = statusResponse.data;
-        if (statusData.status === "completed") {
-          if ("data" in statusData) {
-            return statusData;
-          } else {
-            throw new Error("Crawl job completed but no data was returned");
-          }
-        } else if (
-          ["active", "paused", "pending", "queued", "scraping"].includes(statusData.status)
+        let statusData = statusResponse.data;
+          if (statusData.status === "completed") {
+            if ("data" in statusData) {
+              let data = statusData.data;
+              while ('next' in statusData) {
+                statusResponse = await this.getRequest(statusData.next, headers);
+                statusData = statusResponse.data;
+                data = data.concat(statusData.data);
+              }
+              statusData.data = data;
+              return statusData;
+            } else {
+              throw new Error("Crawl job completed but no data was returned");
+            }
+          } else if (
+          ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
         ) {
           checkInterval = Math.max(checkInterval, 2);
           await new Promise((resolve) =>
@@ -504,21 +531,21 @@ export default class FirecrawlApp {
 }
 
 interface CrawlWatcherEvents {
-  document: CustomEvent<FirecrawlDocument>,
+  document: CustomEvent<FirecrawlDocument<undefined>>,
   done: CustomEvent<{
     status: CrawlStatusResponse["status"];
-    data: FirecrawlDocument[];
+    data: FirecrawlDocument<undefined>[];
   }>,
   error: CustomEvent<{
     status: CrawlStatusResponse["status"],
-    data: FirecrawlDocument[],
+    data: FirecrawlDocument<undefined>[],
     error: string,
   }>,
 }
 
 export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
   private ws: WebSocket;
-  public data: FirecrawlDocument[];
+  public data: FirecrawlDocument<undefined>[];
   public status: CrawlStatusResponse["status"];
 
   constructor(id: string, app: FirecrawlApp) {
@@ -539,7 +566,7 @@ export class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
     
     type DocumentMessage = {
       type: "document",
-      data: FirecrawlDocument,
+      data: FirecrawlDocument<undefined>,
     }
     
     type DoneMessage = { type: "done" }