firecrawl 0.0.30 → 1.2.0

package/tsconfig.json

@@ -11,7 +11,7 @@
     // "disableReferencedProjectLoad": true,             /* Reduce the number of projects loaded automatically by TypeScript. */
 
     /* Language and Environment */
-    "target": "es2016",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
+    "target": "es2020",                                  /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */
     // "lib": [],                                        /* Specify a set of bundled library declaration files that describe the target runtime environment. */
     // "jsx": "preserve",                                /* Specify what JSX code is generated. */
     // "experimentalDecorators": true,                   /* Enable experimental support for legacy experimental decorators. */
@@ -25,9 +25,9 @@
     // "moduleDetection": "auto",                        /* Control what method is used to detect module-format JS files. */
 
     /* Modules */
-    "module": "NodeNext",                                /* Specify what module code is generated. */
+    "module": "commonjs",                                /* Specify what module code is generated. */
     "rootDir": "./src",                                  /* Specify the root folder within your source files. */
-    "moduleResolution": "nodenext",                     /* Specify how TypeScript looks up a file from a given module specifier. */
+    "moduleResolution": "node",                     /* Specify how TypeScript looks up a file from a given module specifier. */
     // "baseUrl": "./",                                  /* Specify the base directory to resolve non-relative module names. */
     // "paths": {},                                      /* Specify a set of entries that re-map imports to additional lookup locations. */
     // "rootDirs": [],                                   /* Allow multiple folders to be treated as one when resolving modules. */

package/types/index.d.ts

@@ -1,7 +1,10 @@
 import { AxiosResponse, AxiosRequestHeaders } from "axios";
 import { z } from "zod";
+import { TypedEventTarget } from "typescript-event-target";
 /**
  * Configuration interface for FirecrawlApp.
+ * @param apiKey - Optional API key for authentication.
+ * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
  */
 export interface FirecrawlAppConfig {
     apiKey?: string | null;
@@ -9,6 +12,7 @@ export interface FirecrawlAppConfig {
 }
 /**
  * Metadata for a Firecrawl document.
+ * Includes various optional properties for document metadata.
  */
 export interface FirecrawlDocumentMetadata {
     title?: string;
@@ -41,145 +45,191 @@ export interface FirecrawlDocumentMetadata {
     articleTag?: string;
     articleSection?: string;
     sourceURL?: string;
-    pageStatusCode?: number;
-    pageError?: string;
+    statusCode?: number;
+    error?: string;
     [key: string]: any;
 }
 /**
  * Document interface for Firecrawl.
+ * Represents a document retrieved or processed by Firecrawl.
  */
 export interface FirecrawlDocument {
-    id?: string;
     url?: string;
-    content: string;
     markdown?: string;
     html?: string;
-    llm_extraction?: Record<string, any>;
-    createdAt?: Date;
-    updatedAt?: Date;
-    type?: string;
-    metadata: FirecrawlDocumentMetadata;
-    childrenLinks?: string[];
-    provider?: string;
-    warning?: string;
-    index?: number;
+    rawHtml?: string;
+    links?: string[];
+    extract?: Record<any, any>;
+    screenshot?: string;
+    metadata?: FirecrawlDocumentMetadata;
+}
+/**
+ * Parameters for scraping operations.
+ * Defines the options and configurations available for scraping web content.
+ */
+export interface ScrapeParams {
+    formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "extract" | "full@scrennshot")[];
+    headers?: Record<string, string>;
+    includeTags?: string[];
+    excludeTags?: string[];
+    onlyMainContent?: boolean;
+    extract?: {
+        prompt?: string;
+        schema?: z.ZodSchema | any;
+        systemPrompt?: string;
+    };
+    waitFor?: number;
+    timeout?: number;
 }
 /**
  * Response interface for scraping operations.
+ * Defines the structure of the response received after a scraping operation.
  */
-export interface ScrapeResponse {
-    success: boolean;
-    data?: FirecrawlDocument;
+export interface ScrapeResponse extends FirecrawlDocument {
+    success: true;
+    warning?: string;
     error?: string;
 }
 /**
- * Response interface for searching operations.
+ * Parameters for crawling operations.
+ * Includes options for both scraping and mapping during a crawl.
  */
-export interface SearchResponse {
-    success: boolean;
-    data?: FirecrawlDocument[];
-    error?: string;
+export interface CrawlParams {
+    includePaths?: string[];
+    excludePaths?: string[];
+    maxDepth?: number;
+    limit?: number;
+    allowBackwardLinks?: boolean;
+    allowExternalLinks?: boolean;
+    ignoreSitemap?: boolean;
+    scrapeOptions?: ScrapeParams;
 }
 /**
  * Response interface for crawling operations.
+ * Defines the structure of the response received after initiating a crawl.
  */
 export interface CrawlResponse {
-    success: boolean;
-    jobId?: string;
-    data?: FirecrawlDocument[];
+    id?: string;
+    url?: string;
+    success: true;
     error?: string;
 }
 /**
  * Response interface for job status checks.
+ * Provides detailed status of a crawl job including progress and results.
  */
-export interface JobStatusResponse {
-    success: boolean;
-    status: string;
-    jobId?: string;
+export interface CrawlStatusResponse {
+    success: true;
+    total: number;
+    completed: number;
+    creditsUsed: number;
+    expiresAt: Date;
+    status: "scraping" | "completed" | "failed";
+    next: string;
     data?: FirecrawlDocument[];
-    partial_data?: FirecrawlDocument[];
     error?: string;
 }
 /**
- * Generic parameter interface.
+ * Parameters for mapping operations.
+ * Defines options for mapping URLs during a crawl.
  */
-export interface Params {
-    [key: string]: any;
-    extractorOptions?: {
-        extractionSchema: z.ZodSchema | any;
-        mode?: "llm-extraction";
-        extractionPrompt?: string;
-    };
+export interface MapParams {
+    search?: string;
+    ignoreSitemap?: boolean;
+    includeSubdomains?: boolean;
+    limit?: number;
+}
+/**
+ * Response interface for mapping operations.
+ * Defines the structure of the response received after a mapping operation.
+ */
+export interface MapResponse {
+    success: true;
+    links?: string[];
+    error?: string;
+}
+/**
+ * Error response interface.
+ * Defines the structure of the response received when an error occurs.
+ */
+export interface ErrorResponse {
+    success: false;
+    error: string;
 }
 /**
  * Main class for interacting with the Firecrawl API.
+ * Provides methods for scraping, searching, crawling, and mapping web content.
  */
 export default class FirecrawlApp {
-    private apiKey;
-    private apiUrl;
+    apiKey: string;
+    apiUrl: string;
     /**
      * Initializes a new instance of the FirecrawlApp class.
-     * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
+     * @param config - Configuration options for the FirecrawlApp instance.
      */
     constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
     /**
      * Scrapes a URL using the Firecrawl API.
-     * @param {string} url - The URL to scrape.
-     * @param {Params | null} params - Additional parameters for the scrape request.
-     * @returns {Promise<ScrapeResponse>} The response from the scrape operation.
+     * @param url - The URL to scrape.
+     * @param params - Additional parameters for the scrape request.
+     * @returns The response from the scrape operation.
      */
-    scrapeUrl(url: string, params?: Params | null): Promise<ScrapeResponse>;
+    scrapeUrl(url: string, params?: ScrapeParams): Promise<ScrapeResponse | ErrorResponse>;
     /**
-     * Searches for a query using the Firecrawl API.
-     * @param {string} query - The query to search for.
-     * @param {Params | null} params - Additional parameters for the search request.
-     * @returns {Promise<SearchResponse>} The response from the search operation.
+     * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+     * @param query - The search query string.
+     * @param params - Additional parameters for the search.
+     * @returns Throws an error advising to use version 0 of the API.
      */
-    search(query: string, params?: Params | null): Promise<SearchResponse>;
+    search(query: string, params?: any): Promise<any>;
     /**
      * Initiates a crawl job for a URL using the Firecrawl API.
-     * @param {string} url - The URL to crawl.
-     * @param {Params | null} params - Additional parameters for the crawl request.
-     * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
-     * @param {number} pollInterval - Time in seconds for job status checks.
-     * @param {string} idempotencyKey - Optional idempotency key for the request.
-     * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
+     * @param url - The URL to crawl.
+     * @param params - Additional parameters for the crawl request.
+     * @param pollInterval - Time in seconds for job status checks.
+     * @param idempotencyKey - Optional idempotency key for the request.
+     * @returns The response from the crawl operation.
      */
-    crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, pollInterval?: number, idempotencyKey?: string): Promise<CrawlResponse | any>;
+    crawlUrl(url: string, params?: CrawlParams, pollInterval?: number, idempotencyKey?: string): Promise<CrawlStatusResponse | ErrorResponse>;
+    asyncCrawlUrl(url: string, params?: CrawlParams, idempotencyKey?: string): Promise<CrawlResponse | ErrorResponse>;
     /**
      * Checks the status of a crawl job using the Firecrawl API.
-     * @param {string} jobId - The job ID of the crawl operation.
-     * @returns {Promise<JobStatusResponse>} The response containing the job status.
+     * @param id - The ID of the crawl operation.
+     * @returns The response containing the job status.
      */
-    checkCrawlStatus(jobId: string): Promise<JobStatusResponse>;
+    checkCrawlStatus(id?: string): Promise<CrawlStatusResponse | ErrorResponse>;
+    crawlUrlAndWatch(url: string, params?: CrawlParams, idempotencyKey?: string): Promise<CrawlWatcher>;
+    mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse>;
     /**
      * Prepares the headers for an API request.
-     * @returns {AxiosRequestHeaders} The prepared headers.
+     * @param idempotencyKey - Optional key to ensure idempotency.
+     * @returns The prepared headers.
      */
     prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders;
     /**
      * Sends a POST request to the specified URL.
-     * @param {string} url - The URL to send the request to.
-     * @param {Params} data - The data to send in the request.
-     * @param {AxiosRequestHeaders} headers - The headers for the request.
-     * @returns {Promise<AxiosResponse>} The response from the POST request.
+     * @param url - The URL to send the request to.
+     * @param data - The data to send in the request.
+     * @param headers - The headers for the request.
+     * @returns The response from the POST request.
      */
-    postRequest(url: string, data: Params, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
+    postRequest(url: string, data: any, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
     /**
      * Sends a GET request to the specified URL.
-     * @param {string} url - The URL to send the request to.
-     * @param {AxiosRequestHeaders} headers - The headers for the request.
-     * @returns {Promise<AxiosResponse>} The response from the GET request.
+     * @param url - The URL to send the request to.
+     * @param headers - The headers for the request.
+     * @returns The response from the GET request.
      */
     getRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
     /**
      * Monitors the status of a crawl job until completion or failure.
-     * @param {string} jobId - The job ID of the crawl operation.
-     * @param {AxiosRequestHeaders} headers - The headers for the request.
-     * @param {number} timeout - Timeout in seconds for job status checks.
-     * @returns {Promise<any>} The final job status or data.
+     * @param id - The ID of the crawl operation.
+     * @param headers - The headers for the request.
+     * @param checkInterval - Interval in seconds for job status checks.
+     * @param checkUrl - Optional URL to check the status (used for v1 API)
+     * @returns The final job status or data.
      */
-    monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<any>;
+    monitorJobStatus(id: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<CrawlStatusResponse>;
     /**
      * Handles errors from API responses.
      * @param {AxiosResponse} response - The response from the API.
@@ -187,3 +237,23 @@ export default class FirecrawlApp {
      */
     handleError(response: AxiosResponse, action: string): void;
 }
+interface CrawlWatcherEvents {
+    document: CustomEvent<FirecrawlDocument>;
+    done: CustomEvent<{
+        status: CrawlStatusResponse["status"];
+        data: FirecrawlDocument[];
+    }>;
+    error: CustomEvent<{
+        status: CrawlStatusResponse["status"];
+        data: FirecrawlDocument[];
+        error: string;
+    }>;
+}
+export declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
+    private ws;
+    data: FirecrawlDocument[];
+    status: CrawlStatusResponse["status"];
+    constructor(id: string, app: FirecrawlApp);
+    close(): void;
+}
+export {};

package/build/index.js

@@ -1,257 +0,0 @@
-var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
-    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
-    return new (P || (P = Promise))(function (resolve, reject) {
-        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
-        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
-        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
-        step((generator = generator.apply(thisArg, _arguments || [])).next());
-    });
-};
-import axios from "axios";
-import { z } from "zod";
-import { zodToJsonSchema } from "zod-to-json-schema";
-/**
- * Main class for interacting with the Firecrawl API.
- */
-export default class FirecrawlApp {
-    /**
-     * Initializes a new instance of the FirecrawlApp class.
-     * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
-     */
-    constructor({ apiKey = null, apiUrl = null }) {
-        this.apiKey = apiKey || "";
-        this.apiUrl = apiUrl || "https://api.firecrawl.dev";
-        if (!this.apiKey) {
-            throw new Error("No API key provided");
-        }
-    }
-    /**
-     * Scrapes a URL using the Firecrawl API.
-     * @param {string} url - The URL to scrape.
-     * @param {Params | null} params - Additional parameters for the scrape request.
-     * @returns {Promise<ScrapeResponse>} The response from the scrape operation.
-     */
-    scrapeUrl(url, params = null) {
-        var _a;
-        return __awaiter(this, void 0, void 0, function* () {
-            const headers = {
-                "Content-Type": "application/json",
-                Authorization: `Bearer ${this.apiKey}`,
-            };
-            let jsonData = Object.assign({ url }, params);
-            if ((_a = params === null || params === void 0 ? void 0 : params.extractorOptions) === null || _a === void 0 ? void 0 : _a.extractionSchema) {
-                let schema = params.extractorOptions.extractionSchema;
-                // Check if schema is an instance of ZodSchema to correctly identify Zod schemas
-                if (schema instanceof z.ZodSchema) {
-                    schema = zodToJsonSchema(schema);
-                }
-                jsonData = Object.assign(Object.assign({}, jsonData), { extractorOptions: Object.assign(Object.assign({}, params.extractorOptions), { extractionSchema: schema, mode: params.extractorOptions.mode || "llm-extraction" }) });
-            }
-            try {
-                const response = yield axios.post(this.apiUrl + "/v0/scrape", jsonData, { headers });
-                if (response.status === 200) {
-                    const responseData = response.data;
-                    if (responseData.success) {
-                        return responseData;
-                    }
-                    else {
-                        throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
-                    }
-                }
-                else {
-                    this.handleError(response, "scrape URL");
-                }
-            }
-            catch (error) {
-                throw new Error(error.message);
-            }
-            return { success: false, error: "Internal server error." };
-        });
-    }
-    /**
-     * Searches for a query using the Firecrawl API.
-     * @param {string} query - The query to search for.
-     * @param {Params | null} params - Additional parameters for the search request.
-     * @returns {Promise<SearchResponse>} The response from the search operation.
-     */
-    search(query, params = null) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const headers = {
-                "Content-Type": "application/json",
-                Authorization: `Bearer ${this.apiKey}`,
-            };
-            let jsonData = { query };
-            if (params) {
-                jsonData = Object.assign(Object.assign({}, jsonData), params);
-            }
-            try {
-                const response = yield axios.post(this.apiUrl + "/v0/search", jsonData, { headers });
-                if (response.status === 200) {
-                    const responseData = response.data;
-                    if (responseData.success) {
-                        return responseData;
-                    }
-                    else {
-                        throw new Error(`Failed to search. Error: ${responseData.error}`);
-                    }
-                }
-                else {
-                    this.handleError(response, "search");
-                }
-            }
-            catch (error) {
-                throw new Error(error.message);
-            }
-            return { success: false, error: "Internal server error." };
-        });
-    }
-    /**
-     * Initiates a crawl job for a URL using the Firecrawl API.
-     * @param {string} url - The URL to crawl.
-     * @param {Params | null} params - Additional parameters for the crawl request.
-     * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
-     * @param {number} pollInterval - Time in seconds for job status checks.
-     * @param {string} idempotencyKey - Optional idempotency key for the request.
-     * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
-     */
-    crawlUrl(url, params = null, waitUntilDone = true, pollInterval = 2, idempotencyKey) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const headers = this.prepareHeaders(idempotencyKey);
-            let jsonData = { url };
-            if (params) {
-                jsonData = Object.assign(Object.assign({}, jsonData), params);
-            }
-            try {
-                const response = yield this.postRequest(this.apiUrl + "/v0/crawl", jsonData, headers);
-                if (response.status === 200) {
-                    const jobId = response.data.jobId;
-                    if (waitUntilDone) {
-                        return this.monitorJobStatus(jobId, headers, pollInterval);
-                    }
-                    else {
-                        return { success: true, jobId };
-                    }
-                }
-                else {
-                    this.handleError(response, "start crawl job");
-                }
-            }
-            catch (error) {
-                console.log(error);
-                throw new Error(error.message);
-            }
-            return { success: false, error: "Internal server error." };
-        });
-    }
-    /**
-     * Checks the status of a crawl job using the Firecrawl API.
-     * @param {string} jobId - The job ID of the crawl operation.
-     * @returns {Promise<JobStatusResponse>} The response containing the job status.
-     */
-    checkCrawlStatus(jobId) {
-        return __awaiter(this, void 0, void 0, function* () {
-            const headers = this.prepareHeaders();
-            try {
-                const response = yield this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
-                if (response.status === 200) {
-                    return {
-                        success: true,
-                        status: response.data.status,
-                        data: response.data.data,
-                        partial_data: !response.data.data
-                            ? response.data.partial_data
-                            : undefined,
-                    };
-                }
-                else {
-                    this.handleError(response, "check crawl status");
-                }
-            }
-            catch (error) {
-                throw new Error(error.message);
-            }
-            return {
-                success: false,
-                status: "unknown",
-                error: "Internal server error.",
-            };
-        });
-    }
-    /**
-     * Prepares the headers for an API request.
-     * @returns {AxiosRequestHeaders} The prepared headers.
-     */
-    prepareHeaders(idempotencyKey) {
-        return Object.assign({ "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}` }, (idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}));
-    }
-    /**
-     * Sends a POST request to the specified URL.
-     * @param {string} url - The URL to send the request to.
-     * @param {Params} data - The data to send in the request.
-     * @param {AxiosRequestHeaders} headers - The headers for the request.
-     * @returns {Promise<AxiosResponse>} The response from the POST request.
-     */
-    postRequest(url, data, headers) {
-        return axios.post(url, data, { headers });
-    }
-    /**
-     * Sends a GET request to the specified URL.
-     * @param {string} url - The URL to send the request to.
-     * @param {AxiosRequestHeaders} headers - The headers for the request.
-     * @returns {Promise<AxiosResponse>} The response from the GET request.
-     */
-    getRequest(url, headers) {
-        return axios.get(url, { headers });
-    }
-    /**
-     * Monitors the status of a crawl job until completion or failure.
-     * @param {string} jobId - The job ID of the crawl operation.
-     * @param {AxiosRequestHeaders} headers - The headers for the request.
-     * @param {number} timeout - Timeout in seconds for job status checks.
-     * @returns {Promise<any>} The final job status or data.
-     */
-    monitorJobStatus(jobId, headers, checkInterval) {
-        return __awaiter(this, void 0, void 0, function* () {
-            while (true) {
-                const statusResponse = yield this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
-                if (statusResponse.status === 200) {
-                    const statusData = statusResponse.data;
-                    if (statusData.status === "completed") {
-                        if ("data" in statusData) {
-                            return statusData.data;
-                        }
-                        else {
-                            throw new Error("Crawl job completed but no data was returned");
-                        }
-                    }
-                    else if (["active", "paused", "pending", "queued"].includes(statusData.status)) {
-                        if (checkInterval < 2) {
-                            checkInterval = 2;
-                        }
-                        yield new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
-                    }
-                    else {
-                        throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`);
-                    }
-                }
-                else {
-                    this.handleError(statusResponse, "check crawl status");
-                }
-            }
-        });
-    }
-    /**
-     * Handles errors from API responses.
-     * @param {AxiosResponse} response - The response from the API.
-     * @param {string} action - The action being performed when the error occurred.
-     */
-    handleError(response, action) {
-        if ([402, 408, 409, 500].includes(response.status)) {
-            const errorMessage = response.data.error || "Unknown error occurred";
-            throw new Error(`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`);
-        }
-        else {
-            throw new Error(`Unexpected error occurred while trying to ${action}. Status code: ${response.status}`);
-        }
-    }
-}

package/build_and_publish.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-
-function build_and_publish {
-    PACKAGE_NAME=$1
-
-    # Replace placeholder with the package name in package.json
-    jq --arg name "$PACKAGE_NAME" '.name = $name' package.json > temp.json && mv temp.json package.json
-
-    # Debug: show modified state
-    echo "Modified package.json for $PACKAGE_NAME:"
-    cat package.json
-
-    # Publish the package using npm
-    npm publish
-
-    # Check if publish was successful
-    if [ $? -ne 0 ]; then
-        echo "Publish failed for $PACKAGE_NAME"
-        exit 1
-    fi
-
-    # Revert the changes to the original placeholder in package.json
-    jq '.name = "PLACEHOLDER_NAME"' package.json > temp.json && mv temp.json package.json
-
-    # Debug: show reverted state
-    echo "Reverted package.json to placeholder:"
-    cat package.json
-}
-
-# Build and publish the first package to npm
-build_and_publish "@mendable/firecrawl-js"
-
-# Build and publish the second package to npm
-build_and_publish "firecrawl"

package/jest.config.cjs

@@ -1,5 +0,0 @@
-/** @type {import('ts-jest').JestConfigWithTsJest} */
-module.exports = {
-  preset: 'ts-jest',
-  testEnvironment: 'node',
-};