@mendable/firecrawl 1.2.2

package/.env.example

@@ -0,0 +1,3 @@
+API_URL=http://localhost:3002
+TEST_API_KEY=fc-YOUR_API_KEY
+

package/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Sideguide Technologies Inc.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/README.md

@@ -0,0 +1,160 @@
+# Firecrawl Node SDK
+
+The Firecrawl Node SDK is a library that allows you to easily scrape and crawl websites, and output the data in a format ready for use with language models (LLMs). It provides a simple and intuitive interface for interacting with the Firecrawl API.
+
+## Installation
+
+To install the Firecrawl Node SDK, you can use npm:
+
+```bash
+npm install @mendable/firecrawl-js
+```
+
+## Usage
+
+1. Get an API key from [firecrawl.dev](https://firecrawl.dev)
+2. Set the API key as an environment variable named `FIRECRAWL_API_KEY` or pass it as a parameter to the `FirecrawlApp` class.
+
+Here's an example of how to use the SDK with error handling:
+
+```js
+import FirecrawlApp, { CrawlParams, CrawlStatusResponse } from '@mendable/firecrawl-js';
+
+const app = new FirecrawlApp({apiKey: "fc-YOUR_API_KEY"});
+
+// Scrape a website
+const scrapeResponse = await app.scrapeUrl('https://firecrawl.dev', {
+  formats: ['markdown', 'html'],
+});
+
+if (scrapeResponse) {
+  console.log(scrapeResponse)
+}
+
+// Crawl a website
+const crawlResponse = await app.crawlUrl('https://firecrawl.dev', {
+  limit: 100,
+  scrapeOptions: {
+    formats: ['markdown', 'html'],
+  }
+})
+
+console.log(crawlResponse)
+```
+
+### Scraping a URL
+
+To scrape a single URL with error handling, use the `scrapeUrl` method. It takes the URL as a parameter and returns the scraped data as a dictionary.
+
+```js
+const url = "https://example.com";
+const scrapedData = await app.scrapeUrl(url);
+```
+
+### Crawling a Website
+
+To crawl a website with error handling, use the `crawlUrl` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
+
+```js
+const crawlResponse = await app.crawlUrl('https://firecrawl.dev', {
+  limit: 100,
+  scrapeOptions: {
+    formats: ['markdown', 'html'],
+  }
+})
+```
+
+
+### Asynchronous Crawl
+
+To initiate an asynchronous crawl of a website, utilize the AsyncCrawlURL method. This method requires the starting URL and optional parameters as inputs. The params argument enables you to define various settings for the asynchronous crawl, such as the maximum number of pages to crawl, permitted domains, and the output format. Upon successful initiation, this method returns an ID, which is essential for subsequently checking the status of the crawl.
+
+```js
+const asyncCrawlResult = await app.asyncCrawlUrl('mendable.ai', { excludePaths: ['blog/*'], limit: 5});
+```
+
+### Checking Crawl Status
+
+To check the status of a crawl job with error handling, use the `checkCrawlStatus` method. It takes the job ID as a parameter and returns the current status of the crawl job`
+
+```js
+const status = await app.checkCrawlStatus(id);
+```
+
+### Extracting structured data from a URL
+
+With LLM extraction, you can easily extract structured data from any URL. We support zod schema to make it easier for you too. Here is how you to use it:
+
+```js
+import FirecrawlApp from "@mendable/firecrawl-js";
+import { z } from "zod";
+
+const app = new FirecrawlApp({
+  apiKey: "fc-YOUR_API_KEY",
+});
+
+// Define schema to extract contents into
+const schema = z.object({
+  top: z
+    .array(
+      z.object({
+        title: z.string(),
+        points: z.number(),
+        by: z.string(),
+        commentsURL: z.string(),
+      })
+    )
+    .length(5)
+    .describe("Top 5 stories on Hacker News"),
+});
+
+const scrapeResult = await app.scrapeUrl("https://firecrawl.dev", {
+  extractorOptions: { extractionSchema: schema },
+});
+
+console.log(scrapeResult.data["llm_extraction"]);
+```
+
+### Map a Website
+
+Use `map_url` to generate a list of URLs from a website. The `params` argument let you customize the mapping process, including options to exclude subdomains or to utilize the sitemap.
+
+```js
+const mapResult = await app.mapUrl('https://example.com') as MapResponse;
+console.log(mapResult)
+```
+
+### Crawl a website with WebSockets
+
+To crawl a website with WebSockets, use the `crawlUrlAndWatch` method. It takes the starting URL and optional parameters as arguments. The `params` argument allows you to specify additional options for the crawl job, such as the maximum number of pages to crawl, allowed domains, and the output format.
+
+```js
+// Crawl a website with WebSockets:
+const watch = await app.crawlUrlAndWatch('mendable.ai', { excludePaths: ['blog/*'], limit: 5});
+
+watch.addEventListener("document", doc => {
+ console.log("DOC", doc.detail);
+});
+
+watch.addEventListener("error", err => {
+ console.error("ERR", err.detail.error);
+});
+
+watch.addEventListener("done", state => {
+ console.log("DONE", state.detail.status);
+});
+```
+
+## Error Handling
+
+The SDK handles errors returned by the Firecrawl API and raises appropriate exceptions. If an error occurs during a request, an exception will be raised with a descriptive error message. The examples above demonstrate how to handle these errors using `try/catch` blocks.
+
+## License
+
+The Firecrawl Node SDK is licensed under the MIT License. This means you are free to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the SDK, subject to the following conditions:
+
+- The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+Please note that while this SDK is MIT licensed, it is part of a larger project which may be under different licensing terms. Always refer to the license information in the root directory of the main project for overall licensing details.

package/build/cjs/index.js

@@ -0,0 +1,354 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CrawlWatcher = void 0;
+const axios_1 = __importDefault(require("axios"));
+const zod_to_json_schema_1 = require("zod-to-json-schema");
+const isows_1 = require("isows");
+const typescript_event_target_1 = require("typescript-event-target");
+/**
+ * Main class for interacting with the Firecrawl API.
+ * Provides methods for scraping, searching, crawling, and mapping web content.
+ */
+class FirecrawlApp {
+    /**
+     * Initializes a new instance of the FirecrawlApp class.
+     * @param config - Configuration options for the FirecrawlApp instance.
+     */
+    constructor({ apiKey = null, apiUrl = null }) {
+        this.apiKey = apiKey || "";
+        this.apiUrl = apiUrl || "https://api.firecrawl.dev";
+    }
+    /**
+     * Scrapes a URL using the Firecrawl API.
+     * @param url - The URL to scrape.
+     * @param params - Additional parameters for the scrape request.
+     * @returns The response from the scrape operation.
+     */
+    async scrapeUrl(url, params) {
+        const headers = {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiKey}`,
+        };
+        let jsonData = { url, ...params };
+        if (jsonData?.extract?.schema) {
+            let schema = jsonData.extract.schema;
+            // Try parsing the schema as a Zod schema
+            try {
+                schema = (0, zod_to_json_schema_1.zodToJsonSchema)(schema);
+            }
+            catch (error) {
+            }
+            jsonData = {
+                ...jsonData,
+                extract: {
+                    ...jsonData.extract,
+                    schema: schema,
+                },
+            };
+        }
+        try {
+            const response = await axios_1.default.post(this.apiUrl + `/v1/scrape`, jsonData, { headers });
+            if (response.status === 200) {
+                const responseData = response.data;
+                if (responseData.success) {
+                    return {
+                        success: true,
+                        warning: responseData.warning,
+                        error: responseData.error,
+                        ...responseData.data
+                    };
+                }
+                else {
+                    throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
+                }
+            }
+            else {
+                this.handleError(response, "scrape URL");
+            }
+        }
+        catch (error) {
+            throw new Error(error.message);
+        }
+        return { success: false, error: "Internal server error." };
+    }
+    /**
+     * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+     * @param query - The search query string.
+     * @param params - Additional parameters for the search.
+     * @returns Throws an error advising to use version 0 of the API.
+     */
+    async search(query, params) {
+        throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0.");
+    }
+    /**
+     * Initiates a crawl job for a URL using the Firecrawl API.
+     * @param url - The URL to crawl.
+     * @param params - Additional parameters for the crawl request.
+     * @param pollInterval - Time in seconds for job status checks.
+     * @param idempotencyKey - Optional idempotency key for the request.
+     * @returns The response from the crawl operation.
+     */
+    async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
+        const headers = this.prepareHeaders(idempotencyKey);
+        let jsonData = { url, ...params };
+        try {
+            const response = await this.postRequest(this.apiUrl + `/v1/crawl`, jsonData, headers);
+            if (response.status === 200) {
+                const id = response.data.id;
+                return this.monitorJobStatus(id, headers, pollInterval);
+            }
+            else {
+                this.handleError(response, "start crawl job");
+            }
+        }
+        catch (error) {
+            if (error.response?.data?.error) {
+                throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`);
+            }
+            else {
+                throw new Error(error.message);
+            }
+        }
+        return { success: false, error: "Internal server error." };
+    }
+    async asyncCrawlUrl(url, params, idempotencyKey) {
+        const headers = this.prepareHeaders(idempotencyKey);
+        let jsonData = { url, ...params };
+        try {
+            const response = await this.postRequest(this.apiUrl + `/v1/crawl`, jsonData, headers);
+            if (response.status === 200) {
+                return response.data;
+            }
+            else {
+                this.handleError(response, "start crawl job");
+            }
+        }
+        catch (error) {
+            if (error.response?.data?.error) {
+                throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`);
+            }
+            else {
+                throw new Error(error.message);
+            }
+        }
+        return { success: false, error: "Internal server error." };
+    }
+    /**
+     * Checks the status of a crawl job using the Firecrawl API.
+     * @param id - The ID of the crawl operation.
+     * @returns The response containing the job status.
+     */
+    async checkCrawlStatus(id) {
+        if (!id) {
+            throw new Error("No crawl ID provided");
+        }
+        const headers = this.prepareHeaders();
+        try {
+            const response = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
+            if (response.status === 200) {
+                return ({
+                    success: true,
+                    status: response.data.status,
+                    total: response.data.total,
+                    completed: response.data.completed,
+                    creditsUsed: response.data.creditsUsed,
+                    expiresAt: new Date(response.data.expiresAt),
+                    next: response.data.next,
+                    data: response.data.data,
+                    error: response.data.error
+                });
+            }
+            else {
+                this.handleError(response, "check crawl status");
+            }
+        }
+        catch (error) {
+            throw new Error(error.message);
+        }
+        return { success: false, error: "Internal server error." };
+    }
+    async crawlUrlAndWatch(url, params, idempotencyKey) {
+        const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
+        if (crawl.success && crawl.id) {
+            const id = crawl.id;
+            return new CrawlWatcher(id, this);
+        }
+        throw new Error("Crawl job failed to start");
+    }
+    async mapUrl(url, params) {
+        const headers = this.prepareHeaders();
+        let jsonData = { url, ...params };
+        try {
+            const response = await this.postRequest(this.apiUrl + `/v1/map`, jsonData, headers);
+            if (response.status === 200) {
+                return response.data;
+            }
+            else {
+                this.handleError(response, "map");
+            }
+        }
+        catch (error) {
+            throw new Error(error.message);
+        }
+        return { success: false, error: "Internal server error." };
+    }
+    /**
+     * Prepares the headers for an API request.
+     * @param idempotencyKey - Optional key to ensure idempotency.
+     * @returns The prepared headers.
+     */
+    prepareHeaders(idempotencyKey) {
+        return {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiKey}`,
+            ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),
+        };
+    }
+    /**
+     * Sends a POST request to the specified URL.
+     * @param url - The URL to send the request to.
+     * @param data - The data to send in the request.
+     * @param headers - The headers for the request.
+     * @returns The response from the POST request.
+     */
+    postRequest(url, data, headers) {
+        return axios_1.default.post(url, data, { headers });
+    }
+    /**
+     * Sends a GET request to the specified URL.
+     * @param url - The URL to send the request to.
+     * @param headers - The headers for the request.
+     * @returns The response from the GET request.
+     */
+    getRequest(url, headers) {
+        return axios_1.default.get(url, { headers });
+    }
+    /**
+     * Monitors the status of a crawl job until completion or failure.
+     * @param id - The ID of the crawl operation.
+     * @param headers - The headers for the request.
+     * @param checkInterval - Interval in seconds for job status checks.
+     * @param checkUrl - Optional URL to check the status (used for v1 API)
+     * @returns The final job status or data.
+     */
+    async monitorJobStatus(id, headers, checkInterval) {
+        while (true) {
+            let statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
+            if (statusResponse.status === 200) {
+                let statusData = statusResponse.data;
+                if (statusData.status === "completed") {
+                    if ("data" in statusData) {
+                        let data = statusData.data;
+                        while ('next' in statusData) {
+                            statusResponse = await this.getRequest(statusData.next, headers);
+                            statusData = statusResponse.data;
+                            data = data.concat(statusData.data);
+                        }
+                        statusData.data = data;
+                        return statusData;
+                    }
+                    else {
+                        throw new Error("Crawl job completed but no data was returned");
+                    }
+                }
+                else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
+                    checkInterval = Math.max(checkInterval, 2);
+                    await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000));
+                }
+                else {
+                    throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`);
+                }
+            }
+            else {
+                this.handleError(statusResponse, "check crawl status");
+            }
+        }
+    }
+    /**
+     * Handles errors from API responses.
+     * @param {AxiosResponse} response - The response from the API.
+     * @param {string} action - The action being performed when the error occurred.
+     */
+    handleError(response, action) {
+        if ([402, 408, 409, 500].includes(response.status)) {
+            const errorMessage = response.data.error || "Unknown error occurred";
+            throw new Error(`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`);
+        }
+        else {
+            throw new Error(`Unexpected error occurred while trying to ${action}. Status code: ${response.status}`);
+        }
+    }
+}
+exports.default = FirecrawlApp;
+class CrawlWatcher extends typescript_event_target_1.TypedEventTarget {
+    constructor(id, app) {
+        super();
+        this.ws = new isows_1.WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
+        this.status = "scraping";
+        this.data = [];
+        const messageHandler = (msg) => {
+            if (msg.type === "done") {
+                this.status = "completed";
+                this.dispatchTypedEvent("done", new CustomEvent("done", {
+                    detail: {
+                        status: this.status,
+                        data: this.data,
+                    },
+                }));
+            }
+            else if (msg.type === "error") {
+                this.status = "failed";
+                this.dispatchTypedEvent("error", new CustomEvent("error", {
+                    detail: {
+                        status: this.status,
+                        data: this.data,
+                        error: msg.error,
+                    },
+                }));
+            }
+            else if (msg.type === "catchup") {
+                this.status = msg.data.status;
+                this.data.push(...(msg.data.data ?? []));
+                for (const doc of this.data) {
+                    this.dispatchTypedEvent("document", new CustomEvent("document", {
+                        detail: doc,
+                    }));
+                }
+            }
+            else if (msg.type === "document") {
+                this.dispatchTypedEvent("document", new CustomEvent("document", {
+                    detail: msg.data,
+                }));
+            }
+        };
+        this.ws.onmessage = ((ev) => {
+            if (typeof ev.data !== "string") {
+                this.ws.close();
+                return;
+            }
+            const msg = JSON.parse(ev.data);
+            messageHandler(msg);
+        }).bind(this);
+        this.ws.onclose = ((ev) => {
+            const msg = JSON.parse(ev.reason);
+            messageHandler(msg);
+        }).bind(this);
+        this.ws.onerror = ((_) => {
+            this.status = "failed";
+            this.dispatchTypedEvent("error", new CustomEvent("error", {
+                detail: {
+                    status: this.status,
+                    data: this.data,
+                    error: "WebSocket error",
+                },
+            }));
+        }).bind(this);
+    }
+    close() {
+        this.ws.close();
+    }
+}
+exports.CrawlWatcher = CrawlWatcher;

package/build/cjs/package.json

@@ -0,0 +1 @@
+{"type": "commonjs"}