@mendable/firecrawl 1.2.2

package/build/esm/index.js

@@ -0,0 +1,346 @@
+import axios from "axios";
+import { zodToJsonSchema } from "zod-to-json-schema";
+import { WebSocket } from "isows";
+import { TypedEventTarget } from "typescript-event-target";
+/**
+ * Main class for interacting with the Firecrawl API.
+ * Provides methods for scraping, searching, crawling, and mapping web content.
+ */
+export default class FirecrawlApp {
+    /**
+     * Initializes a new instance of the FirecrawlApp class.
+     * @param config - Configuration options for the FirecrawlApp instance.
+     */
+    constructor({ apiKey = null, apiUrl = null }) {
+        this.apiKey = apiKey || "";
+        this.apiUrl = apiUrl || "https://api.firecrawl.dev";
+    }
+    /**
+     * Scrapes a URL using the Firecrawl API.
+     * @param url - The URL to scrape.
+     * @param params - Additional parameters for the scrape request.
+     * @returns The response from the scrape operation.
+     */
+    async scrapeUrl(url, params) {
+        const headers = {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiKey}`,
+        };
+        let jsonData = { url, ...params };
+        if (jsonData?.extract?.schema) {
+            let schema = jsonData.extract.schema;
+            // Try parsing the schema as a Zod schema
+            try {
+                schema = zodToJsonSchema(schema);
+            }
+            catch (error) {
+            }
+            jsonData = {
+                ...jsonData,
+                extract: {
+                    ...jsonData.extract,
+                    schema: schema,
+                },
+            };
+        }
+        try {
+            const response = await axios.post(this.apiUrl + `/v1/scrape`, jsonData, { headers });
+            if (response.status === 200) {
+                const responseData = response.data;
+                if (responseData.success) {
+                    return {
+                        success: true,
+                        warning: responseData.warning,
+                        error: responseData.error,
+                        ...responseData.data
+                    };
+                }
+                else {
+                    throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
+                }
+            }
+            else {
+                this.handleError(response, "scrape URL");
+            }
+        }
+        catch (error) {
+            throw new Error(error.message);
+        }
+        return { success: false, error: "Internal server error." };
+    }
+    /**
+     * This method is intended to search for a query using the Firecrawl API. However, it is not supported in version 1 of the API.
+     * @param query - The search query string.
+     * @param params - Additional parameters for the search.
+     * @returns Throws an error advising to use version 0 of the API.
+     */
+    async search(query, params) {
+        throw new Error("Search is not supported in v1, please update FirecrawlApp() initialization to use v0.");
+    }
+    /**
+     * Initiates a crawl job for a URL using the Firecrawl API.
+     * @param url - The URL to crawl.
+     * @param params - Additional parameters for the crawl request.
+     * @param pollInterval - Time in seconds for job status checks.
+     * @param idempotencyKey - Optional idempotency key for the request.
+     * @returns The response from the crawl operation.
+     */
+    async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
+        const headers = this.prepareHeaders(idempotencyKey);
+        let jsonData = { url, ...params };
+        try {
+            const response = await this.postRequest(this.apiUrl + `/v1/crawl`, jsonData, headers);
+            if (response.status === 200) {
+                const id = response.data.id;
+                return this.monitorJobStatus(id, headers, pollInterval);
+            }
+            else {
+                this.handleError(response, "start crawl job");
+            }
+        }
+        catch (error) {
+            if (error.response?.data?.error) {
+                throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`);
+            }
+            else {
+                throw new Error(error.message);
+            }
+        }
+        return { success: false, error: "Internal server error." };
+    }
+    async asyncCrawlUrl(url, params, idempotencyKey) {
+        const headers = this.prepareHeaders(idempotencyKey);
+        let jsonData = { url, ...params };
+        try {
+            const response = await this.postRequest(this.apiUrl + `/v1/crawl`, jsonData, headers);
+            if (response.status === 200) {
+                return response.data;
+            }
+            else {
+                this.handleError(response, "start crawl job");
+            }
+        }
+        catch (error) {
+            if (error.response?.data?.error) {
+                throw new Error(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`);
+            }
+            else {
+                throw new Error(error.message);
+            }
+        }
+        return { success: false, error: "Internal server error." };
+    }
+    /**
+     * Checks the status of a crawl job using the Firecrawl API.
+     * @param id - The ID of the crawl operation.
+     * @returns The response containing the job status.
+     */
+    async checkCrawlStatus(id) {
+        if (!id) {
+            throw new Error("No crawl ID provided");
+        }
+        const headers = this.prepareHeaders();
+        try {
+            const response = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
+            if (response.status === 200) {
+                return ({
+                    success: true,
+                    status: response.data.status,
+                    total: response.data.total,
+                    completed: response.data.completed,
+                    creditsUsed: response.data.creditsUsed,
+                    expiresAt: new Date(response.data.expiresAt),
+                    next: response.data.next,
+                    data: response.data.data,
+                    error: response.data.error
+                });
+            }
+            else {
+                this.handleError(response, "check crawl status");
+            }
+        }
+        catch (error) {
+            throw new Error(error.message);
+        }
+        return { success: false, error: "Internal server error." };
+    }
+    async crawlUrlAndWatch(url, params, idempotencyKey) {
+        const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
+        if (crawl.success && crawl.id) {
+            const id = crawl.id;
+            return new CrawlWatcher(id, this);
+        }
+        throw new Error("Crawl job failed to start");
+    }
+    async mapUrl(url, params) {
+        const headers = this.prepareHeaders();
+        let jsonData = { url, ...params };
+        try {
+            const response = await this.postRequest(this.apiUrl + `/v1/map`, jsonData, headers);
+            if (response.status === 200) {
+                return response.data;
+            }
+            else {
+                this.handleError(response, "map");
+            }
+        }
+        catch (error) {
+            throw new Error(error.message);
+        }
+        return { success: false, error: "Internal server error." };
+    }
+    /**
+     * Prepares the headers for an API request.
+     * @param idempotencyKey - Optional key to ensure idempotency.
+     * @returns The prepared headers.
+     */
+    prepareHeaders(idempotencyKey) {
+        return {
+            "Content-Type": "application/json",
+            Authorization: `Bearer ${this.apiKey}`,
+            ...(idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}),
+        };
+    }
+    /**
+     * Sends a POST request to the specified URL.
+     * @param url - The URL to send the request to.
+     * @param data - The data to send in the request.
+     * @param headers - The headers for the request.
+     * @returns The response from the POST request.
+     */
+    postRequest(url, data, headers) {
+        return axios.post(url, data, { headers });
+    }
+    /**
+     * Sends a GET request to the specified URL.
+     * @param url - The URL to send the request to.
+     * @param headers - The headers for the request.
+     * @returns The response from the GET request.
+     */
+    getRequest(url, headers) {
+        return axios.get(url, { headers });
+    }
+    /**
+     * Monitors the status of a crawl job until completion or failure.
+     * @param id - The ID of the crawl operation.
+     * @param headers - The headers for the request.
+     * @param checkInterval - Interval in seconds for job status checks.
+     * @param checkUrl - Optional URL to check the status (used for v1 API)
+     * @returns The final job status or data.
+     */
+    async monitorJobStatus(id, headers, checkInterval) {
+        while (true) {
+            let statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
+            if (statusResponse.status === 200) {
+                let statusData = statusResponse.data;
+                if (statusData.status === "completed") {
+                    if ("data" in statusData) {
+                        let data = statusData.data;
+                        while ('next' in statusData) {
+                            statusResponse = await this.getRequest(statusData.next, headers);
+                            statusData = statusResponse.data;
+                            data = data.concat(statusData.data);
+                        }
+                        statusData.data = data;
+                        return statusData;
+                    }
+                    else {
+                        throw new Error("Crawl job completed but no data was returned");
+                    }
+                }
+                else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
+                    checkInterval = Math.max(checkInterval, 2);
+                    await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000));
+                }
+                else {
+                    throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`);
+                }
+            }
+            else {
+                this.handleError(statusResponse, "check crawl status");
+            }
+        }
+    }
+    /**
+     * Handles errors from API responses.
+     * @param {AxiosResponse} response - The response from the API.
+     * @param {string} action - The action being performed when the error occurred.
+     */
+    handleError(response, action) {
+        if ([402, 408, 409, 500].includes(response.status)) {
+            const errorMessage = response.data.error || "Unknown error occurred";
+            throw new Error(`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`);
+        }
+        else {
+            throw new Error(`Unexpected error occurred while trying to ${action}. Status code: ${response.status}`);
+        }
+    }
+}
+export class CrawlWatcher extends TypedEventTarget {
+    constructor(id, app) {
+        super();
+        this.ws = new WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
+        this.status = "scraping";
+        this.data = [];
+        const messageHandler = (msg) => {
+            if (msg.type === "done") {
+                this.status = "completed";
+                this.dispatchTypedEvent("done", new CustomEvent("done", {
+                    detail: {
+                        status: this.status,
+                        data: this.data,
+                    },
+                }));
+            }
+            else if (msg.type === "error") {
+                this.status = "failed";
+                this.dispatchTypedEvent("error", new CustomEvent("error", {
+                    detail: {
+                        status: this.status,
+                        data: this.data,
+                        error: msg.error,
+                    },
+                }));
+            }
+            else if (msg.type === "catchup") {
+                this.status = msg.data.status;
+                this.data.push(...(msg.data.data ?? []));
+                for (const doc of this.data) {
+                    this.dispatchTypedEvent("document", new CustomEvent("document", {
+                        detail: doc,
+                    }));
+                }
+            }
+            else if (msg.type === "document") {
+                this.dispatchTypedEvent("document", new CustomEvent("document", {
+                    detail: msg.data,
+                }));
+            }
+        };
+        this.ws.onmessage = ((ev) => {
+            if (typeof ev.data !== "string") {
+                this.ws.close();
+                return;
+            }
+            const msg = JSON.parse(ev.data);
+            messageHandler(msg);
+        }).bind(this);
+        this.ws.onclose = ((ev) => {
+            const msg = JSON.parse(ev.reason);
+            messageHandler(msg);
+        }).bind(this);
+        this.ws.onerror = ((_) => {
+            this.status = "failed";
+            this.dispatchTypedEvent("error", new CustomEvent("error", {
+                detail: {
+                    status: this.status,
+                    data: this.data,
+                    error: "WebSocket error",
+                },
+            }));
+        }).bind(this);
+    }
+    close() {
+        this.ws.close();
+    }
+}

package/build/esm/package.json

@@ -0,0 +1 @@
+{"type": "module"}

package/jest.config.js

@@ -0,0 +1,16 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} **/
+export default {
+  testEnvironment: "node",
+  "moduleNameMapper": {
+    "^(\\.{1,2}/.*)\\.js$": "$1",
+  },
+  "extensionsToTreatAsEsm": [".ts"],
+  "transform": {
+    "^.+\\.(mt|t|cj|j)s$": [
+      "ts-jest",
+      {
+        "useESM": true
+      }
+    ]
+  },
+};

package/package.json

@@ -0,0 +1,64 @@
+{
+  "name": "@mendable/firecrawl",
+  "version": "1.2.2",
+  "description": "JavaScript SDK for Firecrawl API",
+  "main": "build/cjs/index.js",
+  "types": "types/index.d.ts",
+  "type": "module",
+  "exports": {
+    "require": {
+      "types": "./types/index.d.ts",
+      "default": "./build/cjs/index.js"
+    },
+    "import": {
+      "types": "./types/index.d.ts",
+      "default": "./build/esm/index.js"
+    }
+  },
+  "scripts": {
+    "build": "tsc --module commonjs --moduleResolution node10 --outDir build/cjs/ && echo '{\"type\": \"commonjs\"}' > build/cjs/package.json && npx tsc --module NodeNext --moduleResolution NodeNext --outDir build/esm/ && echo '{\"type\": \"module\"}' > build/esm/package.json",
+    "build-and-publish": "npm run build && npm publish --access public",
+    "publish-beta": "npm run build && npm publish --access public --tag beta",
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/v1/**/*.test.ts"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/mendableai/firecrawl.git"
+  },
+  "author": "Mendable.ai",
+  "license": "MIT",
+  "dependencies": {
+    "axios": "^1.6.8",
+    "dotenv": "^16.4.5",
+    "isows": "^1.0.4",
+    "typescript-event-target": "^1.1.1",
+    "uuid": "^9.0.1",
+    "zod": "^3.23.8",
+    "zod-to-json-schema": "^3.23.0"
+  },
+  "bugs": {
+    "url": "https://github.com/mendableai/firecrawl/issues"
+  },
+  "homepage": "https://github.com/mendableai/firecrawl#readme",
+  "devDependencies": {
+    "@jest/globals": "^29.7.0",
+    "@types/axios": "^0.14.0",
+    "@types/dotenv": "^8.2.0",
+    "@types/jest": "^29.5.12",
+    "@types/mocha": "^10.0.6",
+    "@types/node": "^20.12.12",
+    "@types/uuid": "^9.0.8",
+    "jest": "^29.7.0",
+    "ts-jest": "^29.2.2",
+    "typescript": "^5.4.5"
+  },
+  "keywords": [
+    "firecrawl",
+    "mendable",
+    "crawler",
+    "web",
+    "scraper",
+    "api",
+    "sdk"
+  ]
+}