@mendable/firecrawl-js 0.0.30 → 0.0.31

package/build/cjs/index.js

@@ -0,0 +1,271 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const axios_1 = __importDefault(require("axios"));
+const zod_1 = require("zod");
+const zod_to_json_schema_1 = require("zod-to-json-schema");
+/**
+ * Main class for interacting with the Firecrawl API.
+ */
+class FirecrawlApp {
+    /**
+     * Initializes a new instance of the FirecrawlApp class.
+     * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
+     */
+    constructor({ apiKey = null, apiUrl = null }) {
+        this.apiKey = apiKey || "";
+        this.apiUrl = apiUrl || "https://api.firecrawl.dev";
+        if (!this.apiKey) {
+            throw new Error("No API key provided");
+        }
+    }
+    /**
+     * Scrapes a URL using the Firecrawl API.
+     * @param {string} url - The URL to scrape.
+     * @param {Params | null} params - Additional parameters for the scrape request.
+     * @returns {Promise<ScrapeResponse>} The response from the scrape operation.
+     */
+    scrapeUrl(url, params = null) {
+        var _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            const headers = {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${this.apiKey}`,
+            };
+            let jsonData = Object.assign({ url }, params);
+            if ((_a = params === null || params === void 0 ? void 0 : params.extractorOptions) === null || _a === void 0 ? void 0 : _a.extractionSchema) {
+                let schema = params.extractorOptions.extractionSchema;
+                // Check if schema is an instance of ZodSchema to correctly identify Zod schemas
+                if (schema instanceof zod_1.z.ZodSchema) {
+                    schema = (0, zod_to_json_schema_1.zodToJsonSchema)(schema);
+                }
+                jsonData = Object.assign(Object.assign({}, jsonData), { extractorOptions: Object.assign(Object.assign({}, params.extractorOptions), { extractionSchema: schema, mode: params.extractorOptions.mode || "llm-extraction" }) });
+            }
+            try {
+                const response = yield axios_1.default.post(this.apiUrl + "/v0/scrape", jsonData, { headers });
+                if (response.status === 200) {
+                    const responseData = response.data;
+                    if (responseData.success) {
+                        return responseData;
+                    }
+                    else {
+                        throw new Error(`Failed to scrape URL. Error: ${responseData.error}`);
+                    }
+                }
+                else {
+                    this.handleError(response, "scrape URL");
+                }
+            }
+            catch (error) {
+                throw new Error(error.message);
+            }
+            return { success: false, error: "Internal server error." };
+        });
+    }
+    /**
+     * Searches for a query using the Firecrawl API.
+     * @param {string} query - The query to search for.
+     * @param {Params | null} params - Additional parameters for the search request.
+     * @returns {Promise<SearchResponse>} The response from the search operation.
+     */
+    search(query, params = null) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const headers = {
+                "Content-Type": "application/json",
+                Authorization: `Bearer ${this.apiKey}`,
+            };
+            let jsonData = { query };
+            if (params) {
+                jsonData = Object.assign(Object.assign({}, jsonData), params);
+            }
+            try {
+                const response = yield axios_1.default.post(this.apiUrl + "/v0/search", jsonData, { headers });
+                if (response.status === 200) {
+                    const responseData = response.data;
+                    if (responseData.success) {
+                        return responseData;
+                    }
+                    else {
+                        throw new Error(`Failed to search. Error: ${responseData.error}`);
+                    }
+                }
+                else {
+                    this.handleError(response, "search");
+                }
+            }
+            catch (error) {
+                throw new Error(error.message);
+            }
+            return { success: false, error: "Internal server error." };
+        });
+    }
+    /**
+     * Initiates a crawl job for a URL using the Firecrawl API.
+     * @param {string} url - The URL to crawl.
+     * @param {Params | null} params - Additional parameters for the crawl request.
+     * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
+     * @param {number} pollInterval - Time in seconds for job status checks.
+     * @param {string} idempotencyKey - Optional idempotency key for the request.
+     * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
+     */
+    crawlUrl(url, params = null, waitUntilDone = true, pollInterval = 2, idempotencyKey) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const headers = this.prepareHeaders(idempotencyKey);
+            let jsonData = { url };
+            if (params) {
+                jsonData = Object.assign(Object.assign({}, jsonData), params);
+            }
+            try {
+                const response = yield this.postRequest(this.apiUrl + "/v0/crawl", jsonData, headers);
+                if (response.status === 200) {
+                    const jobId = response.data.jobId;
+                    if (waitUntilDone) {
+                        return this.monitorJobStatus(jobId, headers, pollInterval);
+                    }
+                    else {
+                        return { success: true, jobId };
+                    }
+                }
+                else {
+                    this.handleError(response, "start crawl job");
+                }
+            }
+            catch (error) {
+                console.log(error);
+                throw new Error(error.message);
+            }
+            return { success: false, error: "Internal server error." };
+        });
+    }
+    /**
+     * Checks the status of a crawl job using the Firecrawl API.
+     * @param {string} jobId - The job ID of the crawl operation.
+     * @returns {Promise<JobStatusResponse>} The response containing the job status.
+     */
+    checkCrawlStatus(jobId) {
+        return __awaiter(this, void 0, void 0, function* () {
+            const headers = this.prepareHeaders();
+            try {
+                const response = yield this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
+                if (response.status === 200) {
+                    return {
+                        success: true,
+                        status: response.data.status,
+                        current: response.data.current,
+                        current_url: response.data.current_url,
+                        current_step: response.data.current_step,
+                        total: response.data.total,
+                        data: response.data.data,
+                        partial_data: !response.data.data
+                            ? response.data.partial_data
+                            : undefined,
+                    };
+                }
+                else {
+                    this.handleError(response, "check crawl status");
+                }
+            }
+            catch (error) {
+                throw new Error(error.message);
+            }
+            return {
+                success: false,
+                status: "unknown",
+                current: 0,
+                current_url: "",
+                current_step: "",
+                total: 0,
+                error: "Internal server error.",
+            };
+        });
+    }
+    /**
+     * Prepares the headers for an API request.
+     * @returns {AxiosRequestHeaders} The prepared headers.
+     */
+    prepareHeaders(idempotencyKey) {
+        return Object.assign({ "Content-Type": "application/json", Authorization: `Bearer ${this.apiKey}` }, (idempotencyKey ? { "x-idempotency-key": idempotencyKey } : {}));
+    }
+    /**
+     * Sends a POST request to the specified URL.
+     * @param {string} url - The URL to send the request to.
+     * @param {Params} data - The data to send in the request.
+     * @param {AxiosRequestHeaders} headers - The headers for the request.
+     * @returns {Promise<AxiosResponse>} The response from the POST request.
+     */
+    postRequest(url, data, headers) {
+        return axios_1.default.post(url, data, { headers });
+    }
+    /**
+     * Sends a GET request to the specified URL.
+     * @param {string} url - The URL to send the request to.
+     * @param {AxiosRequestHeaders} headers - The headers for the request.
+     * @returns {Promise<AxiosResponse>} The response from the GET request.
+     */
+    getRequest(url, headers) {
+        return axios_1.default.get(url, { headers });
+    }
+    /**
+     * Monitors the status of a crawl job until completion or failure.
+     * @param {string} jobId - The job ID of the crawl operation.
+     * @param {AxiosRequestHeaders} headers - The headers for the request.
+     * @param {number} timeout - Timeout in seconds for job status checks.
+     * @returns {Promise<any>} The final job status or data.
+     */
+    monitorJobStatus(jobId, headers, checkInterval) {
+        return __awaiter(this, void 0, void 0, function* () {
+            while (true) {
+                const statusResponse = yield this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
+                if (statusResponse.status === 200) {
+                    const statusData = statusResponse.data;
+                    if (statusData.status === "completed") {
+                        if ("data" in statusData) {
+                            return statusData.data;
+                        }
+                        else {
+                            throw new Error("Crawl job completed but no data was returned");
+                        }
+                    }
+                    else if (["active", "paused", "pending", "queued"].includes(statusData.status)) {
+                        if (checkInterval < 2) {
+                            checkInterval = 2;
+                        }
+                        yield new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
+                    }
+                    else {
+                        throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`);
+                    }
+                }
+                else {
+                    this.handleError(statusResponse, "check crawl status");
+                }
+            }
+        });
+    }
+    /**
+     * Handles errors from API responses.
+     * @param {AxiosResponse} response - The response from the API.
+     * @param {string} action - The action being performed when the error occurred.
+     */
+    handleError(response, action) {
+        if ([402, 408, 409, 500].includes(response.status)) {
+            const errorMessage = response.data.error || "Unknown error occurred";
+            throw new Error(`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`);
+        }
+        else {
+            throw new Error(`Unexpected error occurred while trying to ${action}. Status code: ${response.status}`);
+        }
+    }
+}
+exports.default = FirecrawlApp;

package/build/cjs/package.json

@@ -0,0 +1 @@
+{"type": "commonjs"}

package/build/{index.js → esm/index.js}

@@ -157,6 +157,10 @@ export default class FirecrawlApp {
                     return {
                         success: true,
                         status: response.data.status,
+                        current: response.data.current,
+                        current_url: response.data.current_url,
+                        current_step: response.data.current_step,
+                        total: response.data.total,
                         data: response.data.data,
                         partial_data: !response.data.data
                             ? response.data.partial_data
@@ -173,6 +177,10 @@ export default class FirecrawlApp {
             return {
                 success: false,
                 status: "unknown",
+                current: 0,
+                current_url: "",
+                current_step: "",
+                total: 0,
                 error: "Internal server error.",
             };
         });

package/build/esm/package.json

@@ -0,0 +1 @@
+{"type": "module"}

package/jest.config.js

@@ -0,0 +1,16 @@
+/** @type {import('ts-jest').JestConfigWithTsJest} **/
+export default {
+  testEnvironment: "node",
+  "moduleNameMapper": {
+    "^(\\.{1,2}/.*)\\.js$": "$1",
+  },
+  "extensionsToTreatAsEsm": [".ts"],
+  "transform": {
+    "^.+\\.(mt|t|cj|j)s$": [
+      "ts-jest",
+      {
+        "useESM": true
+      }
+    ]
+  },
+};

package/package.json

@@ -1,15 +1,19 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "0.0.30",
+  "version": "0.0.31",
   "description": "JavaScript SDK for Firecrawl API",
-  "main": "build/index.js",
+  "main": "build/cjs/index.js",
   "types": "types/index.d.ts",
   "type": "module",
+  "exports": {
+    "require": "./build/cjs/index.js",
+    "import": "./build/esm/index.js"
+  },
   "scripts": {
-    "build": "tsc",
+    "build": "tsc --module commonjs --moduleResolution node10 --outDir build/cjs/ && echo '{\"type\": \"commonjs\"}' > build/cjs/package.json && npx tsc --module NodeNext --moduleResolution NodeNext --outDir build/esm/ && echo '{\"type\": \"module\"}' > build/esm/package.json",
     "build-and-publish": "npm run build && npm publish --access public",
     "publish-beta": "npm run build && npm publish --access public --tag beta",
-    "test": "jest src/__tests__/**/*.test.ts"
+    "test": "NODE_OPTIONS=--experimental-vm-modules jest --verbose src/__tests__/**/*.test.ts"
   },
   "repository": {
     "type": "git",
@@ -37,7 +41,7 @@
     "@types/node": "^20.12.12",
     "@types/uuid": "^9.0.8",
     "jest": "^29.7.0",
-    "ts-jest": "^29.1.2",
+    "ts-jest": "^29.2.2",
     "typescript": "^5.4.5"
   },
   "keywords": [

package/src/__tests__/e2e_withAuth/index.test.ts

@@ -1,7 +1,7 @@
 import FirecrawlApp from '../../index';
 import { v4 as uuidv4 } from 'uuid';
 import dotenv from 'dotenv';
-
+import { describe, test, expect } from '@jest/globals';
 
 dotenv.config();
 
@@ -9,7 +9,7 @@ const TEST_API_KEY = process.env.TEST_API_KEY;
 const API_URL = "http://127.0.0.1:3002";
 
 describe('FirecrawlApp E2E Tests', () => {
-  test.concurrent('should throw error for no API key', () => {
+  test.concurrent('should throw error for no API key', async () => {
     expect(() => {
       new FirecrawlApp({ apiKey: null, apiUrl: API_URL });
     }).toThrow("No API key provided");
@@ -107,12 +107,16 @@ describe('FirecrawlApp E2E Tests', () => {
     while (statusResponse.status === 'active' && checks < maxChecks) {
       await new Promise(resolve => setTimeout(resolve, 1000));
       expect(statusResponse.partial_data).not.toBeNull();
+      expect(statusResponse.current).toBeGreaterThanOrEqual(1);
       statusResponse = await app.checkCrawlStatus(response.jobId);
       checks++;
     }
 
     expect(statusResponse).not.toBeNull();
+    expect(statusResponse.success).toBe(true);
     expect(statusResponse.status).toBe('completed');
+    expect(statusResponse.total).toEqual(statusResponse.current);
+    expect(statusResponse.current_step).not.toBeNull();
     expect(statusResponse?.data?.length).toBeGreaterThan(0);
   }, 35000); // 35 seconds timeout
 

package/src/index.ts

@@ -100,6 +100,10 @@ export interface CrawlResponse {
 export interface JobStatusResponse {
   success: boolean;
   status: string;
+  current?: number;
+  current_url?: string;
+  current_step?: string;
+  total?: number;
   jobId?: string;
   data?: FirecrawlDocument[];
   partial_data?: FirecrawlDocument[];
@@ -287,6 +291,10 @@ export default class FirecrawlApp {
         return {
           success: true,
           status: response.data.status,
+          current: response.data.current,
+          current_url: response.data.current_url,
+          current_step: response.data.current_step,
+          total: response.data.total,
           data: response.data.data,
           partial_data: !response.data.data
             ? response.data.partial_data
@@ -301,6 +309,10 @@ export default class FirecrawlApp {
     return {
       success: false,
       status: "unknown",
+      current: 0,
+      current_url: "",
+      current_step: "",
+      total: 0,
       error: "Internal server error.",
     };
   }

package/build_and_publish.sh

@@ -1,34 +0,0 @@
-#!/bin/bash
-
-function build_and_publish {
-    PACKAGE_NAME=$1
-
-    # Replace placeholder with the package name in package.json
-    jq --arg name "$PACKAGE_NAME" '.name = $name' package.json > temp.json && mv temp.json package.json
-
-    # Debug: show modified state
-    echo "Modified package.json for $PACKAGE_NAME:"
-    cat package.json
-
-    # Publish the package using npm
-    npm publish
-
-    # Check if publish was successful
-    if [ $? -ne 0 ]; then
-        echo "Publish failed for $PACKAGE_NAME"
-        exit 1
-    fi
-
-    # Revert the changes to the original placeholder in package.json
-    jq '.name = "PLACEHOLDER_NAME"' package.json > temp.json && mv temp.json package.json
-
-    # Debug: show reverted state
-    echo "Reverted package.json to placeholder:"
-    cat package.json
-}
-
-# Build and publish the first package to npm
-build_and_publish "@mendable/firecrawl-js"
-
-# Build and publish the second package to npm
-build_and_publish "firecrawl"

package/jest.config.cjs

@@ -1,5 +0,0 @@
-/** @type {import('ts-jest').JestConfigWithTsJest} */
-module.exports = {
-  preset: 'ts-jest',
-  testEnvironment: 'node',
-};

package/types/index.d.ts

@@ -1,189 +0,0 @@
-import { AxiosResponse, AxiosRequestHeaders } from "axios";
-import { z } from "zod";
-/**
- * Configuration interface for FirecrawlApp.
- */
-export interface FirecrawlAppConfig {
-    apiKey?: string | null;
-    apiUrl?: string | null;
-}
-/**
- * Metadata for a Firecrawl document.
- */
-export interface FirecrawlDocumentMetadata {
-    title?: string;
-    description?: string;
-    language?: string;
-    keywords?: string;
-    robots?: string;
-    ogTitle?: string;
-    ogDescription?: string;
-    ogUrl?: string;
-    ogImage?: string;
-    ogAudio?: string;
-    ogDeterminer?: string;
-    ogLocale?: string;
-    ogLocaleAlternate?: string[];
-    ogSiteName?: string;
-    ogVideo?: string;
-    dctermsCreated?: string;
-    dcDateCreated?: string;
-    dcDate?: string;
-    dctermsType?: string;
-    dcType?: string;
-    dctermsAudience?: string;
-    dctermsSubject?: string;
-    dcSubject?: string;
-    dcDescription?: string;
-    dctermsKeywords?: string;
-    modifiedTime?: string;
-    publishedTime?: string;
-    articleTag?: string;
-    articleSection?: string;
-    sourceURL?: string;
-    pageStatusCode?: number;
-    pageError?: string;
-    [key: string]: any;
-}
-/**
- * Document interface for Firecrawl.
- */
-export interface FirecrawlDocument {
-    id?: string;
-    url?: string;
-    content: string;
-    markdown?: string;
-    html?: string;
-    llm_extraction?: Record<string, any>;
-    createdAt?: Date;
-    updatedAt?: Date;
-    type?: string;
-    metadata: FirecrawlDocumentMetadata;
-    childrenLinks?: string[];
-    provider?: string;
-    warning?: string;
-    index?: number;
-}
-/**
- * Response interface for scraping operations.
- */
-export interface ScrapeResponse {
-    success: boolean;
-    data?: FirecrawlDocument;
-    error?: string;
-}
-/**
- * Response interface for searching operations.
- */
-export interface SearchResponse {
-    success: boolean;
-    data?: FirecrawlDocument[];
-    error?: string;
-}
-/**
- * Response interface for crawling operations.
- */
-export interface CrawlResponse {
-    success: boolean;
-    jobId?: string;
-    data?: FirecrawlDocument[];
-    error?: string;
-}
-/**
- * Response interface for job status checks.
- */
-export interface JobStatusResponse {
-    success: boolean;
-    status: string;
-    jobId?: string;
-    data?: FirecrawlDocument[];
-    partial_data?: FirecrawlDocument[];
-    error?: string;
-}
-/**
- * Generic parameter interface.
- */
-export interface Params {
-    [key: string]: any;
-    extractorOptions?: {
-        extractionSchema: z.ZodSchema | any;
-        mode?: "llm-extraction";
-        extractionPrompt?: string;
-    };
-}
-/**
- * Main class for interacting with the Firecrawl API.
- */
-export default class FirecrawlApp {
-    private apiKey;
-    private apiUrl;
-    /**
-     * Initializes a new instance of the FirecrawlApp class.
-     * @param {FirecrawlAppConfig} config - Configuration options for the FirecrawlApp instance.
-     */
-    constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
-    /**
-     * Scrapes a URL using the Firecrawl API.
-     * @param {string} url - The URL to scrape.
-     * @param {Params | null} params - Additional parameters for the scrape request.
-     * @returns {Promise<ScrapeResponse>} The response from the scrape operation.
-     */
-    scrapeUrl(url: string, params?: Params | null): Promise<ScrapeResponse>;
-    /**
-     * Searches for a query using the Firecrawl API.
-     * @param {string} query - The query to search for.
-     * @param {Params | null} params - Additional parameters for the search request.
-     * @returns {Promise<SearchResponse>} The response from the search operation.
-     */
-    search(query: string, params?: Params | null): Promise<SearchResponse>;
-    /**
-     * Initiates a crawl job for a URL using the Firecrawl API.
-     * @param {string} url - The URL to crawl.
-     * @param {Params | null} params - Additional parameters for the crawl request.
-     * @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
-     * @param {number} pollInterval - Time in seconds for job status checks.
-     * @param {string} idempotencyKey - Optional idempotency key for the request.
-     * @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
-     */
-    crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, pollInterval?: number, idempotencyKey?: string): Promise<CrawlResponse | any>;
-    /**
-     * Checks the status of a crawl job using the Firecrawl API.
-     * @param {string} jobId - The job ID of the crawl operation.
-     * @returns {Promise<JobStatusResponse>} The response containing the job status.
-     */
-    checkCrawlStatus(jobId: string): Promise<JobStatusResponse>;
-    /**
-     * Prepares the headers for an API request.
-     * @returns {AxiosRequestHeaders} The prepared headers.
-     */
-    prepareHeaders(idempotencyKey?: string): AxiosRequestHeaders;
-    /**
-     * Sends a POST request to the specified URL.
-     * @param {string} url - The URL to send the request to.
-     * @param {Params} data - The data to send in the request.
-     * @param {AxiosRequestHeaders} headers - The headers for the request.
-     * @returns {Promise<AxiosResponse>} The response from the POST request.
-     */
-    postRequest(url: string, data: Params, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
-    /**
-     * Sends a GET request to the specified URL.
-     * @param {string} url - The URL to send the request to.
-     * @param {AxiosRequestHeaders} headers - The headers for the request.
-     * @returns {Promise<AxiosResponse>} The response from the GET request.
-     */
-    getRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
-    /**
-     * Monitors the status of a crawl job until completion or failure.
-     * @param {string} jobId - The job ID of the crawl operation.
-     * @param {AxiosRequestHeaders} headers - The headers for the request.
-     * @param {number} timeout - Timeout in seconds for job status checks.
-     * @returns {Promise<any>} The final job status or data.
-     */
-    monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<any>;
-    /**
-     * Handles errors from API responses.
-     * @param {AxiosResponse} response - The response from the API.
-     * @param {string} action - The action being performed when the error occurred.
-     */
-    handleError(response: AxiosResponse, action: string): void;
-}