firecrawl 4.25.2 → 4.25.4

package/dist/index.d.ts

@@ -4,7 +4,7 @@ import { AxiosResponse, AxiosRequestHeaders } from 'axios';
 import { EventEmitter } from 'events';
 import { TypedEventTarget } from 'typescript-event-target';
 
-type FormatString = "markdown" | "html" | "rawHtml" | "links" | "images" | "screenshot" | "summary" | "changeTracking" | "json" | "attributes" | "branding" | "audio" | "video" | "pii";
+type FormatString = "markdown" | "html" | "rawHtml" | "links" | "images" | "screenshot" | "summary" | "changeTracking" | "json" | "attributes" | "branding" | "audio" | "video";
 interface Viewport {
     width: number;
     height: number;
@@ -165,34 +165,6 @@ interface RedactPIIOptions {
      */
     replaceStyle?: "tag" | "mask" | "remove";
 }
-type PIISource = "model" | "heuristics" | "unknown";
-interface PIISpan {
-    start: number;
-    end: number;
-    /** Unified entity bucket. Omitted when `kind` doesn't map onto one. */
-    entity?: RedactPIIEntity;
-    /** Granular recognizer label from fire-privacy. */
-    kind: string;
-    source: PIISource;
-    /** Confidence in [0, 1] when supplied. */
-    score?: number;
-}
-/**
- * - ok: redaction completed; redactedMarkdown is the result.
- * - skipped: redaction was not performed; see `reason`.
- * - failed: redaction was attempted but did not produce a usable result.
- */
-type PIIStatus = "ok" | "skipped" | "failed";
-/** Always set when status !== "ok". */
-type PIIReason = "empty_input" | "too_large" | "upstream_skipped" | "service_unavailable" | "timeout" | "error";
-interface PIIBlock {
-    status: PIIStatus;
-    reason?: PIIReason;
-    redactedMarkdown: string | null;
-    spans: PIISpan[];
-    /** Span count per entity bucket. Only non-zero entries are present. */
-    counts: Partial<Record<RedactPIIEntity, number>>;
-}
 type ParseFileData = Blob | File | Buffer | Uint8Array | ArrayBuffer | string;
 interface ParseFile {
     data: ParseFileData;
@@ -410,7 +382,6 @@ interface Document {
     warning?: string;
     changeTracking?: Record<string, unknown>;
     branding?: BrandingProfile;
-    pii?: PIIBlock;
 }
 interface PaginationConfig {
     /** When true (default), automatically follow `next` links and aggregate all documents. */
@@ -918,6 +889,146 @@ interface BrowserListResponse {
     sessions?: BrowserSession[];
     error?: string;
 }
+/**
+ * Source identifiers grouped by namespace. Currently only `arxiv` is
+ * populated; each value is an array of ids in that namespace.
+ */
+type IdMap = Record<string, string[]>;
+/** Per-candidate ranking signals (present on similarity results). */
+interface PaperSignals {
+    /** Raw structural strength (co-citation / coupling counts, or seed overlap). */
+    structural: number;
+    /** Semantic score from the intent abstract search (0 if absent). */
+    semantic: number;
+    /** Citation-graph PageRank of the candidate. */
+    pagerank: number;
+    /** Number of distinct seeds connected to this candidate. */
+    seed_overlap: number;
+}
+/** A ranked paper. `paper_id` is canonical; arXiv lives in `ids`. */
+interface PaperResult {
+    /** Canonical paper id — the Milvus INT64 primary key as a decimal string. */
+    paper_id: string;
+    ids?: IdMap;
+    title: string;
+    abstract: string;
+    /** Final ranking score (post-rerank when enabled). Not normalized. */
+    score: number;
+    /** Present on similarity results. */
+    signals?: PaperSignals;
+}
+interface PaperMetadata {
+    paper_id: string;
+    ids?: IdMap;
+    title: string;
+    abstract: string;
+    /** Comma-joined author names. Omitted if unknown. */
+    authors?: string;
+    /** arXiv categories. Omitted if unknown. */
+    categories?: string[];
+    /** Original creation date string (format varies). Omitted if unknown. */
+    created_date?: string;
+    /** Last-updated date string. Omitted if unknown. */
+    update_date?: string;
+}
+interface Passage {
+    /** In-body passage text (may be markdown, including tables). */
+    text: string;
+    /** Dense similarity score for the passage. */
+    score: number;
+}
+interface SearchPapersResponse {
+    results: PaperResult[];
+}
+interface PaperMetadataResponse {
+    paper: PaperMetadata;
+}
+interface ReadPaperResponse {
+    paper: PaperMetadata;
+    /** Resolved canonical paper id (empty string if not found via id-key). */
+    paper_id: string;
+    /** Echo of the read query. */
+    query: string;
+    /** Top matching in-body passages. */
+    passages: Passage[];
+}
+interface SimilarPapersResponse {
+    /** Ranked related papers; each carries `signals`. */
+    results: PaperResult[];
+    /** Number of resolved candidates considered before truncation to `k`. */
+    pool_size: number;
+    /** True if more resolved candidates existed than were returned. */
+    truncated: boolean;
+    /** Human-readable note when no results are produced. */
+    note?: string | null;
+}
+/** Component scores; each field is present only when that signal contributed. */
+interface GitHubScoreBreakdown {
+    rrf?: number;
+    semantic?: number;
+    lexical?: number;
+    fusion?: number;
+}
+interface GitHubSearchItem {
+    resultType: "github_history" | "repo_readme";
+    /** `owner/name`. */
+    repo: string;
+    url: string;
+    /** History page type (e.g. `issue`, `pull`). Omitted for readmes. */
+    pageType?: string;
+    /** Issue/PR number. Omitted for readmes. */
+    number?: number;
+    /** Number of matched segments/chunks. Omitted when not applicable. */
+    segmentCount?: number;
+    /** Readme URL (readme results). Omitted otherwise. */
+    readmeUrl?: string;
+    /** Short matched excerpt. */
+    snippet: string;
+    /** Full matched content in markdown. Omitted unless available. */
+    contentMd?: string;
+    scores: GitHubScoreBreakdown;
+}
+interface GitHubSearchResponse {
+    results: GitHubSearchItem[];
+}
+/** Options for `research.searchPapers`. */
+interface SearchPapersOptions {
+    /** Number of results to return (1–500, default 40). */
+    k?: number;
+    /** Author substring filter(s); ALL must match (case-insensitive). */
+    authors?: string[];
+    /** arXiv category filter(s) (e.g. `cs.LG`); ALL must match. */
+    categories?: string[];
+    /** Inclusive lower bound on created/updated date (ISO `YYYY-MM-DD`). */
+    from?: string;
+    /** Inclusive upper bound on created/updated date (lexicographic). */
+    to?: string;
+}
+/** Options for `research.getPaper`. */
+interface GetPaperOptions {
+    /** When present, switches to read mode and returns in-body passages. */
+    query?: string;
+    /** Passage count (read mode only; 1–50, default 4). Requires `query`. */
+    k?: number;
+}
+/** Options for `research.similarPapers`. */
+interface SimilarPapersOptions {
+    /** Natural-language intent used to semantically rerank candidates. Required. */
+    intent: string;
+    /** Traversal mode (default `similar`). */
+    mode?: "similar" | "citers" | "references";
+    /** Number of related papers to return (1–500, default 40). */
+    k?: number;
+    /** Apply an additional ZeroEntropy rerank over the fused candidates. */
+    rerank?: boolean;
+    /** Additional seed paper reference(s), same format as `id`. */
+    anchor?: string[];
+}
+/** Options for `research.searchGithub`. */
+interface SearchGithubOptions {
+    /** Number of results to return (1–100, default 20). */
+    k?: number;
+}
 
 interface HttpClientOptions {
     apiKey: string;
@@ -1002,6 +1113,48 @@ declare function listBrowsers(http: HttpClient, args?: {
     status?: "active" | "destroyed";
 }): Promise<BrowserListResponse>;
 
+/**
+ * Client for the v2 research endpoints (arXiv papers + GitHub history/readmes).
+ * Accessed via `firecrawl.research`.
+ */
+declare class ResearchClient {
+    private readonly http;
+    constructor(http: HttpClient);
+    /**
+     * Search papers by abstract relevance.
+     * @param query Natural-language search query.
+     * @param options Optional filters (k, authors, categories, from, to).
+     */
+    searchPapers(query: string, options?: SearchPapersOptions): Promise<SearchPapersResponse>;
+    /**
+     * Get paper metadata (detail mode), or read in-body passages (when `query` is
+     * supplied). `k` is only valid together with `query`.
+     * @param id Paper reference: a canonical `paper_id`, an `arxiv:<id>` key, or a
+     *   bare arXiv id / URL.
+     * @param options Optional `query` (switches to read mode) and `k`.
+     */
+    getPaper(id: string, options?: {
+        query?: undefined;
+        k?: undefined;
+    }): Promise<PaperMetadataResponse>;
+    getPaper(id: string, options: {
+        query: string;
+        k?: number;
+    }): Promise<ReadPaperResponse>;
+    /**
+     * Find related papers via the citation graph.
+     * @param id Primary seed paper reference.
+     * @param options Required `intent` plus optional mode, k, rerank, anchor.
+     */
+    similarPapers(id: string, options: SimilarPapersOptions): Promise<SimilarPapersResponse>;
+    /**
+     * Search GitHub issue/PR history and repository readmes.
+     * @param query Search query.
+     * @param options Optional `k`.
+     */
+    searchGithub(query: string, options?: SearchGithubOptions): Promise<GitHubSearchResponse>;
+}
+
 type JobKind = "crawl" | "batch";
 interface WatcherOptions {
     kind?: JobKind;
@@ -1057,6 +1210,7 @@ type FirecrawlClientInput = FirecrawlClientOptions | string;
  */
 declare class FirecrawlClient {
     private readonly http;
+    private _research?;
     private isCloudService;
     /**
      * Create a v2 client.
@@ -1117,6 +1271,11 @@ declare class FirecrawlClient {
      * @returns Structured search results.
      */
     search(query: string, req?: Omit<SearchRequest, "query">): Promise<SearchData>;
+    /**
+     * Access the v2 research endpoints (arXiv papers + GitHub history/readmes).
+     * Example: `firecrawl.research.searchPapers("diffusion models")`.
+     */
+    get research(): ResearchClient;
     /**
      * Map a site to discover URLs (sitemap-aware).
      * @param url Root URL to map.
@@ -2280,4 +2439,4 @@ declare class Firecrawl extends FirecrawlClient {
     get v1(): FirecrawlApp;
 }
 
-export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type HighlightsFormat, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorSummary, type MonitorTarget, type MonitorWebhookConfig, type PDFAction, type PIIBlock, type PIIReason, type PIISource, type PIISpan, type PIIStatus, type PaginationConfig, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type PressAction, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type RedactPIIEntity, type RedactPIIOptions, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
+export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorSummary, type MonitorTarget, type MonitorWebhookConfig, type PDFAction, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PressAction, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   require_package
-} from "./chunk-XCQC2QCZ.js";
+} from "./chunk-BVQPX6RA.js";
 
 // src/v2/utils/httpClient.ts
 import axios from "axios";
@@ -1301,6 +1301,143 @@ async function getTokenUsageHistorical(http, byApiKey) {
   }
 }
 
+// src/v2/methods/research.ts
+var BASE = "/v2/research";
+function appendParam(params, key, value) {
+  if (value == null) return;
+  if (Array.isArray(value)) {
+    for (const v of value) {
+      if (v != null && String(v).length > 0) params.append(key, String(v));
+    }
+  } else {
+    params.append(key, String(value));
+  }
+}
+function withQuery(path, params) {
+  const qs = params.toString();
+  return qs ? `${path}?${qs}` : path;
+}
+function normalizeResearchError(err, action) {
+  if (err?.isAxiosError) {
+    const status = err.response?.status;
+    const body = err.response?.data;
+    if (body && (body.detail || body.title)) {
+      const message = body.detail || body.title;
+      throw new SdkError(message, status, body.type, body);
+    }
+    throw new SdkError(
+      err.message || `Request failed while trying to ${action}`,
+      status,
+      err.code,
+      body
+    );
+  }
+  throw err;
+}
+var ResearchClient = class {
+  constructor(http) {
+    this.http = http;
+  }
+  http;
+  /**
+   * Search papers by abstract relevance.
+   * @param query Natural-language search query.
+   * @param options Optional filters (k, authors, categories, from, to).
+   */
+  async searchPapers(query, options = {}) {
+    if (!query || !query.trim()) throw new Error("query cannot be empty");
+    if (options.k != null && options.k <= 0)
+      throw new Error("k must be positive");
+    const params = new URLSearchParams();
+    appendParam(params, "query", query);
+    appendParam(params, "k", options.k);
+    appendParam(params, "authors", options.authors);
+    appendParam(params, "categories", options.categories);
+    appendParam(params, "from", options.from);
+    appendParam(params, "to", options.to);
+    try {
+      const res = await this.http.get(
+        withQuery(`${BASE}/papers`, params)
+      );
+      if (res.status !== 200) throwForBadResponse(res, "search papers");
+      return res.data;
+    } catch (err) {
+      return normalizeResearchError(err, "search papers");
+    }
+  }
+  async getPaper(id, options = {}) {
+    if (!id || !id.trim()) throw new Error("id cannot be empty");
+    if (options.k != null && options.query == null)
+      throw new Error("k is only valid together with query");
+    if (options.k != null && options.k <= 0)
+      throw new Error("k must be positive");
+    const params = new URLSearchParams();
+    appendParam(params, "query", options.query);
+    appendParam(params, "k", options.k);
+    try {
+      const res = await this.http.get(
+        withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params)
+      );
+      if (res.status !== 200) throwForBadResponse(res, "get paper");
+      return res.data;
+    } catch (err) {
+      return normalizeResearchError(err, "get paper");
+    }
+  }
+  /**
+   * Find related papers via the citation graph.
+   * @param id Primary seed paper reference.
+   * @param options Required `intent` plus optional mode, k, rerank, anchor.
+   */
+  async similarPapers(id, options) {
+    if (!id || !id.trim()) throw new Error("id cannot be empty");
+    if (!options?.intent || !options.intent.trim())
+      throw new Error("intent cannot be empty");
+    if (options.k != null && options.k <= 0)
+      throw new Error("k must be positive");
+    const params = new URLSearchParams();
+    appendParam(params, "intent", options.intent);
+    appendParam(params, "mode", options.mode);
+    appendParam(params, "k", options.k);
+    if (options.rerank != null) appendParam(params, "rerank", options.rerank);
+    appendParam(params, "anchor", options.anchor);
+    try {
+      const res = await this.http.get(
+        withQuery(
+          `${BASE}/papers/${encodeURIComponent(id)}/similar`,
+          params
+        )
+      );
+      if (res.status !== 200) throwForBadResponse(res, "find similar papers");
+      return res.data;
+    } catch (err) {
+      return normalizeResearchError(err, "find similar papers");
+    }
+  }
+  /**
+   * Search GitHub issue/PR history and repository readmes.
+   * @param query Search query.
+   * @param options Optional `k`.
+   */
+  async searchGithub(query, options = {}) {
+    if (!query || !query.trim()) throw new Error("query cannot be empty");
+    if (options.k != null && options.k <= 0)
+      throw new Error("k must be positive");
+    const params = new URLSearchParams();
+    appendParam(params, "query", query);
+    appendParam(params, "k", options.k);
+    try {
+      const res = await this.http.get(
+        withQuery(`${BASE}/github`, params)
+      );
+      if (res.status !== 200) throwForBadResponse(res, "search github");
+      return res.data;
+    } catch (err) {
+      return normalizeResearchError(err, "search github");
+    }
+  }
+};
+
 // src/v2/methods/monitor.ts
 function queryString(params) {
   if (!params) return "";
@@ -1666,6 +1803,7 @@ var Watcher = class extends EventEmitter {
 import "zod";
 var FirecrawlClient = class {
   http;
+  _research;
   isCloudService(url) {
     return url.includes("api.firecrawl.dev");
   }
@@ -1738,6 +1876,15 @@ var FirecrawlClient = class {
   async search(query, req = {}) {
     return search(this.http, { query, ...req });
   }
+  // Research
+  /**
+   * Access the v2 research endpoints (arXiv papers + GitHub history/readmes).
+   * Example: `firecrawl.research.searchPapers("diffusion models")`.
+   */
+  get research() {
+    if (!this._research) this._research = new ResearchClient(this.http);
+    return this._research;
+  }
   // Map
   /**
    * Map a site to discover URLs (sitemap-aware).
@@ -2097,7 +2244,7 @@ var FirecrawlApp = class {
       if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
         return process.env.npm_package_version;
       }
-      const packageJson = await import("./package-D6422PQU.js");
+      const packageJson = await import("./package-4T5PXLTT.js");
       return packageJson.default.version;
     } catch (error) {
       const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
@@ -3494,6 +3641,7 @@ export {
   FirecrawlApp as FirecrawlAppV1,
   FirecrawlClient,
   JobTimeoutError,
+  ResearchClient,
   SdkError,
   Watcher,
   index_default as default

package/dist/{package-D6422PQU.js → package-4T5PXLTT.js}

@@ -1,4 +1,4 @@
 import {
   require_package
-} from "./chunk-XCQC2QCZ.js";
+} from "./chunk-BVQPX6RA.js";
 export default require_package();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "firecrawl",
-  "version": "4.25.2",
+  "version": "4.25.4",
   "description": "JavaScript SDK for Firecrawl API",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -40,7 +40,7 @@
     "ts-jest": "^29.4.5",
     "tsup": "^8.5.0",
     "typescript": "^5.4.5",
-    "uuid": "^9.0.1"
+    "uuid": "^14.0.0"
   },
   "keywords": [
     "firecrawl",

package/pnpm-workspace.yaml

@@ -0,0 +1,3 @@
+allowBuilds:
+  esbuild: true
+  unrs-resolver: true

package/src/__tests__/e2e/v1/index.test.ts

@@ -26,10 +26,10 @@ describe('FirecrawlApp E2E Tests', () => {
   test.concurrent('should throw error for invalid API key on scrape', async () => {
     if (API_URL.includes('api.firecrawl.dev')) {
       const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-      await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 401");
+      await expect(invalidApp.scrapeUrl('https://firecrawl-test-site.vercel.app')).rejects.toThrow("Unexpected error occurred while trying to scrape URL. Status code: 401");
     } else {
       const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-      await expect(invalidApp.scrapeUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
+      await expect(invalidApp.scrapeUrl('https://firecrawl-test-site.vercel.app')).resolves.not.toThrow();
     }
   });
 
@@ -42,7 +42,7 @@ describe('FirecrawlApp E2E Tests', () => {
   test.concurrent('should return successful response for valid scrape', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
 
-    const response = await app.scrapeUrl('https://roastmywebsite.ai');
+    const response = await app.scrapeUrl('https://firecrawl-test-site.vercel.app');
     if (!response.success) {
       throw new Error(response.error);
     }
@@ -51,7 +51,7 @@ describe('FirecrawlApp E2E Tests', () => {
   test.concurrent('should return successful response with valid API key and options', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.scrapeUrl(
-      'https://roastmywebsite.ai', {
+      'https://firecrawl-test-site.vercel.app', {
       formats: ['markdown', 'html', 'rawHtml', 'screenshot', 'links'],
       headers: { "x-key": "test" },
       includeTags: ['h1'],
@@ -69,7 +69,7 @@ describe('FirecrawlApp E2E Tests', () => {
   test.concurrent('should return successful response with valid API key and screenshot fullPage', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const response = await app.scrapeUrl(
-      'https://roastmywebsite.ai', {
+      'https://firecrawl-test-site.vercel.app', {
       formats: ['screenshot@fullPage'],
     });
     if (!response.success) {
@@ -132,16 +132,16 @@ describe('FirecrawlApp E2E Tests', () => {
   test.concurrent('should throw error for invalid API key on crawl', async () => {
     if (API_URL.includes('api.firecrawl.dev')) {
       const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-      await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
+      await expect(invalidApp.crawlUrl('https://firecrawl-test-site.vercel.app')).rejects.toThrow("Request failed with status code 401");
     } else {
       const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-      await expect(invalidApp.crawlUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
+      await expect(invalidApp.crawlUrl('https://firecrawl-test-site.vercel.app')).resolves.not.toThrow();
     }
   });
 
   test.concurrent('should return successful response for crawl and wait for completion', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const response = await app.crawlUrl('https://roastmywebsite.ai', {}, 30) as CrawlStatusResponse;
+    const response = await app.crawlUrl('https://firecrawl-test-site.vercel.app', {}, 30) as CrawlStatusResponse;
     expect(response).not.toHaveProperty("next"); // wait until done
     expect(response.data.length).toBeGreaterThan(0);
     if (response.data[0]) {
@@ -151,7 +151,7 @@ describe('FirecrawlApp E2E Tests', () => {
 
   test.concurrent('should return successful response for crawl with options and wait for completion', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
-    const response = await app.crawlUrl('https://roastmywebsite.ai', {
+    const response = await app.crawlUrl('https://firecrawl-test-site.vercel.app', {
       excludePaths: ['blog/*'],
       includePaths: ['/'],
       maxDepth: 2,
@@ -183,11 +183,11 @@ describe('FirecrawlApp E2E Tests', () => {
   test.concurrent('should handle idempotency key for crawl', async () => {
     const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL });
     const uniqueIdempotencyKey = uuidv4();
-    const response = await app.asyncCrawlUrl('https://roastmywebsite.ai', {}, uniqueIdempotencyKey) as CrawlResponse;
+    const response = await app.asyncCrawlUrl('https://firecrawl-test-site.vercel.app', {}, uniqueIdempotencyKey) as CrawlResponse;
     expect(response).not.toBeNull();
     expect(response.id).toBeDefined();
 
-    await expect(app.crawlUrl('https://roastmywebsite.ai', {}, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409");
+    await expect(app.crawlUrl('https://firecrawl-test-site.vercel.app', {}, 2, uniqueIdempotencyKey)).rejects.toThrow("Request failed with status code 409");
   });
 
   test.concurrent('should check crawl status', async () => {
@@ -236,10 +236,10 @@ describe('FirecrawlApp E2E Tests', () => {
   test.concurrent('should throw error for invalid API key on map', async () => {
     if (API_URL.includes('api.firecrawl.dev')) {
       const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-      await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).rejects.toThrow("Request failed with status code 401");
+      await expect(invalidApp.mapUrl('https://firecrawl-test-site.vercel.app')).rejects.toThrow("Request failed with status code 401");
     } else {
       const invalidApp = new FirecrawlApp({ apiKey: "invalid_api_key", apiUrl: API_URL });
-      await expect(invalidApp.mapUrl('https://roastmywebsite.ai')).resolves.not.toThrow();
+      await expect(invalidApp.mapUrl('https://firecrawl-test-site.vercel.app')).resolves.not.toThrow();
     }
   });
 
@@ -250,12 +250,12 @@ describe('FirecrawlApp E2E Tests', () => {
   });
 
   test.concurrent('should return successful response for valid map', async () => {
-    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;
+    const app = new FirecrawlApp({ apiKey: TEST_API_KEY, apiUrl: API_URL }); const response = await app.mapUrl('https://firecrawl-test-site.vercel.app') as MapResponse;
     expect(response).not.toBeNull();
 
     expect(response.links?.length).toBeGreaterThan(0);
     expect(response.links?.[0]).toContain("https://");
-    const filteredLinks = response.links?.filter((link: string) => link.includes("roastmywebsite.ai"));
+    const filteredLinks = response.links?.filter((link: string) => link.includes("firecrawl-test-site.vercel.app"));
     expect(filteredLinks?.length).toBeGreaterThan(0);
   }, 30000); // 30 seconds timeout