firecrawl 4.38.0 → 4.40.0

package/src/__tests__/unit/v2/search-country.unit.test.ts

@@ -0,0 +1,32 @@
+import { describe, expect, jest, test } from "@jest/globals";
+import { search } from "../../../v2/methods/search";
+
+function httpMock() {
+  return {
+    post: jest.fn(async () => ({ status: 200, data: { success: true } })),
+  } as any;
+}
+
+describe("v2 search country", () => {
+  test("forwards the country option", async () => {
+    const http = httpMock();
+
+    await search(http, { query: "firecrawl", country: "de" });
+
+    expect(http.post).toHaveBeenCalledWith(
+      "/v2/search",
+      { query: "firecrawl", country: "de" },
+      {},
+    );
+  });
+
+  test("omits country when the caller does not set it", async () => {
+    const http = httpMock();
+
+    await search(http, { query: "firecrawl" });
+
+    expect(http.post).toHaveBeenCalledWith("/v2/search", { query: "firecrawl" }, {});
+    const payload = http.post.mock.calls[0][1] as Record<string, unknown>;
+    expect(payload).not.toHaveProperty("country");
+  });
+});

package/src/__tests__/unit/v2/search-data-property.test.ts

@@ -1,3 +1,4 @@
+import { jest } from "@jest/globals";
 import { search } from "../../../v2/methods/search";
 
 const fakeHttp = {

package/src/__tests__/unit/v2/threat-protection.unit.test.ts

@@ -32,7 +32,17 @@ describe("v2 threatProtection request serialization", () => {
     expect(http.post).toHaveBeenCalledWith(
       "/v2/scrape",
       expect.objectContaining({ url: "https://example.com", threatProtection }),
-      {},
+      { timeoutMs: 330000 },
+    );
+  });
+
+  test("scrape sends domainTools when true", async () => {
+    const http = makeHttp({ success: true, data: {} });
+    await scrape(http, "https://example.com", { domainTools: true });
+    expect(http.post).toHaveBeenCalledWith(
+      "/v2/scrape",
+      expect.objectContaining({ url: "https://example.com", domainTools: true }),
+      expect.anything(),
     );
   });
 
@@ -121,3 +131,10 @@ describe("v2 threatProtection request serialization", () => {
     expect(body).not.toHaveProperty("threatProtection");
   });
 });
+
+
+test.each([false, undefined])("scrape omits domainTools when %s", async (domainTools) => {
+  const http = makeHttp({ success: true, data: {} });
+  await scrape(http, "https://example.com", { domainTools });
+  expect(http.post.mock.calls[0][1]).not.toHaveProperty("domainTools");
+});

package/src/__tests__/unit/v2/tools.unit.test.ts

@@ -0,0 +1,240 @@
+import { afterAll, beforeAll, beforeEach, describe, expect, jest, test } from "@jest/globals";
+import { createServer, type Server } from "node:http";
+import { FirecrawlClient } from "../../../v2/client";
+
+import { scrapeAlexandria } from "../../../v2/methods/tools";
+
+const tool = {
+  id: "particle/podcasts/episodes/search",
+  provider: "particle",
+  capability: "podcasts/episodes/search",
+  name: "Episode search",
+  description: "Find episodes",
+  creditsCost: 15,
+  perRecord: false,
+  options: [{ name: "semantic_search", type: "string" }],
+  response: { about: "Episodes", key: "data", fields: [] },
+  examples: { javascript: "example", python: "example", curl: "example" },
+  matchedBy: ["semantic", "domain"],
+  matchedUrls: ["https://podcasts.apple.com"],
+};
+const productionTool = {
+  id: "benzinga/calendar/ratings",
+  provider: "benzinga",
+  capability: "calendar/ratings",
+  name: "Analyst ratings",
+  description: "Ratings",
+  creditsCost: 5,
+  perRecord: false,
+  label: "Ratings",
+  whenToUse: "Analyst ratings for a ticker",
+  returns: { about: "Ratings" },
+  discovery: { urls: [] },
+  attribution: { required: true },
+  options: [{ name: "tickers", type: "string" }],
+  response: { fields: [] },
+  matchedBy: ["semantic"],
+  matchedUrls: [],
+};
+const termsRequired = {
+  success: false,
+  code: "THIRD_PARTY_DATA_TERMS_REQUIRED",
+  error: "An organization admin must accept the benzinga provider's terms",
+  requiresAction: {
+    type: "accept_terms",
+    terms: "benzinga",
+    version: "C-1.0.0-draft",
+    url: "https://www.firecrawl.dev/app/alexandria/benzinga",
+  },
+};
+const next = {
+  provider: "firecrawl",
+  capability: "find-tools",
+  options: { providers: ["particle"], level: "tools" },
+};
+let server: Server;
+let client: FirecrawlClient;
+const sent: Array<{ body: any; id: string | undefined }> = [];
+let attempts = 0;
+
+beforeEach(() => {
+  sent.length = 0;
+  attempts = 0;
+});
+
+beforeAll(async () => {
+  server = createServer(async (req, res) => {
+    let raw = "";
+    for await (const chunk of req) raw += chunk;
+    const body = JSON.parse(raw);
+    sent.push({ body, id: req.headers["x-request-id"] as string | undefined });
+    res.setHeader("content-type", "application/json");
+    if (req.url === "/v2/search")
+      return res.end(
+        JSON.stringify({
+          success: true,
+          warning: "Example warning",
+          data: {
+            web: [{ url: "https://podcasts.apple.com" }],
+            tools: [tool, productionTool],
+          },
+        }),
+      );
+    if (body.alexandria[0].provider === "retry" && attempts++ === 0) {
+      res.statusCode = 502;
+      return res.end("{}");
+    }
+    if (body.alexandria[0].provider === "benzinga") {
+      res.statusCode = 403;
+      return res.end(JSON.stringify(termsRequired));
+    }
+    if (body.alexandria[0].provider === "denied") {
+      res.statusCode = 402;
+      return res.end(
+        JSON.stringify({
+          success: false,
+          code: "insufficient_credits",
+          error: "Insufficient credits",
+        }),
+      );
+    }
+    const alexandria =
+      body.alexandria[0].provider === "firecrawl"
+        ? [
+            {
+              ...next,
+              creditsCost: 0,
+              data: {
+                level: "providers",
+                items: [{ id: "particle", name: "Particle", next }],
+                total: 1,
+                next: null,
+              },
+            },
+          ]
+        : [
+            {
+              provider: "retry",
+              capability: "a/b",
+              creditsCost: 15,
+              data: { nested: { value: 1 } },
+            },
+            {
+              provider: "x",
+              capability: "b",
+              error: {
+                code: "unavailable",
+                message: "Unavailable",
+                status: 503,
+              },
+            },
+          ];
+    res.end(
+      JSON.stringify({
+        success: true,
+        scrape_id: "scrape-1",
+        data: { alexandria, creditsCost: alexandria.length === 1 ? 0 : 15 },
+      }),
+    );
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const port = (server.address() as { port: number }).port;
+  client = new FirecrawlClient({
+    apiKey: "fc-test",
+    apiUrl: `http://127.0.0.1:${port}`,
+    backoffFactor: 0,
+  });
+});
+afterAll(async () => {
+  await new Promise<void>((resolve) => server.close(() => resolve()));
+});
+
+describe("Alexandria contracts and execution", () => {
+  test("returns complete unified tools and warning beside web results", async () => {
+    const result = await client.search("podcasts", {
+      sources: ["web", { type: "alexandria" }],
+      domainTools: true,
+      limit: 2,
+    });
+    expect(result.tools).toEqual([tool, productionTool]);
+    expect(result.tools?.[1].examples).toBeUndefined();
+    expect(result.tools?.[1].whenToUse).toBe("Analyst ratings for a ticker");
+    expect(result.warning).toBe("Example warning");
+    expect(sent.at(-1)?.body).toMatchObject({
+      domainTools: true,
+      sources: ["web", { type: "alexandria" }],
+    });
+    expect(result).not.toHaveProperty("alexandria");
+  });
+  test("retains one ID through transport retries and per-tool failures", async () => {
+    const result = await client.scrape({
+      alexandria: [{ provider: "retry", capability: "a/b" }],
+      requestId: "same-request",
+    });
+    const retries = sent.filter(
+      (r) => r.body.alexandria?.[0].provider === "retry",
+    );
+    expect(retries).toHaveLength(2);
+    expect(retries.map((r) => r.id)).toEqual(["same-request", "same-request"]);
+    expect(retries[0].body).not.toHaveProperty("requestId");
+    expect(result.requestId).toBe("same-request");
+    expect(result.creditsCost).toBe(15);
+    expect(result.alexandria[1].error?.code).toBe("unavailable");
+  });
+  test("returns an error code and retry identity on failed execution", async () => {
+    await expect(
+      client.scrape({
+        alexandria: { provider: "denied", capability: "a/b" },
+        requestId: "denied-1",
+      }),
+    ).rejects.toMatchObject({
+      status: 402,
+      code: "insufficient_credits",
+      requestId: "denied-1",
+    });
+  });
+  test("exposes requiresAction on a provider terms rejection", async () => {
+    await expect(
+      client.scrape({
+        alexandria: { provider: "benzinga", capability: "calendar/ratings" },
+        requestId: "terms-1",
+      }),
+    ).rejects.toMatchObject({
+      status: 403,
+      code: "THIRD_PARTY_DATA_TERMS_REQUIRED",
+      message: termsRequired.error,
+      requestId: "terms-1",
+      requiresAction: termsRequired.requiresAction,
+    });
+  });
+  test("walks with Find Tools and feeds next directly into scrape", async () => {
+    const found = await client.findTools({ providers: ["particle"], limit: 2 });
+    const result = await client.scrape({ alexandria: found.items[0].next! });
+    expect(result.creditsCost).toBe(0);
+    expect(sent.at(-1)?.id).toBeTruthy();
+    expect(sent.at(-1)?.body.alexandria).toEqual([next]);
+  });
+  test("rejects URL options and queryless browsing before dispatch", async () => {
+    const count = sent.length;
+    await expect(
+      client.scrape({ alexandria: next, url: "https://example.com" } as any),
+    ).rejects.toThrow();
+    await expect(
+      client.search("", { sources: ["alexandria"] }),
+    ).rejects.toThrow("Query cannot be empty");
+    expect(sent).toHaveLength(count);
+  });
+});
+
+
+test.each([[undefined, 80000], [1000, 31000], [100000, 80000]])(
+  "Alexandria timeout %s allows response delivery (%s ms)",
+  async (timeout, timeoutMs) => {
+    const http = { post: jest.fn(async () => ({ status: 200, data: {
+      success: true, data: { alexandria: [], creditsCost: 0 },
+    } })) };
+    await scrapeAlexandria(http as any, [next], { timeout });
+    expect(http.post).toHaveBeenCalledWith("/v2/scrape", expect.anything(),
+      expect.objectContaining({ timeoutMs }));
+  },
+);

package/src/__tests__/unit/v2/v1-aliases.test.ts

@@ -1,3 +1,4 @@
+import { jest } from "@jest/globals";
 import { FirecrawlClient } from "../../../v2/client";
 
 const ALIAS_MAP: Record<string, string> = {

package/src/v2/client.ts

@@ -6,6 +6,7 @@ import {
 } from "./methods/scrape";
 import { parse as parseMethod } from "./methods/parse";
 import { search } from "./methods/search";
+import { scrapeAlexandria, findTools } from "./methods/tools";
 import { developerSearch as developerSearchMethod } from "./methods/developer";
 import { map as mapMethod } from "./methods/map";
 import { feedback as feedbackMethod, searchFeedback as searchFeedbackMethod } from "./methods/feedback";
@@ -26,7 +27,7 @@ import {
   batchScrape as batchWaiter,
 } from "./methods/batch";
 import { startExtract, getExtractStatus, extract as extractWaiter } from "./methods/extract";
-import { startAgent, getAgentStatus, getAgentTrace, getAgentSnapshot, cancelAgent, listAgents, agent as agentWaiter } from "./methods/agent";
+import { startAgent, getAgentStatus, getAgentThread, getAgentTrace, getAgentSnapshot, cancelAgent, listAgents, agent as agentWaiter } from "./methods/agent";
 import {
   browser as browserMethod,
   browserExecute,
@@ -47,6 +48,10 @@ import {
 } from "./methods/monitor";
 import type {
   Document,
+  AlexandriaScrapeRequest,
+  FindToolsOptions,
+  FindToolsData,
+  AlexandriaScrapeData,
   ParseFile,
   ParseOptions,
   ScrapeOptions,
@@ -70,6 +75,7 @@ import type {
   AgentStatusResponse,
   AgentTraceResponse,
   AgentSnapshotResponse,
+  AgentThreadResponse,
   AgentListOptions,
   AgentListResponse,
   CrawlOptions,
@@ -170,11 +176,42 @@ export class FirecrawlClient {
    */
   async scrape<Opts extends ScrapeOptions>(
     url: string,
-    options: Opts
+    options: Opts,
   ): Promise<Omit<Document, "json"> & { json?: InferredJsonFromOptions<Opts> }>;
   async scrape(url: string, options?: ScrapeCallOptions): Promise<Document>;
-  async scrape(url: string, options?: ScrapeCallOptions): Promise<Document> {
-    return scrape(this.http, url, options);
+  async scrape(request: AlexandriaScrapeRequest): Promise<AlexandriaScrapeData>;
+  async scrape(
+    url: string | AlexandriaScrapeRequest,
+    options?: ScrapeCallOptions,
+  ): Promise<Document | AlexandriaScrapeData> {
+    if (typeof url === "string") return scrape(this.http, url, options);
+    if (
+      !url ||
+      options !== undefined ||
+      Object.keys(url).some(
+        (key) =>
+          ![
+            "alexandria",
+            "requestId",
+            "timeout",
+            "integration",
+            "origin",
+          ].includes(key),
+      )
+    ) {
+      throw new Error("Provide an alexandria request without URL scrape options");
+    }
+    const { alexandria, ...opts } = url;
+    return scrapeAlexandria(
+      this.http,
+      Array.isArray(alexandria) ? alexandria : [alexandria],
+      opts,
+    );
+  }
+
+  /** Explore the catalogue without executing the tools it returns. */
+  async findTools(options?: FindToolsOptions): Promise<FindToolsData> {
+    return findTools(this.http, options);
   }
   /**
    * Interact with the browser session associated with a scrape job.
@@ -284,6 +321,9 @@ export class FirecrawlClient {
    * Access the v2 research endpoints — Firecrawl's **research paper index**
    * (~43M paper abstracts) plus GitHub history/readmes.
    *
+   * `research.searchGithub()` is deprecated and stops responding after
+   * 2026-11-03. Use `developerSearch()` instead.
+   *
    * The paper corpus is roughly 90% biomedical and life sciences — PubMed,
    * bioRxiv and medRxiv — with arXiv covering physics, mathematics and
    * computer science.
@@ -566,6 +606,14 @@ export class FirecrawlClient {
   async getAgentSnapshot(jobId: string, snapshotId: string): Promise<AgentSnapshotResponse> {
     return getAgentSnapshot(this.http, jobId, snapshotId);
   }
+  /**
+   * Get a thread and its runs, oldest turn first.
+   * @param threadId Thread id, as returned by startAgent or getAgentStatus.
+   * @param options.includeData Inline each succeeded run's data.
+   */
+  async getAgentThread(threadId: string, options?: { includeData?: boolean }): Promise<AgentThreadResponse> {
+    return getAgentThread(this.http, threadId, options);
+  }
 
   // Browser
   /**

package/src/v2/methods/agent.ts

@@ -1,4 +1,4 @@
-import { type AgentEffort, type AgentListOptions, type AgentListResponse, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentTraceResponse, type AgentWebhookConfig, type AuditMetadata, type ThreatProtectionOptions } from "../types";
+import { type AgentEffort, type AgentExchangeOptions, type AgentListOptions, type AgentListResponse, type AgentMode, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentThreadResponse, type AgentTraceResponse, type AgentWebhookConfig, type AuditMetadata, type ThreatProtectionOptions } from "../types";
 import { HttpClient } from "../utils/httpClient";
 import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
 import { isZodSchema, zodSchemaToJsonSchema } from "../../utils/zodSchemaToJson";
@@ -17,6 +17,9 @@ function prepareAgentPayload(args: {
   webhook?: string | AgentWebhookConfig;
   threatProtection?: ThreatProtectionOptions;
   auditMetadata?: AuditMetadata;
+  threadId?: string;
+  mode?: AgentMode;
+  exchange?: AgentExchangeOptions;
 }): Record<string, unknown> {
   const body: Record<string, unknown> = {};
   if (args.urls) body.urls = args.urls;
@@ -35,6 +38,9 @@ function prepareAgentPayload(args: {
     body.threatProtection = args.threatProtection;
   if (args.auditMetadata != null)
     body.auditMetadata = args.auditMetadata;
+  if (args.threadId != null) body.threadId = args.threadId;
+  if (args.mode != null) body.mode = args.mode;
+  if (args.exchange != null) body.exchange = args.exchange;
   return body;
 }
 
@@ -129,6 +135,22 @@ export async function getAgentSnapshot(
   }
 }
 
+export async function getAgentThread(
+  http: HttpClient,
+  threadId: string,
+  options?: { includeData?: boolean }
+): Promise<AgentThreadResponse> {
+  try {
+    const query = options?.includeData ? "?includeData=true" : "";
+    const res = await http.get<AgentThreadResponse>(`/v2/agent/threads/${threadId}${query}`);
+    if (res.status !== 200) throwForBadResponse(res, "agent thread");
+    return res.data;
+  } catch (err: any) {
+    if (err?.isAxiosError) return normalizeAxiosError(err, "agent thread");
+    throw err;
+  }
+}
+
 export async function cancelAgent(http: HttpClient, jobId: string): Promise<boolean> {
   try {
     const res = await http.delete<{ success: boolean }>(`/v2/agent/${jobId}`);

package/src/v2/methods/research.ts

@@ -159,9 +159,9 @@ export class ResearchClient {
     appendParam(params, "query", options.query);
     appendParam(params, "k", options.k);
     try {
-      const res = await this.http.get<PaperMetadataResponse | ReadPaperResponse>(
-        withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params),
-      );
+      const res = await this.http.get<
+        PaperMetadataResponse | ReadPaperResponse
+      >(withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params));
       if (res.status !== 200) throwForBadResponse(res, "get paper");
       return res.data;
     } catch (err) {
@@ -191,10 +191,7 @@ export class ResearchClient {
     appendParam(params, "anchor", options.anchor);
     try {
       const res = await this.http.get<SimilarPapersResponse>(
-        withQuery(
-          `${BASE}/papers/${encodeURIComponent(id)}/similar`,
-          params,
-        ),
+        withQuery(`${BASE}/papers/${encodeURIComponent(id)}/similar`, params),
       );
       if (res.status !== 200) throwForBadResponse(res, "find similar papers");
       return res.data;
@@ -204,7 +201,16 @@ export class ResearchClient {
   }
 
   /**
-   * Search GitHub issue/PR history and repository readmes.
+   * Search the research index GitHub slice: issue/PR history and repository
+   * readmes.
+   *
+   * @deprecated Use `developerSearch()` on the client instead. This
+   * endpoint stops responding after 2026-11-03. The developer index searches
+   * GitHub issues, pull requests and readmes plus curated documentation
+   * sources, returns matched passages, and adds filters for repo, language,
+   * license and stars. It does not carry over the `scores` breakdown or the
+   * `resultType: "web"` fallback results. See
+   * https://docs.firecrawl.dev/features/developer.
    * @param query Search query.
    * @param options Optional `k`.
    */

package/src/v2/methods/scrape.ts

@@ -88,9 +88,11 @@ export async function scrape(
   if (options) ensureValidScrapeOptions(options);
 
   // autoResume is SDK behavior, never part of the wire payload.
-  const { autoResume, ...requestOptions } = options ?? {};
+  const { autoResume, domainTools, ...requestOptions } = options ?? {};
   const payload: Record<string, unknown> = { url: url.trim() };
   Object.assign(payload, requestOptions);
+  // Default off: omit entirely rather than send `false`.
+  if (domainTools) payload.domainTools = domainTools;
 
   // Per-request timeout. With auto-resume disabled this is exactly the
   // pre-existing behavior (explicit timeout +5s, else the client's own

package/src/v2/methods/search.ts

@@ -27,6 +27,7 @@ function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
   const payload: Record<string, unknown> = {
     query: req.query,
   };
+  if (req.domainTools != null) payload.domainTools = req.domainTools;
   if (req.sources) payload.sources = req.sources;
   if (req.categories) payload.categories = req.categories;
   if (req.includeDomains) payload.includeDomains = req.includeDomains;
@@ -34,6 +35,7 @@ function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
   if (req.limit != null) payload.limit = req.limit;
   if (req.tbs != null) payload.tbs = req.tbs;
   if (req.location != null) payload.location = req.location;
+  if (req.country != null) payload.country = req.country;
   if (req.ignoreInvalidURLs != null)
     payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
   if (req.timeout != null) payload.timeout = req.timeout;
@@ -86,6 +88,7 @@ export async function search(
       success: boolean;
       data?: Record<string, unknown>;
       error?: string;
+      warning?: string;
     }>(
       "/v2/search",
       payload,
@@ -98,10 +101,12 @@ export async function search(
     }
     const data = (res.data.data || {}) as Record<string, any>;
     const out: SearchData = {};
+    if (res.data.warning) out.warning = res.data.warning;
     if (data.web) out.web = transformArray<SearchResultWeb>(data.web);
     if (data.news) out.news = transformArray<SearchResultNews>(data.news);
     if (data.images)
       out.images = transformArray<SearchResultImages>(data.images);
+    if (data.tools) out.tools = data.tools;
     Object.defineProperty(out, "data", {
       get() {
         const parts: string[] = [];
@@ -109,9 +114,11 @@ export async function search(
         if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
         if (out.images?.length)
           parts.push(`.images (${out.images.length} results)`);
+        if (out.tools?.length)
+          parts.push(`.tools (${out.tools.length} results)`);
         const available = parts.length
           ? parts.join(", ")
-          : ".web, .news, or .images";
+          : ".web, .news, .images, or .tools";
         throw new Error(
           `SearchData has no '.data'. Results are grouped by source: ${available}`,
         );