firecrawl 4.39.0 → 4.41.0

package/src/__tests__/unit/v2/tools.unit.test.ts

@@ -0,0 +1,297 @@
+import {
+  afterAll,
+  beforeAll,
+  beforeEach,
+  describe,
+  expect,
+  jest,
+  test,
+} from "@jest/globals";
+import { createServer, type Server } from "node:http";
+import { FirecrawlClient } from "../../../v2/client";
+
+import { scrapeAlexandria } from "../../../v2/methods/tools";
+
+const tool = {
+  id: "particle/podcasts/episodes/search",
+  provider: "particle",
+  capability: "podcasts/episodes/search",
+  name: "Episode search",
+  description: "Find episodes",
+  creditsCost: 15,
+  perRecord: false,
+  options: [{ name: "semantic_search", type: "string" }],
+  response: { about: "Episodes", key: "data", fields: [] },
+  examples: { javascript: "example", python: "example", curl: "example" },
+  matchedBy: ["semantic", "domain"],
+  matchedUrls: ["https://podcasts.apple.com"],
+};
+const productionTool = {
+  id: "benzinga/calendar/ratings",
+  provider: "benzinga",
+  capability: "calendar/ratings",
+  name: "Analyst ratings",
+  description: "Ratings",
+  creditsCost: 5,
+  perRecord: false,
+  label: "Ratings",
+  whenToUse: "Analyst ratings for a ticker",
+  returns: { about: "Ratings" },
+  discovery: { urls: [] },
+  attribution: { required: true },
+  options: [{ name: "tickers", type: "string" }],
+  response: { fields: [] },
+  matchedBy: ["semantic"],
+  matchedUrls: [],
+};
+const termsRequired = {
+  success: false,
+  code: "THIRD_PARTY_DATA_TERMS_REQUIRED",
+  error: "An organization admin must accept the benzinga provider's terms",
+  requiresAction: {
+    type: "accept_terms",
+    terms: "benzinga",
+    version: "C-1.0.0-draft",
+    url: "https://www.firecrawl.dev/app/alexandria/benzinga",
+  },
+};
+const next = {
+  provider: "firecrawl",
+  capability: "find-tools",
+  options: { providers: ["particle"], level: "tools" },
+};
+const compactTools = [tool, productionTool].map(
+  ({ provider, capability, description }) => ({
+    provider,
+    capability,
+    description,
+  }),
+);
+let server: Server;
+let client: FirecrawlClient;
+const sent: Array<{ body: any; id: string | undefined }> = [];
+let attempts = 0;
+
+beforeEach(() => {
+  sent.length = 0;
+  attempts = 0;
+});
+
+beforeAll(async () => {
+  server = createServer(async (req, res) => {
+    let raw = "";
+    for await (const chunk of req) raw += chunk;
+    const body = JSON.parse(raw);
+    sent.push({ body, id: req.headers["x-request-id"] as string | undefined });
+    res.setHeader("content-type", "application/json");
+    if (req.url === "/v2/search")
+      return res.end(
+        JSON.stringify({
+          success: true,
+          warning: "Example warning",
+          data: {
+            web: [{ url: "https://podcasts.apple.com" }],
+            tools:
+              body.toolDetail === "compact"
+                ? compactTools
+                : [tool, productionTool],
+          },
+        }),
+      );
+    if (req.url === "/v2/scrape" && body.url)
+      return res.end(
+        JSON.stringify({
+          success: true,
+          data: { markdown: "Example", tools: compactTools },
+        }),
+      );
+    if (body.alexandria[0].provider === "retry" && attempts++ === 0) {
+      res.statusCode = 502;
+      return res.end("{}");
+    }
+    if (body.alexandria[0].provider === "benzinga") {
+      res.statusCode = 403;
+      return res.end(JSON.stringify(termsRequired));
+    }
+    if (body.alexandria[0].provider === "denied") {
+      res.statusCode = 402;
+      return res.end(
+        JSON.stringify({
+          success: false,
+          code: "insufficient_credits",
+          error: "Insufficient credits",
+        }),
+      );
+    }
+    const alexandria =
+      body.alexandria[0].provider === "firecrawl"
+        ? [
+            {
+              ...next,
+              creditsCost: 0,
+              data: {
+                level: "providers",
+                items: [{ id: "particle", name: "Particle", next }],
+                total: 1,
+                next: null,
+              },
+            },
+          ]
+        : [
+            {
+              provider: "retry",
+              capability: "a/b",
+              creditsCost: 15,
+              data: { nested: { value: 1 } },
+            },
+            {
+              provider: "x",
+              capability: "b",
+              error: {
+                code: "unavailable",
+                message: "Unavailable",
+                status: 503,
+              },
+            },
+          ];
+    res.end(
+      JSON.stringify({
+        success: true,
+        scrape_id: "scrape-1",
+        data: { alexandria, creditsCost: alexandria.length === 1 ? 0 : 15 },
+      }),
+    );
+  });
+  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
+  const port = (server.address() as { port: number }).port;
+  client = new FirecrawlClient({
+    apiKey: "fc-test",
+    apiUrl: `http://127.0.0.1:${port}`,
+    backoffFactor: 0,
+  });
+});
+afterAll(async () => {
+  await new Promise<void>((resolve) => server.close(() => resolve()));
+});
+
+describe("Alexandria contracts and execution", () => {
+  test.each(["compact", "summary", "full"] as const)(
+    "forwards %s discovery detail",
+    async (toolDetail) => {
+      const result = await client.search("records", { toolDetail });
+      if (toolDetail === "compact") expect(result.tools).toEqual(compactTools);
+      expect(sent.at(-1)?.body).toMatchObject({ toolDetail });
+    },
+  );
+  test("URL scrape forwards compact detail and preserves tools", async () => {
+    const result = await client.scrape("https://example.com", {
+      domainTools: true,
+      toolDetail: "compact",
+    });
+    expect(sent.at(-1)?.body).toMatchObject({
+      url: "https://example.com",
+      domainTools: true,
+      toolDetail: "compact",
+    });
+    expect(result.tools).toEqual(compactTools);
+  });
+  test("returns complete unified tools and warning beside web results", async () => {
+    const result = await client.search("podcasts", {
+      sources: ["web", { type: "alexandria" }],
+      domainTools: true,
+      limit: 2,
+    });
+    expect(result.tools).toEqual([tool, productionTool]);
+    expect(result.tools?.[1].examples).toBeUndefined();
+    expect(result.tools?.[1].whenToUse).toBe("Analyst ratings for a ticker");
+    expect(result.warning).toBe("Example warning");
+    expect(sent.at(-1)?.body).toMatchObject({
+      domainTools: true,
+      sources: ["web", { type: "alexandria" }],
+    });
+    expect(result).not.toHaveProperty("alexandria");
+  });
+  test("retains one ID through transport retries and per-tool failures", async () => {
+    const result = await client.scrape({
+      alexandria: [{ provider: "retry", capability: "a/b" }],
+      requestId: "same-request",
+    });
+    const retries = sent.filter(
+      (r) => r.body.alexandria?.[0].provider === "retry",
+    );
+    expect(retries).toHaveLength(2);
+    expect(retries.map((r) => r.id)).toEqual(["same-request", "same-request"]);
+    expect(retries[0].body).not.toHaveProperty("requestId");
+    expect(result.requestId).toBe("same-request");
+    expect(result.creditsCost).toBe(15);
+    expect(result.alexandria[1].error?.code).toBe("unavailable");
+  });
+  test("returns an error code and retry identity on failed execution", async () => {
+    await expect(
+      client.scrape({
+        alexandria: { provider: "denied", capability: "a/b" },
+        requestId: "denied-1",
+      }),
+    ).rejects.toMatchObject({
+      status: 402,
+      code: "insufficient_credits",
+      requestId: "denied-1",
+    });
+  });
+  test("exposes requiresAction on a provider terms rejection", async () => {
+    await expect(
+      client.scrape({
+        alexandria: { provider: "benzinga", capability: "calendar/ratings" },
+        requestId: "terms-1",
+      }),
+    ).rejects.toMatchObject({
+      status: 403,
+      code: "THIRD_PARTY_DATA_TERMS_REQUIRED",
+      message: termsRequired.error,
+      requestId: "terms-1",
+      requiresAction: termsRequired.requiresAction,
+    });
+  });
+  test("walks with Find Tools and feeds next directly into scrape", async () => {
+    const found = await client.findTools({ providers: ["particle"], limit: 2 });
+    const result = await client.scrape({ alexandria: found.items[0].next! });
+    expect(result.creditsCost).toBe(0);
+    expect(sent.at(-1)?.id).toBeTruthy();
+    expect(sent.at(-1)?.body.alexandria).toEqual([next]);
+  });
+  test("rejects URL options and queryless browsing before dispatch", async () => {
+    const count = sent.length;
+    await expect(
+      client.scrape({ alexandria: next, url: "https://example.com" } as any),
+    ).rejects.toThrow();
+    await expect(
+      client.search("", { sources: ["alexandria"] }),
+    ).rejects.toThrow("Query cannot be empty");
+    expect(sent).toHaveLength(count);
+  });
+});
+
+test.each([
+  [undefined, 80000],
+  [1000, 31000],
+  [100000, 80000],
+])(
+  "Alexandria timeout %s allows response delivery (%s ms)",
+  async (timeout, timeoutMs) => {
+    const http = {
+      post: jest.fn(async () => ({
+        status: 200,
+        data: {
+          success: true,
+          data: { alexandria: [], creditsCost: 0 },
+        },
+      })),
+    };
+    await scrapeAlexandria(http as any, [next], { timeout });
+    expect(http.post).toHaveBeenCalledWith(
+      "/v2/scrape",
+      expect.anything(),
+      expect.objectContaining({ timeoutMs }),
+    );
+  },
+);

package/src/v2/client.ts

@@ -6,6 +6,7 @@ import {
 } from "./methods/scrape";
 import { parse as parseMethod } from "./methods/parse";
 import { search } from "./methods/search";
+import { scrapeAlexandria, findTools } from "./methods/tools";
 import { developerSearch as developerSearchMethod } from "./methods/developer";
 import { map as mapMethod } from "./methods/map";
 import { feedback as feedbackMethod, searchFeedback as searchFeedbackMethod } from "./methods/feedback";
@@ -47,6 +48,10 @@ import {
 } from "./methods/monitor";
 import type {
   Document,
+  AlexandriaScrapeRequest,
+  FindToolsOptions,
+  FindToolsData,
+  AlexandriaScrapeData,
   ParseFile,
   ParseOptions,
   ScrapeOptions,
@@ -171,11 +176,42 @@ export class FirecrawlClient {
    */
   async scrape<Opts extends ScrapeOptions>(
     url: string,
-    options: Opts
+    options: Opts,
   ): Promise<Omit<Document, "json"> & { json?: InferredJsonFromOptions<Opts> }>;
   async scrape(url: string, options?: ScrapeCallOptions): Promise<Document>;
-  async scrape(url: string, options?: ScrapeCallOptions): Promise<Document> {
-    return scrape(this.http, url, options);
+  async scrape(request: AlexandriaScrapeRequest): Promise<AlexandriaScrapeData>;
+  async scrape(
+    url: string | AlexandriaScrapeRequest,
+    options?: ScrapeCallOptions,
+  ): Promise<Document | AlexandriaScrapeData> {
+    if (typeof url === "string") return scrape(this.http, url, options);
+    if (
+      !url ||
+      options !== undefined ||
+      Object.keys(url).some(
+        (key) =>
+          ![
+            "alexandria",
+            "requestId",
+            "timeout",
+            "integration",
+            "origin",
+          ].includes(key),
+      )
+    ) {
+      throw new Error("Provide an alexandria request without URL scrape options");
+    }
+    const { alexandria, ...opts } = url;
+    return scrapeAlexandria(
+      this.http,
+      Array.isArray(alexandria) ? alexandria : [alexandria],
+      opts,
+    );
+  }
+
+  /** Explore the catalogue without executing the tools it returns. */
+  async findTools(options?: FindToolsOptions): Promise<FindToolsData> {
+    return findTools(this.http, options);
   }
   /**
    * Interact with the browser session associated with a scrape job.

package/src/v2/methods/extract.ts

@@ -40,7 +40,8 @@ function prepareExtractPayload(args: {
     body.threatProtection = args.threatProtection;
   if (args.scrapeOptions) {
     ensureValidScrapeOptions(args.scrapeOptions);
-    body.scrapeOptions = args.scrapeOptions;
+    const { toolDetail, domainTools, ...scrapeOptions } = args.scrapeOptions;
+    body.scrapeOptions = scrapeOptions;
   }
   return body;
 }

package/src/v2/methods/scrape.ts

@@ -88,9 +88,11 @@ export async function scrape(
   if (options) ensureValidScrapeOptions(options);
 
   // autoResume is SDK behavior, never part of the wire payload.
-  const { autoResume, ...requestOptions } = options ?? {};
+  const { autoResume, domainTools, ...requestOptions } = options ?? {};
   const payload: Record<string, unknown> = { url: url.trim() };
   Object.assign(payload, requestOptions);
+  // Default off: omit entirely rather than send `false`.
+  if (domainTools) payload.domainTools = domainTools;
 
   // Per-request timeout. With auto-resume disabled this is exactly the
   // pre-existing behavior (explicit timeout +5s, else the client's own

package/src/v2/methods/search.ts

@@ -27,6 +27,8 @@ function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
   const payload: Record<string, unknown> = {
     query: req.query,
   };
+  if (req.toolDetail != null) payload.toolDetail = req.toolDetail;
+  if (req.domainTools != null) payload.domainTools = req.domainTools;
   if (req.sources) payload.sources = req.sources;
   if (req.categories) payload.categories = req.categories;
   if (req.includeDomains) payload.includeDomains = req.includeDomains;
@@ -87,6 +89,7 @@ export async function search(
       success: boolean;
       data?: Record<string, unknown>;
       error?: string;
+      warning?: string;
     }>(
       "/v2/search",
       payload,
@@ -99,10 +102,12 @@ export async function search(
     }
     const data = (res.data.data || {}) as Record<string, any>;
     const out: SearchData = {};
+    if (res.data.warning) out.warning = res.data.warning;
     if (data.web) out.web = transformArray<SearchResultWeb>(data.web);
     if (data.news) out.news = transformArray<SearchResultNews>(data.news);
     if (data.images)
       out.images = transformArray<SearchResultImages>(data.images);
+    if (data.tools) out.tools = data.tools;
     Object.defineProperty(out, "data", {
       get() {
         const parts: string[] = [];
@@ -110,9 +115,11 @@ export async function search(
         if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
         if (out.images?.length)
           parts.push(`.images (${out.images.length} results)`);
+        if (out.tools?.length)
+          parts.push(`.tools (${out.tools.length} results)`);
         const available = parts.length
           ? parts.join(", ")
-          : ".web, .news, or .images";
+          : ".web, .news, .images, or .tools";
         throw new Error(
           `SearchData has no '.data'. Results are grouped by source: ${available}`,
         );

package/src/v2/methods/tools.ts

@@ -0,0 +1,137 @@
+import type {
+  AlexandriaCall,
+  AlexandriaOptions,
+  AlexandriaScrapeData,
+  AlexandriaScrapeResult,
+  FindToolsData,
+  FindToolsOptions,
+} from "../types";
+import { SdkError } from "../types";
+import { HttpClient } from "../utils/httpClient";
+import {
+  normalizeAxiosError,
+  throwForBadResponse,
+} from "../utils/errorHandler";
+
+const ALEXANDRIA_MAX_CALLS = 10;
+
+function prepareAlexandriaPayload(
+  calls: AlexandriaCall[],
+  opts: AlexandriaOptions,
+): Record<string, unknown> {
+  if (!Array.isArray(calls) || calls.length === 0) {
+    throw new Error("alexandria requires at least one call");
+  }
+  if (calls.length > ALEXANDRIA_MAX_CALLS) {
+    throw new Error(`alexandria accepts at most ${ALEXANDRIA_MAX_CALLS} calls`);
+  }
+  const alexandria = calls.map((call, index) => {
+    if (!call || typeof call.provider !== "string" || !call.provider.trim()) {
+      throw new Error(`alexandria[${index}].provider cannot be empty`);
+    }
+    if (typeof call.capability !== "string" || !call.capability.trim()) {
+      throw new Error(`alexandria[${index}].capability cannot be empty`);
+    }
+    if (
+      Object.keys(call).some(
+        (key) => !["provider", "capability", "options"].includes(key),
+      )
+    )
+      throw new Error("Unknown alexandria call option");
+    const item: Record<string, unknown> = {
+      provider: call.provider.trim(),
+      capability: call.capability.trim(),
+    };
+    if (call.options != null) {
+      if (typeof call.options !== "object" || Array.isArray(call.options)) {
+        throw new Error(`alexandria[${index}].options must be an object`);
+      }
+      item.options = call.options;
+    }
+    return item;
+  });
+  if (
+    opts.timeout != null &&
+    (!Number.isInteger(opts.timeout) || opts.timeout <= 0)
+  ) {
+    throw new Error("timeout must be a positive integer");
+  }
+  const payload: Record<string, unknown> = { alexandria };
+  if (opts.timeout != null) payload.timeout = opts.timeout;
+  if (opts.integration && opts.integration.trim())
+    payload.integration = opts.integration.trim();
+  if (opts.origin) payload.origin = opts.origin;
+  return payload;
+}
+
+export async function scrapeAlexandria(
+  http: HttpClient,
+  calls: AlexandriaCall[],
+  opts: AlexandriaOptions = {},
+): Promise<AlexandriaScrapeData> {
+  const payload = prepareAlexandriaPayload(calls, opts);
+  const requestId = opts.requestId ?? crypto.randomUUID();
+  if (!/^[A-Za-z0-9._:-]{1,128}$/.test(requestId))
+    throw new Error("Invalid requestId");
+  try {
+    const res = await http.post<{
+      success: boolean;
+      scrape_id?: string;
+      data?: { alexandria?: AlexandriaScrapeResult[]; creditsCost?: number };
+      error?: string;
+    }>("/v2/scrape", payload, {
+      headers: { "x-request-id": requestId },
+      // Allow response delivery after the API's capped execution deadline.
+      timeoutMs: Math.min(opts.timeout ?? 50000, 50000) + 30000,
+    });
+    if (res.status !== 200 || !res.data?.success) {
+      throwForBadResponse(res, "alexandria");
+    }
+    const data = res.data.data;
+    if (
+      !data ||
+      !Array.isArray(data.alexandria) ||
+      typeof data.creditsCost !== "number" ||
+      !Number.isInteger(data.creditsCost) ||
+      data.creditsCost < 0
+    ) {
+      throw new SdkError("Invalid alexandria response");
+    }
+    return {
+      scrapeId: res.data.scrape_id ?? "",
+      requestId,
+      alexandria: data.alexandria,
+      creditsCost: data.creditsCost,
+    };
+  } catch (err: any) {
+    try {
+      if (err?.isAxiosError) normalizeAxiosError(err, "alexandria");
+      throw err;
+    } catch (error) {
+      if (error && typeof error === "object")
+        Object.assign(error, { requestId });
+      throw error;
+    }
+  }
+}
+
+export async function findTools(
+  http: HttpClient,
+  options: FindToolsOptions = {},
+): Promise<FindToolsData> {
+  const result = await scrapeAlexandria(http, [
+    {
+      provider: "firecrawl",
+      capability: "find-tools",
+      options: { ...options },
+    },
+  ]);
+  const item = result.alexandria[0];
+  if (!item) throw new SdkError("Missing Find Tools result");
+  if (item.error)
+    throw Object.assign(
+      new SdkError(item.error.message, item.error.status, item.error.code),
+      { requestId: result.requestId },
+    );
+  return item.data as FindToolsData;
+}