@mendable/firecrawl-js 4.41.0 → 4.42.1

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   require_package
-} from "./chunk-I2DFXQYR.js";
+} from "./chunk-YG554MNR.js";
 
 // src/v2/utils/httpClient.ts
 import axios from "axios";
@@ -18,6 +18,21 @@ function getVersion() {
   }
 }
 
+// src/utils/apiOrigin.ts
+function pinToApiOrigin(apiUrl, url) {
+  const normalized = url.replace(/[\t\n\r]/g, "").replace(/^[\u0000-\u0020]+/, "");
+  if (!/^([a-z][a-z\d+\-.]*:|[\\/]{2})/i.test(normalized)) return url;
+  const base = URL.canParse(apiUrl) ? new URL(apiUrl) : null;
+  if (!base?.host) {
+    throw new Error(`apiUrl must be an absolute URL, got ${JSON.stringify(apiUrl)}`);
+  }
+  const target = new URL(normalized, base);
+  const pinned = new URL(base.origin);
+  pinned.pathname = target.pathname;
+  pinned.search = target.search;
+  return pinned.href;
+}
+
 // src/v2/utils/httpClient.ts
 var HttpClient = class {
   instance;
@@ -58,6 +73,7 @@ var HttpClient = class {
     config.headers = {
       ...config.headers || {}
     };
+    if (config.url) config.url = pinToApiOrigin(this.apiUrl, config.url);
     let lastError;
     for (let attempt = 0; attempt < this.maxRetries; attempt++) {
       try {
@@ -185,6 +201,12 @@ var JobTimeoutError = class extends SdkError {
   }
 };
 
+// src/v2/utils/agentHints.ts
+function agentHintMetadata(body) {
+  const hints = body?.agent_hints;
+  return Array.isArray(hints) && hints.every((hint) => typeof hint === "string") ? { agent_hints: [...hints] } : {};
+}
+
 // src/utils/zodSchemaToJson.ts
 import { zodToJsonSchema as zodToJsonSchemaLib } from "zod-to-json-schema";
 function isZodSchema(value) {
@@ -418,7 +440,7 @@ function throwForBadResponse(resp, action) {
   const status = resp.status;
   const body = resp.data || {};
   const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
-  throw new SdkError(
+  throw Object.assign(new SdkError(
     msg,
     status,
     body?.code,
@@ -426,14 +448,14 @@ function throwForBadResponse(resp, action) {
     void 0,
     body?.chargeId,
     parseRequiresAction(body?.requiresAction)
-  );
+  ), agentHintMetadata(body));
 }
 function normalizeAxiosError(err, action) {
   const status = err.response?.status;
   const body = err.response?.data;
   const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
   const code = body?.code || err.code;
-  throw new SdkError(
+  throw Object.assign(new SdkError(
     message,
     status,
     code,
@@ -441,7 +463,7 @@ function normalizeAxiosError(err, action) {
     void 0,
     body?.chargeId,
     parseRequiresAction(body?.requiresAction)
-  );
+  ), agentHintMetadata(body));
 }
 function isRetryableError(err) {
   if (err instanceof JobTimeoutError) {
@@ -509,7 +531,7 @@ async function scrape(http, url, options, deps = {}) {
       if (res.status !== 200 || !res.data?.success) {
         throwForBadResponse(res, "scrape");
       }
-      return res.data.data || {};
+      return { ...res.data.data, ...agentHintMetadata(res.data) };
     } catch (err) {
       const delayMs = autoResume !== false ? processingContinuesDelayMs(err) : void 0;
       if (delayMs !== void 0 && resumes < RESUME_MAX_ATTEMPTS && resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS) {
@@ -626,7 +648,7 @@ async function parse(http, file, options) {
     if (res.status !== 200 || !res.data?.success) {
       throwForBadResponse(res, "parse");
     }
-    return res.data.data || {};
+    return { ...res.data.data, ...agentHintMetadata(res.data) };
   } catch (err) {
     if (err?.isAxiosError) return normalizeAxiosError(err, "parse");
     throw err;
@@ -700,7 +722,7 @@ async function search(http, request) {
       throwForBadResponse(res, "search");
     }
     const data = res.data.data || {};
-    const out = {};
+    const out = { ...agentHintMetadata(res.data) };
     if (res.data.warning) out.warning = res.data.warning;
     if (data.web) out.web = transformArray(data.web);
     if (data.news) out.news = transformArray(data.news);
@@ -905,7 +927,7 @@ async function map(http, url, options) {
           description: item.description
         });
     }
-    return { id: res.data.id, links };
+    return { id: res.data.id, links, ...agentHintMetadata(res.data) };
   } catch (err) {
     if (err?.isAxiosError) return normalizeAxiosError(err, "map");
     throw err;
@@ -2667,7 +2689,7 @@ var FirecrawlApp = class {
       if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
         return process.env.npm_package_version;
       }
-      const packageJson = await import("./package-V72LKFNW.js");
+      const packageJson = await import("./package-FOKY5K5X.js");
       return packageJson.default.version;
     } catch (error) {
       const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
@@ -2882,7 +2904,7 @@ var FirecrawlApp = class {
       throw new FirecrawlError("No crawl ID provided", 400);
     }
     const headers = this.prepareHeaders();
-    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`, this.apiUrl);
     if (skip !== void 0) {
       targetURL.searchParams.set("skip", skip.toString());
     }
@@ -3127,7 +3149,7 @@ var FirecrawlApp = class {
       throw new FirecrawlError("No batch scrape ID provided", 400);
     }
     const headers = this.prepareHeaders();
-    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`, this.apiUrl);
     if (skip !== void 0) {
       targetURL.searchParams.set("skip", skip.toString());
     }
@@ -3339,7 +3361,7 @@ var FirecrawlApp = class {
    */
   async getRequest(url, headers) {
     try {
-      return await axios2.get(url, { headers });
+      return await axios2.get(pinToApiOrigin(this.apiUrl, url), { headers });
     } catch (error) {
       if (error instanceof AxiosError && error.response) {
         return error.response;

package/dist/{package-V72LKFNW.js → package-FOKY5K5X.js}

@@ -1,4 +1,4 @@
 import {
   require_package
-} from "./chunk-I2DFXQYR.js";
+} from "./chunk-YG554MNR.js";
 export default require_package();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "4.41.0",
+  "version": "4.42.1",
   "description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/pnpm-workspace.yaml

@@ -7,7 +7,9 @@ overrides:
   "rollup@<4.59.0": ">=4.59.0"
   "picomatch@<4.0.4": ">=4.0.4"
   handlebars: ">=4.7.9"
-  "brace-expansion@>=5.0.0 <5.0.9": "5.0.9"
+  # GHSA-q2hr-2g5m-vwhr / GHSA-qhr7-859c-m2p7 / GHSA-6j4f-fj2g-mc7p:
+  # pin the 5.x line only. Exact 5.0.12 cannot resolve to 6.x.
+  "brace-expansion@>=5.0.0 <5.0.12": "5.0.12"
   "js-yaml@>=3.0.0 <3.15.2": "3.15.2"
   "js-yaml@>=4.0.0 <4.3.2": "4.3.2"
   "axios@>=1.0.0 <1.16.0": "1.16.1"

package/src/__tests__/unit/v1/next-url-pinning.test.ts

@@ -0,0 +1,81 @@
+import { describe, test, expect, jest, afterEach } from "@jest/globals";
+import axios from "axios";
+import FirecrawlApp from "../../../v1";
+
+const API_URL = "https://api.firecrawl.dev";
+const API_KEY = "fc-test";
+
+const origins: Array<[string, string]> = [
+  ["same origin", API_URL],
+  ["cross host", "https://evil.example"],
+  ["protocol-relative", "//evil.example"],
+  ["different port", "https://api.firecrawl.dev:8443"],
+  ["different scheme", "http://api.firecrawl.dev"],
+  ["whitespace-prefixed", "\n https://evil.example"],
+];
+
+function mockGet(respond: (url: URL) => unknown) {
+  const sent: Array<{ url: string; authorization: unknown }> = [];
+  jest.spyOn(axios, "get").mockImplementation(async (url: string, config?: any) => {
+    sent.push({ url, authorization: config?.headers?.Authorization });
+    return { status: 200, data: respond(new URL(url)) };
+  });
+  return sent;
+}
+
+function expectPinned(sent: Array<{ url: string; authorization: unknown }>, expectedUrls: string[]) {
+  expect(sent.map(req => req.url)).toEqual(expectedUrls);
+  for (const req of sent) {
+    expect(new URL(req.url).origin).toBe(API_URL);
+    expect(req.url).not.toContain("evil.example");
+    expect(req.authorization).toBe(`Bearer ${API_KEY}`);
+  }
+}
+
+const completedPage = (next: string) => ({
+  success: true,
+  status: "completed",
+  total: 2,
+  completed: 2,
+  expiresAt: "2030-01-01T00:00:00Z",
+  data: [{ markdown: "a" }],
+  next,
+});
+const lastPage = () => ({ success: true, status: "completed", data: [{ markdown: "b" }] });
+
+describe("v1 pins next URLs to the api_url origin", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  describe.each(origins)("%s next", (_label, origin) => {
+    const app = new FirecrawlApp({ apiKey: API_KEY, apiUrl: API_URL });
+
+    test.each([
+      ["checkCrawlStatus", "/v1/crawl/abc"],
+      ["checkBatchScrapeStatus", "/v1/batch/scrape/abc"],
+    ] as const)("%s getAllData follows server next", async (method, path) => {
+      const sent = mockGet(url => (url.searchParams.has("skip") ? lastPage() : completedPage(`${origin}${path}?skip=10`)));
+      const res: any = await app[method]("abc", true);
+      expect(res.data).toHaveLength(2);
+      expectPinned(sent, [`${API_URL}${path}`, `${API_URL}${path}?skip=10`]);
+    });
+
+    test.each([
+      ["checkCrawlStatus", "/v1/crawl/abc"],
+      ["checkBatchScrapeStatus", "/v1/batch/scrape/abc"],
+    ] as const)("%s caller-supplied nextURL", async (method, path) => {
+      const sent = mockGet(lastPage);
+      await app[method]("abc", false, `${origin}${path}?skip=10`);
+      expectPinned(sent, [`${API_URL}${path}?skip=10`]);
+    });
+
+    test("monitorJobStatus follows server next", async () => {
+      const path = "/v1/crawl/abc";
+      const sent = mockGet(url => (url.searchParams.has("skip") ? lastPage() : completedPage(`${origin}${path}?skip=10`)));
+      const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0);
+      expect(res.data).toHaveLength(2);
+      expectPinned(sent, [`${API_URL}${path}`, `${API_URL}${path}?skip=10`]);
+    });
+  });
+});

package/src/__tests__/unit/v2/agent-hints.test.ts

@@ -0,0 +1,116 @@
+import { describe, expect, it, jest } from "@jest/globals";
+import { search } from "../../../v2/methods/search";
+import { scrape } from "../../../v2/methods/scrape";
+import { parse } from "../../../v2/methods/parse";
+import { map } from "../../../v2/methods/map";
+import {
+  normalizeAxiosError,
+  throwForBadResponse,
+} from "../../../v2/utils/errorHandler";
+
+const hints = [
+  "Scrape a selected search result if you need full page content.",
+  "Evaluate this result before submitting feedback.",
+];
+function httpFor(body: object) {
+  return {
+    post: jest.fn<any>().mockResolvedValue({ status: 200, data: body }),
+    postMultipart: jest
+      .fn<any>()
+      .mockResolvedValue({ status: 200, data: body }),
+  } as any;
+}
+
+describe("agent hints survive SDK response unwrapping", () => {
+  it("preserves search hints without adding them to records", async () => {
+    const web = [{ url: "https://example.com" }];
+    const result = await search(
+      httpFor({ success: true, data: { web }, agent_hints: hints }),
+      { query: "test" },
+    );
+    expect(result.web).toEqual(web);
+    expect(result.web[0]).not.toHaveProperty("agent_hints");
+    expect(result.agent_hints).toEqual(hints);
+    expect(JSON.parse(JSON.stringify(result)).agent_hints).toEqual(hints);
+  });
+
+  it("keeps existing search output unchanged when hints are absent", async () => {
+    const result = await search(httpFor({ success: true, data: { web: [] } }), {
+      query: "test",
+    });
+    expect(Object.keys(result)).toEqual(["web"]);
+  });
+
+  it.each(["scrape", "parse"])(
+    "keeps %s hints separate from document content",
+    async (method) => {
+      const http = httpFor({
+        success: true,
+        data: { markdown: "# Page" },
+        agent_hints: hints,
+      });
+      const result =
+        method === "scrape"
+          ? await scrape(http, "https://example.com")
+          : await parse(http, { filename: "page.txt", data: "Page" });
+      expect(result.markdown).toBe("# Page");
+      expect(result.agent_hints).toEqual(hints);
+    },
+  );
+
+  it("preserves map metadata alongside normalized links", async () => {
+    const result = await map(
+      httpFor({
+        success: true,
+        id: "map-id",
+        links: ["https://example.com"],
+        agent_hints: hints,
+      }),
+      "https://example.com",
+    );
+    expect(result).toEqual({
+      id: "map-id",
+      links: [{ url: "https://example.com" }],
+      agent_hints: hints,
+    });
+  });
+
+  it.each([null, "unexpected", [123]])(
+    "ignores malformed optional hints: %j",
+    async (value) => {
+      const result = await search(
+        httpFor({ success: true, data: {}, agent_hints: value }),
+        { query: "test" },
+      );
+      expect(result).not.toHaveProperty("agent_hints");
+    },
+  );
+
+  it("preserves hints on HTTP and Axios errors without changing their status", () => {
+    const response = {
+      status: 400,
+      data: {
+        success: false,
+        code: "INVALID",
+        error: "Invalid request",
+        agent_hints: hints,
+      },
+    } as any;
+    for (const raise of [
+      () => throwForBadResponse(response, "search"),
+      () =>
+        normalizeAxiosError({ response, isAxiosError: true } as any, "search"),
+    ]) {
+      try {
+        raise();
+        throw new Error("Expected SDK error");
+      } catch (error) {
+        expect(error).toMatchObject({
+          status: 400,
+          code: "INVALID",
+          agent_hints: hints,
+        });
+      }
+    }
+  });
+});

package/src/__tests__/unit/v2/agent-thread.unit.test.ts

@@ -161,4 +161,126 @@ describe("v2.agent threads unit", () => {
       "/v2/agent/threads/thread-1?includeData=true",
     );
   });
+  test("startAgent forwards exchange.onTermsRequired and a terms approve", async () => {
+    const post = okPost();
+    const exchange = {
+      onTermsRequired: "ask" as const,
+      approve: { approvalId: "0199aaaa-0000-7000-8000-000000000000" },
+    };
+
+    await startAgent({ post } as any, {
+      prompt: "Continue",
+      threadId: "thread-1",
+      exchange,
+    });
+
+    expect(post).toHaveBeenCalledWith("/v2/agent", {
+      prompt: "Continue",
+      threadId: "thread-1",
+      exchange,
+    });
+  });
+
+  test("startAgent forwards exchange.onTermsRequired skip", async () => {
+    const post = okPost();
+
+    await startAgent({ post } as any, {
+      prompt: "Find the key business contact at exa.ai",
+      exchange: { onTermsRequired: "skip" },
+    });
+
+    expect(post).toHaveBeenCalledWith("/v2/agent", {
+      prompt: "Find the key business contact at exa.ai",
+      exchange: { onTermsRequired: "skip" },
+    });
+  });
+
+  test("getAgentStatus parses skippedProviders, requiresAction and a terms approval", async () => {
+    const approvalId = "0199aaaa-0000-7000-8000-000000000000";
+    const get = jest.fn().mockResolvedValue({
+      status: 200,
+      data: {
+        success: true,
+        status: "completed",
+        data: null,
+        model: "spark-2",
+        expiresAt: "2026-09-02T00:00:00.000Z",
+        exchange: {
+          enabled: true,
+          onTermsRequired: "ask",
+          paidCalls: 0,
+          creditsUsed: null,
+          skippedProviders: [
+            {
+              provider: "apollo",
+              name: "Apollo",
+              capability: "people/search",
+              adds: "verified work emails",
+              reason: "terms_required",
+              version: "F-1.0.0",
+              termsUrl: "https://www.firecrawl.dev/app/alexandria/apollo",
+            },
+          ],
+          requiresAction: {
+            type: "accept_terms",
+            approvalId,
+            providers: [
+              {
+                provider: "apollo",
+                name: "Apollo",
+                version: "F-1.0.0",
+                digest: null,
+                url: "https://www.firecrawl.dev/app/alexandria/apollo",
+                show: {
+                  provider: "firecrawl",
+                  capability: "terms/show",
+                  options: { provider: "apollo" },
+                },
+                accept: {
+                  provider: "firecrawl",
+                  capability: "terms/accept",
+                  options: {
+                    provider: "apollo",
+                    version: "F-1.0.0",
+                    digest: null,
+                    confirmed: true,
+                  },
+                },
+              },
+            ],
+          },
+        },
+        pendingApproval: {
+          id: approvalId,
+          kind: "terms",
+          reason: "Apollo could add verified work emails.",
+          calls: [],
+          terms: [
+            {
+              provider: "apollo",
+              name: "Apollo",
+              version: "F-1.0.0",
+              digest: null,
+              url: "https://www.firecrawl.dev/app/alexandria/apollo",
+            },
+          ],
+          resolution: null,
+        },
+      },
+    });
+
+    const status = await getAgentStatus({ get } as any, "agent-job");
+
+    expect(status.exchange!.onTermsRequired).toBe("ask");
+    expect(status.exchange!.skippedProviders![0].reason).toBe("terms_required");
+    const action = status.exchange!.requiresAction!;
+    expect(action.approvalId).toBe(approvalId);
+    expect(action.providers[0].accept.capability).toBe("terms/accept");
+    expect(action.providers[0].accept.options.digest).toBeNull();
+    const pending = status.pendingApproval!;
+    expect(pending.kind).toBe("terms");
+    if (pending.kind !== "terms") throw new Error("expected a terms approval");
+    expect(pending.calls).toEqual([]);
+    expect(pending.terms[0].provider).toBe("apollo");
+  });
 });

package/src/__tests__/unit/v2/next-url-pinning.test.ts

@@ -0,0 +1,104 @@
+import { describe, test, expect } from "@jest/globals";
+import axios, { type AxiosAdapter } from "axios";
+import { FirecrawlClient } from "../../../v2/client";
+import { pinToApiOrigin } from "../../../utils/apiOrigin";
+
+const API_URL = "https://api.firecrawl.dev";
+const API_KEY = "fc-test";
+
+const origins: Array<[string, string]> = [
+  ["same origin", API_URL],
+  ["cross host", "https://evil.example"],
+  ["protocol-relative", "//evil.example"],
+  ["different port", "https://api.firecrawl.dev:8443"],
+  ["different scheme", "http://api.firecrawl.dev"],
+  ["whitespace-prefixed", "\n https://evil.example"],
+];
+
+function makeClient(firstPage: (next: string) => unknown, lastPage: unknown, next: string) {
+  const client = new FirecrawlClient({ apiKey: API_KEY, apiUrl: API_URL });
+  const sent: Array<{ url: string; authorization: unknown }> = [];
+  const adapter: AxiosAdapter = async config => {
+    const url = axios.getUri(config);
+    sent.push({ url, authorization: config.headers.Authorization });
+    const data = new URL(url).searchParams.has("skip") ? lastPage : firstPage(next);
+    return { data, status: 200, statusText: "OK", headers: {}, config };
+  };
+  (client as any).http.instance.defaults.adapter = adapter;
+  return { client, sent };
+}
+
+function expectPinned(sent: Array<{ url: string; authorization: unknown }>, path: string) {
+  expect(sent).toHaveLength(2);
+  expect(sent[1]!.url).toBe(`${API_URL}${path}?skip=10`);
+  for (const req of sent) {
+    expect(new URL(req.url).origin).toBe(API_URL);
+    expect(req.url).not.toContain("evil.example");
+    expect(req.authorization).toBe(`Bearer ${API_KEY}`);
+  }
+}
+
+describe("v2 pagination pins next URLs to the api_url origin", () => {
+  describe.each(origins)("%s next", (_label, origin) => {
+    test("crawl status", async () => {
+      const path = "/v2/crawl/abc";
+      const { client, sent } = makeClient(
+        next => ({ success: true, status: "completed", completed: 2, total: 2, next, data: [{ markdown: "a" }] }),
+        { success: true, next: null, data: [{ markdown: "b" }] },
+        `${origin}${path}?skip=10`,
+      );
+      const job = await client.getCrawlStatus("abc");
+      expect(job.data).toHaveLength(2);
+      expectPinned(sent, path);
+    });
+
+    test("batch scrape status", async () => {
+      const path = "/v2/batch/scrape/abc";
+      const { client, sent } = makeClient(
+        next => ({ success: true, status: "completed", completed: 2, total: 2, next, data: [{ markdown: "a" }] }),
+        { success: true, next: null, data: [{ markdown: "b" }] },
+        `${origin}${path}?skip=10`,
+      );
+      const job = await client.getBatchScrapeStatus("abc");
+      expect(job.data).toHaveLength(2);
+      expectPinned(sent, path);
+    });
+
+    test("monitor check", async () => {
+      const path = "/v2/monitor/m1/checks/c1";
+      const { client, sent } = makeClient(
+        next => ({ success: true, data: { id: "c1", pages: [{ url: "https://a.example" }], next } }),
+        { success: true, data: { pages: [{ url: "https://b.example" }], next: null } },
+        `${origin}${path}?skip=10`,
+      );
+      const check = await client.getMonitorCheck("m1", "c1");
+      expect(check.pages).toHaveLength(2);
+      expectPinned(sent, path);
+    });
+  });
+});
+
+describe("pinToApiOrigin", () => {
+  test("rewrites absolute URLs onto the api origin, keeping path and query and dropping the fragment", () => {
+    expect(pinToApiOrigin(API_URL, "https://evil.example/v2/crawl/abc?skip=10#frag")).toBe(
+      `${API_URL}/v2/crawl/abc?skip=10`,
+    );
+    expect(pinToApiOrigin("http://localhost:3002", "https://evil.example:444/v2/crawl/abc?skip=10")).toBe(
+      "http://localhost:3002/v2/crawl/abc?skip=10",
+    );
+    expect(pinToApiOrigin(API_URL, "\\\\evil.example/v2/crawl/abc")).toBe(`${API_URL}/v2/crawl/abc`);
+    for (const url of ["\nhttps://evil.example/v2/crawl/abc", "h\tttps://evil.example/v2/crawl/abc", "\u0000 //evil.example/v2/crawl/abc", "/\\evil.example/v2/crawl/abc"]) {
+      expect(pinToApiOrigin(API_URL, url)).toBe(`${API_URL}/v2/crawl/abc`);
+    }
+  });
+
+  test("leaves relative URLs unchanged", () => {
+    expect(pinToApiOrigin(API_URL, "/v2/crawl/abc?skip=10")).toBe("/v2/crawl/abc?skip=10");
+  });
+
+  test("throws when apiUrl is not absolute and the URL needs pinning", () => {
+    expect(() => pinToApiOrigin("api.firecrawl.dev", "https://evil.example/v2/crawl/abc")).toThrow(
+      "apiUrl must be an absolute URL",
+    );
+  });
+});

package/src/utils/apiOrigin.ts

@@ -0,0 +1,19 @@
+/**
+ * Rewrite an absolute or protocol-relative URL onto apiUrl's scheme, host and
+ * port (keeping path and query, dropping any fragment) so credentials never
+ * leave the configured API origin. Relative URLs are returned unchanged.
+ * Throws if apiUrl is not an absolute URL.
+ */
+export function pinToApiOrigin(apiUrl: string, url: string): string {
+  const normalized = url.replace(/[\t\n\r]/g, "").replace(/^[\u0000-\u0020]+/, "");
+  if (!/^([a-z][a-z\d+\-.]*:|[\\/]{2})/i.test(normalized)) return url;
+  const base = URL.canParse(apiUrl) ? new URL(apiUrl) : null;
+  if (!base?.host) {
+    throw new Error(`apiUrl must be an absolute URL, got ${JSON.stringify(apiUrl)}`);
+  }
+  const target = new URL(normalized, base);
+  const pinned = new URL(base.origin);
+  pinned.pathname = target.pathname;
+  pinned.search = target.search;
+  return pinned.href;
+}

package/src/v1/index.ts

@@ -2,6 +2,8 @@ import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from
 import * as zt from "zod";
 import { zodSchemaToJsonSchema } from "../utils/zodSchemaToJson";
 import { TypedEventTarget } from "typescript-event-target";
+import type { RequiresAction } from "../v2/types";
+import { pinToApiOrigin } from "../utils/apiOrigin";
 
 /**
  * Configuration interface for FirecrawlApp.
@@ -439,6 +441,8 @@ export interface CrawlErrorsResponse {
     url: string,
     code?: string,
     error: string,
+    /** Present when the page needs provider terms accepted first. */
+    requiresAction?: RequiresAction,
   }[];
 
   /**
@@ -919,7 +923,7 @@ export default class FirecrawlApp {
     }
 
     const headers: AxiosRequestHeaders = this.prepareHeaders();
-    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`, this.apiUrl);
     if (skip !== undefined) {
       targetURL.searchParams.set("skip", skip.toString());
     }
@@ -1207,7 +1211,7 @@ export default class FirecrawlApp {
     }
 
     const headers: AxiosRequestHeaders = this.prepareHeaders();
-    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`, this.apiUrl);
     if (skip !== undefined) {
       targetURL.searchParams.set("skip", skip.toString());
     }
@@ -1448,7 +1452,7 @@ export default class FirecrawlApp {
     headers: AxiosRequestHeaders
   ): Promise<AxiosResponse> {
     try {
-      return await axios.get(url, { headers });
+      return await axios.get(pinToApiOrigin(this.apiUrl, url), { headers });
     } catch (error) {
       if (error instanceof AxiosError && error.response) {
         return error.response as AxiosResponse;