@mendable/firecrawl-js 4.42.0 → 4.42.1

package/dist/{chunk-OCJTYRX5.js → chunk-YG554MNR.js}

@@ -12,7 +12,7 @@ var require_package = __commonJS({
   "package.json"(exports, module) {
     module.exports = {
       name: "@mendable/firecrawl-js",
-      version: "4.42.0",
+      version: "4.42.1",
       description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
       main: "dist/index.js",
       types: "dist/index.d.ts",

package/dist/index.cjs

@@ -39,7 +39,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@mendable/firecrawl-js",
-      version: "4.42.0",
+      version: "4.42.1",
       description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
       main: "dist/index.js",
       types: "dist/index.d.ts",
@@ -150,6 +150,21 @@ function getVersion() {
   }
 }
 
+// src/utils/apiOrigin.ts
+function pinToApiOrigin(apiUrl, url) {
+  const normalized = url.replace(/[\t\n\r]/g, "").replace(/^[\u0000-\u0020]+/, "");
+  if (!/^([a-z][a-z\d+\-.]*:|[\\/]{2})/i.test(normalized)) return url;
+  const base = URL.canParse(apiUrl) ? new URL(apiUrl) : null;
+  if (!base?.host) {
+    throw new Error(`apiUrl must be an absolute URL, got ${JSON.stringify(apiUrl)}`);
+  }
+  const target = new URL(normalized, base);
+  const pinned = new URL(base.origin);
+  pinned.pathname = target.pathname;
+  pinned.search = target.search;
+  return pinned.href;
+}
+
 // src/v2/utils/httpClient.ts
 var HttpClient = class {
   instance;
@@ -190,6 +205,7 @@ var HttpClient = class {
     config.headers = {
       ...config.headers || {}
     };
+    if (config.url) config.url = pinToApiOrigin(this.apiUrl, config.url);
     let lastError;
     for (let attempt = 0; attempt < this.maxRetries; attempt++) {
       try {
@@ -3020,7 +3036,7 @@ var FirecrawlApp = class {
       throw new FirecrawlError("No crawl ID provided", 400);
     }
     const headers = this.prepareHeaders();
-    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`, this.apiUrl);
     if (skip !== void 0) {
       targetURL.searchParams.set("skip", skip.toString());
     }
@@ -3265,7 +3281,7 @@ var FirecrawlApp = class {
       throw new FirecrawlError("No batch scrape ID provided", 400);
     }
     const headers = this.prepareHeaders();
-    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`, this.apiUrl);
     if (skip !== void 0) {
       targetURL.searchParams.set("skip", skip.toString());
     }
@@ -3477,7 +3493,7 @@ var FirecrawlApp = class {
    */
   async getRequest(url, headers) {
     try {
-      return await import_axios3.default.get(url, { headers });
+      return await import_axios3.default.get(pinToApiOrigin(this.apiUrl, url), { headers });
     } catch (error) {
       if (error instanceof import_axios3.AxiosError && error.response) {
         return error.response;

package/dist/index.d.cts

@@ -1676,6 +1676,8 @@ interface CrawlErrorsResponse$1 {
         url: string;
         code?: string;
         error: string;
+        /** Present when the page needs provider terms accepted first. */
+        requiresAction?: RequiresAction;
     }[];
     robotsBlocked: string[];
 }
@@ -2993,6 +2995,8 @@ interface CrawlErrorsResponse {
         url: string;
         code?: string;
         error: string;
+        /** Present when the page needs provider terms accepted first. */
+        requiresAction?: RequiresAction;
     }[];
     /**
      * URLs blocked by robots.txt

package/dist/index.d.ts

@@ -1676,6 +1676,8 @@ interface CrawlErrorsResponse$1 {
         url: string;
         code?: string;
         error: string;
+        /** Present when the page needs provider terms accepted first. */
+        requiresAction?: RequiresAction;
     }[];
     robotsBlocked: string[];
 }
@@ -2993,6 +2995,8 @@ interface CrawlErrorsResponse {
         url: string;
         code?: string;
         error: string;
+        /** Present when the page needs provider terms accepted first. */
+        requiresAction?: RequiresAction;
     }[];
     /**
      * URLs blocked by robots.txt

package/dist/index.js

@@ -1,6 +1,6 @@
 import {
   require_package
-} from "./chunk-OCJTYRX5.js";
+} from "./chunk-YG554MNR.js";
 
 // src/v2/utils/httpClient.ts
 import axios from "axios";
@@ -18,6 +18,21 @@ function getVersion() {
   }
 }
 
+// src/utils/apiOrigin.ts
+function pinToApiOrigin(apiUrl, url) {
+  const normalized = url.replace(/[\t\n\r]/g, "").replace(/^[\u0000-\u0020]+/, "");
+  if (!/^([a-z][a-z\d+\-.]*:|[\\/]{2})/i.test(normalized)) return url;
+  const base = URL.canParse(apiUrl) ? new URL(apiUrl) : null;
+  if (!base?.host) {
+    throw new Error(`apiUrl must be an absolute URL, got ${JSON.stringify(apiUrl)}`);
+  }
+  const target = new URL(normalized, base);
+  const pinned = new URL(base.origin);
+  pinned.pathname = target.pathname;
+  pinned.search = target.search;
+  return pinned.href;
+}
+
 // src/v2/utils/httpClient.ts
 var HttpClient = class {
   instance;
@@ -58,6 +73,7 @@ var HttpClient = class {
     config.headers = {
       ...config.headers || {}
     };
+    if (config.url) config.url = pinToApiOrigin(this.apiUrl, config.url);
     let lastError;
     for (let attempt = 0; attempt < this.maxRetries; attempt++) {
       try {
@@ -2673,7 +2689,7 @@ var FirecrawlApp = class {
       if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
         return process.env.npm_package_version;
       }
-      const packageJson = await import("./package-ECWDBUY3.js");
+      const packageJson = await import("./package-FOKY5K5X.js");
       return packageJson.default.version;
     } catch (error) {
       const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
@@ -2888,7 +2904,7 @@ var FirecrawlApp = class {
       throw new FirecrawlError("No crawl ID provided", 400);
     }
     const headers = this.prepareHeaders();
-    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`, this.apiUrl);
     if (skip !== void 0) {
       targetURL.searchParams.set("skip", skip.toString());
     }
@@ -3133,7 +3149,7 @@ var FirecrawlApp = class {
       throw new FirecrawlError("No batch scrape ID provided", 400);
     }
     const headers = this.prepareHeaders();
-    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`, this.apiUrl);
     if (skip !== void 0) {
       targetURL.searchParams.set("skip", skip.toString());
     }
@@ -3345,7 +3361,7 @@ var FirecrawlApp = class {
    */
   async getRequest(url, headers) {
     try {
-      return await axios2.get(url, { headers });
+      return await axios2.get(pinToApiOrigin(this.apiUrl, url), { headers });
     } catch (error) {
       if (error instanceof AxiosError && error.response) {
         return error.response;

package/dist/{package-ECWDBUY3.js → package-FOKY5K5X.js}

@@ -1,4 +1,4 @@
 import {
   require_package
-} from "./chunk-OCJTYRX5.js";
+} from "./chunk-YG554MNR.js";
 export default require_package();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@mendable/firecrawl-js",
-  "version": "4.42.0",
+  "version": "4.42.1",
   "description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",

package/pnpm-workspace.yaml

@@ -7,7 +7,9 @@ overrides:
   "rollup@<4.59.0": ">=4.59.0"
   "picomatch@<4.0.4": ">=4.0.4"
   handlebars: ">=4.7.9"
-  "brace-expansion@>=5.0.0 <5.0.9": "5.0.9"
+  # GHSA-q2hr-2g5m-vwhr / GHSA-qhr7-859c-m2p7 / GHSA-6j4f-fj2g-mc7p:
+  # pin the 5.x line only. Exact 5.0.12 cannot resolve to 6.x.
+  "brace-expansion@>=5.0.0 <5.0.12": "5.0.12"
   "js-yaml@>=3.0.0 <3.15.2": "3.15.2"
   "js-yaml@>=4.0.0 <4.3.2": "4.3.2"
   "axios@>=1.0.0 <1.16.0": "1.16.1"

package/src/__tests__/unit/v1/next-url-pinning.test.ts

@@ -0,0 +1,81 @@
+import { describe, test, expect, jest, afterEach } from "@jest/globals";
+import axios from "axios";
+import FirecrawlApp from "../../../v1";
+
+const API_URL = "https://api.firecrawl.dev";
+const API_KEY = "fc-test";
+
+const origins: Array<[string, string]> = [
+  ["same origin", API_URL],
+  ["cross host", "https://evil.example"],
+  ["protocol-relative", "//evil.example"],
+  ["different port", "https://api.firecrawl.dev:8443"],
+  ["different scheme", "http://api.firecrawl.dev"],
+  ["whitespace-prefixed", "\n https://evil.example"],
+];
+
+function mockGet(respond: (url: URL) => unknown) {
+  const sent: Array<{ url: string; authorization: unknown }> = [];
+  jest.spyOn(axios, "get").mockImplementation(async (url: string, config?: any) => {
+    sent.push({ url, authorization: config?.headers?.Authorization });
+    return { status: 200, data: respond(new URL(url)) };
+  });
+  return sent;
+}
+
+function expectPinned(sent: Array<{ url: string; authorization: unknown }>, expectedUrls: string[]) {
+  expect(sent.map(req => req.url)).toEqual(expectedUrls);
+  for (const req of sent) {
+    expect(new URL(req.url).origin).toBe(API_URL);
+    expect(req.url).not.toContain("evil.example");
+    expect(req.authorization).toBe(`Bearer ${API_KEY}`);
+  }
+}
+
+const completedPage = (next: string) => ({
+  success: true,
+  status: "completed",
+  total: 2,
+  completed: 2,
+  expiresAt: "2030-01-01T00:00:00Z",
+  data: [{ markdown: "a" }],
+  next,
+});
+const lastPage = () => ({ success: true, status: "completed", data: [{ markdown: "b" }] });
+
+describe("v1 pins next URLs to the api_url origin", () => {
+  afterEach(() => {
+    jest.restoreAllMocks();
+  });
+
+  describe.each(origins)("%s next", (_label, origin) => {
+    const app = new FirecrawlApp({ apiKey: API_KEY, apiUrl: API_URL });
+
+    test.each([
+      ["checkCrawlStatus", "/v1/crawl/abc"],
+      ["checkBatchScrapeStatus", "/v1/batch/scrape/abc"],
+    ] as const)("%s getAllData follows server next", async (method, path) => {
+      const sent = mockGet(url => (url.searchParams.has("skip") ? lastPage() : completedPage(`${origin}${path}?skip=10`)));
+      const res: any = await app[method]("abc", true);
+      expect(res.data).toHaveLength(2);
+      expectPinned(sent, [`${API_URL}${path}`, `${API_URL}${path}?skip=10`]);
+    });
+
+    test.each([
+      ["checkCrawlStatus", "/v1/crawl/abc"],
+      ["checkBatchScrapeStatus", "/v1/batch/scrape/abc"],
+    ] as const)("%s caller-supplied nextURL", async (method, path) => {
+      const sent = mockGet(lastPage);
+      await app[method]("abc", false, `${origin}${path}?skip=10`);
+      expectPinned(sent, [`${API_URL}${path}?skip=10`]);
+    });
+
+    test("monitorJobStatus follows server next", async () => {
+      const path = "/v1/crawl/abc";
+      const sent = mockGet(url => (url.searchParams.has("skip") ? lastPage() : completedPage(`${origin}${path}?skip=10`)));
+      const res: any = await app.monitorJobStatus("abc", app.prepareHeaders(), 0);
+      expect(res.data).toHaveLength(2);
+      expectPinned(sent, [`${API_URL}${path}`, `${API_URL}${path}?skip=10`]);
+    });
+  });
+});

package/src/__tests__/unit/v2/next-url-pinning.test.ts

@@ -0,0 +1,104 @@
+import { describe, test, expect } from "@jest/globals";
+import axios, { type AxiosAdapter } from "axios";
+import { FirecrawlClient } from "../../../v2/client";
+import { pinToApiOrigin } from "../../../utils/apiOrigin";
+
+const API_URL = "https://api.firecrawl.dev";
+const API_KEY = "fc-test";
+
+const origins: Array<[string, string]> = [
+  ["same origin", API_URL],
+  ["cross host", "https://evil.example"],
+  ["protocol-relative", "//evil.example"],
+  ["different port", "https://api.firecrawl.dev:8443"],
+  ["different scheme", "http://api.firecrawl.dev"],
+  ["whitespace-prefixed", "\n https://evil.example"],
+];
+
+function makeClient(firstPage: (next: string) => unknown, lastPage: unknown, next: string) {
+  const client = new FirecrawlClient({ apiKey: API_KEY, apiUrl: API_URL });
+  const sent: Array<{ url: string; authorization: unknown }> = [];
+  const adapter: AxiosAdapter = async config => {
+    const url = axios.getUri(config);
+    sent.push({ url, authorization: config.headers.Authorization });
+    const data = new URL(url).searchParams.has("skip") ? lastPage : firstPage(next);
+    return { data, status: 200, statusText: "OK", headers: {}, config };
+  };
+  (client as any).http.instance.defaults.adapter = adapter;
+  return { client, sent };
+}
+
+function expectPinned(sent: Array<{ url: string; authorization: unknown }>, path: string) {
+  expect(sent).toHaveLength(2);
+  expect(sent[1]!.url).toBe(`${API_URL}${path}?skip=10`);
+  for (const req of sent) {
+    expect(new URL(req.url).origin).toBe(API_URL);
+    expect(req.url).not.toContain("evil.example");
+    expect(req.authorization).toBe(`Bearer ${API_KEY}`);
+  }
+}
+
+describe("v2 pagination pins next URLs to the api_url origin", () => {
+  describe.each(origins)("%s next", (_label, origin) => {
+    test("crawl status", async () => {
+      const path = "/v2/crawl/abc";
+      const { client, sent } = makeClient(
+        next => ({ success: true, status: "completed", completed: 2, total: 2, next, data: [{ markdown: "a" }] }),
+        { success: true, next: null, data: [{ markdown: "b" }] },
+        `${origin}${path}?skip=10`,
+      );
+      const job = await client.getCrawlStatus("abc");
+      expect(job.data).toHaveLength(2);
+      expectPinned(sent, path);
+    });
+
+    test("batch scrape status", async () => {
+      const path = "/v2/batch/scrape/abc";
+      const { client, sent } = makeClient(
+        next => ({ success: true, status: "completed", completed: 2, total: 2, next, data: [{ markdown: "a" }] }),
+        { success: true, next: null, data: [{ markdown: "b" }] },
+        `${origin}${path}?skip=10`,
+      );
+      const job = await client.getBatchScrapeStatus("abc");
+      expect(job.data).toHaveLength(2);
+      expectPinned(sent, path);
+    });
+
+    test("monitor check", async () => {
+      const path = "/v2/monitor/m1/checks/c1";
+      const { client, sent } = makeClient(
+        next => ({ success: true, data: { id: "c1", pages: [{ url: "https://a.example" }], next } }),
+        { success: true, data: { pages: [{ url: "https://b.example" }], next: null } },
+        `${origin}${path}?skip=10`,
+      );
+      const check = await client.getMonitorCheck("m1", "c1");
+      expect(check.pages).toHaveLength(2);
+      expectPinned(sent, path);
+    });
+  });
+});
+
+describe("pinToApiOrigin", () => {
+  test("rewrites absolute URLs onto the api origin, keeping path and query and dropping the fragment", () => {
+    expect(pinToApiOrigin(API_URL, "https://evil.example/v2/crawl/abc?skip=10#frag")).toBe(
+      `${API_URL}/v2/crawl/abc?skip=10`,
+    );
+    expect(pinToApiOrigin("http://localhost:3002", "https://evil.example:444/v2/crawl/abc?skip=10")).toBe(
+      "http://localhost:3002/v2/crawl/abc?skip=10",
+    );
+    expect(pinToApiOrigin(API_URL, "\\\\evil.example/v2/crawl/abc")).toBe(`${API_URL}/v2/crawl/abc`);
+    for (const url of ["\nhttps://evil.example/v2/crawl/abc", "h\tttps://evil.example/v2/crawl/abc", "\u0000 //evil.example/v2/crawl/abc", "/\\evil.example/v2/crawl/abc"]) {
+      expect(pinToApiOrigin(API_URL, url)).toBe(`${API_URL}/v2/crawl/abc`);
+    }
+  });
+
+  test("leaves relative URLs unchanged", () => {
+    expect(pinToApiOrigin(API_URL, "/v2/crawl/abc?skip=10")).toBe("/v2/crawl/abc?skip=10");
+  });
+
+  test("throws when apiUrl is not absolute and the URL needs pinning", () => {
+    expect(() => pinToApiOrigin("api.firecrawl.dev", "https://evil.example/v2/crawl/abc")).toThrow(
+      "apiUrl must be an absolute URL",
+    );
+  });
+});

package/src/utils/apiOrigin.ts

@@ -0,0 +1,19 @@
+/**
+ * Rewrite an absolute or protocol-relative URL onto apiUrl's scheme, host and
+ * port (keeping path and query, dropping any fragment) so credentials never
+ * leave the configured API origin. Relative URLs are returned unchanged.
+ * Throws if apiUrl is not an absolute URL.
+ */
+export function pinToApiOrigin(apiUrl: string, url: string): string {
+  const normalized = url.replace(/[\t\n\r]/g, "").replace(/^[\u0000-\u0020]+/, "");
+  if (!/^([a-z][a-z\d+\-.]*:|[\\/]{2})/i.test(normalized)) return url;
+  const base = URL.canParse(apiUrl) ? new URL(apiUrl) : null;
+  if (!base?.host) {
+    throw new Error(`apiUrl must be an absolute URL, got ${JSON.stringify(apiUrl)}`);
+  }
+  const target = new URL(normalized, base);
+  const pinned = new URL(base.origin);
+  pinned.pathname = target.pathname;
+  pinned.search = target.search;
+  return pinned.href;
+}

package/src/v1/index.ts

@@ -2,6 +2,8 @@ import axios, { type AxiosResponse, type AxiosRequestHeaders, AxiosError } from
 import * as zt from "zod";
 import { zodSchemaToJsonSchema } from "../utils/zodSchemaToJson";
 import { TypedEventTarget } from "typescript-event-target";
+import type { RequiresAction } from "../v2/types";
+import { pinToApiOrigin } from "../utils/apiOrigin";
 
 /**
  * Configuration interface for FirecrawlApp.
@@ -439,6 +441,8 @@ export interface CrawlErrorsResponse {
     url: string,
     code?: string,
     error: string,
+    /** Present when the page needs provider terms accepted first. */
+    requiresAction?: RequiresAction,
   }[];
 
   /**
@@ -919,7 +923,7 @@ export default class FirecrawlApp {
     }
 
     const headers: AxiosRequestHeaders = this.prepareHeaders();
-    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`, this.apiUrl);
     if (skip !== undefined) {
       targetURL.searchParams.set("skip", skip.toString());
     }
@@ -1207,7 +1211,7 @@ export default class FirecrawlApp {
     }
 
     const headers: AxiosRequestHeaders = this.prepareHeaders();
-    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
+    const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`, this.apiUrl);
     if (skip !== undefined) {
       targetURL.searchParams.set("skip", skip.toString());
     }
@@ -1448,7 +1452,7 @@ export default class FirecrawlApp {
     headers: AxiosRequestHeaders
   ): Promise<AxiosResponse> {
     try {
-      return await axios.get(url, { headers });
+      return await axios.get(pinToApiOrigin(this.apiUrl, url), { headers });
     } catch (error) {
       if (error instanceof AxiosError && error.response) {
         return error.response as AxiosResponse;

package/src/v2/methods/batch.ts

@@ -152,7 +152,7 @@ export async function getBatchScrapeErrors(
   try {
     const res = await http.get<{
       success?: boolean;
-      data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] };
+      data?: CrawlErrorsResponse;
     }>(`/v2/batch/scrape/${jobId}/errors`);
     if (res.status !== 200) throwForBadResponse(res, "get batch scrape errors");
     const payload = res.data?.data ?? (res.data as any);

package/src/v2/methods/crawl.ts

@@ -162,7 +162,7 @@ export async function crawl(http: HttpClient, request: CrawlRequest, pollInterva
 
 export async function getCrawlErrors(http: HttpClient, crawlId: string): Promise<CrawlErrorsResponse> {
   try {
-    const res = await http.get<{ success?: boolean; data?: { errors: Array<Record<string, string>>; robotsBlocked: string[] } }>(`/v2/crawl/${crawlId}/errors`);
+    const res = await http.get<{ success?: boolean; data?: CrawlErrorsResponse }>(`/v2/crawl/${crawlId}/errors`);
     if (res.status !== 200) throwForBadResponse(res, "get crawl errors");
     const payload = res.data?.data ?? (res.data as any);
     return { errors: payload.errors || [], robotsBlocked: payload.robotsBlocked || [] };

package/src/v2/types.ts

@@ -1955,6 +1955,8 @@ export interface CrawlErrorsResponse {
     url: string;
     code?: string;
     error: string;
+    /** Present when the page needs provider terms accepted first. */
+    requiresAction?: RequiresAction;
   }[];
   robotsBlocked: string[];
 }

package/src/v2/utils/httpClient.ts

@@ -4,6 +4,7 @@ import axios, {
   type AxiosResponse,
 } from "axios";
 import { getVersion } from "./getVersion";
+import { pinToApiOrigin } from "../../utils/apiOrigin";
 
 export interface HttpClientOptions {
   apiKey: string;
@@ -64,6 +65,7 @@ export class HttpClient {
     config.headers = {
       ...(config.headers || {}),
     };
+    if (config.url) config.url = pinToApiOrigin(this.apiUrl, config.url);
 
     let lastError: any;
     for (let attempt = 0; attempt < this.maxRetries; attempt++) {