firecrawl 4.18.5 → 4.20.0

package/audit-ci.jsonc

@@ -1,4 +1,7 @@
 {
     "$schema": "https://github.com/IBM/audit-ci/raw/main/docs/schema.json",
-    "low": true
-}
+    "low": true,
+    "allowlist": [
+        "GHSA-w5hq-g745-h8pq"
+    ]
+}

package/dist/{chunk-SEIHZPTI.js → chunk-UXVHRV2G.js}

@@ -8,7 +8,7 @@ var require_package = __commonJS({
   "package.json"(exports, module) {
     module.exports = {
       name: "@mendable/firecrawl-js",
-      version: "4.18.5",
+      version: "4.20.0",
       description: "JavaScript SDK for Firecrawl API",
       main: "dist/index.js",
       types: "dist/index.d.ts",

package/dist/index.cjs

@@ -35,7 +35,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@mendable/firecrawl-js",
-      version: "4.18.5",
+      version: "4.20.0",
       description: "JavaScript SDK for Firecrawl API",
       main: "dist/index.js",
       types: "dist/index.d.ts",
@@ -182,10 +182,10 @@ var HttpClient = class {
         const isPlainObjectBody = !isFormDataBody && cfg.data != null && typeof cfg.data === "object" && !Array.isArray(cfg.data);
         if (isPlainObjectBody && cfg.method && ["post", "put", "patch"].includes(cfg.method.toLowerCase())) {
           const data = cfg.data ?? {};
-          cfg.data = { ...data, origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${version}` };
-          if (typeof data.timeout === "number") {
-            cfg.timeout = data.timeout + 5e3;
-          }
+          cfg.data = {
+            ...data,
+            origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${version}`
+          };
         }
         if (isFormDataBody) {
           cfg.headers = { ...cfg.headers || {} };
@@ -213,16 +213,22 @@ var HttpClient = class {
   sleep(seconds) {
     return new Promise((r) => setTimeout(r, seconds * 1e3));
   }
-  post(endpoint, body, headers) {
-    return this.request({ method: "post", url: endpoint, data: body, headers });
+  post(endpoint, body, options) {
+    return this.request({
+      method: "post",
+      url: endpoint,
+      data: body,
+      headers: options?.headers,
+      timeout: options?.timeoutMs
+    });
   }
-  postMultipart(endpoint, formData, headers, timeoutMs) {
+  postMultipart(endpoint, formData, options) {
     return this.request({
       method: "post",
       url: endpoint,
       data: formData,
-      headers,
-      timeout: timeoutMs
+      headers: options?.headers,
+      timeout: options?.timeoutMs
     });
   }
   get(endpoint, headers) {
@@ -433,7 +439,7 @@ function ensureValidParseOptions(options) {
   if (raw.mobile !== void 0) {
     throw new Error("parse does not support mobile rendering");
   }
-  if (raw.maxAge !== void 0 || raw.minAge !== void 0 || raw.storeInCache !== void 0) {
+  if (raw.maxAge !== void 0 || raw.minAge !== void 0 || raw.storeInCache !== void 0 || raw.lockdown !== void 0) {
     throw new Error("parse does not support cache/index options");
   }
   if (raw.proxy !== void 0 && raw.proxy !== "basic" && raw.proxy !== "auto") {
@@ -488,7 +494,11 @@ async function scrape(http, url, options) {
   const payload = { url: url.trim() };
   if (options) Object.assign(payload, options);
   try {
-    const res = await http.post("/v2/scrape", payload);
+    const res = await http.post(
+      "/v2/scrape",
+      payload,
+      typeof options?.timeout === "number" ? { timeoutMs: options.timeout + 5e3 } : {}
+    );
     if (res.status !== 200 || !res.data?.success) {
       throwForBadResponse(res, "scrape");
     }
@@ -516,12 +526,15 @@ async function interact(http, jobId, args) {
   try {
     const res = await http.post(
       `/v2/scrape/${jobId}/interact`,
-      body
+      body,
+      args.timeout != null ? { timeoutMs: args.timeout * 1e3 + 5e3 } : {}
     );
-    if (res.status !== 200) throwForBadResponse(res, "interact with scrape browser");
+    if (res.status !== 200)
+      throwForBadResponse(res, "interact with scrape browser");
     return res.data;
   } catch (err) {
-    if (err?.isAxiosError) return normalizeAxiosError(err, "interact with scrape browser");
+    if (err?.isAxiosError)
+      return normalizeAxiosError(err, "interact with scrape browser");
     throw err;
   }
 }
@@ -559,7 +572,9 @@ function toUploadBlob(input, contentType) {
     return new Blob([input], { type: contentType });
   }
   if (typeof input === "string") {
-    return new Blob([input], { type: contentType ?? "text/plain; charset=utf-8" });
+    return new Blob([input], {
+      type: contentType ?? "text/plain; charset=utf-8"
+    });
   }
   throw new Error("Unsupported parse file data type");
 }
@@ -587,9 +602,12 @@ async function parse(http, file, options) {
     toUploadBlob(file.data, file.contentType),
     file.filename.trim()
   );
-  const requestTimeoutMs = typeof normalizedOptions.timeout === "number" ? normalizedOptions.timeout + 5e3 : void 0;
   try {
-    const res = await http.postMultipart("/v2/parse", formData, void 0, requestTimeoutMs);
+    const res = await http.postMultipart(
+      "/v2/parse",
+      formData,
+      typeof normalizedOptions.timeout === "number" ? { timeoutMs: normalizedOptions.timeout + 5e3 } : {}
+    );
     if (res.status !== 200 || !res.data?.success) {
       throwForBadResponse(res, "parse");
     }
@@ -603,8 +621,10 @@ async function parse(http, file, options) {
 // src/v2/methods/search.ts
 function prepareSearchPayload(req) {
   if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
-  if (req.limit != null && req.limit <= 0) throw new Error("limit must be positive");
-  if (req.timeout != null && req.timeout <= 0) throw new Error("timeout must be positive");
+  if (req.limit != null && req.limit <= 0)
+    throw new Error("limit must be positive");
+  if (req.timeout != null && req.timeout <= 0)
+    throw new Error("timeout must be positive");
   const payload = {
     query: req.query
   };
@@ -613,9 +633,11 @@ function prepareSearchPayload(req) {
   if (req.limit != null) payload.limit = req.limit;
   if (req.tbs != null) payload.tbs = req.tbs;
   if (req.location != null) payload.location = req.location;
-  if (req.ignoreInvalidURLs != null) payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
+  if (req.ignoreInvalidURLs != null)
+    payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
   if (req.timeout != null) payload.timeout = req.timeout;
-  if (req.integration && req.integration.trim()) payload.integration = req.integration.trim();
+  if (req.integration && req.integration.trim())
+    payload.integration = req.integration.trim();
   if (req.origin) payload.origin = req.origin;
   if (req.scrapeOptions) {
     ensureValidScrapeOptions(req.scrapeOptions);
@@ -641,7 +663,11 @@ function transformArray(arr) {
 async function search(http, request) {
   const payload = prepareSearchPayload(request);
   try {
-    const res = await http.post("/v2/search", payload);
+    const res = await http.post(
+      "/v2/search",
+      payload,
+      typeof request.timeout === "number" ? { timeoutMs: request.timeout + 5e3 } : {}
+    );
     if (res.status !== 200 || !res.data?.success) {
       throwForBadResponse(res, "search");
     }
@@ -649,7 +675,8 @@ async function search(http, request) {
     const out = {};
     if (data.web) out.web = transformArray(data.web);
     if (data.news) out.news = transformArray(data.news);
-    if (data.images) out.images = transformArray(data.images);
+    if (data.images)
+      out.images = transformArray(data.images);
     return out;
   } catch (err) {
     if (err?.isAxiosError) return normalizeAxiosError(err, "search");
@@ -664,11 +691,14 @@ function prepareMapPayload(url, options) {
   if (options) {
     if (options.sitemap != null) payload.sitemap = options.sitemap;
     if (options.search != null) payload.search = options.search;
-    if (options.includeSubdomains != null) payload.includeSubdomains = options.includeSubdomains;
-    if (options.ignoreQueryParameters != null) payload.ignoreQueryParameters = options.ignoreQueryParameters;
+    if (options.includeSubdomains != null)
+      payload.includeSubdomains = options.includeSubdomains;
+    if (options.ignoreQueryParameters != null)
+      payload.ignoreQueryParameters = options.ignoreQueryParameters;
     if (options.limit != null) payload.limit = options.limit;
     if (options.timeout != null) payload.timeout = options.timeout;
-    if (options.integration != null && options.integration.trim()) payload.integration = options.integration.trim();
+    if (options.integration != null && options.integration.trim())
+      payload.integration = options.integration.trim();
     if (options.origin) payload.origin = options.origin;
     if (options.location != null) payload.location = options.location;
   }
@@ -677,7 +707,11 @@ function prepareMapPayload(url, options) {
 async function map(http, url, options) {
   const payload = prepareMapPayload(url, options);
   try {
-    const res = await http.post("/v2/map", payload);
+    const res = await http.post(
+      "/v2/map",
+      payload,
+      typeof options?.timeout === "number" ? { timeoutMs: options.timeout + 5e3 } : {}
+    );
     if (res.status !== 200 || !res.data?.success) {
       throwForBadResponse(res, "map");
     }
@@ -685,7 +719,12 @@ async function map(http, url, options) {
     const links = [];
     for (const item of linksIn) {
       if (typeof item === "string") links.push({ url: item });
-      else if (item && typeof item === "object") links.push({ url: item.url, title: item.title, description: item.description });
+      else if (item && typeof item === "object")
+        links.push({
+          url: item.url,
+          title: item.title,
+          description: item.description
+        });
     }
     return { links };
   } catch (err) {
@@ -897,7 +936,8 @@ async function startBatchScrape(http, urls, {
   integration,
   origin
 } = {}) {
-  if (!Array.isArray(urls) || urls.length === 0) throw new Error("URLs list cannot be empty");
+  if (!Array.isArray(urls) || urls.length === 0)
+    throw new Error("URLs list cannot be empty");
   const payload = { urls };
   if (options) {
     ensureValidScrapeOptions(options);
@@ -908,22 +948,30 @@ async function startBatchScrape(http, urls, {
   if (ignoreInvalidURLs != null) payload.ignoreInvalidURLs = ignoreInvalidURLs;
   if (maxConcurrency != null) payload.maxConcurrency = maxConcurrency;
   if (zeroDataRetention != null) payload.zeroDataRetention = zeroDataRetention;
-  if (integration != null && integration.trim()) payload.integration = integration.trim();
+  if (integration != null && integration.trim())
+    payload.integration = integration.trim();
   if (origin) payload.origin = origin;
   try {
     const headers = http.prepareHeaders(idempotencyKey);
-    const res = await http.post("/v2/batch/scrape", payload, headers);
-    if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "start batch scrape");
-    return { id: res.data.id, url: res.data.url, invalidURLs: res.data.invalidURLs || void 0 };
+    const res = await http.post("/v2/batch/scrape", payload, { headers });
+    if (res.status !== 200 || !res.data?.success)
+      throwForBadResponse(res, "start batch scrape");
+    return {
+      id: res.data.id,
+      url: res.data.url,
+      invalidURLs: res.data.invalidURLs || void 0
+    };
   } catch (err) {
-    if (err?.isAxiosError) return normalizeAxiosError(err, "start batch scrape");
+    if (err?.isAxiosError)
+      return normalizeAxiosError(err, "start batch scrape");
     throw err;
   }
 }
 async function getBatchScrapeStatus(http, jobId, pagination) {
   try {
     const res = await http.get(`/v2/batch/scrape/${jobId}`);
-    if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get batch scrape status");
+    if (res.status !== 200 || !res.data?.success)
+      throwForBadResponse(res, "get batch scrape status");
     const body = res.data;
     const initialDocs = body.data || [];
     const auto = pagination?.autoPaginate ?? true;
@@ -939,7 +987,12 @@ async function getBatchScrapeStatus(http, jobId, pagination) {
         data: initialDocs
       };
     }
-    const aggregated = await fetchAllPages(http, body.next, initialDocs, pagination);
+    const aggregated = await fetchAllPages(
+      http,
+      body.next,
+      initialDocs,
+      pagination
+    );
     return {
       id: jobId,
       status: body.status,
@@ -951,17 +1004,21 @@ async function getBatchScrapeStatus(http, jobId, pagination) {
       data: aggregated
     };
   } catch (err) {
-    if (err?.isAxiosError) return normalizeAxiosError(err, "get batch scrape status");
+    if (err?.isAxiosError)
+      return normalizeAxiosError(err, "get batch scrape status");
     throw err;
   }
 }
 async function cancelBatchScrape(http, jobId) {
   try {
-    const res = await http.delete(`/v2/batch/scrape/${jobId}`);
+    const res = await http.delete(
+      `/v2/batch/scrape/${jobId}`
+    );
     if (res.status !== 200) throwForBadResponse(res, "cancel batch scrape");
     return res.data?.status === "cancelled";
   } catch (err) {
-    if (err?.isAxiosError) return normalizeAxiosError(err, "cancel batch scrape");
+    if (err?.isAxiosError)
+      return normalizeAxiosError(err, "cancel batch scrape");
     throw err;
   }
 }
@@ -970,9 +1027,13 @@ async function getBatchScrapeErrors(http, jobId) {
     const res = await http.get(`/v2/batch/scrape/${jobId}/errors`);
     if (res.status !== 200) throwForBadResponse(res, "get batch scrape errors");
     const payload = res.data?.data ?? res.data;
-    return { errors: payload.errors || [], robotsBlocked: payload.robotsBlocked || [] };
+    return {
+      errors: payload.errors || [],
+      robotsBlocked: payload.robotsBlocked || []
+    };
   } catch (err) {
-    if (err?.isAxiosError) return normalizeAxiosError(err, "get batch scrape errors");
+    if (err?.isAxiosError)
+      return normalizeAxiosError(err, "get batch scrape errors");
     throw err;
   }
 }
@@ -1007,7 +1068,12 @@ async function waitForBatchCompletion(http, jobId, pollInterval = 2, timeout) {
 }
 async function batchScrape(http, urls, opts = {}) {
   const start = await startBatchScrape(http, urls, opts);
-  return waitForBatchCompletion(http, start.id, opts.pollInterval ?? 2, opts.timeout);
+  return waitForBatchCompletion(
+    http,
+    start.id,
+    opts.pollInterval ?? 2,
+    opts.timeout
+  );
 }
 
 // src/v2/methods/extract.ts
@@ -1146,7 +1212,8 @@ async function browser(http, args = {}) {
     if (res.status !== 200) throwForBadResponse(res, "create browser session");
     return res.data;
   } catch (err) {
-    if (err?.isAxiosError) return normalizeAxiosError(err, "create browser session");
+    if (err?.isAxiosError)
+      return normalizeAxiosError(err, "create browser session");
     throw err;
   }
 }
@@ -1159,12 +1226,14 @@ async function browserExecute(http, sessionId, args) {
   try {
     const res = await http.post(
       `/v2/browser/${sessionId}/execute`,
-      body
+      body,
+      args.timeout != null ? { timeoutMs: args.timeout * 1e3 + 5e3 } : {}
     );
     if (res.status !== 200) throwForBadResponse(res, "execute browser code");
     return res.data;
   } catch (err) {
-    if (err?.isAxiosError) return normalizeAxiosError(err, "execute browser code");
+    if (err?.isAxiosError)
+      return normalizeAxiosError(err, "execute browser code");
     throw err;
   }
 }
@@ -1176,7 +1245,8 @@ async function deleteBrowser(http, sessionId) {
     if (res.status !== 200) throwForBadResponse(res, "delete browser session");
     return res.data;
   } catch (err) {
-    if (err?.isAxiosError) return normalizeAxiosError(err, "delete browser session");
+    if (err?.isAxiosError)
+      return normalizeAxiosError(err, "delete browser session");
     throw err;
   }
 }
@@ -1188,7 +1258,8 @@ async function listBrowsers(http, args = {}) {
     if (res.status !== 200) throwForBadResponse(res, "list browser sessions");
     return res.data;
   } catch (err) {
-    if (err?.isAxiosError) return normalizeAxiosError(err, "list browser sessions");
+    if (err?.isAxiosError)
+      return normalizeAxiosError(err, "list browser sessions");
     throw err;
   }
 }

package/dist/index.d.cts

@@ -124,6 +124,7 @@ interface ScrapeOptions {
     maxAge?: number;
     minAge?: number;
     storeInCache?: boolean;
+    lockdown?: boolean;
     profile?: {
         name: string;
         saveChanges?: boolean;
@@ -137,7 +138,7 @@ interface ParseFile {
     filename: string;
     contentType?: string;
 }
-type ParseOptions = Omit<ScrapeOptions, 'formats' | 'waitFor' | 'mobile' | 'actions' | 'location' | 'maxAge' | 'minAge' | 'storeInCache' | 'proxy'> & {
+type ParseOptions = Omit<ScrapeOptions, 'formats' | 'waitFor' | 'mobile' | 'actions' | 'location' | 'maxAge' | 'minAge' | 'storeInCache' | 'lockdown' | 'proxy'> & {
     formats?: ParseFormatOption[];
     proxy?: 'basic' | 'auto';
 };
@@ -648,6 +649,10 @@ interface HttpClientOptions {
     maxRetries?: number;
     backoffFactor?: number;
 }
+interface RequestOptions {
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+}
 declare class HttpClient {
     private instance;
     private readonly apiKey;
@@ -659,8 +664,8 @@ declare class HttpClient {
     getApiKey(): string;
     private request;
     private sleep;
-    post<T = any>(endpoint: string, body: Record<string, unknown>, headers?: Record<string, string>): Promise<AxiosResponse<T, any, {}>>;
-    postMultipart<T = any>(endpoint: string, formData: FormData, headers?: Record<string, string>, timeoutMs?: number): Promise<AxiosResponse<T, any, {}>>;
+    post<T = any>(endpoint: string, body: Record<string, unknown>, options?: RequestOptions): Promise<AxiosResponse<T, any, {}>>;
+    postMultipart<T = any>(endpoint: string, formData: FormData, options?: RequestOptions): Promise<AxiosResponse<T, any, {}>>;
     get<T = any>(endpoint: string, headers?: Record<string, string>): Promise<AxiosResponse<T, any, {}>>;
     delete<T = any>(endpoint: string, headers?: Record<string, string>): Promise<AxiosResponse<T, any, {}>>;
     prepareHeaders(idempotencyKey?: string): Record<string, string>;

package/dist/index.d.ts

@@ -124,6 +124,7 @@ interface ScrapeOptions {
     maxAge?: number;
     minAge?: number;
     storeInCache?: boolean;
+    lockdown?: boolean;
     profile?: {
         name: string;
         saveChanges?: boolean;
@@ -137,7 +138,7 @@ interface ParseFile {
     filename: string;
     contentType?: string;
 }
-type ParseOptions = Omit<ScrapeOptions, 'formats' | 'waitFor' | 'mobile' | 'actions' | 'location' | 'maxAge' | 'minAge' | 'storeInCache' | 'proxy'> & {
+type ParseOptions = Omit<ScrapeOptions, 'formats' | 'waitFor' | 'mobile' | 'actions' | 'location' | 'maxAge' | 'minAge' | 'storeInCache' | 'lockdown' | 'proxy'> & {
     formats?: ParseFormatOption[];
     proxy?: 'basic' | 'auto';
 };
@@ -648,6 +649,10 @@ interface HttpClientOptions {
     maxRetries?: number;
     backoffFactor?: number;
 }
+interface RequestOptions {
+    headers?: Record<string, string>;
+    timeoutMs?: number;
+}
 declare class HttpClient {
     private instance;
     private readonly apiKey;
@@ -659,8 +664,8 @@ declare class HttpClient {
     getApiKey(): string;
     private request;
     private sleep;
-    post<T = any>(endpoint: string, body: Record<string, unknown>, headers?: Record<string, string>): Promise<AxiosResponse<T, any, {}>>;
-    postMultipart<T = any>(endpoint: string, formData: FormData, headers?: Record<string, string>, timeoutMs?: number): Promise<AxiosResponse<T, any, {}>>;
+    post<T = any>(endpoint: string, body: Record<string, unknown>, options?: RequestOptions): Promise<AxiosResponse<T, any, {}>>;
+    postMultipart<T = any>(endpoint: string, formData: FormData, options?: RequestOptions): Promise<AxiosResponse<T, any, {}>>;
     get<T = any>(endpoint: string, headers?: Record<string, string>): Promise<AxiosResponse<T, any, {}>>;
     delete<T = any>(endpoint: string, headers?: Record<string, string>): Promise<AxiosResponse<T, any, {}>>;
     prepareHeaders(idempotencyKey?: string): Record<string, string>;