@mendable/firecrawl-js 4.47.0 → 4.48.0

package/README.md

@@ -37,6 +37,16 @@ const crawlResponse = await app.crawl('https://firecrawl.dev', {
 console.log(crawlResponse);
 ```
 
+### Request origin
+
+Requests carry an `origin` tag that identifies the client. Pass the `origin` option to set it. Set `origin` on a single call to override it for that call, except for `app.research` methods, which always use the client origin.
+
+```js
+const app = new Firecrawl({ apiKey: 'fc-YOUR_API_KEY', origin: 'js-sdk.my-wrapper@1.0.0' });
+```
+
+By default the tag is `js-sdk@<version>`. When the client is created by code inside an installed package (a wrapper library), the default is `js-sdk.<package>@<version>` instead, for example `js-sdk.scope-name@<version>`. Application code, REPLs and bundled files without `node_modules` in their path keep the plain default. Detection runs once when the client is created and sends only the package name, lowercased and sanitized into a slug. Set `FIRECRAWL_DISABLE_CALLER_ORIGIN=1` to always use the plain default.
+
 ### Scraping a URL
 
 To scrape a single URL with error handling, use the `scrape` method. It takes the URL as a parameter and returns the scraped data.

package/dist/{chunk-H75FJ6LW.js → chunk-GD463K4D.js}

@@ -1,6 +1,6 @@
 // package.json
 var name = "@mendable/firecrawl-js";
-var version = "4.47.0";
+var version = "4.48.0";
 var description = "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts";
 var main = "dist/index.js";
 var types = "dist/index.d.ts";

package/dist/index.cjs

@@ -39,7 +39,7 @@ var require_package = __commonJS({
   "package.json"(exports2, module2) {
     module2.exports = {
       name: "@mendable/firecrawl-js",
-      version: "4.47.0",
+      version: "4.48.0",
       description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
       main: "dist/index.js",
       types: "dist/index.d.ts",
@@ -161,11 +161,13 @@ var HttpClient = class {
   apiUrl;
   maxRetries;
   backoffFactor;
+  origin;
   constructor(options) {
     this.apiKey = options.apiKey;
     this.apiUrl = options.apiUrl.replace(/\/$/, "");
     this.maxRetries = options.maxRetries ?? 3;
     this.backoffFactor = options.backoffFactor ?? 0.5;
+    this.origin = options.origin || `js-sdk@${getVersion()}`;
     this.instance = import_axios.default.create({
       baseURL: this.apiUrl,
       timeout: options.timeoutMs ?? 3e5,
@@ -184,13 +186,16 @@ var HttpClient = class {
   getApiKey() {
     return this.apiKey;
   }
+  /** The attribution origin stamped into requests that do not set their own. */
+  getOrigin() {
+    return this.origin;
+  }
   /** The client-wide default request timeout, for callers that need to
    * extend a specific request beyond it (e.g. scrape auto-resume). */
   getTimeoutMs() {
     return this.instance.defaults.timeout ?? 3e5;
   }
   async request(config) {
-    const version2 = getVersion();
     config.headers = {
       ...config.headers || {}
     };
@@ -205,7 +210,7 @@ var HttpClient = class {
           const data = cfg.data ?? {};
           cfg.data = {
             ...data,
-            origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${version2}`
+            origin: typeof data.origin === "string" && data.origin ? data.origin : this.origin
           };
         }
         if (isFormDataBody) {
@@ -788,10 +793,9 @@ async function parse(http, file, options) {
     throw new Error("file data cannot be empty");
   }
   if (options) ensureValidParseOptions(options);
-  const version2 = getVersion();
   const normalizedOptions = {
     ...options ?? {},
-    origin: typeof options?.origin === "string" && options.origin.includes("mcp") ? options.origin : options?.origin ?? `js-sdk@${version2}`
+    origin: options?.origin || http.getOrigin()
   };
   const formData = new FormData();
   formData.append("options", JSON.stringify(normalizedOptions));
@@ -1909,8 +1913,8 @@ function appendParam(params, key, value) {
     params.append(key, String(value));
   }
 }
-function withQuery(path, params) {
-  params.append("origin", `js-sdk@${getVersion()}`);
+function withQuery(origin, path, params) {
+  params.append("origin", origin);
   const qs = params.toString();
   return qs ? `${path}?${qs}` : path;
 }
@@ -1965,7 +1969,7 @@ var ResearchClient = class {
     appendParam(params, "to", options.to);
     try {
       const res = await this.http.get(
-        withQuery(`${BASE}/papers`, params)
+        withQuery(this.http.getOrigin(), `${BASE}/papers`, params)
       );
       if (res.status !== 200) throwForBadResponse(res, "search papers");
       return res.data;
@@ -1983,7 +1987,13 @@ var ResearchClient = class {
     appendParam(params, "query", options.query);
     appendParam(params, "k", options.k);
     try {
-      const res = await this.http.get(withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params));
+      const res = await this.http.get(
+        withQuery(
+          this.http.getOrigin(),
+          `${BASE}/papers/${encodeURIComponent(id)}`,
+          params
+        )
+      );
       if (res.status !== 200) throwForBadResponse(res, "get paper");
       return res.data;
     } catch (err) {
@@ -2009,7 +2019,11 @@ var ResearchClient = class {
     appendParam(params, "anchor", options.anchor);
     try {
       const res = await this.http.get(
-        withQuery(`${BASE}/papers/${encodeURIComponent(id)}/similar`, params)
+        withQuery(
+          this.http.getOrigin(),
+          `${BASE}/papers/${encodeURIComponent(id)}/similar`,
+          params
+        )
       );
       if (res.status !== 200) throwForBadResponse(res, "find similar papers");
       return res.data;
@@ -2040,7 +2054,7 @@ var ResearchClient = class {
     appendParam(params, "k", options.k);
     try {
       const res = await this.http.get(
-        withQuery(`${BASE}/github`, params)
+        withQuery(this.http.getOrigin(), `${BASE}/github`, params)
       );
       if (res.status !== 200) throwForBadResponse(res, "search github");
       return res.data;
@@ -2170,6 +2184,67 @@ async function getMonitorCheck(http, monitorId, checkId, options) {
   }
 }
 
+// src/utils/callerOrigin.ts
+var NODE_MODULES = "/node_modules/";
+function packageOf(file) {
+  let path = file.replace(/^file:\/\//, "").replace(/\\/g, "/");
+  try {
+    path = decodeURIComponent(path);
+  } catch {
+  }
+  const at = path.lastIndexOf(NODE_MODULES);
+  if (at < 0) return null;
+  const [first, second] = path.slice(at + NODE_MODULES.length).split("/");
+  const name = first?.startsWith("@") ? second ? `${first}/${second}` : "" : first;
+  return name && !name.startsWith(".") ? name : null;
+}
+function frameFile(line) {
+  const text = line.trim().replace(/^at /, "");
+  const open = text.indexOf(" (");
+  const location = open >= 0 && text.endsWith(")") ? text.slice(open + 2, -1) : text;
+  const match = /^(.*):\d+:\d+$/.exec(location);
+  return match && !match[1].startsWith("node:") ? match[1] : null;
+}
+function slugify(name) {
+  return name.toLowerCase().replace(/[^a-z0-9]+/g, "-").replace(/^-+|-+$/g, "").slice(0, 48).replace(/-+$/, "");
+}
+function detectCallerPackage(stack) {
+  try {
+    if (process.env.FIRECRAWL_DISABLE_CALLER_ORIGIN === "1") return null;
+    if (stack === void 0) {
+      const limit = Error.stackTraceLimit;
+      Error.stackTraceLimit = 50;
+      try {
+        stack = new Error().stack ?? "";
+      } finally {
+        Error.stackTraceLimit = limit;
+      }
+    }
+    let ownFile = null;
+    let ownPackage = null;
+    for (const line of stack.split("\n")) {
+      if (!/^\s*at /.test(line)) continue;
+      const file = frameFile(line);
+      if (!file) continue;
+      if (ownFile === null) {
+        ownFile = file;
+        ownPackage = packageOf(file);
+      }
+      if (file === ownFile) continue;
+      const name = packageOf(file);
+      if (name && name === ownPackage) continue;
+      const slug = name ? slugify(name) : "";
+      return slug && !slug.includes("playground") ? slug : null;
+    }
+  } catch {
+  }
+  return null;
+}
+function defaultOrigin(version2) {
+  const caller = detectCallerPackage();
+  return caller ? `js-sdk.${caller}@${version2}` : `js-sdk@${version2}`;
+}
+
 // src/v2/watcher.ts
 var import_events = require("events");
 var hasGlobalWebSocket = () => {
@@ -2432,7 +2507,8 @@ var FirecrawlClient = class {
       apiUrl,
       timeoutMs: opts.timeoutMs,
       maxRetries: opts.maxRetries,
-      backoffFactor: opts.backoffFactor
+      backoffFactor: opts.backoffFactor,
+      origin: opts.origin || defaultOrigin(getVersion())
     });
   }
   async scrape(url, options) {
@@ -2958,6 +3034,8 @@ var FirecrawlApp = class {
   apiKey;
   apiUrl;
   version = "1.25.1";
+  clientOrigin;
+  callerPackage;
   isCloudService(url) {
     return url.includes("api.firecrawl.dev");
   }
@@ -2983,15 +3061,22 @@ var FirecrawlApp = class {
    * Initializes a new instance of the FirecrawlApp class.
    * @param config - Configuration options for the FirecrawlApp instance.
    */
-  constructor({ apiKey = null, apiUrl = null }) {
+  constructor({ apiKey = null, apiUrl = null, origin }) {
     const baseUrl = apiUrl || "https://api.firecrawl.dev";
     if (this.isCloudService(baseUrl) && typeof apiKey !== "string") {
       throw new FirecrawlError("No API key provided", 401);
     }
     this.apiKey = apiKey || "";
     this.apiUrl = baseUrl;
+    this.clientOrigin = origin || void 0;
+    this.callerPackage = this.clientOrigin ? null : detectCallerPackage();
     this.init();
   }
+  resolveOrigin(params) {
+    if (typeof params?.origin === "string" && params.origin) return params.origin;
+    if (this.clientOrigin) return this.clientOrigin;
+    return this.callerPackage ? `js-sdk.${this.callerPackage}@${this.version}` : `js-sdk@${this.version}`;
+  }
   /**
    * Scrapes a URL using the Firecrawl API.
    * @param url - The URL to scrape.
@@ -3003,7 +3088,7 @@ var FirecrawlApp = class {
       "Content-Type": "application/json",
       Authorization: `Bearer ${this.apiKey}`
     };
-    let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
+    let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
     if (jsonData?.extract?.schema) {
       jsonData = {
         ...jsonData,
@@ -3067,7 +3152,7 @@ var FirecrawlApp = class {
       lang: params?.lang ?? "en",
       country: params?.country ?? "us",
       location: params?.location,
-      origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}`,
+      origin: this.resolveOrigin(params),
       timeout: params?.timeout ?? 6e4,
       scrapeOptions: params?.scrapeOptions ?? { formats: [] }
     };
@@ -3122,7 +3207,7 @@ var FirecrawlApp = class {
    */
   async crawlUrl(url, params, pollInterval = 2, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
+    let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -3146,7 +3231,7 @@ var FirecrawlApp = class {
   }
   async asyncCrawlUrl(url, params, idempotencyKey) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
+    let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/crawl`,
@@ -3305,7 +3390,7 @@ var FirecrawlApp = class {
    */
   async mapUrl(url, params) {
     const headers = this.prepareHeaders();
-    let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
+    let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/map`,
@@ -3334,7 +3419,7 @@ var FirecrawlApp = class {
    */
   async batchScrapeUrls(urls, params, pollInterval = 2, idempotencyKey, webhook, ignoreInvalidURLs, maxConcurrency) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, maxConcurrency, ...params, origin: this.resolveOrigin(params) };
     if (jsonData?.extract?.schema) {
       jsonData = {
         ...jsonData,
@@ -3376,7 +3461,7 @@ var FirecrawlApp = class {
   }
   async asyncBatchScrapeUrls(urls, params, idempotencyKey, webhook, ignoreInvalidURLs) {
     const headers = this.prepareHeaders(idempotencyKey);
-    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
+    let jsonData = { urls, webhook, ignoreInvalidURLs, ...params, origin: this.resolveOrigin(params) };
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/batch/scrape`,
@@ -3520,7 +3605,7 @@ var FirecrawlApp = class {
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
+        { ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
         headers
       );
       if (response.status === 200) {
@@ -3573,7 +3658,7 @@ var FirecrawlApp = class {
     try {
       const response = await this.postRequest(
         this.apiUrl + `/v1/extract`,
-        { ...jsonData, schema: jsonSchema, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` },
+        { ...jsonData, schema: jsonSchema, origin: this.resolveOrigin(params) },
         headers
       );
       if (response.status === 200) {
@@ -3855,7 +3940,7 @@ var FirecrawlApp = class {
    */
   async asyncDeepResearch(query, params) {
     const headers = this.prepareHeaders();
-    let jsonData = { query, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
+    let jsonData = { query, ...params, origin: this.resolveOrigin(params) };
     if (jsonData?.jsonOptions?.schema) {
       jsonData = {
         ...jsonData,
@@ -3974,7 +4059,7 @@ var FirecrawlApp = class {
   async __asyncDeepResearch(topic, params) {
     const headers = this.prepareHeaders();
     try {
-      let jsonData = { topic, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
+      let jsonData = { topic, ...params, origin: this.resolveOrigin(params) };
       const response = await this.postRequest(
         `${this.apiUrl}/v1/deep-research`,
         jsonData,
@@ -4074,7 +4159,7 @@ var FirecrawlApp = class {
    */
   async asyncGenerateLLMsText(url, params) {
     const headers = this.prepareHeaders();
-    let jsonData = { url, ...params, origin: typeof params?.origin === "string" && params.origin.includes("mcp") ? params.origin : `js-sdk@${this.version}` };
+    let jsonData = { url, ...params, origin: this.resolveOrigin(params) };
     try {
       const response = await this.postRequest(
         `${this.apiUrl}/v1/llmstxt`,
@@ -4342,7 +4427,8 @@ var Firecrawl = class extends FirecrawlClient {
     super(resolved);
     this._v1Opts = {
       apiKey: resolved.apiKey,
-      apiUrl: resolved.apiUrl
+      apiUrl: resolved.apiUrl,
+      origin: this.http.getOrigin()
     };
   }
   /** Access the legacy v1 client (instantiated on first access). */

package/dist/index.d.cts

@@ -2019,6 +2019,7 @@ interface HttpClientOptions {
     timeoutMs?: number;
     maxRetries?: number;
     backoffFactor?: number;
+    origin?: string;
 }
 interface RequestOptions {
     headers?: Record<string, string>;
@@ -2030,9 +2031,12 @@ declare class HttpClient {
     private readonly apiUrl;
     private readonly maxRetries;
     private readonly backoffFactor;
+    private readonly origin;
     constructor(options: HttpClientOptions);
     getApiUrl(): string;
     getApiKey(): string;
+    /** The attribution origin stamped into requests that do not set their own. */
+    getOrigin(): string;
     /** The client-wide default request timeout, for callers that need to
      * extend a specific request beyond it (e.g. scrape auto-resume). */
     getTimeoutMs(): number;
@@ -2265,6 +2269,12 @@ interface FirecrawlClientOptions {
     maxRetries?: number;
     /** Exponential backoff factor for retries (optional). */
     backoffFactor?: number;
+    /**
+     * Attribution tag stamped into request bodies. Defaults to `js-sdk@<version>`,
+     * or `js-sdk.<package>@<version>` when an installed package creates the client
+     * (set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to keep the plain default).
+     */
+    origin?: string;
 }
 /** Accepts a plain API key string or a full options object. */
 type FirecrawlClientInput = FirecrawlClientOptions | string;
@@ -2272,7 +2282,7 @@ type FirecrawlClientInput = FirecrawlClientOptions | string;
  * Firecrawl v2 client. Provides typed access to all v2 endpoints and utilities.
  */
 declare class FirecrawlClient {
-    private readonly http;
+    protected readonly http: HttpClient;
     private _research?;
     private isCloudService;
     /**
@@ -2662,10 +2672,12 @@ declare class FirecrawlClient {
  * Configuration interface for FirecrawlApp.
  * @param apiKey - Optional API key for authentication.
  * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
+ * @param origin - Optional attribution tag stamped into request bodies.
  */
 interface FirecrawlAppConfig {
     apiKey?: string | null;
     apiUrl?: string | null;
+    origin?: string;
 }
 /**
  * Metadata for a Firecrawl document.
@@ -3262,6 +3274,8 @@ declare class FirecrawlApp {
     apiKey: string;
     apiUrl: string;
     version: string;
+    private readonly clientOrigin?;
+    private readonly callerPackage;
     private isCloudService;
     private getVersion;
     private init;
@@ -3269,7 +3283,8 @@ declare class FirecrawlApp {
      * Initializes a new instance of the FirecrawlApp class.
      * @param config - Configuration options for the FirecrawlApp instance.
      */
-    constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
+    constructor({ apiKey, apiUrl, origin }: FirecrawlAppConfig);
+    private resolveOrigin;
     /**
      * Scrapes a URL using the Firecrawl API.
      * @param url - The URL to scrape.

package/dist/index.d.ts

@@ -2019,6 +2019,7 @@ interface HttpClientOptions {
     timeoutMs?: number;
     maxRetries?: number;
     backoffFactor?: number;
+    origin?: string;
 }
 interface RequestOptions {
     headers?: Record<string, string>;
@@ -2030,9 +2031,12 @@ declare class HttpClient {
     private readonly apiUrl;
     private readonly maxRetries;
     private readonly backoffFactor;
+    private readonly origin;
     constructor(options: HttpClientOptions);
     getApiUrl(): string;
     getApiKey(): string;
+    /** The attribution origin stamped into requests that do not set their own. */
+    getOrigin(): string;
     /** The client-wide default request timeout, for callers that need to
      * extend a specific request beyond it (e.g. scrape auto-resume). */
     getTimeoutMs(): number;
@@ -2265,6 +2269,12 @@ interface FirecrawlClientOptions {
     maxRetries?: number;
     /** Exponential backoff factor for retries (optional). */
     backoffFactor?: number;
+    /**
+     * Attribution tag stamped into request bodies. Defaults to `js-sdk@<version>`,
+     * or `js-sdk.<package>@<version>` when an installed package creates the client
+     * (set FIRECRAWL_DISABLE_CALLER_ORIGIN=1 to keep the plain default).
+     */
+    origin?: string;
 }
 /** Accepts a plain API key string or a full options object. */
 type FirecrawlClientInput = FirecrawlClientOptions | string;
@@ -2272,7 +2282,7 @@ type FirecrawlClientInput = FirecrawlClientOptions | string;
  * Firecrawl v2 client. Provides typed access to all v2 endpoints and utilities.
  */
 declare class FirecrawlClient {
-    private readonly http;
+    protected readonly http: HttpClient;
     private _research?;
     private isCloudService;
     /**
@@ -2662,10 +2672,12 @@ declare class FirecrawlClient {
  * Configuration interface for FirecrawlApp.
  * @param apiKey - Optional API key for authentication.
  * @param apiUrl - Optional base URL of the API; defaults to 'https://api.firecrawl.dev'.
+ * @param origin - Optional attribution tag stamped into request bodies.
  */
 interface FirecrawlAppConfig {
     apiKey?: string | null;
     apiUrl?: string | null;
+    origin?: string;
 }
 /**
  * Metadata for a Firecrawl document.
@@ -3262,6 +3274,8 @@ declare class FirecrawlApp {
     apiKey: string;
     apiUrl: string;
     version: string;
+    private readonly clientOrigin?;
+    private readonly callerPackage;
     private isCloudService;
     private getVersion;
     private init;
@@ -3269,7 +3283,8 @@ declare class FirecrawlApp {
      * Initializes a new instance of the FirecrawlApp class.
      * @param config - Configuration options for the FirecrawlApp instance.
      */
-    constructor({ apiKey, apiUrl }: FirecrawlAppConfig);
+    constructor({ apiKey, apiUrl, origin }: FirecrawlAppConfig);
+    private resolveOrigin;
     /**
      * Scrapes a URL using the Firecrawl API.
      * @param url - The URL to scrape.