@mendable/firecrawl-js 4.46.0 → 4.48.0

package/src/v2/methods/parse.ts

@@ -6,7 +6,6 @@ import {
   throwForBadResponse,
   normalizeAxiosError,
 } from "../utils/errorHandler";
-import { getVersion } from "../utils/getVersion";
 
 function toUploadBlob(input: ParseFile["data"], contentType?: string): Blob {
   if (typeof Blob !== "undefined" && input instanceof Blob) {
@@ -57,13 +56,9 @@ export async function parse(
 
   if (options) ensureValidParseOptions(options);
 
-  const version = getVersion();
   const normalizedOptions: ParseOptions = {
     ...(options ?? {}),
-    origin:
-      typeof options?.origin === "string" && options.origin.includes("mcp")
-        ? options.origin
-        : (options?.origin ?? `js-sdk@${version}`),
+    origin: options?.origin || http.getOrigin(),
   };
 
   const formData = new FormData();

package/src/v2/methods/research.ts

@@ -12,7 +12,6 @@ import type {
 import { SdkError } from "../types";
 import { HttpClient } from "../utils/httpClient";
 import { throwForBadResponse } from "../utils/errorHandler";
-import { getVersion } from "../utils/getVersion";
 
 const BASE = "/v2/search/research";
 
@@ -32,10 +31,14 @@ function appendParam(
   }
 }
 
-function withQuery(path: string, params: URLSearchParams): string {
+function withQuery(
+  origin: string,
+  path: string,
+  params: URLSearchParams,
+): string {
   // Research endpoints are GETs, so the POST-body origin injection in
   // HttpClient never applies — attach it as a query param instead.
-  params.append("origin", `js-sdk@${getVersion()}`);
+  params.append("origin", origin);
   const qs = params.toString();
   return qs ? `${path}?${qs}` : path;
 }
@@ -121,7 +124,7 @@ export class ResearchClient {
     appendParam(params, "to", options.to);
     try {
       const res = await this.http.get<SearchPapersResponse>(
-        withQuery(`${BASE}/papers`, params),
+        withQuery(this.http.getOrigin(), `${BASE}/papers`, params),
       );
       if (res.status !== 200) throwForBadResponse(res, "search papers");
       return res.data;
@@ -161,7 +164,13 @@ export class ResearchClient {
     try {
       const res = await this.http.get<
         PaperMetadataResponse | ReadPaperResponse
-      >(withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params));
+      >(
+        withQuery(
+          this.http.getOrigin(),
+          `${BASE}/papers/${encodeURIComponent(id)}`,
+          params,
+        ),
+      );
       if (res.status !== 200) throwForBadResponse(res, "get paper");
       return res.data;
     } catch (err) {
@@ -191,7 +200,11 @@ export class ResearchClient {
     appendParam(params, "anchor", options.anchor);
     try {
       const res = await this.http.get<SimilarPapersResponse>(
-        withQuery(`${BASE}/papers/${encodeURIComponent(id)}/similar`, params),
+        withQuery(
+          this.http.getOrigin(),
+          `${BASE}/papers/${encodeURIComponent(id)}/similar`,
+          params,
+        ),
       );
       if (res.status !== 200) throwForBadResponse(res, "find similar papers");
       return res.data;
@@ -226,7 +239,7 @@ export class ResearchClient {
     appendParam(params, "k", options.k);
     try {
       const res = await this.http.get<GitHubSearchResponse>(
-        withQuery(`${BASE}/github`, params),
+        withQuery(this.http.getOrigin(), `${BASE}/github`, params),
       );
       if (res.status !== 200) throwForBadResponse(res, "search github");
       return res.data;

package/src/v2/methods/search.ts

@@ -9,7 +9,7 @@ import {
 } from "../types";
 import { HttpClient } from "../utils/httpClient";
 import { agentHintMetadata } from "../utils/agentHints";
-import { ensureValidScrapeOptions } from "../utils/validation";
+import { ensureNoV1Options, ensureValidScrapeOptions } from "../utils/validation";
 import {
   throwForBadResponse,
   normalizeAxiosError,
@@ -17,6 +17,7 @@ import {
 
 function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
   if (!req.query || !req.query.trim()) throw new Error("Query cannot be empty");
+  ensureNoV1Options(req, "search");
   if (req.limit != null && req.limit <= 0)
     throw new Error("limit must be positive");
   if (req.timeout != null && req.timeout <= 0)

package/src/v2/types.ts

@@ -915,10 +915,10 @@ export interface FindToolsOptions {
 }
 
 export interface FindToolsData {
-  level: "providers" | "groups" | "tools";
+  level: "categories" | "providers" | "groups" | "tools";
   items: Array<{
     id: string;
-    name: string;
+    name?: string;
     next?: AlexandriaCall;
     execute?: Pick<AlexandriaCall, "provider" | "capability">;
     [key: string]: unknown;

package/src/v2/utils/errorHandler.ts

@@ -2,12 +2,33 @@ import { type AxiosError, type AxiosResponse } from "axios";
 import { SdkError, JobTimeoutError, parseRequiresAction } from "../types";
 import { agentHintMetadata } from "./agentHints";
 
+/** Names the fields a validation error rejected, so a bare "Bad Request" says what to change. */
+function describeValidationIssues(details: unknown): string | undefined {
+  if (!Array.isArray(details)) return undefined;
+  const parts = details.slice(0, 3).flatMap((issue) => {
+    if (!issue || typeof issue !== "object") return [];
+    const { path, message, keys } = issue as { path?: unknown; message?: unknown; keys?: unknown };
+    const what = Array.isArray(keys) && keys.length > 0
+      ? `unrecognized key(s) ${keys.join(", ")}`
+      : typeof message === "string" ? message : undefined;
+    if (!what) return [];
+    return Array.isArray(path) && path.length > 0 ? [`${path.join(".")}: ${what}`] : [what];
+  });
+  return parts.length > 0 ? parts.join("; ") : undefined;
+}
+
+function withIssues(message: unknown, details: unknown): string {
+  const text = typeof message === "string" ? message : JSON.stringify(message);
+  const issues = describeValidationIssues(details);
+  return issues && !text.includes(issues) ? `${text} (${issues})` : text;
+}
+
 export function throwForBadResponse(resp: AxiosResponse, action: string): never {
   const status = resp.status;
   const body = resp.data || {};
   const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
   throw Object.assign(new SdkError(
-    msg,
+    withIssues(msg, body?.details),
     status,
     body?.code,
     body?.details,
@@ -23,7 +44,7 @@ export function normalizeAxiosError(err: AxiosError, action: string): never {
   const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
   const code = (body?.code as string) || err.code;
   throw Object.assign(new SdkError(
-    message,
+    withIssues(message, body?.details),
     status,
     code,
     body?.details ?? body,

package/src/v2/utils/getVersion.ts

@@ -1,14 +1,6 @@
-export function getVersion(): string {
-  try {
-    if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
-      return process.env.npm_package_version as string;
-    }
+import { version } from "../../../package.json";
 
-    // eslint-disable-next-line @typescript-eslint/no-var-requires
-    const pkg = require("../../../package.json");
-    return (pkg?.version as string) || "3.x.x";
-  } catch {
-    return "3.x.x";
-  }
+/** The SDK's own version, inlined at build time so a host app's npm_package_version never replaces it. */
+export function getVersion(): string {
+  return version;
 }
-

package/src/v2/utils/httpClient.ts

@@ -12,6 +12,7 @@ export interface HttpClientOptions {
   timeoutMs?: number;
   maxRetries?: number;
   backoffFactor?: number; // seconds factor for 0.5, 1, 2...
+  origin?: string;
 }
 
 export interface RequestOptions {
@@ -25,12 +26,14 @@ export class HttpClient {
   private readonly apiUrl: string;
   private readonly maxRetries: number;
   private readonly backoffFactor: number;
+  private readonly origin: string;
 
   constructor(options: HttpClientOptions) {
     this.apiKey = options.apiKey;
     this.apiUrl = options.apiUrl.replace(/\/$/, "");
     this.maxRetries = options.maxRetries ?? 3;
     this.backoffFactor = options.backoffFactor ?? 0.5;
+    this.origin = options.origin || `js-sdk@${getVersion()}`;
     this.instance = axios.create({
       baseURL: this.apiUrl,
       timeout: options.timeoutMs ?? 300000,
@@ -52,6 +55,11 @@ export class HttpClient {
     return this.apiKey;
   }
 
+  /** The attribution origin stamped into requests that do not set their own. */
+  getOrigin(): string {
+    return this.origin;
+  }
+
   /** The client-wide default request timeout, for callers that need to
    * extend a specific request beyond it (e.g. scrape auto-resume). */
   getTimeoutMs(): number {
@@ -61,7 +69,6 @@ export class HttpClient {
   private async request<T = any>(
     config: AxiosRequestConfig,
   ): Promise<AxiosResponse<T>> {
-    const version = getVersion();
     config.headers = {
       ...(config.headers || {}),
     };
@@ -89,9 +96,9 @@ export class HttpClient {
           cfg.data = {
             ...data,
             origin:
-              typeof data.origin === "string" && data.origin.includes("mcp")
+              typeof data.origin === "string" && data.origin
                 ? data.origin
-                : `js-sdk@${version}`,
+                : this.origin,
           };
         }
 

package/src/v2/utils/pagination.ts

@@ -39,7 +39,10 @@ export async function fetchAllPages<T = Document>(
       docs.push(d as T);
     }
     if (maxResults != null && docs.length >= maxResults) break;
-    current = (payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null) as string | null;
+    const next = (payload.next ?? (Array.isArray(payload.data) ? null : payload.data?.next) ?? null) as string | null;
+    // A running job's cursor can point back at itself until new documents land; re-requesting it here only spins.
+    if (next === current) break;
+    current = next;
     pageCount += 1;
   }
   return docs;

package/src/v2/utils/validation.ts

@@ -86,6 +86,34 @@ export function ensureValidFormats(formats?: FormatOption[]): void {
   }
 }
 
+const V1_SCRAPE_OPTION_REPLACEMENTS: Record<string, string> = {
+  pageOptions: "use scrapeOptions",
+  includeLinks: "add 'links' to scrapeOptions.formats",
+  parsePDF: "use scrapeOptions.parsers",
+};
+
+const V1_OPTION_REPLACEMENTS: Record<"crawl" | "search", Record<string, string>> = {
+  crawl: {
+    crawlerOptions: "move its fields to the top level",
+    maxDepth: "use maxDiscoveryDepth",
+    allowBackwardLinks: "use crawlEntireDomain",
+    ignoreSitemap: "use sitemap: 'skip'",
+    ...V1_SCRAPE_OPTION_REPLACEMENTS,
+  },
+  search: V1_SCRAPE_OPTION_REPLACEMENTS,
+};
+
+/** Throws on v1 option names, which the v2 payload builders would otherwise drop without a word. */
+export function ensureNoV1Options(options: object | undefined, method: "crawl" | "search"): void {
+  if (!options) return;
+  const replacements = V1_OPTION_REPLACEMENTS[method];
+  const found = Object.keys(replacements).filter((key) => (options as Record<string, unknown>)[key] != null);
+  if (found.length === 0) return;
+  throw new Error(
+    `${method}: ${found.map((key) => `${key} is a v1 option, ${replacements[key]}`).join("; ")}`,
+  );
+}
+
 export function ensureValidScrapeOptions(options?: ScrapeOptions): void {
   if (!options) return;
   if (options.timeout != null && options.timeout <= 0) {

package/dist/chunk-R4XT7PME.js

@@ -1,93 +0,0 @@
-var __getOwnPropNames = Object.getOwnPropertyNames;
-var __commonJS = (cb, mod) => function __require() {
-  try {
-    return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
-  } catch (e) {
-    throw mod = 0, e;
-  }
-};
-
-// package.json
-var require_package = __commonJS({
-  "package.json"(exports, module) {
-    module.exports = {
-      name: "@mendable/firecrawl-js",
-      version: "4.46.0",
-      description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
-      main: "dist/index.js",
-      types: "dist/index.d.ts",
-      exports: {
-        "./package.json": "./package.json",
-        ".": {
-          import: "./dist/index.js",
-          default: "./dist/index.cjs"
-        }
-      },
-      type: "module",
-      scripts: {
-        build: "tsup",
-        "build-and-publish": "pnpm run build && pnpm publish --access public",
-        "publish-beta": "pnpm run build && pnpm publish --access public --tag beta",
-        test: "vitest run --reporter=verbose src/__tests__/e2e/v2",
-        "test:unit": "vitest run --reporter=verbose src/__tests__/unit"
-      },
-      repository: {
-        type: "git",
-        url: "git+https://github.com/firecrawl/firecrawl.git"
-      },
-      author: "Mendable.ai",
-      license: "MIT",
-      dependencies: {
-        axios: "1.20.0",
-        firecrawl: "4.30.1",
-        "typescript-event-target": "^1.1.1",
-        zod: "^3.23.8",
-        "zod-to-json-schema": "^3.23.0"
-      },
-      bugs: {
-        url: "https://github.com/firecrawl/firecrawl/issues"
-      },
-      homepage: "https://github.com/firecrawl/firecrawl#readme",
-      devDependencies: {
-        "@types/dotenv": "^8.2.0",
-        "@types/node": "^20.12.12",
-        "@types/uuid": "^9.0.8",
-        dotenv: "^16.4.5",
-        tsup: "^8.5.0",
-        typescript: "^5.4.5",
-        uuid: "^14.0.0",
-        vitest: "^4.1.11"
-      },
-      keywords: [
-        "firecrawl",
-        "mendable",
-        "crawler",
-        "web",
-        "scraper",
-        "api",
-        "sdk",
-        "web-search",
-        "literature-search",
-        "research",
-        "scientific-papers",
-        "academic-search",
-        "biomedical",
-        "life-sciences",
-        "pubmed",
-        "biorxiv",
-        "medrxiv",
-        "arxiv",
-        "preprints",
-        "citations",
-        "bioinformatics"
-      ],
-      engines: {
-        node: ">=22.0.0"
-      }
-    };
-  }
-});
-
-export {
-  require_package
-};

package/dist/package-EI4Q33HD.js

@@ -1,4 +0,0 @@
-import {
-  require_package
-} from "./chunk-R4XT7PME.js";
-export default require_package();