firecrawl 4.31.1 → 4.32.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -166,6 +166,73 @@ const mapResult = await app.map('https://example.com');
166
166
  console.log(mapResult);
167
167
  ```
168
168
 
169
+ ### Search
170
+
171
+ Use `search` to search the web and optionally scrape the results in the same call.
172
+
173
+ ```js
174
+ const results = await app.search('what is retrieval augmented generation?', { limit: 5 });
175
+ for (const result of results.web ?? []) {
176
+ console.log(result.url, '-', result.title);
177
+ }
178
+
179
+ // Scrape every result as part of the search:
180
+ const scraped = await app.search('firecrawl changelog', {
181
+ limit: 3,
182
+ scrapeOptions: { formats: ['markdown'] },
183
+ });
184
+ ```
185
+
186
+ Results are grouped by source: `.web`, `.news`, `.images` and `.developer`.
187
+
188
+ Use `categories` to narrow web search to a kind of site:
189
+
190
+ ```js
191
+ const results = await app.search('nanopore basecalling accuracy', {
192
+ categories: ['research'],
193
+ });
194
+ ```
195
+
196
+ > **`categories: ['research']` is a website filter, not the paper index.** It
197
+ > restricts ordinary web search to roughly 14 academic domains (arxiv.org,
198
+ > pubmed.ncbi.nlm.nih.gov, nature.com, biorxiv.org, ...) and returns web page
199
+ > results. To search papers themselves, use `research.searchPapers` below.
200
+
201
+ ### Research / paper search
202
+
203
+ Use `app.research` to search Firecrawl's research paper index: ~43M paper
204
+ abstracts, roughly 90% biomedical and life sciences (PubMed, bioRxiv, medRxiv),
205
+ plus arXiv for physics, mathematics and computer science.
206
+
207
+ ```js
208
+ // Search the paper index (semantic search over abstracts):
209
+ const papers = await app.research.searchPapers(
210
+ 'CRISPR base editing off-target effects in primary human T cells',
211
+ { k: 10 },
212
+ );
213
+ for (const paper of papers.results) {
214
+ console.log(paper.primaryId, '-', paper.title);
215
+ }
216
+
217
+ // Inspect one paper's metadata (accepts pmid:, pmcid:, doi: or arxiv: ids):
218
+ const paper = await app.research.getPaper('pmid:<id>');
219
+
220
+ // Read the passages inside a paper that answer a specific question:
221
+ const read = await app.research.getPaper('pmid:<id>', {
222
+ query: 'what was the primary endpoint and the reported hazard ratio?',
223
+ k: 4,
224
+ });
225
+
226
+ // Expand along the citation graph, re-ranked for your stated intent:
227
+ const related = await app.research.similarPapers('pmid:<id>', {
228
+ intent: 'replication attempts in larger cohorts',
229
+ k: 20,
230
+ });
231
+ ```
232
+
233
+ A companion `app.research.searchGithub` searches indexed GitHub issue/PR history
234
+ and repository readmes.
235
+
169
236
  ### Scrape-bound interactive browsing (v2)
170
237
 
171
238
  Use a scrape job ID to keep interacting with the replayed browser context:
@@ -12,8 +12,8 @@ var require_package = __commonJS({
12
12
  "package.json"(exports, module) {
13
13
  module.exports = {
14
14
  name: "@mendable/firecrawl-js",
15
- version: "4.31.1",
16
- description: "JavaScript SDK for Firecrawl API",
15
+ version: "4.32.1",
16
+ description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
17
17
  main: "dist/index.js",
18
18
  types: "dist/index.d.ts",
19
19
  exports: {
@@ -69,7 +69,21 @@ var require_package = __commonJS({
69
69
  "web",
70
70
  "scraper",
71
71
  "api",
72
- "sdk"
72
+ "sdk",
73
+ "web-search",
74
+ "literature-search",
75
+ "research",
76
+ "scientific-papers",
77
+ "academic-search",
78
+ "biomedical",
79
+ "life-sciences",
80
+ "pubmed",
81
+ "biorxiv",
82
+ "medrxiv",
83
+ "arxiv",
84
+ "preprints",
85
+ "citations",
86
+ "bioinformatics"
73
87
  ],
74
88
  engines: {
75
89
  node: ">=22.0.0"
package/dist/index.cjs CHANGED
@@ -39,8 +39,8 @@ var require_package = __commonJS({
39
39
  "package.json"(exports2, module2) {
40
40
  module2.exports = {
41
41
  name: "@mendable/firecrawl-js",
42
- version: "4.31.1",
43
- description: "JavaScript SDK for Firecrawl API",
42
+ version: "4.32.1",
43
+ description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
44
44
  main: "dist/index.js",
45
45
  types: "dist/index.d.ts",
46
46
  exports: {
@@ -96,7 +96,21 @@ var require_package = __commonJS({
96
96
  "web",
97
97
  "scraper",
98
98
  "api",
99
- "sdk"
99
+ "sdk",
100
+ "web-search",
101
+ "literature-search",
102
+ "research",
103
+ "scientific-papers",
104
+ "academic-search",
105
+ "biomedical",
106
+ "life-sciences",
107
+ "pubmed",
108
+ "biorxiv",
109
+ "medrxiv",
110
+ "arxiv",
111
+ "preprints",
112
+ "citations",
113
+ "bioinformatics"
100
114
  ],
101
115
  engines: {
102
116
  node: ">=22.0.0"
@@ -746,12 +760,16 @@ async function search(http, request) {
746
760
  if (data.news) out.news = transformArray(data.news);
747
761
  if (data.images)
748
762
  out.images = transformArray(data.images);
763
+ if (data.developer)
764
+ out.developer = transformArray(data.developer);
749
765
  Object.defineProperty(out, "data", {
750
766
  get() {
751
767
  const parts = [];
752
768
  if (out.web?.length) parts.push(`.web (${out.web.length} results)`);
753
769
  if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
754
770
  if (out.images?.length) parts.push(`.images (${out.images.length} results)`);
771
+ if (out.developer?.length)
772
+ parts.push(`.developer (${out.developer.length} results)`);
755
773
  const available = parts.length ? parts.join(", ") : ".web, .news, or .images";
756
774
  throw new Error(
757
775
  `SearchData has no '.data'. Results are grouped by source: ${available}`
@@ -1515,9 +1533,20 @@ var ResearchClient = class {
1515
1533
  }
1516
1534
  http;
1517
1535
  /**
1518
- * Search papers by abstract relevance.
1519
- * @param query Natural-language search query.
1520
- * @param options Optional filters (k, authors, categories, from, to).
1536
+ * Search the research paper index by abstract relevance.
1537
+ *
1538
+ * Queries ~43M paper abstracts: PubMed, bioRxiv and medRxiv (about 90% of the
1539
+ * corpus — biomedical and life sciences) plus arXiv (physics, mathematics,
1540
+ * computer science). Semantic search over abstracts, not keyword matching.
1541
+ *
1542
+ * This is **not** `search({ categories: ["research"] })`, which only narrows
1543
+ * ordinary web search to ~14 academic websites.
1544
+ *
1545
+ * @param query Natural-language search query, e.g. `"CRISPR base editing
1546
+ * off-target effects in primary human T cells"`.
1547
+ * @param options Optional filters (k, authors, categories, from, to). Note
1548
+ * `categories` here filters *paper* subject categories (e.g. `"q-bio.GN"`)
1549
+ * and is unrelated to the `categories` option of `search()`.
1521
1550
  */
1522
1551
  async searchPapers(query, options = {}) {
1523
1552
  if (!query || !query.trim()) throw new Error("query cannot be empty");
@@ -2067,8 +2096,19 @@ var FirecrawlClient = class {
2067
2096
  }
2068
2097
  // Research
2069
2098
  /**
2070
- * Access the v2 research endpoints (arXiv papers + GitHub history/readmes).
2071
- * Example: `firecrawl.research.searchPapers("diffusion models")`.
2099
+ * Access the v2 research endpoints — Firecrawl's **research paper index**
2100
+ * (~43M paper abstracts) plus GitHub history/readmes.
2101
+ *
2102
+ * The paper corpus is roughly 90% biomedical and life sciences — PubMed,
2103
+ * bioRxiv and medRxiv — with arXiv covering physics, mathematics and
2104
+ * computer science.
2105
+ *
2106
+ * ⚠️ Not the same as `search({ categories: ["research"] })`, which is only a
2107
+ * website/domain filter over ordinary web search (~14 academic domains,
2108
+ * returning page snippets). Use `research.searchPapers()` for literature
2109
+ * search.
2110
+ *
2111
+ * Example: `firecrawl.research.searchPapers("GLP-1 receptor agonists cardiovascular outcomes")`.
2072
2112
  */
2073
2113
  get research() {
2074
2114
  if (!this._research) this._research = new ResearchClient(this.http);
@@ -3259,7 +3299,7 @@ var FirecrawlApp = class {
3259
3299
  * @param onActivity - Optional callback to receive activity updates in real-time.
3260
3300
  * @param onSource - Optional callback to receive source updates in real-time.
3261
3301
  * @returns The final research results.
3262
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
3302
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
3263
3303
  */
3264
3304
  async deepResearch(query, params, onActivity, onSource) {
3265
3305
  try {
@@ -3316,7 +3356,7 @@ var FirecrawlApp = class {
3316
3356
  * Initiates a deep research operation on a given query without polling.
3317
3357
  * @param params - Parameters for the deep research operation.
3318
3358
  * @returns The response containing the research job ID.
3319
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
3359
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
3320
3360
  */
3321
3361
  async asyncDeepResearch(query, params) {
3322
3362
  const headers = this.prepareHeaders();
@@ -3354,7 +3394,7 @@ var FirecrawlApp = class {
3354
3394
  * Checks the status of a deep research operation.
3355
3395
  * @param id - The ID of the deep research operation.
3356
3396
  * @returns The current status and results of the research operation.
3357
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
3397
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
3358
3398
  */
3359
3399
  async checkDeepResearchStatus(id) {
3360
3400
  const headers = this.prepareHeaders();
package/dist/index.d.cts CHANGED
@@ -139,6 +139,10 @@ interface ScrapeOptions {
139
139
  useMock?: string;
140
140
  blockAds?: boolean;
141
141
  proxy?: "basic" | "stealth" | "enhanced" | "auto" | string;
142
+ /**
143
+ * Maximum age, in milliseconds, of indexed content that may be reused.
144
+ * Set to `0` to bypass index reuse.
145
+ */
142
146
  maxAge?: number;
143
147
  minAge?: number;
144
148
  storeInCache?: boolean;
@@ -423,6 +427,7 @@ interface MenuProfile {
423
427
  interface DocumentMetadata {
424
428
  title?: string;
425
429
  description?: string;
430
+ /** URL reported by the selected scrape engine. */
426
431
  url?: string;
427
432
  language?: string;
428
433
  keywords?: string | string[];
@@ -452,7 +457,9 @@ interface DocumentMetadata {
452
457
  publishedTime?: string;
453
458
  articleTag?: string;
454
459
  articleSection?: string;
460
+ /** URL requested for the scrape. */
455
461
  sourceURL?: string;
462
+ /** HTTP status code reported for the scrape response. */
456
463
  statusCode?: number;
457
464
  scrapeId?: string;
458
465
  numPages?: number;
@@ -508,6 +515,7 @@ interface SearchResultWeb {
508
515
  url: string;
509
516
  title?: string;
510
517
  description?: string;
518
+ position?: number;
511
519
  category?: string;
512
520
  }
513
521
  interface SearchResultNews {
@@ -531,16 +539,47 @@ interface SearchData {
531
539
  web?: Array<SearchResultWeb | Document>;
532
540
  news?: Array<SearchResultNews | Document>;
533
541
  images?: Array<SearchResultImages | Document>;
542
+ developer?: Array<SearchResultWeb | Document>;
534
543
  }
544
+ /**
545
+ * Narrows ordinary **web search**. A category does not switch `search()` to a
546
+ * different index.
547
+ *
548
+ * - `github` — restrict web results to github.com (a `site:` filter).
549
+ * - `research` — restrict web results to a fixed list of ~14 academic
550
+ * *websites* (arxiv.org, pubmed.ncbi.nlm.nih.gov, nature.com, science.org,
551
+ * ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). It returns
552
+ * ordinary web page results from those domains, **not** paper records.
553
+ * - `pdf` — restrict results to PDFs (adds `filetype:pdf`).
554
+ * - `developer` — add developer results (issues, pull requests, READMEs and
555
+ * documentation) under `.developer`.
556
+ *
557
+ * ⚠️ `categories: ["research"]` is **not** Firecrawl's research paper index.
558
+ * To search papers themselves — ~43M abstracts, roughly 90% biomedical
559
+ * (PubMed, bioRxiv, medRxiv) plus arXiv — with full abstracts, in-body passage
560
+ * reads and citation-graph expansion, use `firecrawl.research.searchPapers()`
561
+ * (plus `getPaper()` and `similarPapers()`), which call `/v2/search/research`.
562
+ *
563
+ * Rule of thumb: literature search → `research.searchPapers()`; web pages that
564
+ * happen to live on academic domains → `search({ categories: ["research"] })`.
565
+ */
535
566
  interface CategoryOption {
536
- type: "github" | "research" | "pdf";
567
+ type: "github" | "research" | "pdf" | "developer";
537
568
  }
538
569
  interface SearchRequest {
539
570
  query: string;
540
571
  sources?: Array<"web" | "news" | "images" | {
541
572
  type: "web" | "news" | "images";
542
573
  }>;
543
- categories?: Array<"github" | "research" | "pdf" | CategoryOption>;
574
+ /**
575
+ * Narrow web search by category. See {@link CategoryOption}.
576
+ *
577
+ * ⚠️ `"research"` is a website/domain filter over ordinary web search (~14
578
+ * academic domains), **not** the research paper index. For literature search
579
+ * over ~43M paper abstracts (PubMed / bioRxiv / medRxiv / arXiv) use
580
+ * `firecrawl.research.searchPapers()` instead.
581
+ */
582
+ categories?: Array<"github" | "research" | "pdf" | "developer" | CategoryOption>;
544
583
  includeDomains?: string[];
545
584
  excludeDomains?: string[];
546
585
  limit?: number;
@@ -1086,8 +1125,20 @@ interface BrowserListResponse {
1086
1125
  error?: string;
1087
1126
  }
1088
1127
  /**
1089
- * Source identifiers grouped by namespace. Currently only `arxiv` is
1090
- * populated; each value is an array of ids in that namespace.
1128
+ * Source identifiers grouped by namespace. Each key is a namespace and each
1129
+ * value is an array of ids in that namespace.
1130
+ *
1131
+ * Which namespaces appear depends on the paper's source. Biomedical records
1132
+ * (PubMed, bioRxiv, medRxiv) typically carry `pmid`, `pmcid` and/or `doi`;
1133
+ * arXiv records carry `arxiv`. The set is open — treat this as a lookup by
1134
+ * namespace rather than a fixed schema, and prefer {@link PaperResult.primaryId}
1135
+ * when you just need one citable identifier.
1136
+ *
1137
+ * @example
1138
+ * ```ts
1139
+ * const doi = paper.ids?.doi?.[0];
1140
+ * const pmid = paper.ids?.pmid?.[0];
1141
+ * ```
1091
1142
  */
1092
1143
  type IdMap = Record<string, string[]>;
1093
1144
  /** Per-candidate ranking signals (present on similarity results). */
@@ -1101,7 +1152,7 @@ interface PaperSignals {
1101
1152
  /** Number of distinct seeds connected to this candidate. */
1102
1153
  seedOverlap: number;
1103
1154
  }
1104
- /** A ranked paper. `paperId` is canonical; arXiv lives in `ids`. */
1155
+ /** A ranked paper. `paperId` is canonical; source ids (pmid, pmcid, doi, arxiv, ...) live in `ids`. */
1105
1156
  interface PaperResult {
1106
1157
  /** Canonical paper id — the Milvus INT64 primary key as a decimal string. */
1107
1158
  paperId: string;
@@ -1173,7 +1224,7 @@ interface GitHubScoreBreakdown {
1173
1224
  rerank?: number;
1174
1225
  }
1175
1226
  interface GitHubSearchItem {
1176
- resultType: "github_history" | "repo_readme" | "web";
1227
+ resultType?: "github_history" | "repo_readme" | "web";
1177
1228
  /** `owner/name`; empty for web results whose URL is not a repo page. */
1178
1229
  repo: string;
1179
1230
  url: string;
@@ -1324,23 +1375,55 @@ declare function listBrowsers(http: HttpClient, args?: {
1324
1375
  }): Promise<BrowserListResponse>;
1325
1376
 
1326
1377
  /**
1327
- * Client for the v2 research endpoints (arXiv papers + GitHub history/readmes).
1378
+ * Client for the v2 research endpoints — Firecrawl's **research paper index**
1379
+ * (~43M paper abstracts) plus GitHub history/readmes.
1380
+ *
1381
+ * The paper corpus is roughly 90% biomedical and life sciences — PubMed,
1382
+ * bioRxiv and medRxiv — with arXiv covering physics, mathematics and computer
1383
+ * science. Supports abstract search, paper metadata, in-body passage reads and
1384
+ * citation-graph expansion.
1385
+ *
1386
+ * ⚠️ This is **not** the same as `search({ categories: ["research"] })`. That
1387
+ * option is a website/domain filter on ordinary web search: it narrows
1388
+ * Google-style results to ~14 academic domains and returns page snippets. The
1389
+ * methods here query the paper index itself and return ranked paper records.
1390
+ *
1328
1391
  * Accessed via `firecrawl.research`.
1392
+ *
1393
+ * @example
1394
+ * ```ts
1395
+ * const res = await firecrawl.research.searchPapers(
1396
+ * "tau aggregation inhibitors in Alzheimer's disease",
1397
+ * { k: 10 },
1398
+ * );
1399
+ * ```
1329
1400
  */
1330
1401
  declare class ResearchClient {
1331
1402
  private readonly http;
1332
1403
  constructor(http: HttpClient);
1333
1404
  /**
1334
- * Search papers by abstract relevance.
1335
- * @param query Natural-language search query.
1336
- * @param options Optional filters (k, authors, categories, from, to).
1405
+ * Search the research paper index by abstract relevance.
1406
+ *
1407
+ * Queries ~43M paper abstracts: PubMed, bioRxiv and medRxiv (about 90% of the
1408
+ * corpus — biomedical and life sciences) plus arXiv (physics, mathematics,
1409
+ * computer science). Semantic search over abstracts, not keyword matching.
1410
+ *
1411
+ * This is **not** `search({ categories: ["research"] })`, which only narrows
1412
+ * ordinary web search to ~14 academic websites.
1413
+ *
1414
+ * @param query Natural-language search query, e.g. `"CRISPR base editing
1415
+ * off-target effects in primary human T cells"`.
1416
+ * @param options Optional filters (k, authors, categories, from, to). Note
1417
+ * `categories` here filters *paper* subject categories (e.g. `"q-bio.GN"`)
1418
+ * and is unrelated to the `categories` option of `search()`.
1337
1419
  */
1338
1420
  searchPapers(query: string, options?: SearchPapersOptions): Promise<SearchPapersResponse>;
1339
1421
  /**
1340
1422
  * Get paper metadata (detail mode), or read in-body passages (when `query` is
1341
1423
  * supplied). `k` is only valid together with `query`.
1342
- * @param id Paper reference: a canonical `paper_id`, an `arxiv:<id>` key, or a
1343
- * bare arXiv id / URL.
1424
+ * @param id Paper reference: a canonical `paperId`, or a namespaced id key
1425
+ * such as `pmid:<id>`, `pmcid:<id>`, `doi:<doi>` or `arxiv:<id>`. Bare
1426
+ * arXiv ids and arXiv URLs are also accepted.
1344
1427
  * @param options Optional `query` (switches to read mode) and `k`.
1345
1428
  */
1346
1429
  getPaper(id: string, options?: {
@@ -1495,8 +1578,19 @@ declare class FirecrawlClient {
1495
1578
  */
1496
1579
  searchFeedback(jobId: string, request: SearchFeedbackRequest): Promise<FeedbackResponse>;
1497
1580
  /**
1498
- * Access the v2 research endpoints (arXiv papers + GitHub history/readmes).
1499
- * Example: `firecrawl.research.searchPapers("diffusion models")`.
1581
+ * Access the v2 research endpoints — Firecrawl's **research paper index**
1582
+ * (~43M paper abstracts) plus GitHub history/readmes.
1583
+ *
1584
+ * The paper corpus is roughly 90% biomedical and life sciences — PubMed,
1585
+ * bioRxiv and medRxiv — with arXiv covering physics, mathematics and
1586
+ * computer science.
1587
+ *
1588
+ * ⚠️ Not the same as `search({ categories: ["research"] })`, which is only a
1589
+ * website/domain filter over ordinary web search (~14 academic domains,
1590
+ * returning page snippets). Use `research.searchPapers()` for literature
1591
+ * search.
1592
+ *
1593
+ * Example: `firecrawl.research.searchPapers("GLP-1 receptor agonists cardiovascular outcomes")`.
1500
1594
  */
1501
1595
  get research(): ResearchClient;
1502
1596
  /**
@@ -2525,7 +2619,7 @@ declare class FirecrawlApp {
2525
2619
  * @param onActivity - Optional callback to receive activity updates in real-time.
2526
2620
  * @param onSource - Optional callback to receive source updates in real-time.
2527
2621
  * @returns The final research results.
2528
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
2622
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
2529
2623
  */
2530
2624
  deepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>, onActivity?: (activity: {
2531
2625
  type: string;
@@ -2543,14 +2637,14 @@ declare class FirecrawlApp {
2543
2637
  * Initiates a deep research operation on a given query without polling.
2544
2638
  * @param params - Parameters for the deep research operation.
2545
2639
  * @returns The response containing the research job ID.
2546
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
2640
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
2547
2641
  */
2548
2642
  asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse>;
2549
2643
  /**
2550
2644
  * Checks the status of a deep research operation.
2551
2645
  * @param id - The ID of the deep research operation.
2552
2646
  * @returns The current status and results of the research operation.
2553
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
2647
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
2554
2648
  */
2555
2649
  checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse>;
2556
2650
  /**
package/dist/index.d.ts CHANGED
@@ -139,6 +139,10 @@ interface ScrapeOptions {
139
139
  useMock?: string;
140
140
  blockAds?: boolean;
141
141
  proxy?: "basic" | "stealth" | "enhanced" | "auto" | string;
142
+ /**
143
+ * Maximum age, in milliseconds, of indexed content that may be reused.
144
+ * Set to `0` to bypass index reuse.
145
+ */
142
146
  maxAge?: number;
143
147
  minAge?: number;
144
148
  storeInCache?: boolean;
@@ -423,6 +427,7 @@ interface MenuProfile {
423
427
  interface DocumentMetadata {
424
428
  title?: string;
425
429
  description?: string;
430
+ /** URL reported by the selected scrape engine. */
426
431
  url?: string;
427
432
  language?: string;
428
433
  keywords?: string | string[];
@@ -452,7 +457,9 @@ interface DocumentMetadata {
452
457
  publishedTime?: string;
453
458
  articleTag?: string;
454
459
  articleSection?: string;
460
+ /** URL requested for the scrape. */
455
461
  sourceURL?: string;
462
+ /** HTTP status code reported for the scrape response. */
456
463
  statusCode?: number;
457
464
  scrapeId?: string;
458
465
  numPages?: number;
@@ -508,6 +515,7 @@ interface SearchResultWeb {
508
515
  url: string;
509
516
  title?: string;
510
517
  description?: string;
518
+ position?: number;
511
519
  category?: string;
512
520
  }
513
521
  interface SearchResultNews {
@@ -531,16 +539,47 @@ interface SearchData {
531
539
  web?: Array<SearchResultWeb | Document>;
532
540
  news?: Array<SearchResultNews | Document>;
533
541
  images?: Array<SearchResultImages | Document>;
542
+ developer?: Array<SearchResultWeb | Document>;
534
543
  }
544
+ /**
545
+ * Narrows ordinary **web search**. A category does not switch `search()` to a
546
+ * different index.
547
+ *
548
+ * - `github` — restrict web results to github.com (a `site:` filter).
549
+ * - `research` — restrict web results to a fixed list of ~14 academic
550
+ * *websites* (arxiv.org, pubmed.ncbi.nlm.nih.gov, nature.com, science.org,
551
+ * ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). It returns
552
+ * ordinary web page results from those domains, **not** paper records.
553
+ * - `pdf` — restrict results to PDFs (adds `filetype:pdf`).
554
+ * - `developer` — add developer results (issues, pull requests, READMEs and
555
+ * documentation) under `.developer`.
556
+ *
557
+ * ⚠️ `categories: ["research"]` is **not** Firecrawl's research paper index.
558
+ * To search papers themselves — ~43M abstracts, roughly 90% biomedical
559
+ * (PubMed, bioRxiv, medRxiv) plus arXiv — with full abstracts, in-body passage
560
+ * reads and citation-graph expansion, use `firecrawl.research.searchPapers()`
561
+ * (plus `getPaper()` and `similarPapers()`), which call `/v2/search/research`.
562
+ *
563
+ * Rule of thumb: literature search → `research.searchPapers()`; web pages that
564
+ * happen to live on academic domains → `search({ categories: ["research"] })`.
565
+ */
535
566
  interface CategoryOption {
536
- type: "github" | "research" | "pdf";
567
+ type: "github" | "research" | "pdf" | "developer";
537
568
  }
538
569
  interface SearchRequest {
539
570
  query: string;
540
571
  sources?: Array<"web" | "news" | "images" | {
541
572
  type: "web" | "news" | "images";
542
573
  }>;
543
- categories?: Array<"github" | "research" | "pdf" | CategoryOption>;
574
+ /**
575
+ * Narrow web search by category. See {@link CategoryOption}.
576
+ *
577
+ * ⚠️ `"research"` is a website/domain filter over ordinary web search (~14
578
+ * academic domains), **not** the research paper index. For literature search
579
+ * over ~43M paper abstracts (PubMed / bioRxiv / medRxiv / arXiv) use
580
+ * `firecrawl.research.searchPapers()` instead.
581
+ */
582
+ categories?: Array<"github" | "research" | "pdf" | "developer" | CategoryOption>;
544
583
  includeDomains?: string[];
545
584
  excludeDomains?: string[];
546
585
  limit?: number;
@@ -1086,8 +1125,20 @@ interface BrowserListResponse {
1086
1125
  error?: string;
1087
1126
  }
1088
1127
  /**
1089
- * Source identifiers grouped by namespace. Currently only `arxiv` is
1090
- * populated; each value is an array of ids in that namespace.
1128
+ * Source identifiers grouped by namespace. Each key is a namespace and each
1129
+ * value is an array of ids in that namespace.
1130
+ *
1131
+ * Which namespaces appear depends on the paper's source. Biomedical records
1132
+ * (PubMed, bioRxiv, medRxiv) typically carry `pmid`, `pmcid` and/or `doi`;
1133
+ * arXiv records carry `arxiv`. The set is open — treat this as a lookup by
1134
+ * namespace rather than a fixed schema, and prefer {@link PaperResult.primaryId}
1135
+ * when you just need one citable identifier.
1136
+ *
1137
+ * @example
1138
+ * ```ts
1139
+ * const doi = paper.ids?.doi?.[0];
1140
+ * const pmid = paper.ids?.pmid?.[0];
1141
+ * ```
1091
1142
  */
1092
1143
  type IdMap = Record<string, string[]>;
1093
1144
  /** Per-candidate ranking signals (present on similarity results). */
@@ -1101,7 +1152,7 @@ interface PaperSignals {
1101
1152
  /** Number of distinct seeds connected to this candidate. */
1102
1153
  seedOverlap: number;
1103
1154
  }
1104
- /** A ranked paper. `paperId` is canonical; arXiv lives in `ids`. */
1155
+ /** A ranked paper. `paperId` is canonical; source ids (pmid, pmcid, doi, arxiv, ...) live in `ids`. */
1105
1156
  interface PaperResult {
1106
1157
  /** Canonical paper id — the Milvus INT64 primary key as a decimal string. */
1107
1158
  paperId: string;
@@ -1173,7 +1224,7 @@ interface GitHubScoreBreakdown {
1173
1224
  rerank?: number;
1174
1225
  }
1175
1226
  interface GitHubSearchItem {
1176
- resultType: "github_history" | "repo_readme" | "web";
1227
+ resultType?: "github_history" | "repo_readme" | "web";
1177
1228
  /** `owner/name`; empty for web results whose URL is not a repo page. */
1178
1229
  repo: string;
1179
1230
  url: string;
@@ -1324,23 +1375,55 @@ declare function listBrowsers(http: HttpClient, args?: {
1324
1375
  }): Promise<BrowserListResponse>;
1325
1376
 
1326
1377
  /**
1327
- * Client for the v2 research endpoints (arXiv papers + GitHub history/readmes).
1378
+ * Client for the v2 research endpoints — Firecrawl's **research paper index**
1379
+ * (~43M paper abstracts) plus GitHub history/readmes.
1380
+ *
1381
+ * The paper corpus is roughly 90% biomedical and life sciences — PubMed,
1382
+ * bioRxiv and medRxiv — with arXiv covering physics, mathematics and computer
1383
+ * science. Supports abstract search, paper metadata, in-body passage reads and
1384
+ * citation-graph expansion.
1385
+ *
1386
+ * ⚠️ This is **not** the same as `search({ categories: ["research"] })`. That
1387
+ * option is a website/domain filter on ordinary web search: it narrows
1388
+ * Google-style results to ~14 academic domains and returns page snippets. The
1389
+ * methods here query the paper index itself and return ranked paper records.
1390
+ *
1328
1391
  * Accessed via `firecrawl.research`.
1392
+ *
1393
+ * @example
1394
+ * ```ts
1395
+ * const res = await firecrawl.research.searchPapers(
1396
+ * "tau aggregation inhibitors in Alzheimer's disease",
1397
+ * { k: 10 },
1398
+ * );
1399
+ * ```
1329
1400
  */
1330
1401
  declare class ResearchClient {
1331
1402
  private readonly http;
1332
1403
  constructor(http: HttpClient);
1333
1404
  /**
1334
- * Search papers by abstract relevance.
1335
- * @param query Natural-language search query.
1336
- * @param options Optional filters (k, authors, categories, from, to).
1405
+ * Search the research paper index by abstract relevance.
1406
+ *
1407
+ * Queries ~43M paper abstracts: PubMed, bioRxiv and medRxiv (about 90% of the
1408
+ * corpus — biomedical and life sciences) plus arXiv (physics, mathematics,
1409
+ * computer science). Semantic search over abstracts, not keyword matching.
1410
+ *
1411
+ * This is **not** `search({ categories: ["research"] })`, which only narrows
1412
+ * ordinary web search to ~14 academic websites.
1413
+ *
1414
+ * @param query Natural-language search query, e.g. `"CRISPR base editing
1415
+ * off-target effects in primary human T cells"`.
1416
+ * @param options Optional filters (k, authors, categories, from, to). Note
1417
+ * `categories` here filters *paper* subject categories (e.g. `"q-bio.GN"`)
1418
+ * and is unrelated to the `categories` option of `search()`.
1337
1419
  */
1338
1420
  searchPapers(query: string, options?: SearchPapersOptions): Promise<SearchPapersResponse>;
1339
1421
  /**
1340
1422
  * Get paper metadata (detail mode), or read in-body passages (when `query` is
1341
1423
  * supplied). `k` is only valid together with `query`.
1342
- * @param id Paper reference: a canonical `paper_id`, an `arxiv:<id>` key, or a
1343
- * bare arXiv id / URL.
1424
+ * @param id Paper reference: a canonical `paperId`, or a namespaced id key
1425
+ * such as `pmid:<id>`, `pmcid:<id>`, `doi:<doi>` or `arxiv:<id>`. Bare
1426
+ * arXiv ids and arXiv URLs are also accepted.
1344
1427
  * @param options Optional `query` (switches to read mode) and `k`.
1345
1428
  */
1346
1429
  getPaper(id: string, options?: {
@@ -1495,8 +1578,19 @@ declare class FirecrawlClient {
1495
1578
  */
1496
1579
  searchFeedback(jobId: string, request: SearchFeedbackRequest): Promise<FeedbackResponse>;
1497
1580
  /**
1498
- * Access the v2 research endpoints (arXiv papers + GitHub history/readmes).
1499
- * Example: `firecrawl.research.searchPapers("diffusion models")`.
1581
+ * Access the v2 research endpoints — Firecrawl's **research paper index**
1582
+ * (~43M paper abstracts) plus GitHub history/readmes.
1583
+ *
1584
+ * The paper corpus is roughly 90% biomedical and life sciences — PubMed,
1585
+ * bioRxiv and medRxiv — with arXiv covering physics, mathematics and
1586
+ * computer science.
1587
+ *
1588
+ * ⚠️ Not the same as `search({ categories: ["research"] })`, which is only a
1589
+ * website/domain filter over ordinary web search (~14 academic domains,
1590
+ * returning page snippets). Use `research.searchPapers()` for literature
1591
+ * search.
1592
+ *
1593
+ * Example: `firecrawl.research.searchPapers("GLP-1 receptor agonists cardiovascular outcomes")`.
1500
1594
  */
1501
1595
  get research(): ResearchClient;
1502
1596
  /**
@@ -2525,7 +2619,7 @@ declare class FirecrawlApp {
2525
2619
  * @param onActivity - Optional callback to receive activity updates in real-time.
2526
2620
  * @param onSource - Optional callback to receive source updates in real-time.
2527
2621
  * @returns The final research results.
2528
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
2622
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
2529
2623
  */
2530
2624
  deepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>, onActivity?: (activity: {
2531
2625
  type: string;
@@ -2543,14 +2637,14 @@ declare class FirecrawlApp {
2543
2637
  * Initiates a deep research operation on a given query without polling.
2544
2638
  * @param params - Parameters for the deep research operation.
2545
2639
  * @returns The response containing the research job ID.
2546
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
2640
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
2547
2641
  */
2548
2642
  asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse>;
2549
2643
  /**
2550
2644
  * Checks the status of a deep research operation.
2551
2645
  * @param id - The ID of the deep research operation.
2552
2646
  * @returns The current status and results of the research operation.
2553
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
2647
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
2554
2648
  */
2555
2649
  checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse>;
2556
2650
  /**
package/dist/index.js CHANGED
@@ -1,6 +1,6 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-BEF2NAMB.js";
3
+ } from "./chunk-S7CQ7Q2W.js";
4
4
 
5
5
  // src/v2/utils/httpClient.ts
6
6
  import axios from "axios";
@@ -629,12 +629,16 @@ async function search(http, request) {
629
629
  if (data.news) out.news = transformArray(data.news);
630
630
  if (data.images)
631
631
  out.images = transformArray(data.images);
632
+ if (data.developer)
633
+ out.developer = transformArray(data.developer);
632
634
  Object.defineProperty(out, "data", {
633
635
  get() {
634
636
  const parts = [];
635
637
  if (out.web?.length) parts.push(`.web (${out.web.length} results)`);
636
638
  if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
637
639
  if (out.images?.length) parts.push(`.images (${out.images.length} results)`);
640
+ if (out.developer?.length)
641
+ parts.push(`.developer (${out.developer.length} results)`);
638
642
  const available = parts.length ? parts.join(", ") : ".web, .news, or .images";
639
643
  throw new Error(
640
644
  `SearchData has no '.data'. Results are grouped by source: ${available}`
@@ -1398,9 +1402,20 @@ var ResearchClient = class {
1398
1402
  }
1399
1403
  http;
1400
1404
  /**
1401
- * Search papers by abstract relevance.
1402
- * @param query Natural-language search query.
1403
- * @param options Optional filters (k, authors, categories, from, to).
1405
+ * Search the research paper index by abstract relevance.
1406
+ *
1407
+ * Queries ~43M paper abstracts: PubMed, bioRxiv and medRxiv (about 90% of the
1408
+ * corpus — biomedical and life sciences) plus arXiv (physics, mathematics,
1409
+ * computer science). Semantic search over abstracts, not keyword matching.
1410
+ *
1411
+ * This is **not** `search({ categories: ["research"] })`, which only narrows
1412
+ * ordinary web search to ~14 academic websites.
1413
+ *
1414
+ * @param query Natural-language search query, e.g. `"CRISPR base editing
1415
+ * off-target effects in primary human T cells"`.
1416
+ * @param options Optional filters (k, authors, categories, from, to). Note
1417
+ * `categories` here filters *paper* subject categories (e.g. `"q-bio.GN"`)
1418
+ * and is unrelated to the `categories` option of `search()`.
1404
1419
  */
1405
1420
  async searchPapers(query, options = {}) {
1406
1421
  if (!query || !query.trim()) throw new Error("query cannot be empty");
@@ -1950,8 +1965,19 @@ var FirecrawlClient = class {
1950
1965
  }
1951
1966
  // Research
1952
1967
  /**
1953
- * Access the v2 research endpoints (arXiv papers + GitHub history/readmes).
1954
- * Example: `firecrawl.research.searchPapers("diffusion models")`.
1968
+ * Access the v2 research endpoints — Firecrawl's **research paper index**
1969
+ * (~43M paper abstracts) plus GitHub history/readmes.
1970
+ *
1971
+ * The paper corpus is roughly 90% biomedical and life sciences — PubMed,
1972
+ * bioRxiv and medRxiv — with arXiv covering physics, mathematics and
1973
+ * computer science.
1974
+ *
1975
+ * ⚠️ Not the same as `search({ categories: ["research"] })`, which is only a
1976
+ * website/domain filter over ordinary web search (~14 academic domains,
1977
+ * returning page snippets). Use `research.searchPapers()` for literature
1978
+ * search.
1979
+ *
1980
+ * Example: `firecrawl.research.searchPapers("GLP-1 receptor agonists cardiovascular outcomes")`.
1955
1981
  */
1956
1982
  get research() {
1957
1983
  if (!this._research) this._research = new ResearchClient(this.http);
@@ -2316,7 +2342,7 @@ var FirecrawlApp = class {
2316
2342
  if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
2317
2343
  return process.env.npm_package_version;
2318
2344
  }
2319
- const packageJson = await import("./package-GIVYQAT4.js");
2345
+ const packageJson = await import("./package-KQCMZM5L.js");
2320
2346
  return packageJson.default.version;
2321
2347
  } catch (error) {
2322
2348
  const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
@@ -3142,7 +3168,7 @@ var FirecrawlApp = class {
3142
3168
  * @param onActivity - Optional callback to receive activity updates in real-time.
3143
3169
  * @param onSource - Optional callback to receive source updates in real-time.
3144
3170
  * @returns The final research results.
3145
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
3171
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
3146
3172
  */
3147
3173
  async deepResearch(query, params, onActivity, onSource) {
3148
3174
  try {
@@ -3199,7 +3225,7 @@ var FirecrawlApp = class {
3199
3225
  * Initiates a deep research operation on a given query without polling.
3200
3226
  * @param params - Parameters for the deep research operation.
3201
3227
  * @returns The response containing the research job ID.
3202
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
3228
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
3203
3229
  */
3204
3230
  async asyncDeepResearch(query, params) {
3205
3231
  const headers = this.prepareHeaders();
@@ -3237,7 +3263,7 @@ var FirecrawlApp = class {
3237
3263
  * Checks the status of a deep research operation.
3238
3264
  * @param id - The ID of the deep research operation.
3239
3265
  * @returns The current status and results of the research operation.
3240
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
3266
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
3241
3267
  */
3242
3268
  async checkDeepResearchStatus(id) {
3243
3269
  const headers = this.prepareHeaders();
@@ -1,4 +1,4 @@
1
1
  import {
2
2
  require_package
3
- } from "./chunk-BEF2NAMB.js";
3
+ } from "./chunk-S7CQ7Q2W.js";
4
4
  export default require_package();
package/package.json CHANGED
@@ -1,7 +1,7 @@
1
1
  {
2
2
  "name": "firecrawl",
3
- "version": "4.31.1",
4
- "description": "JavaScript SDK for Firecrawl API",
3
+ "version": "4.32.1",
4
+ "description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
5
5
  "main": "dist/index.js",
6
6
  "types": "dist/index.d.ts",
7
7
  "exports": {
@@ -49,7 +49,21 @@
49
49
  "web",
50
50
  "scraper",
51
51
  "api",
52
- "sdk"
52
+ "sdk",
53
+ "web-search",
54
+ "literature-search",
55
+ "research",
56
+ "scientific-papers",
57
+ "academic-search",
58
+ "biomedical",
59
+ "life-sciences",
60
+ "pubmed",
61
+ "biorxiv",
62
+ "medrxiv",
63
+ "arxiv",
64
+ "preprints",
65
+ "citations",
66
+ "bioinformatics"
53
67
  ],
54
68
  "engines": {
55
69
  "node": ">=22.0.0"
@@ -7,8 +7,9 @@ overrides:
7
7
  "rollup@<4.59.0": ">=4.59.0"
8
8
  "picomatch@<4.0.4": ">=4.0.4"
9
9
  handlebars: ">=4.7.9"
10
- "brace-expansion@>=5.0.0 <5.0.8": "5.0.8"
11
- "js-yaml@>=3.0.0 <3.15.0": "3.15.0"
10
+ "brace-expansion@>=5.0.0 <5.0.9": "5.0.9"
11
+ "js-yaml@>=3.0.0 <3.15.1": "3.15.1"
12
+ "js-yaml@>=4.0.0 <4.3.1": "4.3.1"
12
13
  "axios@>=1.0.0 <1.16.0": "1.16.1"
13
14
  "follow-redirects@<1.16.0": ">=1.16.0 <2.0.0"
14
15
  "form-data@>=4.0.0 <4.0.6": "4.0.6"
@@ -0,0 +1,65 @@
1
+ import { describe, expect, jest, test } from "@jest/globals";
2
+ import { search } from "../../../v2/methods/search";
3
+
4
+ const developerResult = {
5
+ title: "https://github.com/firecrawl/firecrawl/issues/1",
6
+ url: "https://github.com/firecrawl/firecrawl/issues/1",
7
+ description: "passage",
8
+ position: 1,
9
+ category: "developer",
10
+ };
11
+
12
+ function httpWith(data: Record<string, unknown>) {
13
+ return {
14
+ post: jest.fn(async () => ({ status: 200, data: { success: true, data } })),
15
+ } as any;
16
+ }
17
+
18
+ describe("v2 search developer category", () => {
19
+ test("forwards the developer category", async () => {
20
+ const http = httpWith({});
21
+
22
+ await search(http, { query: "firecrawl", categories: ["developer"] });
23
+
24
+ expect(http.post).toHaveBeenCalledWith(
25
+ "/v2/search",
26
+ { query: "firecrawl", categories: ["developer"] },
27
+ {},
28
+ );
29
+ });
30
+
31
+ test("returns developer results", async () => {
32
+ const http = httpWith({
33
+ web: [{ title: "W", url: "http://a.com" }],
34
+ developer: [developerResult],
35
+ });
36
+
37
+ const result = await search(http, {
38
+ query: "firecrawl",
39
+ categories: [{ type: "developer" }],
40
+ });
41
+
42
+ expect(result.web).toHaveLength(1);
43
+ expect(result.developer).toHaveLength(1);
44
+ expect(result.developer?.[0]).toEqual(developerResult);
45
+ });
46
+
47
+ test("omits developer when the response has none", async () => {
48
+ const http = httpWith({ web: [] });
49
+
50
+ const result = await search(http, { query: "firecrawl" });
51
+
52
+ expect(result.developer).toBeUndefined();
53
+ });
54
+
55
+ test("lists developer in the .data error", async () => {
56
+ const http = httpWith({ developer: [developerResult] });
57
+
58
+ const result = await search(http, {
59
+ query: "firecrawl",
60
+ categories: ["developer"],
61
+ });
62
+
63
+ expect(() => (result as any).data).toThrow(/\.developer \(1 results\)/);
64
+ });
65
+ });
package/src/v1/index.ts CHANGED
@@ -1642,7 +1642,7 @@ export default class FirecrawlApp {
1642
1642
  * @param onActivity - Optional callback to receive activity updates in real-time.
1643
1643
  * @param onSource - Optional callback to receive source updates in real-time.
1644
1644
  * @returns The final research results.
1645
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
1645
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
1646
1646
  */
1647
1647
  async deepResearch(
1648
1648
  query: string,
@@ -1730,7 +1730,7 @@ export default class FirecrawlApp {
1730
1730
  * Initiates a deep research operation on a given query without polling.
1731
1731
  * @param params - Parameters for the deep research operation.
1732
1732
  * @returns The response containing the research job ID.
1733
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
1733
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
1734
1734
  */
1735
1735
  async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
1736
1736
  const headers = this.prepareHeaders();
@@ -1772,7 +1772,7 @@ export default class FirecrawlApp {
1772
1772
  * Checks the status of a deep research operation.
1773
1773
  * @param id - The ID of the deep research operation.
1774
1774
  * @returns The current status and results of the research operation.
1775
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
1775
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
1776
1776
  */
1777
1777
  async checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {
1778
1778
  const headers = this.prepareHeaders();
package/src/v2/client.ts CHANGED
@@ -261,8 +261,19 @@ export class FirecrawlClient {
261
261
 
262
262
  // Research
263
263
  /**
264
- * Access the v2 research endpoints (arXiv papers + GitHub history/readmes).
265
- * Example: `firecrawl.research.searchPapers("diffusion models")`.
264
+ * Access the v2 research endpoints — Firecrawl's **research paper index**
265
+ * (~43M paper abstracts) plus GitHub history/readmes.
266
+ *
267
+ * The paper corpus is roughly 90% biomedical and life sciences — PubMed,
268
+ * bioRxiv and medRxiv — with arXiv covering physics, mathematics and
269
+ * computer science.
270
+ *
271
+ * ⚠️ Not the same as `search({ categories: ["research"] })`, which is only a
272
+ * website/domain filter over ordinary web search (~14 academic domains,
273
+ * returning page snippets). Use `research.searchPapers()` for literature
274
+ * search.
275
+ *
276
+ * Example: `firecrawl.research.searchPapers("GLP-1 receptor agonists cardiovascular outcomes")`.
266
277
  */
267
278
  get research(): ResearchClient {
268
279
  if (!this._research) this._research = new ResearchClient(this.http);
@@ -63,16 +63,47 @@ function normalizeResearchError(err: any, action: string): never {
63
63
  }
64
64
 
65
65
  /**
66
- * Client for the v2 research endpoints (arXiv papers + GitHub history/readmes).
66
+ * Client for the v2 research endpoints — Firecrawl's **research paper index**
67
+ * (~43M paper abstracts) plus GitHub history/readmes.
68
+ *
69
+ * The paper corpus is roughly 90% biomedical and life sciences — PubMed,
70
+ * bioRxiv and medRxiv — with arXiv covering physics, mathematics and computer
71
+ * science. Supports abstract search, paper metadata, in-body passage reads and
72
+ * citation-graph expansion.
73
+ *
74
+ * ⚠️ This is **not** the same as `search({ categories: ["research"] })`. That
75
+ * option is a website/domain filter on ordinary web search: it narrows
76
+ * Google-style results to ~14 academic domains and returns page snippets. The
77
+ * methods here query the paper index itself and return ranked paper records.
78
+ *
67
79
  * Accessed via `firecrawl.research`.
80
+ *
81
+ * @example
82
+ * ```ts
83
+ * const res = await firecrawl.research.searchPapers(
84
+ * "tau aggregation inhibitors in Alzheimer's disease",
85
+ * { k: 10 },
86
+ * );
87
+ * ```
68
88
  */
69
89
  export class ResearchClient {
70
90
  constructor(private readonly http: HttpClient) {}
71
91
 
72
92
  /**
73
- * Search papers by abstract relevance.
74
- * @param query Natural-language search query.
75
- * @param options Optional filters (k, authors, categories, from, to).
93
+ * Search the research paper index by abstract relevance.
94
+ *
95
+ * Queries ~43M paper abstracts: PubMed, bioRxiv and medRxiv (about 90% of the
96
+ * corpus — biomedical and life sciences) plus arXiv (physics, mathematics,
97
+ * computer science). Semantic search over abstracts, not keyword matching.
98
+ *
99
+ * This is **not** `search({ categories: ["research"] })`, which only narrows
100
+ * ordinary web search to ~14 academic websites.
101
+ *
102
+ * @param query Natural-language search query, e.g. `"CRISPR base editing
103
+ * off-target effects in primary human T cells"`.
104
+ * @param options Optional filters (k, authors, categories, from, to). Note
105
+ * `categories` here filters *paper* subject categories (e.g. `"q-bio.GN"`)
106
+ * and is unrelated to the `categories` option of `search()`.
76
107
  */
77
108
  async searchPapers(
78
109
  query: string,
@@ -102,8 +133,9 @@ export class ResearchClient {
102
133
  /**
103
134
  * Get paper metadata (detail mode), or read in-body passages (when `query` is
104
135
  * supplied). `k` is only valid together with `query`.
105
- * @param id Paper reference: a canonical `paper_id`, an `arxiv:<id>` key, or a
106
- * bare arXiv id / URL.
136
+ * @param id Paper reference: a canonical `paperId`, or a namespaced id key
137
+ * such as `pmid:<id>`, `pmcid:<id>`, `doi:<doi>` or `arxiv:<id>`. Bare
138
+ * arXiv ids and arXiv URLs are also accepted.
107
139
  * @param options Optional `query` (switches to read mode) and `k`.
108
140
  */
109
141
  async getPaper(
@@ -102,12 +102,16 @@ export async function search(
102
102
  if (data.news) out.news = transformArray<SearchResultNews>(data.news);
103
103
  if (data.images)
104
104
  out.images = transformArray<SearchResultImages>(data.images);
105
+ if (data.developer)
106
+ out.developer = transformArray<SearchResultWeb>(data.developer);
105
107
  Object.defineProperty(out, "data", {
106
108
  get() {
107
109
  const parts: string[] = [];
108
110
  if (out.web?.length) parts.push(`.web (${out.web.length} results)`);
109
111
  if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
110
112
  if (out.images?.length) parts.push(`.images (${out.images.length} results)`);
113
+ if (out.developer?.length)
114
+ parts.push(`.developer (${out.developer.length} results)`);
111
115
  const available = parts.length ? parts.join(", ") : ".web, .news, or .images";
112
116
  throw new Error(
113
117
  `SearchData has no '.data'. Results are grouped by source: ${available}`,
package/src/v2/types.ts CHANGED
@@ -207,6 +207,10 @@ export interface ScrapeOptions {
207
207
  useMock?: string;
208
208
  blockAds?: boolean;
209
209
  proxy?: "basic" | "stealth" | "enhanced" | "auto" | string;
210
+ /**
211
+ * Maximum age, in milliseconds, of indexed content that may be reused.
212
+ * Set to `0` to bypass index reuse.
213
+ */
210
214
  maxAge?: number;
211
215
  minAge?: number;
212
216
  storeInCache?: boolean;
@@ -558,6 +562,7 @@ export interface DocumentMetadata {
558
562
  // Common metadata fields
559
563
  title?: string;
560
564
  description?: string;
565
+ /** URL reported by the selected scrape engine. */
561
566
  url?: string;
562
567
  language?: string;
563
568
  keywords?: string | string[];
@@ -594,7 +599,9 @@ export interface DocumentMetadata {
594
599
  articleSection?: string;
595
600
 
596
601
  // Response-level metadata
602
+ /** URL requested for the scrape. */
597
603
  sourceURL?: string;
604
+ /** HTTP status code reported for the scrape response. */
598
605
  statusCode?: number;
599
606
  scrapeId?: string;
600
607
  numPages?: number;
@@ -657,6 +664,7 @@ export interface SearchResultWeb {
657
664
  url: string;
658
665
  title?: string;
659
666
  description?: string;
667
+ position?: number;
660
668
  category?: string;
661
669
  }
662
670
 
@@ -683,10 +691,33 @@ export interface SearchData {
683
691
  web?: Array<SearchResultWeb | Document>;
684
692
  news?: Array<SearchResultNews | Document>;
685
693
  images?: Array<SearchResultImages | Document>;
694
+ developer?: Array<SearchResultWeb | Document>;
686
695
  }
687
696
 
697
+ /**
698
+ * Narrows ordinary **web search**. A category does not switch `search()` to a
699
+ * different index.
700
+ *
701
+ * - `github` — restrict web results to github.com (a `site:` filter).
702
+ * - `research` — restrict web results to a fixed list of ~14 academic
703
+ * *websites* (arxiv.org, pubmed.ncbi.nlm.nih.gov, nature.com, science.org,
704
+ * ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). It returns
705
+ * ordinary web page results from those domains, **not** paper records.
706
+ * - `pdf` — restrict results to PDFs (adds `filetype:pdf`).
707
+ * - `developer` — add developer results (issues, pull requests, READMEs and
708
+ * documentation) under `.developer`.
709
+ *
710
+ * ⚠️ `categories: ["research"]` is **not** Firecrawl's research paper index.
711
+ * To search papers themselves — ~43M abstracts, roughly 90% biomedical
712
+ * (PubMed, bioRxiv, medRxiv) plus arXiv — with full abstracts, in-body passage
713
+ * reads and citation-graph expansion, use `firecrawl.research.searchPapers()`
714
+ * (plus `getPaper()` and `similarPapers()`), which call `/v2/search/research`.
715
+ *
716
+ * Rule of thumb: literature search → `research.searchPapers()`; web pages that
717
+ * happen to live on academic domains → `search({ categories: ["research"] })`.
718
+ */
688
719
  export interface CategoryOption {
689
- type: "github" | "research" | "pdf";
720
+ type: "github" | "research" | "pdf" | "developer";
690
721
  }
691
722
 
692
723
  export interface SearchRequest {
@@ -694,7 +725,17 @@ export interface SearchRequest {
694
725
  sources?: Array<
695
726
  "web" | "news" | "images" | { type: "web" | "news" | "images" }
696
727
  >;
697
- categories?: Array<"github" | "research" | "pdf" | CategoryOption>;
728
+ /**
729
+ * Narrow web search by category. See {@link CategoryOption}.
730
+ *
731
+ * ⚠️ `"research"` is a website/domain filter over ordinary web search (~14
732
+ * academic domains), **not** the research paper index. For literature search
733
+ * over ~43M paper abstracts (PubMed / bioRxiv / medRxiv / arXiv) use
734
+ * `firecrawl.research.searchPapers()` instead.
735
+ */
736
+ categories?: Array<
737
+ "github" | "research" | "pdf" | "developer" | CategoryOption
738
+ >;
698
739
  includeDomains?: string[];
699
740
  excludeDomains?: string[];
700
741
  limit?: number;
@@ -1351,8 +1392,20 @@ export interface BrowserListResponse {
1351
1392
  // ---------- Research (v2) ----------
1352
1393
 
1353
1394
  /**
1354
- * Source identifiers grouped by namespace. Currently only `arxiv` is
1355
- * populated; each value is an array of ids in that namespace.
1395
+ * Source identifiers grouped by namespace. Each key is a namespace and each
1396
+ * value is an array of ids in that namespace.
1397
+ *
1398
+ * Which namespaces appear depends on the paper's source. Biomedical records
1399
+ * (PubMed, bioRxiv, medRxiv) typically carry `pmid`, `pmcid` and/or `doi`;
1400
+ * arXiv records carry `arxiv`. The set is open — treat this as a lookup by
1401
+ * namespace rather than a fixed schema, and prefer {@link PaperResult.primaryId}
1402
+ * when you just need one citable identifier.
1403
+ *
1404
+ * @example
1405
+ * ```ts
1406
+ * const doi = paper.ids?.doi?.[0];
1407
+ * const pmid = paper.ids?.pmid?.[0];
1408
+ * ```
1356
1409
  */
1357
1410
  export type IdMap = Record<string, string[]>;
1358
1411
 
@@ -1368,7 +1421,7 @@ export interface PaperSignals {
1368
1421
  seedOverlap: number;
1369
1422
  }
1370
1423
 
1371
- /** A ranked paper. `paperId` is canonical; arXiv lives in `ids`. */
1424
+ /** A ranked paper. `paperId` is canonical; source ids (pmid, pmcid, doi, arxiv, ...) live in `ids`. */
1372
1425
  export interface PaperResult {
1373
1426
  /** Canonical paper id — the Milvus INT64 primary key as a decimal string. */
1374
1427
  paperId: string;
@@ -1448,7 +1501,7 @@ export interface GitHubScoreBreakdown {
1448
1501
  }
1449
1502
 
1450
1503
  export interface GitHubSearchItem {
1451
- resultType: "github_history" | "repo_readme" | "web";
1504
+ resultType?: "github_history" | "repo_readme" | "web";
1452
1505
  /** `owner/name`; empty for web results whose URL is not a repo page. */
1453
1506
  repo: string;
1454
1507
  url: string;