firecrawl 4.34.1 → 4.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -1
- package/dist/{chunk-DWBMD5N5.js → chunk-666CR5HH.js} +1 -1
- package/dist/index.cjs +45 -6
- package/dist/index.d.cts +83 -4
- package/dist/index.d.ts +83 -4
- package/dist/index.js +46 -7
- package/dist/{package-XAWX5C5Y.js → package-ZOVFJE7Z.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/developer.test.ts +118 -0
- package/src/__tests__/unit/v2/search-developer.unit.test.ts +11 -21
- package/src/__tests__/unit/v2/validation.test.ts +3 -2
- package/src/v2/client.ts +16 -0
- package/src/v2/methods/developer.ts +50 -0
- package/src/v2/methods/search.ts +5 -6
- package/src/v2/types.ts +80 -3
package/README.md
CHANGED
|
@@ -183,7 +183,8 @@ const scraped = await app.search('firecrawl changelog', {
|
|
|
183
183
|
});
|
|
184
184
|
```
|
|
185
185
|
|
|
186
|
-
Results are grouped by source: `.web`, `.news
|
|
186
|
+
Results are grouped by source: `.web`, `.news` and `.images`. Developer
|
|
187
|
+
category results are served inside `.web`.
|
|
187
188
|
|
|
188
189
|
Use `categories` to narrow web search to a kind of site:
|
|
189
190
|
|
|
@@ -198,6 +199,36 @@ const results = await app.search('nanopore basecalling accuracy', {
|
|
|
198
199
|
> pubmed.ncbi.nlm.nih.gov, nature.com, biorxiv.org, ...) and returns web page
|
|
199
200
|
> results. To search papers themselves, use `research.searchPapers` below.
|
|
200
201
|
|
|
202
|
+
### Developer search
|
|
203
|
+
|
|
204
|
+
Use `developerSearch` for the dedicated developer index and its complete filter
|
|
205
|
+
and response contract. Generic `search(..., { categories: ['developer'] })`
|
|
206
|
+
returns developer results inside `.web` in the ordinary web-result shape
|
|
207
|
+
(first passage as the description) and does not accept these filters.
|
|
208
|
+
|
|
209
|
+
```js
|
|
210
|
+
const evidence = await app.developerSearch('configure retry backoff', {
|
|
211
|
+
repos: ['firecrawl/firecrawl'],
|
|
212
|
+
types: ['issue', 'pull_request', 'readme'],
|
|
213
|
+
passages: 3,
|
|
214
|
+
language: 'TypeScript',
|
|
215
|
+
license: 'MIT',
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
for (const result of evidence.results) {
|
|
219
|
+
// Result kind is the id prefix; the API intentionally omits a type field.
|
|
220
|
+
console.log(result.id, result.license);
|
|
221
|
+
for (const passage of result.passages) {
|
|
222
|
+
console.log(passage.text, passage.citation_url);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
console.log(evidence.repos); // indexed-status echoes for requested repos
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
`developerSearch` also supports `sources`, `topic`, `minStars`, `maxStars`,
|
|
229
|
+
`archived`, `fork`, and `skills: 'only'`. Supplying both `repos` and `sources`
|
|
230
|
+
OR-combines GitHub-backed and documentation results.
|
|
231
|
+
|
|
201
232
|
### Research / paper search
|
|
202
233
|
|
|
203
234
|
Use `app.research` to search Firecrawl's research paper index: ~43M paper
|
|
@@ -12,7 +12,7 @@ var require_package = __commonJS({
|
|
|
12
12
|
"package.json"(exports, module) {
|
|
13
13
|
module.exports = {
|
|
14
14
|
name: "@mendable/firecrawl-js",
|
|
15
|
-
version: "4.
|
|
15
|
+
version: "4.35.0",
|
|
16
16
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
17
17
|
main: "dist/index.js",
|
|
18
18
|
types: "dist/index.d.ts",
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.
|
|
42
|
+
version: "4.35.0",
|
|
43
43
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -760,16 +760,13 @@ async function search(http, request) {
|
|
|
760
760
|
if (data.news) out.news = transformArray(data.news);
|
|
761
761
|
if (data.images)
|
|
762
762
|
out.images = transformArray(data.images);
|
|
763
|
-
if (data.developer)
|
|
764
|
-
out.developer = transformArray(data.developer);
|
|
765
763
|
Object.defineProperty(out, "data", {
|
|
766
764
|
get() {
|
|
767
765
|
const parts = [];
|
|
768
766
|
if (out.web?.length) parts.push(`.web (${out.web.length} results)`);
|
|
769
767
|
if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
|
|
770
|
-
if (out.images?.length)
|
|
771
|
-
|
|
772
|
-
parts.push(`.developer (${out.developer.length} results)`);
|
|
768
|
+
if (out.images?.length)
|
|
769
|
+
parts.push(`.images (${out.images.length} results)`);
|
|
773
770
|
const available = parts.length ? parts.join(", ") : ".web, .news, or .images";
|
|
774
771
|
throw new Error(
|
|
775
772
|
`SearchData has no '.data'. Results are grouped by source: ${available}`
|
|
@@ -785,6 +782,39 @@ async function search(http, request) {
|
|
|
785
782
|
}
|
|
786
783
|
}
|
|
787
784
|
|
|
785
|
+
// src/v2/methods/developer.ts
|
|
786
|
+
var ENDPOINT = "/v2/search/developer";
|
|
787
|
+
async function developerSearch(http, query, options = {}) {
|
|
788
|
+
if (!query || !query.trim()) throw new Error("query cannot be empty");
|
|
789
|
+
try {
|
|
790
|
+
const response = await http.post(ENDPOINT, {
|
|
791
|
+
query,
|
|
792
|
+
...options.k !== void 0 ? { k: options.k } : {},
|
|
793
|
+
...options.passages !== void 0 ? { passages: options.passages } : {},
|
|
794
|
+
...options.types !== void 0 ? { types: options.types } : {},
|
|
795
|
+
...options.repos !== void 0 ? { repos: options.repos } : {},
|
|
796
|
+
...options.sources !== void 0 ? { sources: options.sources } : {},
|
|
797
|
+
...options.language !== void 0 ? { language: options.language } : {},
|
|
798
|
+
...options.topic !== void 0 ? { topic: options.topic } : {},
|
|
799
|
+
...options.license !== void 0 ? { license: options.license } : {},
|
|
800
|
+
...options.minStars !== void 0 ? { min_stars: options.minStars } : {},
|
|
801
|
+
...options.maxStars !== void 0 ? { max_stars: options.maxStars } : {},
|
|
802
|
+
...options.archived !== void 0 ? { archived: options.archived } : {},
|
|
803
|
+
...options.fork !== void 0 ? { fork: options.fork } : {},
|
|
804
|
+
...options.skills !== void 0 ? { skills: options.skills } : {}
|
|
805
|
+
});
|
|
806
|
+
if (response.status !== 200 || !response.data?.success) {
|
|
807
|
+
throwForBadResponse(response, "search developer sources");
|
|
808
|
+
}
|
|
809
|
+
return response.data;
|
|
810
|
+
} catch (error) {
|
|
811
|
+
if (error?.isAxiosError) {
|
|
812
|
+
return normalizeAxiosError(error, "search developer sources");
|
|
813
|
+
}
|
|
814
|
+
throw error;
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
|
|
788
818
|
// src/v2/methods/map.ts
|
|
789
819
|
function prepareMapPayload(url, options) {
|
|
790
820
|
if (!url || !url.trim()) throw new Error("URL cannot be empty");
|
|
@@ -2077,6 +2107,15 @@ var FirecrawlClient = class {
|
|
|
2077
2107
|
async search(query, req = {}) {
|
|
2078
2108
|
return search(this.http, { query, ...req });
|
|
2079
2109
|
}
|
|
2110
|
+
/**
|
|
2111
|
+
* Search the dedicated developer index with repository, documentation,
|
|
2112
|
+
* artifact, language, topic, license, star, archive, and fork filters.
|
|
2113
|
+
* Unlike `search(query, { categories: ["developer"] })`, this returns all
|
|
2114
|
+
* requested passages, citations, license disclosures, and indexing echoes.
|
|
2115
|
+
*/
|
|
2116
|
+
async developerSearch(query, options = {}) {
|
|
2117
|
+
return developerSearch(this.http, query, options);
|
|
2118
|
+
}
|
|
2080
2119
|
/**
|
|
2081
2120
|
* Submit feedback for a v2 job.
|
|
2082
2121
|
* @param request Feedback payload with endpoint, job id, rating, and supporting signals.
|
package/dist/index.d.cts
CHANGED
|
@@ -129,6 +129,14 @@ type PDFParser = {
|
|
|
129
129
|
pageMarkdown?: boolean;
|
|
130
130
|
/** Include per-page typed layout blocks (bounding boxes, block types, reading order). */
|
|
131
131
|
blocks?: boolean;
|
|
132
|
+
/**
|
|
133
|
+
* Join PDF pages in `document.markdown` with `\n\n---\n\n<!-- page N -->\n\n`,
|
|
134
|
+
* where N is the 1-based physical page of the content that follows. Markers
|
|
135
|
+
* appear between pages only (no leading marker for page 1), and numbering may
|
|
136
|
+
* skip pages merged by cross-page stitching — use `pages: true` when every
|
|
137
|
+
* physical page is needed. No new response field.
|
|
138
|
+
*/
|
|
139
|
+
pageMarkers?: boolean;
|
|
132
140
|
};
|
|
133
141
|
interface PdfPage {
|
|
134
142
|
pageNumber: number;
|
|
@@ -551,6 +559,71 @@ interface PaginationConfig {
|
|
|
551
559
|
/** Maximum time to spend fetching additional pages (in seconds). */
|
|
552
560
|
maxWaitTime?: number;
|
|
553
561
|
}
|
|
562
|
+
type DeveloperSearchType = "doc" | "issue" | "pull_request" | "readme";
|
|
563
|
+
interface DeveloperSearchOptions {
|
|
564
|
+
/** Total ranked results, 1–100 (default 10). */
|
|
565
|
+
k?: number;
|
|
566
|
+
/** Passages per result, 1–5 (default 1). */
|
|
567
|
+
passages?: number;
|
|
568
|
+
/** Result kinds to search (default all). */
|
|
569
|
+
types?: DeveloperSearchType[];
|
|
570
|
+
/** GitHub owner/name filters, at most 20. */
|
|
571
|
+
repos?: string[];
|
|
572
|
+
/** Developer documentation source IDs, at most 20. */
|
|
573
|
+
sources?: string[];
|
|
574
|
+
/** One case-insensitive GitHub Linguist primary language. */
|
|
575
|
+
language?: string;
|
|
576
|
+
/** Repository topics that must all match, at most 8. */
|
|
577
|
+
topic?: string[];
|
|
578
|
+
/** One case-insensitive SPDX repository license identifier. */
|
|
579
|
+
license?: string;
|
|
580
|
+
/** Minimum repository stars. */
|
|
581
|
+
minStars?: number;
|
|
582
|
+
/** Maximum repository stars. */
|
|
583
|
+
maxStars?: number;
|
|
584
|
+
/** Filter by repository archived status. */
|
|
585
|
+
archived?: boolean;
|
|
586
|
+
/** Filter by repository fork status. */
|
|
587
|
+
fork?: boolean;
|
|
588
|
+
/** Return packaged agent-skill evidence only. */
|
|
589
|
+
skills?: "only";
|
|
590
|
+
}
|
|
591
|
+
interface DeveloperSearchLicenseDisclosure {
|
|
592
|
+
state: "licensed" | "known_absent" | "unknown";
|
|
593
|
+
spdx_id: string | null;
|
|
594
|
+
}
|
|
595
|
+
interface DeveloperSearchPassage {
|
|
596
|
+
text: string;
|
|
597
|
+
citation_url?: string;
|
|
598
|
+
}
|
|
599
|
+
interface DeveloperSearchResult {
|
|
600
|
+
id: string;
|
|
601
|
+
url: string;
|
|
602
|
+
title?: string;
|
|
603
|
+
passages: DeveloperSearchPassage[];
|
|
604
|
+
license?: DeveloperSearchLicenseDisclosure | string;
|
|
605
|
+
}
|
|
606
|
+
interface DeveloperSearchRepoStatus {
|
|
607
|
+
repo: string;
|
|
608
|
+
indexed: boolean;
|
|
609
|
+
types: {
|
|
610
|
+
issue: boolean;
|
|
611
|
+
pullRequest: boolean;
|
|
612
|
+
readme: boolean;
|
|
613
|
+
};
|
|
614
|
+
}
|
|
615
|
+
interface DeveloperSearchSourceStatus {
|
|
616
|
+
source: string;
|
|
617
|
+
indexed: boolean;
|
|
618
|
+
}
|
|
619
|
+
interface DeveloperSearchResponse {
|
|
620
|
+
success: boolean;
|
|
621
|
+
results: DeveloperSearchResult[];
|
|
622
|
+
/** Present only when repos were requested. */
|
|
623
|
+
repos?: DeveloperSearchRepoStatus[];
|
|
624
|
+
/** Present only when sources were requested. */
|
|
625
|
+
sources?: DeveloperSearchSourceStatus[];
|
|
626
|
+
}
|
|
554
627
|
interface SearchResultWeb {
|
|
555
628
|
url: string;
|
|
556
629
|
title?: string;
|
|
@@ -579,7 +652,6 @@ interface SearchData {
|
|
|
579
652
|
web?: Array<SearchResultWeb | Document>;
|
|
580
653
|
news?: Array<SearchResultNews | Document>;
|
|
581
654
|
images?: Array<SearchResultImages | Document>;
|
|
582
|
-
developer?: Array<SearchResultWeb | Document>;
|
|
583
655
|
}
|
|
584
656
|
/**
|
|
585
657
|
* Narrows ordinary **web search**. A category does not switch `search()` to a
|
|
@@ -591,8 +663,8 @@ interface SearchData {
|
|
|
591
663
|
* ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). It returns
|
|
592
664
|
* ordinary web page results from those domains, **not** paper records.
|
|
593
665
|
* - `pdf` — restrict results to PDFs (adds `filetype:pdf`).
|
|
594
|
-
* - `developer` —
|
|
595
|
-
* documentation)
|
|
666
|
+
* - `developer` — developer-index results (issues, pull requests, READMEs and
|
|
667
|
+
* documentation) served in `web`; cannot be combined with other categories.
|
|
596
668
|
*
|
|
597
669
|
* ⚠️ `categories: ["research"]` is **not** Firecrawl's research paper index.
|
|
598
670
|
* To search papers themselves — ~43M abstracts, roughly 90% biomedical
|
|
@@ -1609,6 +1681,13 @@ declare class FirecrawlClient {
|
|
|
1609
1681
|
* @returns Structured search results.
|
|
1610
1682
|
*/
|
|
1611
1683
|
search(query: string, req?: Omit<SearchRequest, "query">): Promise<SearchData>;
|
|
1684
|
+
/**
|
|
1685
|
+
* Search the dedicated developer index with repository, documentation,
|
|
1686
|
+
* artifact, language, topic, license, star, archive, and fork filters.
|
|
1687
|
+
* Unlike `search(query, { categories: ["developer"] })`, this returns all
|
|
1688
|
+
* requested passages, citations, license disclosures, and indexing echoes.
|
|
1689
|
+
*/
|
|
1690
|
+
developerSearch(query: string, options?: DeveloperSearchOptions): Promise<DeveloperSearchResponse>;
|
|
1612
1691
|
/**
|
|
1613
1692
|
* Submit feedback for a v2 job.
|
|
1614
1693
|
* @param request Feedback payload with endpoint, job id, rating, and supporting signals.
|
|
@@ -2806,4 +2885,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
2806
2885
|
get v1(): FirecrawlApp;
|
|
2807
2886
|
}
|
|
2808
2887
|
|
|
2809
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
|
2888
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
package/dist/index.d.ts
CHANGED
|
@@ -129,6 +129,14 @@ type PDFParser = {
|
|
|
129
129
|
pageMarkdown?: boolean;
|
|
130
130
|
/** Include per-page typed layout blocks (bounding boxes, block types, reading order). */
|
|
131
131
|
blocks?: boolean;
|
|
132
|
+
/**
|
|
133
|
+
* Join PDF pages in `document.markdown` with `\n\n---\n\n<!-- page N -->\n\n`,
|
|
134
|
+
* where N is the 1-based physical page of the content that follows. Markers
|
|
135
|
+
* appear between pages only (no leading marker for page 1), and numbering may
|
|
136
|
+
* skip pages merged by cross-page stitching — use `pages: true` when every
|
|
137
|
+
* physical page is needed. No new response field.
|
|
138
|
+
*/
|
|
139
|
+
pageMarkers?: boolean;
|
|
132
140
|
};
|
|
133
141
|
interface PdfPage {
|
|
134
142
|
pageNumber: number;
|
|
@@ -551,6 +559,71 @@ interface PaginationConfig {
|
|
|
551
559
|
/** Maximum time to spend fetching additional pages (in seconds). */
|
|
552
560
|
maxWaitTime?: number;
|
|
553
561
|
}
|
|
562
|
+
type DeveloperSearchType = "doc" | "issue" | "pull_request" | "readme";
|
|
563
|
+
interface DeveloperSearchOptions {
|
|
564
|
+
/** Total ranked results, 1–100 (default 10). */
|
|
565
|
+
k?: number;
|
|
566
|
+
/** Passages per result, 1–5 (default 1). */
|
|
567
|
+
passages?: number;
|
|
568
|
+
/** Result kinds to search (default all). */
|
|
569
|
+
types?: DeveloperSearchType[];
|
|
570
|
+
/** GitHub owner/name filters, at most 20. */
|
|
571
|
+
repos?: string[];
|
|
572
|
+
/** Developer documentation source IDs, at most 20. */
|
|
573
|
+
sources?: string[];
|
|
574
|
+
/** One case-insensitive GitHub Linguist primary language. */
|
|
575
|
+
language?: string;
|
|
576
|
+
/** Repository topics that must all match, at most 8. */
|
|
577
|
+
topic?: string[];
|
|
578
|
+
/** One case-insensitive SPDX repository license identifier. */
|
|
579
|
+
license?: string;
|
|
580
|
+
/** Minimum repository stars. */
|
|
581
|
+
minStars?: number;
|
|
582
|
+
/** Maximum repository stars. */
|
|
583
|
+
maxStars?: number;
|
|
584
|
+
/** Filter by repository archived status. */
|
|
585
|
+
archived?: boolean;
|
|
586
|
+
/** Filter by repository fork status. */
|
|
587
|
+
fork?: boolean;
|
|
588
|
+
/** Return packaged agent-skill evidence only. */
|
|
589
|
+
skills?: "only";
|
|
590
|
+
}
|
|
591
|
+
interface DeveloperSearchLicenseDisclosure {
|
|
592
|
+
state: "licensed" | "known_absent" | "unknown";
|
|
593
|
+
spdx_id: string | null;
|
|
594
|
+
}
|
|
595
|
+
interface DeveloperSearchPassage {
|
|
596
|
+
text: string;
|
|
597
|
+
citation_url?: string;
|
|
598
|
+
}
|
|
599
|
+
interface DeveloperSearchResult {
|
|
600
|
+
id: string;
|
|
601
|
+
url: string;
|
|
602
|
+
title?: string;
|
|
603
|
+
passages: DeveloperSearchPassage[];
|
|
604
|
+
license?: DeveloperSearchLicenseDisclosure | string;
|
|
605
|
+
}
|
|
606
|
+
interface DeveloperSearchRepoStatus {
|
|
607
|
+
repo: string;
|
|
608
|
+
indexed: boolean;
|
|
609
|
+
types: {
|
|
610
|
+
issue: boolean;
|
|
611
|
+
pullRequest: boolean;
|
|
612
|
+
readme: boolean;
|
|
613
|
+
};
|
|
614
|
+
}
|
|
615
|
+
interface DeveloperSearchSourceStatus {
|
|
616
|
+
source: string;
|
|
617
|
+
indexed: boolean;
|
|
618
|
+
}
|
|
619
|
+
interface DeveloperSearchResponse {
|
|
620
|
+
success: boolean;
|
|
621
|
+
results: DeveloperSearchResult[];
|
|
622
|
+
/** Present only when repos were requested. */
|
|
623
|
+
repos?: DeveloperSearchRepoStatus[];
|
|
624
|
+
/** Present only when sources were requested. */
|
|
625
|
+
sources?: DeveloperSearchSourceStatus[];
|
|
626
|
+
}
|
|
554
627
|
interface SearchResultWeb {
|
|
555
628
|
url: string;
|
|
556
629
|
title?: string;
|
|
@@ -579,7 +652,6 @@ interface SearchData {
|
|
|
579
652
|
web?: Array<SearchResultWeb | Document>;
|
|
580
653
|
news?: Array<SearchResultNews | Document>;
|
|
581
654
|
images?: Array<SearchResultImages | Document>;
|
|
582
|
-
developer?: Array<SearchResultWeb | Document>;
|
|
583
655
|
}
|
|
584
656
|
/**
|
|
585
657
|
* Narrows ordinary **web search**. A category does not switch `search()` to a
|
|
@@ -591,8 +663,8 @@ interface SearchData {
|
|
|
591
663
|
* ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). It returns
|
|
592
664
|
* ordinary web page results from those domains, **not** paper records.
|
|
593
665
|
* - `pdf` — restrict results to PDFs (adds `filetype:pdf`).
|
|
594
|
-
* - `developer` —
|
|
595
|
-
* documentation)
|
|
666
|
+
* - `developer` — developer-index results (issues, pull requests, READMEs and
|
|
667
|
+
* documentation) served in `web`; cannot be combined with other categories.
|
|
596
668
|
*
|
|
597
669
|
* ⚠️ `categories: ["research"]` is **not** Firecrawl's research paper index.
|
|
598
670
|
* To search papers themselves — ~43M abstracts, roughly 90% biomedical
|
|
@@ -1609,6 +1681,13 @@ declare class FirecrawlClient {
|
|
|
1609
1681
|
* @returns Structured search results.
|
|
1610
1682
|
*/
|
|
1611
1683
|
search(query: string, req?: Omit<SearchRequest, "query">): Promise<SearchData>;
|
|
1684
|
+
/**
|
|
1685
|
+
* Search the dedicated developer index with repository, documentation,
|
|
1686
|
+
* artifact, language, topic, license, star, archive, and fork filters.
|
|
1687
|
+
* Unlike `search(query, { categories: ["developer"] })`, this returns all
|
|
1688
|
+
* requested passages, citations, license disclosures, and indexing echoes.
|
|
1689
|
+
*/
|
|
1690
|
+
developerSearch(query: string, options?: DeveloperSearchOptions): Promise<DeveloperSearchResponse>;
|
|
1612
1691
|
/**
|
|
1613
1692
|
* Submit feedback for a v2 job.
|
|
1614
1693
|
* @param request Feedback payload with endpoint, job id, rating, and supporting signals.
|
|
@@ -2806,4 +2885,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
2806
2885
|
get v1(): FirecrawlApp;
|
|
2807
2886
|
}
|
|
2808
2887
|
|
|
2809
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
|
2888
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-666CR5HH.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -629,16 +629,13 @@ async function search(http, request) {
|
|
|
629
629
|
if (data.news) out.news = transformArray(data.news);
|
|
630
630
|
if (data.images)
|
|
631
631
|
out.images = transformArray(data.images);
|
|
632
|
-
if (data.developer)
|
|
633
|
-
out.developer = transformArray(data.developer);
|
|
634
632
|
Object.defineProperty(out, "data", {
|
|
635
633
|
get() {
|
|
636
634
|
const parts = [];
|
|
637
635
|
if (out.web?.length) parts.push(`.web (${out.web.length} results)`);
|
|
638
636
|
if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
|
|
639
|
-
if (out.images?.length)
|
|
640
|
-
|
|
641
|
-
parts.push(`.developer (${out.developer.length} results)`);
|
|
637
|
+
if (out.images?.length)
|
|
638
|
+
parts.push(`.images (${out.images.length} results)`);
|
|
642
639
|
const available = parts.length ? parts.join(", ") : ".web, .news, or .images";
|
|
643
640
|
throw new Error(
|
|
644
641
|
`SearchData has no '.data'. Results are grouped by source: ${available}`
|
|
@@ -654,6 +651,39 @@ async function search(http, request) {
|
|
|
654
651
|
}
|
|
655
652
|
}
|
|
656
653
|
|
|
654
|
+
// src/v2/methods/developer.ts
|
|
655
|
+
var ENDPOINT = "/v2/search/developer";
|
|
656
|
+
async function developerSearch(http, query, options = {}) {
|
|
657
|
+
if (!query || !query.trim()) throw new Error("query cannot be empty");
|
|
658
|
+
try {
|
|
659
|
+
const response = await http.post(ENDPOINT, {
|
|
660
|
+
query,
|
|
661
|
+
...options.k !== void 0 ? { k: options.k } : {},
|
|
662
|
+
...options.passages !== void 0 ? { passages: options.passages } : {},
|
|
663
|
+
...options.types !== void 0 ? { types: options.types } : {},
|
|
664
|
+
...options.repos !== void 0 ? { repos: options.repos } : {},
|
|
665
|
+
...options.sources !== void 0 ? { sources: options.sources } : {},
|
|
666
|
+
...options.language !== void 0 ? { language: options.language } : {},
|
|
667
|
+
...options.topic !== void 0 ? { topic: options.topic } : {},
|
|
668
|
+
...options.license !== void 0 ? { license: options.license } : {},
|
|
669
|
+
...options.minStars !== void 0 ? { min_stars: options.minStars } : {},
|
|
670
|
+
...options.maxStars !== void 0 ? { max_stars: options.maxStars } : {},
|
|
671
|
+
...options.archived !== void 0 ? { archived: options.archived } : {},
|
|
672
|
+
...options.fork !== void 0 ? { fork: options.fork } : {},
|
|
673
|
+
...options.skills !== void 0 ? { skills: options.skills } : {}
|
|
674
|
+
});
|
|
675
|
+
if (response.status !== 200 || !response.data?.success) {
|
|
676
|
+
throwForBadResponse(response, "search developer sources");
|
|
677
|
+
}
|
|
678
|
+
return response.data;
|
|
679
|
+
} catch (error) {
|
|
680
|
+
if (error?.isAxiosError) {
|
|
681
|
+
return normalizeAxiosError(error, "search developer sources");
|
|
682
|
+
}
|
|
683
|
+
throw error;
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
657
687
|
// src/v2/methods/map.ts
|
|
658
688
|
function prepareMapPayload(url, options) {
|
|
659
689
|
if (!url || !url.trim()) throw new Error("URL cannot be empty");
|
|
@@ -1946,6 +1976,15 @@ var FirecrawlClient = class {
|
|
|
1946
1976
|
async search(query, req = {}) {
|
|
1947
1977
|
return search(this.http, { query, ...req });
|
|
1948
1978
|
}
|
|
1979
|
+
/**
|
|
1980
|
+
* Search the dedicated developer index with repository, documentation,
|
|
1981
|
+
* artifact, language, topic, license, star, archive, and fork filters.
|
|
1982
|
+
* Unlike `search(query, { categories: ["developer"] })`, this returns all
|
|
1983
|
+
* requested passages, citations, license disclosures, and indexing echoes.
|
|
1984
|
+
*/
|
|
1985
|
+
async developerSearch(query, options = {}) {
|
|
1986
|
+
return developerSearch(this.http, query, options);
|
|
1987
|
+
}
|
|
1949
1988
|
/**
|
|
1950
1989
|
* Submit feedback for a v2 job.
|
|
1951
1990
|
* @param request Feedback payload with endpoint, job id, rating, and supporting signals.
|
|
@@ -2342,7 +2381,7 @@ var FirecrawlApp = class {
|
|
|
2342
2381
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2343
2382
|
return process.env.npm_package_version;
|
|
2344
2383
|
}
|
|
2345
|
-
const packageJson = await import("./package-
|
|
2384
|
+
const packageJson = await import("./package-ZOVFJE7Z.js");
|
|
2346
2385
|
return packageJson.default.version;
|
|
2347
2386
|
} catch (error) {
|
|
2348
2387
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.35.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { describe, expect, jest, test } from "@jest/globals";
|
|
2
|
+
import { FirecrawlClient } from "../../../v2/client";
|
|
3
|
+
import { SdkError } from "../../../v2/types";
|
|
4
|
+
|
|
5
|
+
const response = {
|
|
6
|
+
success: true,
|
|
7
|
+
results: [
|
|
8
|
+
{
|
|
9
|
+
id: "issue:firecrawl/firecrawl#1",
|
|
10
|
+
url: "https://github.com/firecrawl/firecrawl/issues/1",
|
|
11
|
+
passages: [
|
|
12
|
+
{
|
|
13
|
+
text: "evidence",
|
|
14
|
+
citation_url:
|
|
15
|
+
"https://github.com/firecrawl/firecrawl/issues/1#issuecomment-1",
|
|
16
|
+
},
|
|
17
|
+
],
|
|
18
|
+
license: { state: "licensed" as const, spdx_id: "MIT" },
|
|
19
|
+
},
|
|
20
|
+
],
|
|
21
|
+
repos: [
|
|
22
|
+
{
|
|
23
|
+
repo: "firecrawl/firecrawl",
|
|
24
|
+
indexed: true,
|
|
25
|
+
types: { issue: true, pullRequest: true, readme: true },
|
|
26
|
+
},
|
|
27
|
+
],
|
|
28
|
+
sources: [{ source: "firecrawl", indexed: false }],
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
describe("developerSearch", () => {
|
|
32
|
+
test("forwards every dedicated developer filter", async () => {
|
|
33
|
+
const http = {
|
|
34
|
+
post: jest.fn(async () => ({ status: 200, data: response })),
|
|
35
|
+
} as any;
|
|
36
|
+
|
|
37
|
+
const client = new FirecrawlClient({
|
|
38
|
+
apiKey: "test",
|
|
39
|
+
apiUrl: "http://localhost",
|
|
40
|
+
});
|
|
41
|
+
(client as any).http = http;
|
|
42
|
+
|
|
43
|
+
const result = await client.developerSearch("retry configuration", {
|
|
44
|
+
k: 20,
|
|
45
|
+
passages: 5,
|
|
46
|
+
types: ["doc", "issue", "pull_request", "readme"],
|
|
47
|
+
repos: ["firecrawl/firecrawl"],
|
|
48
|
+
sources: ["firecrawl"],
|
|
49
|
+
language: "TypeScript",
|
|
50
|
+
topic: ["web-scraping"],
|
|
51
|
+
license: "MIT",
|
|
52
|
+
minStars: 100,
|
|
53
|
+
maxStars: 50000,
|
|
54
|
+
archived: false,
|
|
55
|
+
fork: true,
|
|
56
|
+
skills: "only",
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
expect(http.post).toHaveBeenCalledWith("/v2/search/developer", {
|
|
60
|
+
query: "retry configuration",
|
|
61
|
+
k: 20,
|
|
62
|
+
passages: 5,
|
|
63
|
+
types: ["doc", "issue", "pull_request", "readme"],
|
|
64
|
+
repos: ["firecrawl/firecrawl"],
|
|
65
|
+
sources: ["firecrawl"],
|
|
66
|
+
language: "TypeScript",
|
|
67
|
+
topic: ["web-scraping"],
|
|
68
|
+
license: "MIT",
|
|
69
|
+
min_stars: 100,
|
|
70
|
+
max_stars: 50000,
|
|
71
|
+
archived: false,
|
|
72
|
+
fork: true,
|
|
73
|
+
skills: "only",
|
|
74
|
+
});
|
|
75
|
+
expect(result).toEqual(response);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test("normalizes transport errors to SdkError", async () => {
|
|
79
|
+
const http = {
|
|
80
|
+
post: jest.fn(async () => {
|
|
81
|
+
throw {
|
|
82
|
+
isAxiosError: true,
|
|
83
|
+
code: "ECONNABORTED",
|
|
84
|
+
message: "request timed out",
|
|
85
|
+
};
|
|
86
|
+
}),
|
|
87
|
+
} as any;
|
|
88
|
+
const client = new FirecrawlClient({
|
|
89
|
+
apiKey: "test",
|
|
90
|
+
apiUrl: "http://localhost",
|
|
91
|
+
});
|
|
92
|
+
(client as any).http = http;
|
|
93
|
+
|
|
94
|
+
await expect(
|
|
95
|
+
client.developerSearch("network failure"),
|
|
96
|
+
).rejects.toBeInstanceOf(SdkError);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("accepts the flattened SPDX license response shape", async () => {
|
|
100
|
+
const flattened = {
|
|
101
|
+
...response,
|
|
102
|
+
results: [{ ...response.results[0], license: "MIT" }],
|
|
103
|
+
};
|
|
104
|
+
const http = {
|
|
105
|
+
post: jest.fn(async () => ({ status: 200, data: flattened })),
|
|
106
|
+
} as any;
|
|
107
|
+
|
|
108
|
+
const client = new FirecrawlClient({
|
|
109
|
+
apiKey: "test",
|
|
110
|
+
apiUrl: "http://localhost",
|
|
111
|
+
});
|
|
112
|
+
(client as any).http = http;
|
|
113
|
+
|
|
114
|
+
await expect(client.developerSearch("license shape")).resolves.toEqual(
|
|
115
|
+
flattened,
|
|
116
|
+
);
|
|
117
|
+
});
|
|
118
|
+
});
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { describe, expect, jest, test } from "@jest/globals";
|
|
2
2
|
import { search } from "../../../v2/methods/search";
|
|
3
|
+
import type { SearchResultWeb } from "../../../v2/types";
|
|
3
4
|
|
|
4
|
-
|
|
5
|
-
|
|
5
|
+
// Developer results arrive in the exact web schema.
|
|
6
|
+
const developerResult: SearchResultWeb = {
|
|
7
|
+
title: "Retry requests",
|
|
6
8
|
url: "https://github.com/firecrawl/firecrawl/issues/1",
|
|
7
9
|
description: "passage",
|
|
8
10
|
position: 1,
|
|
@@ -28,38 +30,26 @@ describe("v2 search developer category", () => {
|
|
|
28
30
|
);
|
|
29
31
|
});
|
|
30
32
|
|
|
31
|
-
test("returns developer results", async () => {
|
|
32
|
-
const http = httpWith({
|
|
33
|
-
web: [{ title: "W", url: "http://a.com" }],
|
|
34
|
-
developer: [developerResult],
|
|
35
|
-
});
|
|
33
|
+
test("returns developer results inside web", async () => {
|
|
34
|
+
const http = httpWith({ web: [developerResult] });
|
|
36
35
|
|
|
37
36
|
const result = await search(http, {
|
|
38
37
|
query: "firecrawl",
|
|
39
38
|
categories: [{ type: "developer" }],
|
|
40
39
|
});
|
|
41
40
|
|
|
42
|
-
expect(result.web).
|
|
43
|
-
expect(result
|
|
44
|
-
expect(result.developer?.[0]).toEqual(developerResult);
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
test("omits developer when the response has none", async () => {
|
|
48
|
-
const http = httpWith({ web: [] });
|
|
49
|
-
|
|
50
|
-
const result = await search(http, { query: "firecrawl" });
|
|
51
|
-
|
|
52
|
-
expect(result.developer).toBeUndefined();
|
|
41
|
+
expect(result.web).toEqual([developerResult]);
|
|
42
|
+
expect(result).not.toHaveProperty("developer");
|
|
53
43
|
});
|
|
54
44
|
|
|
55
|
-
test("lists developer in the .data error", async () => {
|
|
56
|
-
const http = httpWith({
|
|
45
|
+
test("lists developer results as web results in the .data error", async () => {
|
|
46
|
+
const http = httpWith({ web: [developerResult] });
|
|
57
47
|
|
|
58
48
|
const result = await search(http, {
|
|
59
49
|
query: "firecrawl",
|
|
60
50
|
categories: ["developer"],
|
|
61
51
|
});
|
|
62
52
|
|
|
63
|
-
expect(() => (result as any).data).toThrow(/\.
|
|
53
|
+
expect(() => (result as any).data).toThrow(/\.web \(1 results\)/);
|
|
64
54
|
});
|
|
65
55
|
});
|
|
@@ -89,9 +89,9 @@ describe("v2 utils: validation", () => {
|
|
|
89
89
|
expect(options.parsers).toEqual(before);
|
|
90
90
|
});
|
|
91
91
|
|
|
92
|
-
test("ensureValidScrapeOptions: leaves PDF parser pages and
|
|
92
|
+
test("ensureValidScrapeOptions: leaves PDF parser pages, blocks, and pageMarkers options untouched", () => {
|
|
93
93
|
const options = {
|
|
94
|
-
parsers: [{ type: "pdf", mode: "auto", blocks: true, pages: true }],
|
|
94
|
+
parsers: [{ type: "pdf", mode: "auto", blocks: true, pages: true, pageMarkers: true }],
|
|
95
95
|
} as any;
|
|
96
96
|
expect(() => ensureValidScrapeOptions(options)).not.toThrow();
|
|
97
97
|
expect(options.parsers[0]).toEqual({
|
|
@@ -99,6 +99,7 @@ describe("v2 utils: validation", () => {
|
|
|
99
99
|
mode: "auto",
|
|
100
100
|
blocks: true,
|
|
101
101
|
pages: true,
|
|
102
|
+
pageMarkers: true,
|
|
102
103
|
});
|
|
103
104
|
});
|
|
104
105
|
|
package/src/v2/client.ts
CHANGED
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
} from "./methods/scrape";
|
|
7
7
|
import { parse as parseMethod } from "./methods/parse";
|
|
8
8
|
import { search } from "./methods/search";
|
|
9
|
+
import { developerSearch as developerSearchMethod } from "./methods/developer";
|
|
9
10
|
import { map as mapMethod } from "./methods/map";
|
|
10
11
|
import { feedback as feedbackMethod, searchFeedback as searchFeedbackMethod } from "./methods/feedback";
|
|
11
12
|
import {
|
|
@@ -51,6 +52,8 @@ import type {
|
|
|
51
52
|
ScrapeOptions,
|
|
52
53
|
SearchData,
|
|
53
54
|
SearchRequest,
|
|
55
|
+
DeveloperSearchOptions,
|
|
56
|
+
DeveloperSearchResponse,
|
|
54
57
|
EndpointFeedbackRequest,
|
|
55
58
|
FeedbackResponse,
|
|
56
59
|
SearchFeedbackRequest,
|
|
@@ -240,6 +243,19 @@ export class FirecrawlClient {
|
|
|
240
243
|
return search(this.http, { query, ...req });
|
|
241
244
|
}
|
|
242
245
|
|
|
246
|
+
/**
|
|
247
|
+
* Search the dedicated developer index with repository, documentation,
|
|
248
|
+
* artifact, language, topic, license, star, archive, and fork filters.
|
|
249
|
+
* Unlike `search(query, { categories: ["developer"] })`, this returns all
|
|
250
|
+
* requested passages, citations, license disclosures, and indexing echoes.
|
|
251
|
+
*/
|
|
252
|
+
async developerSearch(
|
|
253
|
+
query: string,
|
|
254
|
+
options: DeveloperSearchOptions = {},
|
|
255
|
+
): Promise<DeveloperSearchResponse> {
|
|
256
|
+
return developerSearchMethod(this.http, query, options);
|
|
257
|
+
}
|
|
258
|
+
|
|
243
259
|
/**
|
|
244
260
|
* Submit feedback for a v2 job.
|
|
245
261
|
* @param request Feedback payload with endpoint, job id, rating, and supporting signals.
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { DeveloperSearchOptions, DeveloperSearchResponse } from "../types";
|
|
2
|
+
import { HttpClient } from "../utils/httpClient";
|
|
3
|
+
import {
|
|
4
|
+
normalizeAxiosError,
|
|
5
|
+
throwForBadResponse,
|
|
6
|
+
} from "../utils/errorHandler";
|
|
7
|
+
|
|
8
|
+
const ENDPOINT = "/v2/search/developer";
|
|
9
|
+
|
|
10
|
+
/** Search the dedicated developer index with its complete filter surface. */
|
|
11
|
+
export async function developerSearch(
|
|
12
|
+
http: HttpClient,
|
|
13
|
+
query: string,
|
|
14
|
+
options: DeveloperSearchOptions = {},
|
|
15
|
+
): Promise<DeveloperSearchResponse> {
|
|
16
|
+
if (!query || !query.trim()) throw new Error("query cannot be empty");
|
|
17
|
+
|
|
18
|
+
try {
|
|
19
|
+
const response = await http.post<DeveloperSearchResponse>(ENDPOINT, {
|
|
20
|
+
query,
|
|
21
|
+
...(options.k !== undefined ? { k: options.k } : {}),
|
|
22
|
+
...(options.passages !== undefined ? { passages: options.passages } : {}),
|
|
23
|
+
...(options.types !== undefined ? { types: options.types } : {}),
|
|
24
|
+
...(options.repos !== undefined ? { repos: options.repos } : {}),
|
|
25
|
+
...(options.sources !== undefined ? { sources: options.sources } : {}),
|
|
26
|
+
...(options.language !== undefined ? { language: options.language } : {}),
|
|
27
|
+
...(options.topic !== undefined ? { topic: options.topic } : {}),
|
|
28
|
+
...(options.license !== undefined ? { license: options.license } : {}),
|
|
29
|
+
...(options.minStars !== undefined
|
|
30
|
+
? { min_stars: options.minStars }
|
|
31
|
+
: {}),
|
|
32
|
+
...(options.maxStars !== undefined
|
|
33
|
+
? { max_stars: options.maxStars }
|
|
34
|
+
: {}),
|
|
35
|
+
...(options.archived !== undefined ? { archived: options.archived } : {}),
|
|
36
|
+
...(options.fork !== undefined ? { fork: options.fork } : {}),
|
|
37
|
+
...(options.skills !== undefined ? { skills: options.skills } : {}),
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
if (response.status !== 200 || !response.data?.success) {
|
|
41
|
+
throwForBadResponse(response, "search developer sources");
|
|
42
|
+
}
|
|
43
|
+
return response.data;
|
|
44
|
+
} catch (error: any) {
|
|
45
|
+
if (error?.isAxiosError) {
|
|
46
|
+
return normalizeAxiosError(error, "search developer sources");
|
|
47
|
+
}
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
}
|
package/src/v2/methods/search.ts
CHANGED
|
@@ -102,17 +102,16 @@ export async function search(
|
|
|
102
102
|
if (data.news) out.news = transformArray<SearchResultNews>(data.news);
|
|
103
103
|
if (data.images)
|
|
104
104
|
out.images = transformArray<SearchResultImages>(data.images);
|
|
105
|
-
if (data.developer)
|
|
106
|
-
out.developer = transformArray<SearchResultWeb>(data.developer);
|
|
107
105
|
Object.defineProperty(out, "data", {
|
|
108
106
|
get() {
|
|
109
107
|
const parts: string[] = [];
|
|
110
108
|
if (out.web?.length) parts.push(`.web (${out.web.length} results)`);
|
|
111
109
|
if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
|
|
112
|
-
if (out.images?.length)
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
110
|
+
if (out.images?.length)
|
|
111
|
+
parts.push(`.images (${out.images.length} results)`);
|
|
112
|
+
const available = parts.length
|
|
113
|
+
? parts.join(", ")
|
|
114
|
+
: ".web, .news, or .images";
|
|
116
115
|
throw new Error(
|
|
117
116
|
`SearchData has no '.data'. Results are grouped by source: ${available}`,
|
|
118
117
|
);
|
package/src/v2/types.ts
CHANGED
|
@@ -199,6 +199,14 @@ export type PDFParser = {
|
|
|
199
199
|
pageMarkdown?: boolean;
|
|
200
200
|
/** Include per-page typed layout blocks (bounding boxes, block types, reading order). */
|
|
201
201
|
blocks?: boolean;
|
|
202
|
+
/**
|
|
203
|
+
* Join PDF pages in `document.markdown` with `\n\n---\n\n<!-- page N -->\n\n`,
|
|
204
|
+
* where N is the 1-based physical page of the content that follows. Markers
|
|
205
|
+
* appear between pages only (no leading marker for page 1), and numbering may
|
|
206
|
+
* skip pages merged by cross-page stitching — use `pages: true` when every
|
|
207
|
+
* physical page is needed. No new response field.
|
|
208
|
+
*/
|
|
209
|
+
pageMarkers?: boolean;
|
|
202
210
|
};
|
|
203
211
|
|
|
204
212
|
export interface PdfPage {
|
|
@@ -707,6 +715,76 @@ export interface PaginationConfig {
|
|
|
707
715
|
maxWaitTime?: number;
|
|
708
716
|
}
|
|
709
717
|
|
|
718
|
+
export type DeveloperSearchType = "doc" | "issue" | "pull_request" | "readme";
|
|
719
|
+
|
|
720
|
+
export interface DeveloperSearchOptions {
|
|
721
|
+
/** Total ranked results, 1–100 (default 10). */
|
|
722
|
+
k?: number;
|
|
723
|
+
/** Passages per result, 1–5 (default 1). */
|
|
724
|
+
passages?: number;
|
|
725
|
+
/** Result kinds to search (default all). */
|
|
726
|
+
types?: DeveloperSearchType[];
|
|
727
|
+
/** GitHub owner/name filters, at most 20. */
|
|
728
|
+
repos?: string[];
|
|
729
|
+
/** Developer documentation source IDs, at most 20. */
|
|
730
|
+
sources?: string[];
|
|
731
|
+
/** One case-insensitive GitHub Linguist primary language. */
|
|
732
|
+
language?: string;
|
|
733
|
+
/** Repository topics that must all match, at most 8. */
|
|
734
|
+
topic?: string[];
|
|
735
|
+
/** One case-insensitive SPDX repository license identifier. */
|
|
736
|
+
license?: string;
|
|
737
|
+
/** Minimum repository stars. */
|
|
738
|
+
minStars?: number;
|
|
739
|
+
/** Maximum repository stars. */
|
|
740
|
+
maxStars?: number;
|
|
741
|
+
/** Filter by repository archived status. */
|
|
742
|
+
archived?: boolean;
|
|
743
|
+
/** Filter by repository fork status. */
|
|
744
|
+
fork?: boolean;
|
|
745
|
+
/** Return packaged agent-skill evidence only. */
|
|
746
|
+
skills?: "only";
|
|
747
|
+
}
|
|
748
|
+
|
|
749
|
+
export interface DeveloperSearchLicenseDisclosure {
|
|
750
|
+
state: "licensed" | "known_absent" | "unknown";
|
|
751
|
+
spdx_id: string | null;
|
|
752
|
+
}
|
|
753
|
+
|
|
754
|
+
export interface DeveloperSearchPassage {
|
|
755
|
+
text: string;
|
|
756
|
+
citation_url?: string;
|
|
757
|
+
}
|
|
758
|
+
|
|
759
|
+
export interface DeveloperSearchResult {
|
|
760
|
+
id: string;
|
|
761
|
+
url: string;
|
|
762
|
+
title?: string;
|
|
763
|
+
passages: DeveloperSearchPassage[];
|
|
764
|
+
// Accept both shapes while the API flattens license objects to SPDX strings.
|
|
765
|
+
license?: DeveloperSearchLicenseDisclosure | string;
|
|
766
|
+
}
|
|
767
|
+
|
|
768
|
+
export interface DeveloperSearchRepoStatus {
|
|
769
|
+
repo: string;
|
|
770
|
+
indexed: boolean;
|
|
771
|
+
types: { issue: boolean; pullRequest: boolean; readme: boolean };
|
|
772
|
+
}
|
|
773
|
+
|
|
774
|
+
export interface DeveloperSearchSourceStatus {
|
|
775
|
+
source: string;
|
|
776
|
+
indexed: boolean;
|
|
777
|
+
}
|
|
778
|
+
|
|
779
|
+
export interface DeveloperSearchResponse {
|
|
780
|
+
success: boolean;
|
|
781
|
+
results: DeveloperSearchResult[];
|
|
782
|
+
/** Present only when repos were requested. */
|
|
783
|
+
repos?: DeveloperSearchRepoStatus[];
|
|
784
|
+
/** Present only when sources were requested. */
|
|
785
|
+
sources?: DeveloperSearchSourceStatus[];
|
|
786
|
+
}
|
|
787
|
+
|
|
710
788
|
export interface SearchResultWeb {
|
|
711
789
|
url: string;
|
|
712
790
|
title?: string;
|
|
@@ -738,7 +816,6 @@ export interface SearchData {
|
|
|
738
816
|
web?: Array<SearchResultWeb | Document>;
|
|
739
817
|
news?: Array<SearchResultNews | Document>;
|
|
740
818
|
images?: Array<SearchResultImages | Document>;
|
|
741
|
-
developer?: Array<SearchResultWeb | Document>;
|
|
742
819
|
}
|
|
743
820
|
|
|
744
821
|
/**
|
|
@@ -751,8 +828,8 @@ export interface SearchData {
|
|
|
751
828
|
* ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). It returns
|
|
752
829
|
* ordinary web page results from those domains, **not** paper records.
|
|
753
830
|
* - `pdf` — restrict results to PDFs (adds `filetype:pdf`).
|
|
754
|
-
* - `developer` —
|
|
755
|
-
* documentation)
|
|
831
|
+
* - `developer` — developer-index results (issues, pull requests, READMEs and
|
|
832
|
+
* documentation) served in `web`; cannot be combined with other categories.
|
|
756
833
|
*
|
|
757
834
|
* ⚠️ `categories: ["research"]` is **not** Firecrawl's research paper index.
|
|
758
835
|
* To search papers themselves — ~43M abstracts, roughly 90% biomedical
|