firecrawl 4.34.1 → 4.34.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +32 -1
- package/dist/{chunk-DWBMD5N5.js → chunk-PFIWJELS.js} +1 -1
- package/dist/index.cjs +45 -6
- package/dist/index.d.cts +75 -4
- package/dist/index.d.ts +75 -4
- package/dist/index.js +46 -7
- package/dist/{package-XAWX5C5Y.js → package-H4IIQQQK.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/developer.test.ts +118 -0
- package/src/__tests__/unit/v2/search-developer.unit.test.ts +11 -21
- package/src/v2/client.ts +16 -0
- package/src/v2/methods/developer.ts +50 -0
- package/src/v2/methods/search.ts +5 -6
- package/src/v2/types.ts +72 -3
package/README.md
CHANGED
|
@@ -183,7 +183,8 @@ const scraped = await app.search('firecrawl changelog', {
|
|
|
183
183
|
});
|
|
184
184
|
```
|
|
185
185
|
|
|
186
|
-
Results are grouped by source: `.web`, `.news
|
|
186
|
+
Results are grouped by source: `.web`, `.news` and `.images`. Developer
|
|
187
|
+
category results are served inside `.web`.
|
|
187
188
|
|
|
188
189
|
Use `categories` to narrow web search to a kind of site:
|
|
189
190
|
|
|
@@ -198,6 +199,36 @@ const results = await app.search('nanopore basecalling accuracy', {
|
|
|
198
199
|
> pubmed.ncbi.nlm.nih.gov, nature.com, biorxiv.org, ...) and returns web page
|
|
199
200
|
> results. To search papers themselves, use `research.searchPapers` below.
|
|
200
201
|
|
|
202
|
+
### Developer search
|
|
203
|
+
|
|
204
|
+
Use `developerSearch` for the dedicated developer index and its complete filter
|
|
205
|
+
and response contract. Generic `search(..., { categories: ['developer'] })`
|
|
206
|
+
returns developer results inside `.web` in the ordinary web-result shape
|
|
207
|
+
(first passage as the description) and does not accept these filters.
|
|
208
|
+
|
|
209
|
+
```js
|
|
210
|
+
const evidence = await app.developerSearch('configure retry backoff', {
|
|
211
|
+
repos: ['firecrawl/firecrawl'],
|
|
212
|
+
types: ['issue', 'pull_request', 'readme'],
|
|
213
|
+
passages: 3,
|
|
214
|
+
language: 'TypeScript',
|
|
215
|
+
license: 'MIT',
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
for (const result of evidence.results) {
|
|
219
|
+
// Result kind is the id prefix; the API intentionally omits a type field.
|
|
220
|
+
console.log(result.id, result.license);
|
|
221
|
+
for (const passage of result.passages) {
|
|
222
|
+
console.log(passage.text, passage.citation_url);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
console.log(evidence.repos); // indexed-status echoes for requested repos
|
|
226
|
+
```
|
|
227
|
+
|
|
228
|
+
`developerSearch` also supports `sources`, `topic`, `minStars`, `maxStars`,
|
|
229
|
+
`archived`, `fork`, and `skills: 'only'`. Supplying both `repos` and `sources`
|
|
230
|
+
OR-combines GitHub-backed and documentation results.
|
|
231
|
+
|
|
201
232
|
### Research / paper search
|
|
202
233
|
|
|
203
234
|
Use `app.research` to search Firecrawl's research paper index: ~43M paper
|
|
@@ -12,7 +12,7 @@ var require_package = __commonJS({
|
|
|
12
12
|
"package.json"(exports, module) {
|
|
13
13
|
module.exports = {
|
|
14
14
|
name: "@mendable/firecrawl-js",
|
|
15
|
-
version: "4.34.
|
|
15
|
+
version: "4.34.2",
|
|
16
16
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
17
17
|
main: "dist/index.js",
|
|
18
18
|
types: "dist/index.d.ts",
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.34.
|
|
42
|
+
version: "4.34.2",
|
|
43
43
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -760,16 +760,13 @@ async function search(http, request) {
|
|
|
760
760
|
if (data.news) out.news = transformArray(data.news);
|
|
761
761
|
if (data.images)
|
|
762
762
|
out.images = transformArray(data.images);
|
|
763
|
-
if (data.developer)
|
|
764
|
-
out.developer = transformArray(data.developer);
|
|
765
763
|
Object.defineProperty(out, "data", {
|
|
766
764
|
get() {
|
|
767
765
|
const parts = [];
|
|
768
766
|
if (out.web?.length) parts.push(`.web (${out.web.length} results)`);
|
|
769
767
|
if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
|
|
770
|
-
if (out.images?.length)
|
|
771
|
-
|
|
772
|
-
parts.push(`.developer (${out.developer.length} results)`);
|
|
768
|
+
if (out.images?.length)
|
|
769
|
+
parts.push(`.images (${out.images.length} results)`);
|
|
773
770
|
const available = parts.length ? parts.join(", ") : ".web, .news, or .images";
|
|
774
771
|
throw new Error(
|
|
775
772
|
`SearchData has no '.data'. Results are grouped by source: ${available}`
|
|
@@ -785,6 +782,39 @@ async function search(http, request) {
|
|
|
785
782
|
}
|
|
786
783
|
}
|
|
787
784
|
|
|
785
|
+
// src/v2/methods/developer.ts
|
|
786
|
+
var ENDPOINT = "/v2/search/developer";
|
|
787
|
+
async function developerSearch(http, query, options = {}) {
|
|
788
|
+
if (!query || !query.trim()) throw new Error("query cannot be empty");
|
|
789
|
+
try {
|
|
790
|
+
const response = await http.post(ENDPOINT, {
|
|
791
|
+
query,
|
|
792
|
+
...options.k !== void 0 ? { k: options.k } : {},
|
|
793
|
+
...options.passages !== void 0 ? { passages: options.passages } : {},
|
|
794
|
+
...options.types !== void 0 ? { types: options.types } : {},
|
|
795
|
+
...options.repos !== void 0 ? { repos: options.repos } : {},
|
|
796
|
+
...options.sources !== void 0 ? { sources: options.sources } : {},
|
|
797
|
+
...options.language !== void 0 ? { language: options.language } : {},
|
|
798
|
+
...options.topic !== void 0 ? { topic: options.topic } : {},
|
|
799
|
+
...options.license !== void 0 ? { license: options.license } : {},
|
|
800
|
+
...options.minStars !== void 0 ? { min_stars: options.minStars } : {},
|
|
801
|
+
...options.maxStars !== void 0 ? { max_stars: options.maxStars } : {},
|
|
802
|
+
...options.archived !== void 0 ? { archived: options.archived } : {},
|
|
803
|
+
...options.fork !== void 0 ? { fork: options.fork } : {},
|
|
804
|
+
...options.skills !== void 0 ? { skills: options.skills } : {}
|
|
805
|
+
});
|
|
806
|
+
if (response.status !== 200 || !response.data?.success) {
|
|
807
|
+
throwForBadResponse(response, "search developer sources");
|
|
808
|
+
}
|
|
809
|
+
return response.data;
|
|
810
|
+
} catch (error) {
|
|
811
|
+
if (error?.isAxiosError) {
|
|
812
|
+
return normalizeAxiosError(error, "search developer sources");
|
|
813
|
+
}
|
|
814
|
+
throw error;
|
|
815
|
+
}
|
|
816
|
+
}
|
|
817
|
+
|
|
788
818
|
// src/v2/methods/map.ts
|
|
789
819
|
function prepareMapPayload(url, options) {
|
|
790
820
|
if (!url || !url.trim()) throw new Error("URL cannot be empty");
|
|
@@ -2077,6 +2107,15 @@ var FirecrawlClient = class {
|
|
|
2077
2107
|
async search(query, req = {}) {
|
|
2078
2108
|
return search(this.http, { query, ...req });
|
|
2079
2109
|
}
|
|
2110
|
+
/**
|
|
2111
|
+
* Search the dedicated developer index with repository, documentation,
|
|
2112
|
+
* artifact, language, topic, license, star, archive, and fork filters.
|
|
2113
|
+
* Unlike `search(query, { categories: ["developer"] })`, this returns all
|
|
2114
|
+
* requested passages, citations, license disclosures, and indexing echoes.
|
|
2115
|
+
*/
|
|
2116
|
+
async developerSearch(query, options = {}) {
|
|
2117
|
+
return developerSearch(this.http, query, options);
|
|
2118
|
+
}
|
|
2080
2119
|
/**
|
|
2081
2120
|
* Submit feedback for a v2 job.
|
|
2082
2121
|
* @param request Feedback payload with endpoint, job id, rating, and supporting signals.
|
package/dist/index.d.cts
CHANGED
|
@@ -551,6 +551,71 @@ interface PaginationConfig {
|
|
|
551
551
|
/** Maximum time to spend fetching additional pages (in seconds). */
|
|
552
552
|
maxWaitTime?: number;
|
|
553
553
|
}
|
|
554
|
+
type DeveloperSearchType = "doc" | "issue" | "pull_request" | "readme";
|
|
555
|
+
interface DeveloperSearchOptions {
|
|
556
|
+
/** Total ranked results, 1–100 (default 10). */
|
|
557
|
+
k?: number;
|
|
558
|
+
/** Passages per result, 1–5 (default 1). */
|
|
559
|
+
passages?: number;
|
|
560
|
+
/** Result kinds to search (default all). */
|
|
561
|
+
types?: DeveloperSearchType[];
|
|
562
|
+
/** GitHub owner/name filters, at most 20. */
|
|
563
|
+
repos?: string[];
|
|
564
|
+
/** Developer documentation source IDs, at most 20. */
|
|
565
|
+
sources?: string[];
|
|
566
|
+
/** One case-insensitive GitHub Linguist primary language. */
|
|
567
|
+
language?: string;
|
|
568
|
+
/** Repository topics that must all match, at most 8. */
|
|
569
|
+
topic?: string[];
|
|
570
|
+
/** One case-insensitive SPDX repository license identifier. */
|
|
571
|
+
license?: string;
|
|
572
|
+
/** Minimum repository stars. */
|
|
573
|
+
minStars?: number;
|
|
574
|
+
/** Maximum repository stars. */
|
|
575
|
+
maxStars?: number;
|
|
576
|
+
/** Filter by repository archived status. */
|
|
577
|
+
archived?: boolean;
|
|
578
|
+
/** Filter by repository fork status. */
|
|
579
|
+
fork?: boolean;
|
|
580
|
+
/** Return packaged agent-skill evidence only. */
|
|
581
|
+
skills?: "only";
|
|
582
|
+
}
|
|
583
|
+
interface DeveloperSearchLicenseDisclosure {
|
|
584
|
+
state: "licensed" | "known_absent" | "unknown";
|
|
585
|
+
spdx_id: string | null;
|
|
586
|
+
}
|
|
587
|
+
interface DeveloperSearchPassage {
|
|
588
|
+
text: string;
|
|
589
|
+
citation_url?: string;
|
|
590
|
+
}
|
|
591
|
+
interface DeveloperSearchResult {
|
|
592
|
+
id: string;
|
|
593
|
+
url: string;
|
|
594
|
+
title?: string;
|
|
595
|
+
passages: DeveloperSearchPassage[];
|
|
596
|
+
license?: DeveloperSearchLicenseDisclosure | string;
|
|
597
|
+
}
|
|
598
|
+
interface DeveloperSearchRepoStatus {
|
|
599
|
+
repo: string;
|
|
600
|
+
indexed: boolean;
|
|
601
|
+
types: {
|
|
602
|
+
issue: boolean;
|
|
603
|
+
pullRequest: boolean;
|
|
604
|
+
readme: boolean;
|
|
605
|
+
};
|
|
606
|
+
}
|
|
607
|
+
interface DeveloperSearchSourceStatus {
|
|
608
|
+
source: string;
|
|
609
|
+
indexed: boolean;
|
|
610
|
+
}
|
|
611
|
+
interface DeveloperSearchResponse {
|
|
612
|
+
success: boolean;
|
|
613
|
+
results: DeveloperSearchResult[];
|
|
614
|
+
/** Present only when repos were requested. */
|
|
615
|
+
repos?: DeveloperSearchRepoStatus[];
|
|
616
|
+
/** Present only when sources were requested. */
|
|
617
|
+
sources?: DeveloperSearchSourceStatus[];
|
|
618
|
+
}
|
|
554
619
|
interface SearchResultWeb {
|
|
555
620
|
url: string;
|
|
556
621
|
title?: string;
|
|
@@ -579,7 +644,6 @@ interface SearchData {
|
|
|
579
644
|
web?: Array<SearchResultWeb | Document>;
|
|
580
645
|
news?: Array<SearchResultNews | Document>;
|
|
581
646
|
images?: Array<SearchResultImages | Document>;
|
|
582
|
-
developer?: Array<SearchResultWeb | Document>;
|
|
583
647
|
}
|
|
584
648
|
/**
|
|
585
649
|
* Narrows ordinary **web search**. A category does not switch `search()` to a
|
|
@@ -591,8 +655,8 @@ interface SearchData {
|
|
|
591
655
|
* ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). It returns
|
|
592
656
|
* ordinary web page results from those domains, **not** paper records.
|
|
593
657
|
* - `pdf` — restrict results to PDFs (adds `filetype:pdf`).
|
|
594
|
-
* - `developer` —
|
|
595
|
-
* documentation)
|
|
658
|
+
* - `developer` — developer-index results (issues, pull requests, READMEs and
|
|
659
|
+
* documentation) served in `web`; cannot be combined with other categories.
|
|
596
660
|
*
|
|
597
661
|
* ⚠️ `categories: ["research"]` is **not** Firecrawl's research paper index.
|
|
598
662
|
* To search papers themselves — ~43M abstracts, roughly 90% biomedical
|
|
@@ -1609,6 +1673,13 @@ declare class FirecrawlClient {
|
|
|
1609
1673
|
* @returns Structured search results.
|
|
1610
1674
|
*/
|
|
1611
1675
|
search(query: string, req?: Omit<SearchRequest, "query">): Promise<SearchData>;
|
|
1676
|
+
/**
|
|
1677
|
+
* Search the dedicated developer index with repository, documentation,
|
|
1678
|
+
* artifact, language, topic, license, star, archive, and fork filters.
|
|
1679
|
+
* Unlike `search(query, { categories: ["developer"] })`, this returns all
|
|
1680
|
+
* requested passages, citations, license disclosures, and indexing echoes.
|
|
1681
|
+
*/
|
|
1682
|
+
developerSearch(query: string, options?: DeveloperSearchOptions): Promise<DeveloperSearchResponse>;
|
|
1612
1683
|
/**
|
|
1613
1684
|
* Submit feedback for a v2 job.
|
|
1614
1685
|
* @param request Feedback payload with endpoint, job id, rating, and supporting signals.
|
|
@@ -2806,4 +2877,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
2806
2877
|
get v1(): FirecrawlApp;
|
|
2807
2878
|
}
|
|
2808
2879
|
|
|
2809
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
|
2880
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
package/dist/index.d.ts
CHANGED
|
@@ -551,6 +551,71 @@ interface PaginationConfig {
|
|
|
551
551
|
/** Maximum time to spend fetching additional pages (in seconds). */
|
|
552
552
|
maxWaitTime?: number;
|
|
553
553
|
}
|
|
554
|
+
type DeveloperSearchType = "doc" | "issue" | "pull_request" | "readme";
|
|
555
|
+
interface DeveloperSearchOptions {
|
|
556
|
+
/** Total ranked results, 1–100 (default 10). */
|
|
557
|
+
k?: number;
|
|
558
|
+
/** Passages per result, 1–5 (default 1). */
|
|
559
|
+
passages?: number;
|
|
560
|
+
/** Result kinds to search (default all). */
|
|
561
|
+
types?: DeveloperSearchType[];
|
|
562
|
+
/** GitHub owner/name filters, at most 20. */
|
|
563
|
+
repos?: string[];
|
|
564
|
+
/** Developer documentation source IDs, at most 20. */
|
|
565
|
+
sources?: string[];
|
|
566
|
+
/** One case-insensitive GitHub Linguist primary language. */
|
|
567
|
+
language?: string;
|
|
568
|
+
/** Repository topics that must all match, at most 8. */
|
|
569
|
+
topic?: string[];
|
|
570
|
+
/** One case-insensitive SPDX repository license identifier. */
|
|
571
|
+
license?: string;
|
|
572
|
+
/** Minimum repository stars. */
|
|
573
|
+
minStars?: number;
|
|
574
|
+
/** Maximum repository stars. */
|
|
575
|
+
maxStars?: number;
|
|
576
|
+
/** Filter by repository archived status. */
|
|
577
|
+
archived?: boolean;
|
|
578
|
+
/** Filter by repository fork status. */
|
|
579
|
+
fork?: boolean;
|
|
580
|
+
/** Return packaged agent-skill evidence only. */
|
|
581
|
+
skills?: "only";
|
|
582
|
+
}
|
|
583
|
+
interface DeveloperSearchLicenseDisclosure {
|
|
584
|
+
state: "licensed" | "known_absent" | "unknown";
|
|
585
|
+
spdx_id: string | null;
|
|
586
|
+
}
|
|
587
|
+
interface DeveloperSearchPassage {
|
|
588
|
+
text: string;
|
|
589
|
+
citation_url?: string;
|
|
590
|
+
}
|
|
591
|
+
interface DeveloperSearchResult {
|
|
592
|
+
id: string;
|
|
593
|
+
url: string;
|
|
594
|
+
title?: string;
|
|
595
|
+
passages: DeveloperSearchPassage[];
|
|
596
|
+
license?: DeveloperSearchLicenseDisclosure | string;
|
|
597
|
+
}
|
|
598
|
+
interface DeveloperSearchRepoStatus {
|
|
599
|
+
repo: string;
|
|
600
|
+
indexed: boolean;
|
|
601
|
+
types: {
|
|
602
|
+
issue: boolean;
|
|
603
|
+
pullRequest: boolean;
|
|
604
|
+
readme: boolean;
|
|
605
|
+
};
|
|
606
|
+
}
|
|
607
|
+
interface DeveloperSearchSourceStatus {
|
|
608
|
+
source: string;
|
|
609
|
+
indexed: boolean;
|
|
610
|
+
}
|
|
611
|
+
interface DeveloperSearchResponse {
|
|
612
|
+
success: boolean;
|
|
613
|
+
results: DeveloperSearchResult[];
|
|
614
|
+
/** Present only when repos were requested. */
|
|
615
|
+
repos?: DeveloperSearchRepoStatus[];
|
|
616
|
+
/** Present only when sources were requested. */
|
|
617
|
+
sources?: DeveloperSearchSourceStatus[];
|
|
618
|
+
}
|
|
554
619
|
interface SearchResultWeb {
|
|
555
620
|
url: string;
|
|
556
621
|
title?: string;
|
|
@@ -579,7 +644,6 @@ interface SearchData {
|
|
|
579
644
|
web?: Array<SearchResultWeb | Document>;
|
|
580
645
|
news?: Array<SearchResultNews | Document>;
|
|
581
646
|
images?: Array<SearchResultImages | Document>;
|
|
582
|
-
developer?: Array<SearchResultWeb | Document>;
|
|
583
647
|
}
|
|
584
648
|
/**
|
|
585
649
|
* Narrows ordinary **web search**. A category does not switch `search()` to a
|
|
@@ -591,8 +655,8 @@ interface SearchData {
|
|
|
591
655
|
* ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). It returns
|
|
592
656
|
* ordinary web page results from those domains, **not** paper records.
|
|
593
657
|
* - `pdf` — restrict results to PDFs (adds `filetype:pdf`).
|
|
594
|
-
* - `developer` —
|
|
595
|
-
* documentation)
|
|
658
|
+
* - `developer` — developer-index results (issues, pull requests, READMEs and
|
|
659
|
+
* documentation) served in `web`; cannot be combined with other categories.
|
|
596
660
|
*
|
|
597
661
|
* ⚠️ `categories: ["research"]` is **not** Firecrawl's research paper index.
|
|
598
662
|
* To search papers themselves — ~43M abstracts, roughly 90% biomedical
|
|
@@ -1609,6 +1673,13 @@ declare class FirecrawlClient {
|
|
|
1609
1673
|
* @returns Structured search results.
|
|
1610
1674
|
*/
|
|
1611
1675
|
search(query: string, req?: Omit<SearchRequest, "query">): Promise<SearchData>;
|
|
1676
|
+
/**
|
|
1677
|
+
* Search the dedicated developer index with repository, documentation,
|
|
1678
|
+
* artifact, language, topic, license, star, archive, and fork filters.
|
|
1679
|
+
* Unlike `search(query, { categories: ["developer"] })`, this returns all
|
|
1680
|
+
* requested passages, citations, license disclosures, and indexing echoes.
|
|
1681
|
+
*/
|
|
1682
|
+
developerSearch(query: string, options?: DeveloperSearchOptions): Promise<DeveloperSearchResponse>;
|
|
1612
1683
|
/**
|
|
1613
1684
|
* Submit feedback for a v2 job.
|
|
1614
1685
|
* @param request Feedback payload with endpoint, job id, rating, and supporting signals.
|
|
@@ -2806,4 +2877,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
2806
2877
|
get v1(): FirecrawlApp;
|
|
2807
2878
|
}
|
|
2808
2879
|
|
|
2809
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
|
2880
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-PFIWJELS.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -629,16 +629,13 @@ async function search(http, request) {
|
|
|
629
629
|
if (data.news) out.news = transformArray(data.news);
|
|
630
630
|
if (data.images)
|
|
631
631
|
out.images = transformArray(data.images);
|
|
632
|
-
if (data.developer)
|
|
633
|
-
out.developer = transformArray(data.developer);
|
|
634
632
|
Object.defineProperty(out, "data", {
|
|
635
633
|
get() {
|
|
636
634
|
const parts = [];
|
|
637
635
|
if (out.web?.length) parts.push(`.web (${out.web.length} results)`);
|
|
638
636
|
if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
|
|
639
|
-
if (out.images?.length)
|
|
640
|
-
|
|
641
|
-
parts.push(`.developer (${out.developer.length} results)`);
|
|
637
|
+
if (out.images?.length)
|
|
638
|
+
parts.push(`.images (${out.images.length} results)`);
|
|
642
639
|
const available = parts.length ? parts.join(", ") : ".web, .news, or .images";
|
|
643
640
|
throw new Error(
|
|
644
641
|
`SearchData has no '.data'. Results are grouped by source: ${available}`
|
|
@@ -654,6 +651,39 @@ async function search(http, request) {
|
|
|
654
651
|
}
|
|
655
652
|
}
|
|
656
653
|
|
|
654
|
+
// src/v2/methods/developer.ts
|
|
655
|
+
var ENDPOINT = "/v2/search/developer";
|
|
656
|
+
async function developerSearch(http, query, options = {}) {
|
|
657
|
+
if (!query || !query.trim()) throw new Error("query cannot be empty");
|
|
658
|
+
try {
|
|
659
|
+
const response = await http.post(ENDPOINT, {
|
|
660
|
+
query,
|
|
661
|
+
...options.k !== void 0 ? { k: options.k } : {},
|
|
662
|
+
...options.passages !== void 0 ? { passages: options.passages } : {},
|
|
663
|
+
...options.types !== void 0 ? { types: options.types } : {},
|
|
664
|
+
...options.repos !== void 0 ? { repos: options.repos } : {},
|
|
665
|
+
...options.sources !== void 0 ? { sources: options.sources } : {},
|
|
666
|
+
...options.language !== void 0 ? { language: options.language } : {},
|
|
667
|
+
...options.topic !== void 0 ? { topic: options.topic } : {},
|
|
668
|
+
...options.license !== void 0 ? { license: options.license } : {},
|
|
669
|
+
...options.minStars !== void 0 ? { min_stars: options.minStars } : {},
|
|
670
|
+
...options.maxStars !== void 0 ? { max_stars: options.maxStars } : {},
|
|
671
|
+
...options.archived !== void 0 ? { archived: options.archived } : {},
|
|
672
|
+
...options.fork !== void 0 ? { fork: options.fork } : {},
|
|
673
|
+
...options.skills !== void 0 ? { skills: options.skills } : {}
|
|
674
|
+
});
|
|
675
|
+
if (response.status !== 200 || !response.data?.success) {
|
|
676
|
+
throwForBadResponse(response, "search developer sources");
|
|
677
|
+
}
|
|
678
|
+
return response.data;
|
|
679
|
+
} catch (error) {
|
|
680
|
+
if (error?.isAxiosError) {
|
|
681
|
+
return normalizeAxiosError(error, "search developer sources");
|
|
682
|
+
}
|
|
683
|
+
throw error;
|
|
684
|
+
}
|
|
685
|
+
}
|
|
686
|
+
|
|
657
687
|
// src/v2/methods/map.ts
|
|
658
688
|
function prepareMapPayload(url, options) {
|
|
659
689
|
if (!url || !url.trim()) throw new Error("URL cannot be empty");
|
|
@@ -1946,6 +1976,15 @@ var FirecrawlClient = class {
|
|
|
1946
1976
|
async search(query, req = {}) {
|
|
1947
1977
|
return search(this.http, { query, ...req });
|
|
1948
1978
|
}
|
|
1979
|
+
/**
|
|
1980
|
+
* Search the dedicated developer index with repository, documentation,
|
|
1981
|
+
* artifact, language, topic, license, star, archive, and fork filters.
|
|
1982
|
+
* Unlike `search(query, { categories: ["developer"] })`, this returns all
|
|
1983
|
+
* requested passages, citations, license disclosures, and indexing echoes.
|
|
1984
|
+
*/
|
|
1985
|
+
async developerSearch(query, options = {}) {
|
|
1986
|
+
return developerSearch(this.http, query, options);
|
|
1987
|
+
}
|
|
1949
1988
|
/**
|
|
1950
1989
|
* Submit feedback for a v2 job.
|
|
1951
1990
|
* @param request Feedback payload with endpoint, job id, rating, and supporting signals.
|
|
@@ -2342,7 +2381,7 @@ var FirecrawlApp = class {
|
|
|
2342
2381
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2343
2382
|
return process.env.npm_package_version;
|
|
2344
2383
|
}
|
|
2345
|
-
const packageJson = await import("./package-
|
|
2384
|
+
const packageJson = await import("./package-H4IIQQQK.js");
|
|
2346
2385
|
return packageJson.default.version;
|
|
2347
2386
|
} catch (error) {
|
|
2348
2387
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.34.
|
|
3
|
+
"version": "4.34.2",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
import { describe, expect, jest, test } from "@jest/globals";
|
|
2
|
+
import { FirecrawlClient } from "../../../v2/client";
|
|
3
|
+
import { SdkError } from "../../../v2/types";
|
|
4
|
+
|
|
5
|
+
const response = {
|
|
6
|
+
success: true,
|
|
7
|
+
results: [
|
|
8
|
+
{
|
|
9
|
+
id: "issue:firecrawl/firecrawl#1",
|
|
10
|
+
url: "https://github.com/firecrawl/firecrawl/issues/1",
|
|
11
|
+
passages: [
|
|
12
|
+
{
|
|
13
|
+
text: "evidence",
|
|
14
|
+
citation_url:
|
|
15
|
+
"https://github.com/firecrawl/firecrawl/issues/1#issuecomment-1",
|
|
16
|
+
},
|
|
17
|
+
],
|
|
18
|
+
license: { state: "licensed" as const, spdx_id: "MIT" },
|
|
19
|
+
},
|
|
20
|
+
],
|
|
21
|
+
repos: [
|
|
22
|
+
{
|
|
23
|
+
repo: "firecrawl/firecrawl",
|
|
24
|
+
indexed: true,
|
|
25
|
+
types: { issue: true, pullRequest: true, readme: true },
|
|
26
|
+
},
|
|
27
|
+
],
|
|
28
|
+
sources: [{ source: "firecrawl", indexed: false }],
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
describe("developerSearch", () => {
|
|
32
|
+
test("forwards every dedicated developer filter", async () => {
|
|
33
|
+
const http = {
|
|
34
|
+
post: jest.fn(async () => ({ status: 200, data: response })),
|
|
35
|
+
} as any;
|
|
36
|
+
|
|
37
|
+
const client = new FirecrawlClient({
|
|
38
|
+
apiKey: "test",
|
|
39
|
+
apiUrl: "http://localhost",
|
|
40
|
+
});
|
|
41
|
+
(client as any).http = http;
|
|
42
|
+
|
|
43
|
+
const result = await client.developerSearch("retry configuration", {
|
|
44
|
+
k: 20,
|
|
45
|
+
passages: 5,
|
|
46
|
+
types: ["doc", "issue", "pull_request", "readme"],
|
|
47
|
+
repos: ["firecrawl/firecrawl"],
|
|
48
|
+
sources: ["firecrawl"],
|
|
49
|
+
language: "TypeScript",
|
|
50
|
+
topic: ["web-scraping"],
|
|
51
|
+
license: "MIT",
|
|
52
|
+
minStars: 100,
|
|
53
|
+
maxStars: 50000,
|
|
54
|
+
archived: false,
|
|
55
|
+
fork: true,
|
|
56
|
+
skills: "only",
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
expect(http.post).toHaveBeenCalledWith("/v2/search/developer", {
|
|
60
|
+
query: "retry configuration",
|
|
61
|
+
k: 20,
|
|
62
|
+
passages: 5,
|
|
63
|
+
types: ["doc", "issue", "pull_request", "readme"],
|
|
64
|
+
repos: ["firecrawl/firecrawl"],
|
|
65
|
+
sources: ["firecrawl"],
|
|
66
|
+
language: "TypeScript",
|
|
67
|
+
topic: ["web-scraping"],
|
|
68
|
+
license: "MIT",
|
|
69
|
+
min_stars: 100,
|
|
70
|
+
max_stars: 50000,
|
|
71
|
+
archived: false,
|
|
72
|
+
fork: true,
|
|
73
|
+
skills: "only",
|
|
74
|
+
});
|
|
75
|
+
expect(result).toEqual(response);
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test("normalizes transport errors to SdkError", async () => {
|
|
79
|
+
const http = {
|
|
80
|
+
post: jest.fn(async () => {
|
|
81
|
+
throw {
|
|
82
|
+
isAxiosError: true,
|
|
83
|
+
code: "ECONNABORTED",
|
|
84
|
+
message: "request timed out",
|
|
85
|
+
};
|
|
86
|
+
}),
|
|
87
|
+
} as any;
|
|
88
|
+
const client = new FirecrawlClient({
|
|
89
|
+
apiKey: "test",
|
|
90
|
+
apiUrl: "http://localhost",
|
|
91
|
+
});
|
|
92
|
+
(client as any).http = http;
|
|
93
|
+
|
|
94
|
+
await expect(
|
|
95
|
+
client.developerSearch("network failure"),
|
|
96
|
+
).rejects.toBeInstanceOf(SdkError);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("accepts the flattened SPDX license response shape", async () => {
|
|
100
|
+
const flattened = {
|
|
101
|
+
...response,
|
|
102
|
+
results: [{ ...response.results[0], license: "MIT" }],
|
|
103
|
+
};
|
|
104
|
+
const http = {
|
|
105
|
+
post: jest.fn(async () => ({ status: 200, data: flattened })),
|
|
106
|
+
} as any;
|
|
107
|
+
|
|
108
|
+
const client = new FirecrawlClient({
|
|
109
|
+
apiKey: "test",
|
|
110
|
+
apiUrl: "http://localhost",
|
|
111
|
+
});
|
|
112
|
+
(client as any).http = http;
|
|
113
|
+
|
|
114
|
+
await expect(client.developerSearch("license shape")).resolves.toEqual(
|
|
115
|
+
flattened,
|
|
116
|
+
);
|
|
117
|
+
});
|
|
118
|
+
});
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
import { describe, expect, jest, test } from "@jest/globals";
|
|
2
2
|
import { search } from "../../../v2/methods/search";
|
|
3
|
+
import type { SearchResultWeb } from "../../../v2/types";
|
|
3
4
|
|
|
4
|
-
|
|
5
|
-
|
|
5
|
+
// Developer results arrive in the exact web schema.
|
|
6
|
+
const developerResult: SearchResultWeb = {
|
|
7
|
+
title: "Retry requests",
|
|
6
8
|
url: "https://github.com/firecrawl/firecrawl/issues/1",
|
|
7
9
|
description: "passage",
|
|
8
10
|
position: 1,
|
|
@@ -28,38 +30,26 @@ describe("v2 search developer category", () => {
|
|
|
28
30
|
);
|
|
29
31
|
});
|
|
30
32
|
|
|
31
|
-
test("returns developer results", async () => {
|
|
32
|
-
const http = httpWith({
|
|
33
|
-
web: [{ title: "W", url: "http://a.com" }],
|
|
34
|
-
developer: [developerResult],
|
|
35
|
-
});
|
|
33
|
+
test("returns developer results inside web", async () => {
|
|
34
|
+
const http = httpWith({ web: [developerResult] });
|
|
36
35
|
|
|
37
36
|
const result = await search(http, {
|
|
38
37
|
query: "firecrawl",
|
|
39
38
|
categories: [{ type: "developer" }],
|
|
40
39
|
});
|
|
41
40
|
|
|
42
|
-
expect(result.web).
|
|
43
|
-
expect(result
|
|
44
|
-
expect(result.developer?.[0]).toEqual(developerResult);
|
|
45
|
-
});
|
|
46
|
-
|
|
47
|
-
test("omits developer when the response has none", async () => {
|
|
48
|
-
const http = httpWith({ web: [] });
|
|
49
|
-
|
|
50
|
-
const result = await search(http, { query: "firecrawl" });
|
|
51
|
-
|
|
52
|
-
expect(result.developer).toBeUndefined();
|
|
41
|
+
expect(result.web).toEqual([developerResult]);
|
|
42
|
+
expect(result).not.toHaveProperty("developer");
|
|
53
43
|
});
|
|
54
44
|
|
|
55
|
-
test("lists developer in the .data error", async () => {
|
|
56
|
-
const http = httpWith({
|
|
45
|
+
test("lists developer results as web results in the .data error", async () => {
|
|
46
|
+
const http = httpWith({ web: [developerResult] });
|
|
57
47
|
|
|
58
48
|
const result = await search(http, {
|
|
59
49
|
query: "firecrawl",
|
|
60
50
|
categories: ["developer"],
|
|
61
51
|
});
|
|
62
52
|
|
|
63
|
-
expect(() => (result as any).data).toThrow(/\.
|
|
53
|
+
expect(() => (result as any).data).toThrow(/\.web \(1 results\)/);
|
|
64
54
|
});
|
|
65
55
|
});
|
package/src/v2/client.ts
CHANGED
|
@@ -6,6 +6,7 @@ import {
|
|
|
6
6
|
} from "./methods/scrape";
|
|
7
7
|
import { parse as parseMethod } from "./methods/parse";
|
|
8
8
|
import { search } from "./methods/search";
|
|
9
|
+
import { developerSearch as developerSearchMethod } from "./methods/developer";
|
|
9
10
|
import { map as mapMethod } from "./methods/map";
|
|
10
11
|
import { feedback as feedbackMethod, searchFeedback as searchFeedbackMethod } from "./methods/feedback";
|
|
11
12
|
import {
|
|
@@ -51,6 +52,8 @@ import type {
|
|
|
51
52
|
ScrapeOptions,
|
|
52
53
|
SearchData,
|
|
53
54
|
SearchRequest,
|
|
55
|
+
DeveloperSearchOptions,
|
|
56
|
+
DeveloperSearchResponse,
|
|
54
57
|
EndpointFeedbackRequest,
|
|
55
58
|
FeedbackResponse,
|
|
56
59
|
SearchFeedbackRequest,
|
|
@@ -240,6 +243,19 @@ export class FirecrawlClient {
|
|
|
240
243
|
return search(this.http, { query, ...req });
|
|
241
244
|
}
|
|
242
245
|
|
|
246
|
+
/**
|
|
247
|
+
* Search the dedicated developer index with repository, documentation,
|
|
248
|
+
* artifact, language, topic, license, star, archive, and fork filters.
|
|
249
|
+
* Unlike `search(query, { categories: ["developer"] })`, this returns all
|
|
250
|
+
* requested passages, citations, license disclosures, and indexing echoes.
|
|
251
|
+
*/
|
|
252
|
+
async developerSearch(
|
|
253
|
+
query: string,
|
|
254
|
+
options: DeveloperSearchOptions = {},
|
|
255
|
+
): Promise<DeveloperSearchResponse> {
|
|
256
|
+
return developerSearchMethod(this.http, query, options);
|
|
257
|
+
}
|
|
258
|
+
|
|
243
259
|
/**
|
|
244
260
|
* Submit feedback for a v2 job.
|
|
245
261
|
* @param request Feedback payload with endpoint, job id, rating, and supporting signals.
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
import type { DeveloperSearchOptions, DeveloperSearchResponse } from "../types";
|
|
2
|
+
import { HttpClient } from "../utils/httpClient";
|
|
3
|
+
import {
|
|
4
|
+
normalizeAxiosError,
|
|
5
|
+
throwForBadResponse,
|
|
6
|
+
} from "../utils/errorHandler";
|
|
7
|
+
|
|
8
|
+
const ENDPOINT = "/v2/search/developer";
|
|
9
|
+
|
|
10
|
+
/** Search the dedicated developer index with its complete filter surface. */
|
|
11
|
+
export async function developerSearch(
|
|
12
|
+
http: HttpClient,
|
|
13
|
+
query: string,
|
|
14
|
+
options: DeveloperSearchOptions = {},
|
|
15
|
+
): Promise<DeveloperSearchResponse> {
|
|
16
|
+
if (!query || !query.trim()) throw new Error("query cannot be empty");
|
|
17
|
+
|
|
18
|
+
try {
|
|
19
|
+
const response = await http.post<DeveloperSearchResponse>(ENDPOINT, {
|
|
20
|
+
query,
|
|
21
|
+
...(options.k !== undefined ? { k: options.k } : {}),
|
|
22
|
+
...(options.passages !== undefined ? { passages: options.passages } : {}),
|
|
23
|
+
...(options.types !== undefined ? { types: options.types } : {}),
|
|
24
|
+
...(options.repos !== undefined ? { repos: options.repos } : {}),
|
|
25
|
+
...(options.sources !== undefined ? { sources: options.sources } : {}),
|
|
26
|
+
...(options.language !== undefined ? { language: options.language } : {}),
|
|
27
|
+
...(options.topic !== undefined ? { topic: options.topic } : {}),
|
|
28
|
+
...(options.license !== undefined ? { license: options.license } : {}),
|
|
29
|
+
...(options.minStars !== undefined
|
|
30
|
+
? { min_stars: options.minStars }
|
|
31
|
+
: {}),
|
|
32
|
+
...(options.maxStars !== undefined
|
|
33
|
+
? { max_stars: options.maxStars }
|
|
34
|
+
: {}),
|
|
35
|
+
...(options.archived !== undefined ? { archived: options.archived } : {}),
|
|
36
|
+
...(options.fork !== undefined ? { fork: options.fork } : {}),
|
|
37
|
+
...(options.skills !== undefined ? { skills: options.skills } : {}),
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
if (response.status !== 200 || !response.data?.success) {
|
|
41
|
+
throwForBadResponse(response, "search developer sources");
|
|
42
|
+
}
|
|
43
|
+
return response.data;
|
|
44
|
+
} catch (error: any) {
|
|
45
|
+
if (error?.isAxiosError) {
|
|
46
|
+
return normalizeAxiosError(error, "search developer sources");
|
|
47
|
+
}
|
|
48
|
+
throw error;
|
|
49
|
+
}
|
|
50
|
+
}
|
package/src/v2/methods/search.ts
CHANGED
|
@@ -102,17 +102,16 @@ export async function search(
|
|
|
102
102
|
if (data.news) out.news = transformArray<SearchResultNews>(data.news);
|
|
103
103
|
if (data.images)
|
|
104
104
|
out.images = transformArray<SearchResultImages>(data.images);
|
|
105
|
-
if (data.developer)
|
|
106
|
-
out.developer = transformArray<SearchResultWeb>(data.developer);
|
|
107
105
|
Object.defineProperty(out, "data", {
|
|
108
106
|
get() {
|
|
109
107
|
const parts: string[] = [];
|
|
110
108
|
if (out.web?.length) parts.push(`.web (${out.web.length} results)`);
|
|
111
109
|
if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
|
|
112
|
-
if (out.images?.length)
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
110
|
+
if (out.images?.length)
|
|
111
|
+
parts.push(`.images (${out.images.length} results)`);
|
|
112
|
+
const available = parts.length
|
|
113
|
+
? parts.join(", ")
|
|
114
|
+
: ".web, .news, or .images";
|
|
116
115
|
throw new Error(
|
|
117
116
|
`SearchData has no '.data'. Results are grouped by source: ${available}`,
|
|
118
117
|
);
|
package/src/v2/types.ts
CHANGED
|
@@ -707,6 +707,76 @@ export interface PaginationConfig {
|
|
|
707
707
|
maxWaitTime?: number;
|
|
708
708
|
}
|
|
709
709
|
|
|
710
|
+
export type DeveloperSearchType = "doc" | "issue" | "pull_request" | "readme";
|
|
711
|
+
|
|
712
|
+
export interface DeveloperSearchOptions {
|
|
713
|
+
/** Total ranked results, 1–100 (default 10). */
|
|
714
|
+
k?: number;
|
|
715
|
+
/** Passages per result, 1–5 (default 1). */
|
|
716
|
+
passages?: number;
|
|
717
|
+
/** Result kinds to search (default all). */
|
|
718
|
+
types?: DeveloperSearchType[];
|
|
719
|
+
/** GitHub owner/name filters, at most 20. */
|
|
720
|
+
repos?: string[];
|
|
721
|
+
/** Developer documentation source IDs, at most 20. */
|
|
722
|
+
sources?: string[];
|
|
723
|
+
/** One case-insensitive GitHub Linguist primary language. */
|
|
724
|
+
language?: string;
|
|
725
|
+
/** Repository topics that must all match, at most 8. */
|
|
726
|
+
topic?: string[];
|
|
727
|
+
/** One case-insensitive SPDX repository license identifier. */
|
|
728
|
+
license?: string;
|
|
729
|
+
/** Minimum repository stars. */
|
|
730
|
+
minStars?: number;
|
|
731
|
+
/** Maximum repository stars. */
|
|
732
|
+
maxStars?: number;
|
|
733
|
+
/** Filter by repository archived status. */
|
|
734
|
+
archived?: boolean;
|
|
735
|
+
/** Filter by repository fork status. */
|
|
736
|
+
fork?: boolean;
|
|
737
|
+
/** Return packaged agent-skill evidence only. */
|
|
738
|
+
skills?: "only";
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
export interface DeveloperSearchLicenseDisclosure {
|
|
742
|
+
state: "licensed" | "known_absent" | "unknown";
|
|
743
|
+
spdx_id: string | null;
|
|
744
|
+
}
|
|
745
|
+
|
|
746
|
+
export interface DeveloperSearchPassage {
|
|
747
|
+
text: string;
|
|
748
|
+
citation_url?: string;
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
export interface DeveloperSearchResult {
|
|
752
|
+
id: string;
|
|
753
|
+
url: string;
|
|
754
|
+
title?: string;
|
|
755
|
+
passages: DeveloperSearchPassage[];
|
|
756
|
+
// Accept both shapes while the API flattens license objects to SPDX strings.
|
|
757
|
+
license?: DeveloperSearchLicenseDisclosure | string;
|
|
758
|
+
}
|
|
759
|
+
|
|
760
|
+
export interface DeveloperSearchRepoStatus {
|
|
761
|
+
repo: string;
|
|
762
|
+
indexed: boolean;
|
|
763
|
+
types: { issue: boolean; pullRequest: boolean; readme: boolean };
|
|
764
|
+
}
|
|
765
|
+
|
|
766
|
+
export interface DeveloperSearchSourceStatus {
|
|
767
|
+
source: string;
|
|
768
|
+
indexed: boolean;
|
|
769
|
+
}
|
|
770
|
+
|
|
771
|
+
export interface DeveloperSearchResponse {
|
|
772
|
+
success: boolean;
|
|
773
|
+
results: DeveloperSearchResult[];
|
|
774
|
+
/** Present only when repos were requested. */
|
|
775
|
+
repos?: DeveloperSearchRepoStatus[];
|
|
776
|
+
/** Present only when sources were requested. */
|
|
777
|
+
sources?: DeveloperSearchSourceStatus[];
|
|
778
|
+
}
|
|
779
|
+
|
|
710
780
|
export interface SearchResultWeb {
|
|
711
781
|
url: string;
|
|
712
782
|
title?: string;
|
|
@@ -738,7 +808,6 @@ export interface SearchData {
|
|
|
738
808
|
web?: Array<SearchResultWeb | Document>;
|
|
739
809
|
news?: Array<SearchResultNews | Document>;
|
|
740
810
|
images?: Array<SearchResultImages | Document>;
|
|
741
|
-
developer?: Array<SearchResultWeb | Document>;
|
|
742
811
|
}
|
|
743
812
|
|
|
744
813
|
/**
|
|
@@ -751,8 +820,8 @@ export interface SearchData {
|
|
|
751
820
|
* ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). It returns
|
|
752
821
|
* ordinary web page results from those domains, **not** paper records.
|
|
753
822
|
* - `pdf` — restrict results to PDFs (adds `filetype:pdf`).
|
|
754
|
-
* - `developer` —
|
|
755
|
-
* documentation)
|
|
823
|
+
* - `developer` — developer-index results (issues, pull requests, READMEs and
|
|
824
|
+
* documentation) served in `web`; cannot be combined with other categories.
|
|
756
825
|
*
|
|
757
826
|
* ⚠️ `categories: ["research"]` is **not** Firecrawl's research paper index.
|
|
758
827
|
* To search papers themselves — ~43M abstracts, roughly 90% biomedical
|