firecrawl 4.37.0 → 4.39.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -2
- package/dist/{chunk-LFX7WDGT.js → chunk-4DPCCKWG.js} +1 -1
- package/dist/index.cjs +61 -9
- package/dist/index.d.cts +202 -4
- package/dist/index.d.ts +202 -4
- package/dist/index.js +62 -10
- package/dist/{package-SHMK2LJE.js → package-D2QZMRW7.js} +1 -1
- package/package.json +1 -1
- package/pnpm-workspace.yaml +1 -0
- package/src/__tests__/e2e/v2/batch.test.ts +13 -11
- package/src/__tests__/e2e/v2/crawl.test.ts +24 -20
- package/src/__tests__/e2e/v2/extract.test.ts +32 -7
- package/src/__tests__/e2e/v2/map.test.ts +4 -3
- package/src/__tests__/e2e/v2/parse.test.ts +4 -3
- package/src/__tests__/e2e/v2/scrape.test.ts +10 -9
- package/src/__tests__/e2e/v2/search.test.ts +9 -8
- package/src/__tests__/e2e/v2/usage.test.ts +6 -5
- package/src/__tests__/e2e/v2/utils/rateLimit.ts +257 -0
- package/src/__tests__/e2e/v2/watcher.test.ts +4 -3
- package/src/__tests__/unit/v2/agent-effort-trace.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/agent-list.unit.test.ts +68 -0
- package/src/__tests__/unit/v2/agent-thread.unit.test.ts +164 -0
- package/src/__tests__/unit/v2/e2eRateLimitRetry.test.ts +501 -0
- package/src/__tests__/unit/v2/extract.unit.test.ts +1 -1
- package/src/__tests__/unit/v2/scrapeAutoResume.test.ts +1 -1
- package/src/__tests__/unit/v2/search-country.unit.test.ts +32 -0
- package/src/__tests__/unit/v2/search-data-property.test.ts +1 -0
- package/src/__tests__/unit/v2/v1-aliases.test.ts +1 -0
- package/src/v2/client.ts +26 -1
- package/src/v2/methods/agent.ts +35 -1
- package/src/v2/methods/research.ts +14 -8
- package/src/v2/methods/search.ts +1 -0
- package/src/v2/types.ts +195 -34
package/dist/index.d.ts
CHANGED
|
@@ -698,6 +698,8 @@ interface SearchRequest {
|
|
|
698
698
|
limit?: number;
|
|
699
699
|
tbs?: string;
|
|
700
700
|
location?: string;
|
|
701
|
+
/** ISO 3166-1 alpha-2 country code used to geo-target the search results. */
|
|
702
|
+
country?: string;
|
|
701
703
|
ignoreInvalidURLs?: boolean;
|
|
702
704
|
timeout?: number;
|
|
703
705
|
/** Generate query-relevant highlights for search results. Defaults to true. */
|
|
@@ -1087,10 +1089,66 @@ interface ExtractResponse$1 {
|
|
|
1087
1089
|
expiresAt?: string;
|
|
1088
1090
|
creditsUsed?: number;
|
|
1089
1091
|
}
|
|
1092
|
+
/** Conversation mode for an agent run. Defaults to "extract" server-side. */
|
|
1093
|
+
type AgentMode = "extract" | "chat";
|
|
1094
|
+
/** Options forwarded verbatim to the agent; the server owns every default. */
|
|
1095
|
+
interface AgentExchangeOptions {
|
|
1096
|
+
enabled?: boolean;
|
|
1097
|
+
/** At most 5. */
|
|
1098
|
+
toolkits?: string[];
|
|
1099
|
+
maxCalls?: number;
|
|
1100
|
+
requireApproval?: boolean;
|
|
1101
|
+
/** Answers a pendingApproval from the previous turn of the thread. */
|
|
1102
|
+
approve?: {
|
|
1103
|
+
approvalId: string;
|
|
1104
|
+
callIds?: string[];
|
|
1105
|
+
always?: boolean;
|
|
1106
|
+
};
|
|
1107
|
+
decline?: {
|
|
1108
|
+
approvalId: string;
|
|
1109
|
+
};
|
|
1110
|
+
}
|
|
1111
|
+
/** Per-run summary reported on a status response. */
|
|
1112
|
+
interface AgentExchangeSummary {
|
|
1113
|
+
enabled: boolean;
|
|
1114
|
+
/** What the run resolved to after thread inheritance, not what it requested. */
|
|
1115
|
+
toolkits?: string[];
|
|
1116
|
+
requireApproval?: boolean;
|
|
1117
|
+
paidCalls: number;
|
|
1118
|
+
creditsUsed: number | null;
|
|
1119
|
+
}
|
|
1120
|
+
/** A follow-up the agent offers for the next turn of the thread. */
|
|
1121
|
+
interface AgentSuggestion {
|
|
1122
|
+
label: string;
|
|
1123
|
+
prompt: string;
|
|
1124
|
+
}
|
|
1125
|
+
/** A turn that ended waiting for the caller to allow or refuse paid calls. */
|
|
1126
|
+
interface PendingApproval {
|
|
1127
|
+
id: string;
|
|
1128
|
+
reason: string;
|
|
1129
|
+
calls: {
|
|
1130
|
+
id: string;
|
|
1131
|
+
provider: string;
|
|
1132
|
+
capability: string;
|
|
1133
|
+
input: Record<string, unknown>;
|
|
1134
|
+
more?: Record<string, unknown>[];
|
|
1135
|
+
creditsEstimate: number | null;
|
|
1136
|
+
}[];
|
|
1137
|
+
resolution: null | {
|
|
1138
|
+
approved: boolean;
|
|
1139
|
+
callIds: string[];
|
|
1140
|
+
always: boolean;
|
|
1141
|
+
byRunId: string;
|
|
1142
|
+
};
|
|
1143
|
+
}
|
|
1090
1144
|
interface AgentResponse {
|
|
1091
1145
|
success: boolean;
|
|
1092
1146
|
id: string;
|
|
1093
1147
|
error?: string;
|
|
1148
|
+
/** Thread this run belongs to; pass it back to continue the conversation. */
|
|
1149
|
+
threadId?: string;
|
|
1150
|
+
/** 1-based position of this run in its thread. */
|
|
1151
|
+
threadTurn?: number;
|
|
1094
1152
|
}
|
|
1095
1153
|
interface AgentStatusResponse {
|
|
1096
1154
|
success: boolean;
|
|
@@ -1110,9 +1168,90 @@ interface AgentStatusResponse {
|
|
|
1110
1168
|
effort?: "low" | "medium" | "high";
|
|
1111
1169
|
expiresAt: string;
|
|
1112
1170
|
creditsUsed?: number;
|
|
1171
|
+
threadId?: string;
|
|
1172
|
+
threadTurn?: number;
|
|
1173
|
+
mode?: AgentMode;
|
|
1174
|
+
/** Assistant text reply. Chat-mode runs answer here instead of in `data`. */
|
|
1175
|
+
message?: string;
|
|
1176
|
+
suggestions?: AgentSuggestion[];
|
|
1177
|
+
pendingApproval?: PendingApproval;
|
|
1178
|
+
exchange?: AgentExchangeSummary;
|
|
1179
|
+
}
|
|
1180
|
+
/** A single run of a thread, as returned by getAgentThread. */
|
|
1181
|
+
interface AgentThreadRun {
|
|
1182
|
+
id: string;
|
|
1183
|
+
turn: number;
|
|
1184
|
+
mode: AgentMode;
|
|
1185
|
+
prompt: string;
|
|
1186
|
+
urls?: string[];
|
|
1187
|
+
schema?: unknown;
|
|
1188
|
+
effort?: "low" | "medium" | "high";
|
|
1189
|
+
status: "processing" | "succeeded" | "failed" | "cancelled" | "refused" | "credit_limit_reached";
|
|
1190
|
+
createdAt: string;
|
|
1191
|
+
finishedAt: string | null;
|
|
1192
|
+
creditsUsed: number | null;
|
|
1193
|
+
message: string | null;
|
|
1194
|
+
/** Only present when the request asked for includeData. */
|
|
1195
|
+
data?: unknown;
|
|
1196
|
+
suggestions?: AgentSuggestion[] | null;
|
|
1197
|
+
pendingApproval?: PendingApproval | null;
|
|
1198
|
+
exchange?: AgentExchangeSummary | null;
|
|
1199
|
+
}
|
|
1200
|
+
interface AgentThread {
|
|
1201
|
+
id: string;
|
|
1202
|
+
createdAt: string;
|
|
1203
|
+
updatedAt: string;
|
|
1204
|
+
status: "idle" | "running";
|
|
1205
|
+
runs: AgentThreadRun[];
|
|
1206
|
+
}
|
|
1207
|
+
interface AgentThreadResponse {
|
|
1208
|
+
success: boolean;
|
|
1209
|
+
thread?: AgentThread;
|
|
1210
|
+
error?: string;
|
|
1113
1211
|
}
|
|
1114
1212
|
/** Reasoning effort for agent jobs. Every level runs spark-2. */
|
|
1115
1213
|
type AgentEffort = "low" | "medium" | "high";
|
|
1214
|
+
/** A single agent run as returned by the agent list endpoint. */
|
|
1215
|
+
interface AgentListItem {
|
|
1216
|
+
id: string;
|
|
1217
|
+
createdAt: string;
|
|
1218
|
+
targetHint: string;
|
|
1219
|
+
origin: string;
|
|
1220
|
+
integration?: string;
|
|
1221
|
+
settings: {
|
|
1222
|
+
hidden: boolean;
|
|
1223
|
+
starred: boolean;
|
|
1224
|
+
label?: string;
|
|
1225
|
+
};
|
|
1226
|
+
status: "processing" | "completed" | "failed";
|
|
1227
|
+
options?: {
|
|
1228
|
+
urls?: string[];
|
|
1229
|
+
prompt: string;
|
|
1230
|
+
schema?: unknown;
|
|
1231
|
+
/**
|
|
1232
|
+
* Server-provided model name. Widened past the request-side union on
|
|
1233
|
+
* purpose: new models ship without an SDK release, so pinning this to
|
|
1234
|
+
* known names makes every future model a type error at the call site.
|
|
1235
|
+
*/
|
|
1236
|
+
model: "spark-1-pro" | "spark-1-mini" | "spark-2" | (string & {});
|
|
1237
|
+
effort?: AgentEffort;
|
|
1238
|
+
};
|
|
1239
|
+
}
|
|
1240
|
+
interface AgentListResponse {
|
|
1241
|
+
success: boolean;
|
|
1242
|
+
agents?: AgentListItem[];
|
|
1243
|
+
/**
|
|
1244
|
+
* Absolute URL of the next page (pass its `before` value to listAgents to
|
|
1245
|
+
* continue). Only present when more pages exist.
|
|
1246
|
+
*/
|
|
1247
|
+
next?: string;
|
|
1248
|
+
error?: string;
|
|
1249
|
+
}
|
|
1250
|
+
/** Options for listing agent runs. */
|
|
1251
|
+
interface AgentListOptions {
|
|
1252
|
+
/** Only return agent runs created before this unix millisecond timestamp. */
|
|
1253
|
+
before?: number;
|
|
1254
|
+
}
|
|
1116
1255
|
type AgentTraceAgentRole = "orchestrator" | "subagent" | "browser" | "system";
|
|
1117
1256
|
interface AgentTraceAgentIdentity {
|
|
1118
1257
|
id: string;
|
|
@@ -1464,7 +1603,14 @@ interface SimilarPapersResponse {
|
|
|
1464
1603
|
/** Human-readable note when no results are produced. */
|
|
1465
1604
|
note?: string | null;
|
|
1466
1605
|
}
|
|
1467
|
-
/**
|
|
1606
|
+
/**
|
|
1607
|
+
* Component scores; each field is present only when that signal contributed.
|
|
1608
|
+
*
|
|
1609
|
+
* @deprecated Use the developer index instead. The research index GitHub
|
|
1610
|
+
* search stops responding after 2026-11-03. The developer index does not
|
|
1611
|
+
* expose a score breakdown. See
|
|
1612
|
+
* https://docs.firecrawl.dev/features/developer.
|
|
1613
|
+
*/
|
|
1468
1614
|
interface GitHubScoreBreakdown {
|
|
1469
1615
|
rrf?: number;
|
|
1470
1616
|
semantic?: number;
|
|
@@ -1472,6 +1618,11 @@ interface GitHubScoreBreakdown {
|
|
|
1472
1618
|
fusion?: number;
|
|
1473
1619
|
rerank?: number;
|
|
1474
1620
|
}
|
|
1621
|
+
/**
|
|
1622
|
+
* @deprecated Use the developer index instead. The research index GitHub
|
|
1623
|
+
* search stops responding after 2026-11-03. See
|
|
1624
|
+
* https://docs.firecrawl.dev/features/developer.
|
|
1625
|
+
*/
|
|
1475
1626
|
interface GitHubSearchItem {
|
|
1476
1627
|
resultType?: "github_history" | "repo_readme" | "web";
|
|
1477
1628
|
/** `owner/name`; empty for web results whose URL is not a repo page. */
|
|
@@ -1493,9 +1644,18 @@ interface GitHubSearchItem {
|
|
|
1493
1644
|
contentMd?: string;
|
|
1494
1645
|
scores: GitHubScoreBreakdown;
|
|
1495
1646
|
}
|
|
1647
|
+
/**
|
|
1648
|
+
* @deprecated Use the developer index instead. The research index GitHub
|
|
1649
|
+
* search stops responding after 2026-11-03. See
|
|
1650
|
+
* https://docs.firecrawl.dev/features/developer.
|
|
1651
|
+
*/
|
|
1496
1652
|
interface GitHubSearchResponse {
|
|
1497
1653
|
success: boolean;
|
|
1498
1654
|
results: GitHubSearchItem[];
|
|
1655
|
+
/** Deprecation notice while the sunset window is live. */
|
|
1656
|
+
warnings?: string[];
|
|
1657
|
+
/** Replacement endpoint path, while the sunset window is live. */
|
|
1658
|
+
replacement?: string;
|
|
1499
1659
|
}
|
|
1500
1660
|
/** Options for `research.searchPapers`. */
|
|
1501
1661
|
interface SearchPapersOptions {
|
|
@@ -1530,7 +1690,13 @@ interface SimilarPapersOptions {
|
|
|
1530
1690
|
/** Additional seed paper reference(s), same format as `id`. */
|
|
1531
1691
|
anchor?: string[];
|
|
1532
1692
|
}
|
|
1533
|
-
/**
|
|
1693
|
+
/**
|
|
1694
|
+
* Options for `research.searchGithub`.
|
|
1695
|
+
*
|
|
1696
|
+
* @deprecated Use the developer index instead. The research index GitHub
|
|
1697
|
+
* search stops responding after 2026-11-03. See
|
|
1698
|
+
* https://docs.firecrawl.dev/features/developer.
|
|
1699
|
+
*/
|
|
1534
1700
|
interface SearchGithubOptions {
|
|
1535
1701
|
/** Number of results to return (1–100, default 20). */
|
|
1536
1702
|
k?: number;
|
|
@@ -1625,6 +1791,9 @@ declare function prepareAgentPayload(args: {
|
|
|
1625
1791
|
webhook?: string | AgentWebhookConfig;
|
|
1626
1792
|
threatProtection?: ThreatProtectionOptions;
|
|
1627
1793
|
auditMetadata?: AuditMetadata$1;
|
|
1794
|
+
threadId?: string;
|
|
1795
|
+
mode?: AgentMode;
|
|
1796
|
+
exchange?: AgentExchangeOptions;
|
|
1628
1797
|
}): Record<string, unknown>;
|
|
1629
1798
|
declare function startAgent(http: HttpClient, args: Parameters<typeof prepareAgentPayload>[0]): Promise<AgentResponse>;
|
|
1630
1799
|
|
|
@@ -1715,7 +1884,16 @@ declare class ResearchClient {
|
|
|
1715
1884
|
*/
|
|
1716
1885
|
similarPapers(id: string, options: SimilarPapersOptions): Promise<SimilarPapersResponse>;
|
|
1717
1886
|
/**
|
|
1718
|
-
* Search GitHub issue/PR history and repository
|
|
1887
|
+
* Search the research index GitHub slice: issue/PR history and repository
|
|
1888
|
+
* readmes.
|
|
1889
|
+
*
|
|
1890
|
+
* @deprecated Use `developerSearch()` on the client instead. This
|
|
1891
|
+
* endpoint stops responding after 2026-11-03. The developer index searches
|
|
1892
|
+
* GitHub issues, pull requests and readmes plus curated documentation
|
|
1893
|
+
* sources, returns matched passages, and adds filters for repo, language,
|
|
1894
|
+
* license and stars. It does not carry over the `scores` breakdown or the
|
|
1895
|
+
* `resultType: "web"` fallback results. See
|
|
1896
|
+
* https://docs.firecrawl.dev/features/developer.
|
|
1719
1897
|
* @param query Search query.
|
|
1720
1898
|
* @param options Optional `k`.
|
|
1721
1899
|
*/
|
|
@@ -1862,6 +2040,9 @@ declare class FirecrawlClient {
|
|
|
1862
2040
|
* Access the v2 research endpoints — Firecrawl's **research paper index**
|
|
1863
2041
|
* (~43M paper abstracts) plus GitHub history/readmes.
|
|
1864
2042
|
*
|
|
2043
|
+
* `research.searchGithub()` is deprecated and stops responding after
|
|
2044
|
+
* 2026-11-03. Use `developerSearch()` instead.
|
|
2045
|
+
*
|
|
1865
2046
|
* The paper corpus is roughly 90% biomedical and life sciences — PubMed,
|
|
1866
2047
|
* bioRxiv and medRxiv — with arXiv covering physics, mathematics and
|
|
1867
2048
|
* computer science.
|
|
@@ -2026,6 +2207,15 @@ declare class FirecrawlClient {
|
|
|
2026
2207
|
* @param jobId Agent job id.
|
|
2027
2208
|
*/
|
|
2028
2209
|
getAgentStatus(jobId: string): Promise<AgentStatusResponse>;
|
|
2210
|
+
/**
|
|
2211
|
+
* List agent runs, most recent first.
|
|
2212
|
+
*
|
|
2213
|
+
* Pages are fixed at 20 runs. To fetch the next page, pass the `before`
|
|
2214
|
+
* value from the previous page's `next` URL. This method does not
|
|
2215
|
+
* auto-paginate.
|
|
2216
|
+
* @param options.before Only return runs created before this unix ms timestamp.
|
|
2217
|
+
*/
|
|
2218
|
+
listAgents(options?: AgentListOptions): Promise<AgentListResponse>;
|
|
2029
2219
|
/**
|
|
2030
2220
|
* Convenience waiter: start an agent and poll until it finishes.
|
|
2031
2221
|
* @param args Agent request plus waiter controls (pollInterval, timeout seconds).
|
|
@@ -2055,6 +2245,14 @@ declare class FirecrawlClient {
|
|
|
2055
2245
|
* @param snapshotId Snapshot id from an artifact.updated trace event.
|
|
2056
2246
|
*/
|
|
2057
2247
|
getAgentSnapshot(jobId: string, snapshotId: string): Promise<AgentSnapshotResponse>;
|
|
2248
|
+
/**
|
|
2249
|
+
* Get a thread and its runs, oldest turn first.
|
|
2250
|
+
* @param threadId Thread id, as returned by startAgent or getAgentStatus.
|
|
2251
|
+
* @param options.includeData Inline each succeeded run's data.
|
|
2252
|
+
*/
|
|
2253
|
+
getAgentThread(threadId: string, options?: {
|
|
2254
|
+
includeData?: boolean;
|
|
2255
|
+
}): Promise<AgentThreadResponse>;
|
|
2058
2256
|
/**
|
|
2059
2257
|
* Create a new browser session.
|
|
2060
2258
|
* @param args Session options (ttl, activityTtl, streamWebView, profile).
|
|
@@ -3057,4 +3255,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
3057
3255
|
get v1(): FirecrawlApp;
|
|
3058
3256
|
}
|
|
3059
3257
|
|
|
3060
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentEffort, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentTraceActiveBrowserSession, type AgentTraceAgentFinishedEvent, type AgentTraceAgentIdentity, type AgentTraceAgentRole, type AgentTraceAgentStartedEvent, type AgentTraceArtifactChange, type AgentTraceArtifactUpdatedEvent, type AgentTraceBrowserSessionFinishedEvent, type AgentTraceBrowserSessionStartedEvent, type AgentTraceError, type AgentTraceErrorOccurredEvent, type AgentTraceEvent, type AgentTraceProgressReportedEvent, type AgentTraceReasoningSummaryEvent, type AgentTraceResponse, type AgentTraceRunCancelRequestedEvent, type AgentTraceRunFinishedEvent, type AgentTraceRunStartedEvent, type AgentTraceToolCallFinishedEvent, type AgentTraceToolCallStartedEvent, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
|
3258
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentEffort, type AgentExchangeOptions, type AgentExchangeSummary, type AgentListItem, type AgentListOptions, type AgentListResponse, type AgentMode, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentSuggestion, type AgentThread, type AgentThreadResponse, type AgentThreadRun, type AgentTraceActiveBrowserSession, type AgentTraceAgentFinishedEvent, type AgentTraceAgentIdentity, type AgentTraceAgentRole, type AgentTraceAgentStartedEvent, type AgentTraceArtifactChange, type AgentTraceArtifactUpdatedEvent, type AgentTraceBrowserSessionFinishedEvent, type AgentTraceBrowserSessionStartedEvent, type AgentTraceError, type AgentTraceErrorOccurredEvent, type AgentTraceEvent, type AgentTraceProgressReportedEvent, type AgentTraceReasoningSummaryEvent, type AgentTraceResponse, type AgentTraceRunCancelRequestedEvent, type AgentTraceRunFinishedEvent, type AgentTraceRunStartedEvent, type AgentTraceToolCallFinishedEvent, type AgentTraceToolCallStartedEvent, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PendingApproval, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-4DPCCKWG.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -620,6 +620,7 @@ function prepareSearchPayload(req) {
|
|
|
620
620
|
if (req.limit != null) payload.limit = req.limit;
|
|
621
621
|
if (req.tbs != null) payload.tbs = req.tbs;
|
|
622
622
|
if (req.location != null) payload.location = req.location;
|
|
623
|
+
if (req.country != null) payload.country = req.country;
|
|
623
624
|
if (req.ignoreInvalidURLs != null)
|
|
624
625
|
payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
|
|
625
626
|
if (req.timeout != null) payload.timeout = req.timeout;
|
|
@@ -1240,6 +1241,9 @@ function prepareAgentPayload(args) {
|
|
|
1240
1241
|
body.threatProtection = args.threatProtection;
|
|
1241
1242
|
if (args.auditMetadata != null)
|
|
1242
1243
|
body.auditMetadata = args.auditMetadata;
|
|
1244
|
+
if (args.threadId != null) body.threadId = args.threadId;
|
|
1245
|
+
if (args.mode != null) body.mode = args.mode;
|
|
1246
|
+
if (args.exchange != null) body.exchange = args.exchange;
|
|
1243
1247
|
return body;
|
|
1244
1248
|
}
|
|
1245
1249
|
async function startAgent(http, args) {
|
|
@@ -1263,6 +1267,17 @@ async function getAgentStatus(http, jobId) {
|
|
|
1263
1267
|
throw err;
|
|
1264
1268
|
}
|
|
1265
1269
|
}
|
|
1270
|
+
async function listAgents(http, options) {
|
|
1271
|
+
try {
|
|
1272
|
+
const query = options?.before !== void 0 ? `?before=${options.before}` : "";
|
|
1273
|
+
const res = await http.get(`/v2/agent${query}`);
|
|
1274
|
+
if (res.status !== 200) throwForBadResponse(res, "list agents");
|
|
1275
|
+
return res.data;
|
|
1276
|
+
} catch (err) {
|
|
1277
|
+
if (err?.isAxiosError) return normalizeAxiosError(err, "list agents");
|
|
1278
|
+
throw err;
|
|
1279
|
+
}
|
|
1280
|
+
}
|
|
1266
1281
|
async function waitAgent(http, jobId, pollInterval = 2, timeout) {
|
|
1267
1282
|
const start = Date.now();
|
|
1268
1283
|
while (true) {
|
|
@@ -1299,6 +1314,17 @@ async function getAgentSnapshot(http, jobId, snapshotId) {
|
|
|
1299
1314
|
throw err;
|
|
1300
1315
|
}
|
|
1301
1316
|
}
|
|
1317
|
+
async function getAgentThread(http, threadId, options) {
|
|
1318
|
+
try {
|
|
1319
|
+
const query = options?.includeData ? "?includeData=true" : "";
|
|
1320
|
+
const res = await http.get(`/v2/agent/threads/${threadId}${query}`);
|
|
1321
|
+
if (res.status !== 200) throwForBadResponse(res, "agent thread");
|
|
1322
|
+
return res.data;
|
|
1323
|
+
} catch (err) {
|
|
1324
|
+
if (err?.isAxiosError) return normalizeAxiosError(err, "agent thread");
|
|
1325
|
+
throw err;
|
|
1326
|
+
}
|
|
1327
|
+
}
|
|
1302
1328
|
async function cancelAgent(http, jobId) {
|
|
1303
1329
|
try {
|
|
1304
1330
|
const res = await http.delete(`/v2/agent/${jobId}`);
|
|
@@ -1539,9 +1565,7 @@ var ResearchClient = class {
|
|
|
1539
1565
|
appendParam(params, "query", options.query);
|
|
1540
1566
|
appendParam(params, "k", options.k);
|
|
1541
1567
|
try {
|
|
1542
|
-
const res = await this.http.get(
|
|
1543
|
-
withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params)
|
|
1544
|
-
);
|
|
1568
|
+
const res = await this.http.get(withQuery(`${BASE}/papers/${encodeURIComponent(id)}`, params));
|
|
1545
1569
|
if (res.status !== 200) throwForBadResponse(res, "get paper");
|
|
1546
1570
|
return res.data;
|
|
1547
1571
|
} catch (err) {
|
|
@@ -1567,10 +1591,7 @@ var ResearchClient = class {
|
|
|
1567
1591
|
appendParam(params, "anchor", options.anchor);
|
|
1568
1592
|
try {
|
|
1569
1593
|
const res = await this.http.get(
|
|
1570
|
-
withQuery(
|
|
1571
|
-
`${BASE}/papers/${encodeURIComponent(id)}/similar`,
|
|
1572
|
-
params
|
|
1573
|
-
)
|
|
1594
|
+
withQuery(`${BASE}/papers/${encodeURIComponent(id)}/similar`, params)
|
|
1574
1595
|
);
|
|
1575
1596
|
if (res.status !== 200) throwForBadResponse(res, "find similar papers");
|
|
1576
1597
|
return res.data;
|
|
@@ -1579,7 +1600,16 @@ var ResearchClient = class {
|
|
|
1579
1600
|
}
|
|
1580
1601
|
}
|
|
1581
1602
|
/**
|
|
1582
|
-
* Search GitHub issue/PR history and repository
|
|
1603
|
+
* Search the research index GitHub slice: issue/PR history and repository
|
|
1604
|
+
* readmes.
|
|
1605
|
+
*
|
|
1606
|
+
* @deprecated Use `developerSearch()` on the client instead. This
|
|
1607
|
+
* endpoint stops responding after 2026-11-03. The developer index searches
|
|
1608
|
+
* GitHub issues, pull requests and readmes plus curated documentation
|
|
1609
|
+
* sources, returns matched passages, and adds filters for repo, language,
|
|
1610
|
+
* license and stars. It does not carry over the `scores` breakdown or the
|
|
1611
|
+
* `resultType: "web"` fallback results. See
|
|
1612
|
+
* https://docs.firecrawl.dev/features/developer.
|
|
1583
1613
|
* @param query Search query.
|
|
1584
1614
|
* @param options Optional `k`.
|
|
1585
1615
|
*/
|
|
@@ -2068,6 +2098,9 @@ var FirecrawlClient = class {
|
|
|
2068
2098
|
* Access the v2 research endpoints — Firecrawl's **research paper index**
|
|
2069
2099
|
* (~43M paper abstracts) plus GitHub history/readmes.
|
|
2070
2100
|
*
|
|
2101
|
+
* `research.searchGithub()` is deprecated and stops responding after
|
|
2102
|
+
* 2026-11-03. Use `developerSearch()` instead.
|
|
2103
|
+
*
|
|
2071
2104
|
* The paper corpus is roughly 90% biomedical and life sciences — PubMed,
|
|
2072
2105
|
* bioRxiv and medRxiv — with arXiv covering physics, mathematics and
|
|
2073
2106
|
* computer science.
|
|
@@ -2284,6 +2317,17 @@ var FirecrawlClient = class {
|
|
|
2284
2317
|
async getAgentStatus(jobId) {
|
|
2285
2318
|
return getAgentStatus(this.http, jobId);
|
|
2286
2319
|
}
|
|
2320
|
+
/**
|
|
2321
|
+
* List agent runs, most recent first.
|
|
2322
|
+
*
|
|
2323
|
+
* Pages are fixed at 20 runs. To fetch the next page, pass the `before`
|
|
2324
|
+
* value from the previous page's `next` URL. This method does not
|
|
2325
|
+
* auto-paginate.
|
|
2326
|
+
* @param options.before Only return runs created before this unix ms timestamp.
|
|
2327
|
+
*/
|
|
2328
|
+
async listAgents(options) {
|
|
2329
|
+
return listAgents(this.http, options);
|
|
2330
|
+
}
|
|
2287
2331
|
/**
|
|
2288
2332
|
* Convenience waiter: start an agent and poll until it finishes.
|
|
2289
2333
|
* @param args Agent request plus waiter controls (pollInterval, timeout seconds).
|
|
@@ -2316,6 +2360,14 @@ var FirecrawlClient = class {
|
|
|
2316
2360
|
async getAgentSnapshot(jobId, snapshotId) {
|
|
2317
2361
|
return getAgentSnapshot(this.http, jobId, snapshotId);
|
|
2318
2362
|
}
|
|
2363
|
+
/**
|
|
2364
|
+
* Get a thread and its runs, oldest turn first.
|
|
2365
|
+
* @param threadId Thread id, as returned by startAgent or getAgentStatus.
|
|
2366
|
+
* @param options.includeData Inline each succeeded run's data.
|
|
2367
|
+
*/
|
|
2368
|
+
async getAgentThread(threadId, options) {
|
|
2369
|
+
return getAgentThread(this.http, threadId, options);
|
|
2370
|
+
}
|
|
2319
2371
|
// Browser
|
|
2320
2372
|
/**
|
|
2321
2373
|
* Create a new browser session.
|
|
@@ -2458,7 +2510,7 @@ var FirecrawlApp = class {
|
|
|
2458
2510
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2459
2511
|
return process.env.npm_package_version;
|
|
2460
2512
|
}
|
|
2461
|
-
const packageJson = await import("./package-
|
|
2513
|
+
const packageJson = await import("./package-D2QZMRW7.js");
|
|
2462
2514
|
return packageJson.default.version;
|
|
2463
2515
|
} catch (error) {
|
|
2464
2516
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.39.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
package/pnpm-workspace.yaml
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
import Firecrawl from "../../../index";
|
|
5
5
|
import { config } from "dotenv";
|
|
6
6
|
import { getIdentity, getApiUrl } from "./utils/idmux";
|
|
7
|
+
import { testTimeoutMs, waitForJob, withRateLimitRetry } from "./utils/rateLimit";
|
|
7
8
|
import { describe, test, expect, beforeAll } from "@jest/globals";
|
|
8
9
|
|
|
9
10
|
config();
|
|
@@ -13,7 +14,7 @@ let client: Firecrawl;
|
|
|
13
14
|
|
|
14
15
|
beforeAll(async () => {
|
|
15
16
|
const { apiKey } = await getIdentity({ name: "js-e2e-batch" });
|
|
16
|
-
client = new Firecrawl({ apiKey, apiUrl: API_URL });
|
|
17
|
+
client = withRateLimitRetry(new Firecrawl({ apiKey, apiUrl: API_URL }));
|
|
17
18
|
});
|
|
18
19
|
|
|
19
20
|
describe("v2.batch e2e", () => {
|
|
@@ -22,19 +23,21 @@ describe("v2.batch e2e", () => {
|
|
|
22
23
|
"https://docs.firecrawl.dev",
|
|
23
24
|
"https://firecrawl.dev",
|
|
24
25
|
];
|
|
25
|
-
const
|
|
26
|
+
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] } });
|
|
27
|
+
const job = await waitForJob(() => client.getBatchScrapeStatus(start.id), { pollInterval: 1, timeout: 210 });
|
|
26
28
|
expect(["completed", "failed"]).toContain(job.status);
|
|
27
29
|
expect(job.completed).toBeGreaterThanOrEqual(0);
|
|
28
30
|
expect(job.total).toBeGreaterThanOrEqual(0);
|
|
29
31
|
expect(Array.isArray(job.data)).toBe(true);
|
|
30
|
-
},
|
|
32
|
+
}, testTimeoutMs(362_000));
|
|
31
33
|
|
|
32
34
|
test("batch scrape with wait returns job id for error retrieval", async () => {
|
|
33
35
|
const urls = [
|
|
34
36
|
"https://docs.firecrawl.dev",
|
|
35
37
|
"https://firecrawl.dev",
|
|
36
38
|
];
|
|
37
|
-
const
|
|
39
|
+
const start = await client.startBatchScrape(urls, { options: { formats: ["markdown"] } });
|
|
40
|
+
const job = await waitForJob(() => client.getBatchScrapeStatus(start.id), { pollInterval: 1, timeout: 210 });
|
|
38
41
|
// Verify job has id field
|
|
39
42
|
expect(job.id).toBeDefined();
|
|
40
43
|
expect(typeof job.id).toBe("string");
|
|
@@ -44,7 +47,7 @@ describe("v2.batch e2e", () => {
|
|
|
44
47
|
expect(errors).toHaveProperty("robotsBlocked");
|
|
45
48
|
expect(Array.isArray(errors.errors)).toBe(true);
|
|
46
49
|
expect(Array.isArray(errors.robotsBlocked)).toBe(true);
|
|
47
|
-
},
|
|
50
|
+
}, testTimeoutMs(512_000));
|
|
48
51
|
|
|
49
52
|
test("start batch minimal and status", async () => {
|
|
50
53
|
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
|
|
@@ -57,11 +60,11 @@ describe("v2.batch e2e", () => {
|
|
|
57
60
|
// Verify status includes id field
|
|
58
61
|
expect(status.id).toBeDefined();
|
|
59
62
|
expect(status.id).toBe(start.id);
|
|
60
|
-
},
|
|
63
|
+
}, testTimeoutMs(151_000));
|
|
61
64
|
|
|
62
65
|
test("wait batch with all params", async () => {
|
|
63
66
|
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
|
|
64
|
-
const
|
|
67
|
+
const start = await client.startBatchScrape(urls, {
|
|
65
68
|
options: {
|
|
66
69
|
formats: [
|
|
67
70
|
"markdown",
|
|
@@ -74,14 +77,13 @@ describe("v2.batch e2e", () => {
|
|
|
74
77
|
ignoreInvalidURLs: true,
|
|
75
78
|
maxConcurrency: 2,
|
|
76
79
|
zeroDataRetention: false,
|
|
77
|
-
pollInterval: 1,
|
|
78
|
-
timeout: 180,
|
|
79
80
|
});
|
|
81
|
+
const job = await waitForJob(() => client.getBatchScrapeStatus(start.id), { pollInterval: 1, timeout: 210 });
|
|
80
82
|
expect(["completed", "failed", "cancelled"]).toContain(job.status);
|
|
81
83
|
expect(job.completed).toBeGreaterThanOrEqual(0);
|
|
82
84
|
expect(job.total).toBeGreaterThanOrEqual(0);
|
|
83
85
|
expect(Array.isArray(job.data)).toBe(true);
|
|
84
|
-
},
|
|
86
|
+
}, testTimeoutMs(362_000));
|
|
85
87
|
|
|
86
88
|
test("cancel batch", async () => {
|
|
87
89
|
const urls = ["https://docs.firecrawl.dev", "https://firecrawl.dev"];
|
|
@@ -89,6 +91,6 @@ describe("v2.batch e2e", () => {
|
|
|
89
91
|
expect(typeof start.id).toBe("string");
|
|
90
92
|
const cancelled = await client.cancelBatchScrape(start.id);
|
|
91
93
|
expect(cancelled).toBe(true);
|
|
92
|
-
},
|
|
94
|
+
}, testTimeoutMs(151_000));
|
|
93
95
|
});
|
|
94
96
|
|