firecrawl 4.40.0 → 4.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-OJ4YJQ4J.js → chunk-I2DFXQYR.js} +1 -1
- package/dist/index.cjs +4 -2
- package/dist/index.d.cts +36 -16
- package/dist/index.d.ts +36 -16
- package/dist/index.js +5 -3
- package/dist/{package-F6O7J2XI.js → package-V72LKFNW.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/extract.unit.test.ts +23 -0
- package/src/__tests__/unit/v2/tools.unit.test.ts +66 -9
- package/src/v2/methods/extract.ts +2 -1
- package/src/v2/methods/search.ts +1 -0
- package/src/v2/types.ts +36 -14
|
@@ -12,7 +12,7 @@ var require_package = __commonJS({
|
|
|
12
12
|
"package.json"(exports, module) {
|
|
13
13
|
module.exports = {
|
|
14
14
|
name: "@mendable/firecrawl-js",
|
|
15
|
-
version: "4.
|
|
15
|
+
version: "4.41.0",
|
|
16
16
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
17
17
|
main: "dist/index.js",
|
|
18
18
|
types: "dist/index.d.ts",
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.
|
|
42
|
+
version: "4.41.0",
|
|
43
43
|
description: "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -779,6 +779,7 @@ function prepareSearchPayload(req) {
|
|
|
779
779
|
const payload = {
|
|
780
780
|
query: req.query
|
|
781
781
|
};
|
|
782
|
+
if (req.toolDetail != null) payload.toolDetail = req.toolDetail;
|
|
782
783
|
if (req.domainTools != null) payload.domainTools = req.domainTools;
|
|
783
784
|
if (req.sources) payload.sources = req.sources;
|
|
784
785
|
if (req.categories) payload.categories = req.categories;
|
|
@@ -1447,7 +1448,8 @@ function prepareExtractPayload(args) {
|
|
|
1447
1448
|
body.threatProtection = args.threatProtection;
|
|
1448
1449
|
if (args.scrapeOptions) {
|
|
1449
1450
|
ensureValidScrapeOptions(args.scrapeOptions);
|
|
1450
|
-
|
|
1451
|
+
const { toolDetail, domainTools, ...scrapeOptions } = args.scrapeOptions;
|
|
1452
|
+
body.scrapeOptions = scrapeOptions;
|
|
1451
1453
|
}
|
|
1452
1454
|
return body;
|
|
1453
1455
|
}
|
package/dist/index.d.cts
CHANGED
|
@@ -118,6 +118,15 @@ type ActionOption = WaitAction | ScreenshotAction | ClickAction | WriteAction |
|
|
|
118
118
|
interface AuditMetadata$1 {
|
|
119
119
|
username: string;
|
|
120
120
|
}
|
|
121
|
+
/**
|
|
122
|
+
* OCR raster images (PNG, JPEG, JPEG 2000, TIFF, GIF, BMP, WebP, AVIF) as
|
|
123
|
+
* one-page documents. Part of the default parsers list next to "pdf"; takes
|
|
124
|
+
* no options, so the string "image" is equivalent. Omit it from an explicit
|
|
125
|
+
* list to keep image URLs failing as unsupported files.
|
|
126
|
+
*/
|
|
127
|
+
type ImageParser = {
|
|
128
|
+
type: "image";
|
|
129
|
+
};
|
|
121
130
|
type PDFParser = {
|
|
122
131
|
type: "pdf";
|
|
123
132
|
mode?: "fast" | "auto" | "ocr";
|
|
@@ -174,7 +183,7 @@ interface ScrapeOptions {
|
|
|
174
183
|
timeout?: number;
|
|
175
184
|
waitFor?: number;
|
|
176
185
|
mobile?: boolean;
|
|
177
|
-
parsers?: Array<string | PDFParser>;
|
|
186
|
+
parsers?: Array<string | PDFParser | ImageParser>;
|
|
178
187
|
actions?: ActionOption[];
|
|
179
188
|
location?: LocationConfig$1;
|
|
180
189
|
skipTlsVerification?: boolean;
|
|
@@ -200,8 +209,10 @@ interface ScrapeOptions {
|
|
|
200
209
|
};
|
|
201
210
|
integration?: string;
|
|
202
211
|
origin?: string;
|
|
203
|
-
/** Include domain-matched Alexandria
|
|
212
|
+
/** Include domain-matched Alexandria tools for this URL in `tools`. Default off. */
|
|
204
213
|
domainTools?: boolean;
|
|
214
|
+
/** Summary by default; compact returns identity and description, full includes contracts. */
|
|
215
|
+
toolDetail?: "compact" | "summary" | "full";
|
|
205
216
|
}
|
|
206
217
|
type RedactPIIEntity = "PERSON" | "EMAIL" | "PHONE" | "LOCATION" | "FINANCIAL" | "SECRET";
|
|
207
218
|
interface RedactPIIOptions {
|
|
@@ -227,8 +238,14 @@ interface RedactPIIOptions {
|
|
|
227
238
|
* fields you provide replace the team policy's values.
|
|
228
239
|
*/
|
|
229
240
|
interface ThreatProtectionOptions {
|
|
230
|
-
/**
|
|
231
|
-
|
|
241
|
+
/**
|
|
242
|
+
* "off" disables scanning for this request; "manual-only" enforces only the
|
|
243
|
+
* blacklist / whitelist / blocked TLDs (no provider scan, no scan fee);
|
|
244
|
+
* "normal" scans with Google Web Risk; "zscaler" classifies through your
|
|
245
|
+
* organization's Zscaler connection. Enforced teams may raise the mode per
|
|
246
|
+
* request but never lower it.
|
|
247
|
+
*/
|
|
248
|
+
mode?: "off" | "manual-only" | "normal" | "zscaler";
|
|
232
249
|
/** Block verdicts at or above this risk score (integer 0-100). */
|
|
233
250
|
riskScoreThreshold?: number;
|
|
234
251
|
/** Exact domains or globs like "*.example.com" to always block (max 1000). */
|
|
@@ -246,7 +263,7 @@ interface ParseFile {
|
|
|
246
263
|
filename: string;
|
|
247
264
|
contentType?: string;
|
|
248
265
|
}
|
|
249
|
-
type ParseOptions = Omit<ScrapeOptions, "formats" | "waitFor" | "mobile" | "actions" | "location" | "maxAge" | "minAge" | "storeInCache" | "lockdown" | "proxy" | "threatProtection"> & {
|
|
266
|
+
type ParseOptions = Omit<ScrapeOptions, "formats" | "waitFor" | "mobile" | "actions" | "location" | "maxAge" | "minAge" | "storeInCache" | "lockdown" | "proxy" | "threatProtection" | "toolDetail"> & {
|
|
250
267
|
formats?: ParseFormatOption[];
|
|
251
268
|
proxy?: "basic" | "auto";
|
|
252
269
|
};
|
|
@@ -662,14 +679,15 @@ interface SearchData {
|
|
|
662
679
|
}
|
|
663
680
|
/** A complete tool contract returned by semantic or contextual discovery. */
|
|
664
681
|
interface DiscoveredTool {
|
|
682
|
+
next?: AlexandriaCall;
|
|
665
683
|
id?: string;
|
|
666
684
|
provider: string;
|
|
667
685
|
capability: string;
|
|
668
|
-
name
|
|
686
|
+
name?: string;
|
|
669
687
|
description: string;
|
|
670
|
-
creditsCost
|
|
671
|
-
perRecord
|
|
672
|
-
options
|
|
688
|
+
creditsCost?: number;
|
|
689
|
+
perRecord?: boolean;
|
|
690
|
+
options?: Array<{
|
|
673
691
|
name: string;
|
|
674
692
|
type: string;
|
|
675
693
|
required?: boolean;
|
|
@@ -797,8 +815,10 @@ interface CategoryOption {
|
|
|
797
815
|
}
|
|
798
816
|
interface SearchRequest {
|
|
799
817
|
query: string;
|
|
800
|
-
/** Include domain-matched
|
|
818
|
+
/** Include domain-matched tools in tools alongside semantic matches. */
|
|
801
819
|
domainTools?: boolean;
|
|
820
|
+
/** Compact by default; summary adds metadata, full includes contracts. */
|
|
821
|
+
toolDetail?: "compact" | "summary" | "full";
|
|
802
822
|
sources?: Array<"web" | "news" | "images" | "alexandria" | {
|
|
803
823
|
type: "web" | "news" | "images" | "alexandria";
|
|
804
824
|
}>;
|
|
@@ -822,7 +842,7 @@ interface SearchRequest {
|
|
|
822
842
|
timeout?: number;
|
|
823
843
|
/** Generate query-relevant highlights for search results. Defaults to true. */
|
|
824
844
|
highlights?: boolean;
|
|
825
|
-
scrapeOptions?: ScrapeOptions
|
|
845
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
826
846
|
/**
|
|
827
847
|
* Enterprise search options. Use `["zdr"]` for end-to-end Zero Data
|
|
828
848
|
* Retention or `["anon"]` for anonymized search. Must be enabled for
|
|
@@ -850,7 +870,7 @@ interface CrawlOptions {
|
|
|
850
870
|
delay?: number | null;
|
|
851
871
|
maxConcurrency?: number | null;
|
|
852
872
|
webhook?: string | WebhookConfig | null;
|
|
853
|
-
scrapeOptions?: ScrapeOptions | null;
|
|
873
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail"> | null;
|
|
854
874
|
regexOnFullURL?: boolean;
|
|
855
875
|
zeroDataRetention?: boolean;
|
|
856
876
|
integration?: string;
|
|
@@ -871,7 +891,7 @@ interface CrawlJob {
|
|
|
871
891
|
data: Document[];
|
|
872
892
|
}
|
|
873
893
|
interface BatchScrapeOptions {
|
|
874
|
-
options?: ScrapeOptions
|
|
894
|
+
options?: Omit<ScrapeOptions, "toolDetail">;
|
|
875
895
|
webhook?: string | WebhookConfig;
|
|
876
896
|
appendToId?: string;
|
|
877
897
|
ignoreInvalidURLs?: boolean;
|
|
@@ -1005,14 +1025,14 @@ interface MonitorScrapeTarget {
|
|
|
1005
1025
|
id?: string;
|
|
1006
1026
|
type: "scrape";
|
|
1007
1027
|
urls: string[];
|
|
1008
|
-
scrapeOptions?: ScrapeOptions
|
|
1028
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
1009
1029
|
}
|
|
1010
1030
|
interface MonitorCrawlTarget {
|
|
1011
1031
|
id?: string;
|
|
1012
1032
|
type: "crawl";
|
|
1013
1033
|
url: string;
|
|
1014
1034
|
crawlOptions?: CrawlOptions;
|
|
1015
|
-
scrapeOptions?: ScrapeOptions
|
|
1035
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
1016
1036
|
}
|
|
1017
1037
|
interface MonitorSearchTarget {
|
|
1018
1038
|
id?: string;
|
|
@@ -3389,4 +3409,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
3389
3409
|
get v1(): FirecrawlApp;
|
|
3390
3410
|
}
|
|
3391
3411
|
|
|
3392
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentEffort, type AgentExchangeOptions, type AgentExchangeSummary, type AgentListItem, type AgentListOptions, type AgentListResponse, type AgentMode, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentSuggestion, type AgentThread, type AgentThreadResponse, type AgentThreadRun, type AgentTraceActiveBrowserSession, type AgentTraceAgentFinishedEvent, type AgentTraceAgentIdentity, type AgentTraceAgentRole, type AgentTraceAgentStartedEvent, type AgentTraceArtifactChange, type AgentTraceArtifactUpdatedEvent, type AgentTraceBrowserSessionFinishedEvent, type AgentTraceBrowserSessionStartedEvent, type AgentTraceError, type AgentTraceErrorOccurredEvent, type AgentTraceEvent, type AgentTraceProgressReportedEvent, type AgentTraceReasoningSummaryEvent, type AgentTraceResponse, type AgentTraceRunCancelRequestedEvent, type AgentTraceRunFinishedEvent, type AgentTraceRunStartedEvent, type AgentTraceToolCallFinishedEvent, type AgentTraceToolCallStartedEvent, type AgentWebhookConfig, type AgentWebhookEvent, type AlexandriaCall, type AlexandriaOptions, type AlexandriaScrapeData, type AlexandriaScrapeError, type AlexandriaScrapeRequest, type AlexandriaScrapeResult, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type DiscoveredTool, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, type FindToolsData, type FindToolsOptions, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PendingApproval, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, type RequiresAction, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default, parseRequiresAction };
|
|
3412
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentEffort, type AgentExchangeOptions, type AgentExchangeSummary, type AgentListItem, type AgentListOptions, type AgentListResponse, type AgentMode, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentSuggestion, type AgentThread, type AgentThreadResponse, type AgentThreadRun, type AgentTraceActiveBrowserSession, type AgentTraceAgentFinishedEvent, type AgentTraceAgentIdentity, type AgentTraceAgentRole, type AgentTraceAgentStartedEvent, type AgentTraceArtifactChange, type AgentTraceArtifactUpdatedEvent, type AgentTraceBrowserSessionFinishedEvent, type AgentTraceBrowserSessionStartedEvent, type AgentTraceError, type AgentTraceErrorOccurredEvent, type AgentTraceEvent, type AgentTraceProgressReportedEvent, type AgentTraceReasoningSummaryEvent, type AgentTraceResponse, type AgentTraceRunCancelRequestedEvent, type AgentTraceRunFinishedEvent, type AgentTraceRunStartedEvent, type AgentTraceToolCallFinishedEvent, type AgentTraceToolCallStartedEvent, type AgentWebhookConfig, type AgentWebhookEvent, type AlexandriaCall, type AlexandriaOptions, type AlexandriaScrapeData, type AlexandriaScrapeError, type AlexandriaScrapeRequest, type AlexandriaScrapeResult, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type DiscoveredTool, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, type FindToolsData, type FindToolsOptions, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, type ImageParser, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PendingApproval, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, type RequiresAction, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default, parseRequiresAction };
|
package/dist/index.d.ts
CHANGED
|
@@ -118,6 +118,15 @@ type ActionOption = WaitAction | ScreenshotAction | ClickAction | WriteAction |
|
|
|
118
118
|
interface AuditMetadata$1 {
|
|
119
119
|
username: string;
|
|
120
120
|
}
|
|
121
|
+
/**
|
|
122
|
+
* OCR raster images (PNG, JPEG, JPEG 2000, TIFF, GIF, BMP, WebP, AVIF) as
|
|
123
|
+
* one-page documents. Part of the default parsers list next to "pdf"; takes
|
|
124
|
+
* no options, so the string "image" is equivalent. Omit it from an explicit
|
|
125
|
+
* list to keep image URLs failing as unsupported files.
|
|
126
|
+
*/
|
|
127
|
+
type ImageParser = {
|
|
128
|
+
type: "image";
|
|
129
|
+
};
|
|
121
130
|
type PDFParser = {
|
|
122
131
|
type: "pdf";
|
|
123
132
|
mode?: "fast" | "auto" | "ocr";
|
|
@@ -174,7 +183,7 @@ interface ScrapeOptions {
|
|
|
174
183
|
timeout?: number;
|
|
175
184
|
waitFor?: number;
|
|
176
185
|
mobile?: boolean;
|
|
177
|
-
parsers?: Array<string | PDFParser>;
|
|
186
|
+
parsers?: Array<string | PDFParser | ImageParser>;
|
|
178
187
|
actions?: ActionOption[];
|
|
179
188
|
location?: LocationConfig$1;
|
|
180
189
|
skipTlsVerification?: boolean;
|
|
@@ -200,8 +209,10 @@ interface ScrapeOptions {
|
|
|
200
209
|
};
|
|
201
210
|
integration?: string;
|
|
202
211
|
origin?: string;
|
|
203
|
-
/** Include domain-matched Alexandria
|
|
212
|
+
/** Include domain-matched Alexandria tools for this URL in `tools`. Default off. */
|
|
204
213
|
domainTools?: boolean;
|
|
214
|
+
/** Summary by default; compact returns identity and description, full includes contracts. */
|
|
215
|
+
toolDetail?: "compact" | "summary" | "full";
|
|
205
216
|
}
|
|
206
217
|
type RedactPIIEntity = "PERSON" | "EMAIL" | "PHONE" | "LOCATION" | "FINANCIAL" | "SECRET";
|
|
207
218
|
interface RedactPIIOptions {
|
|
@@ -227,8 +238,14 @@ interface RedactPIIOptions {
|
|
|
227
238
|
* fields you provide replace the team policy's values.
|
|
228
239
|
*/
|
|
229
240
|
interface ThreatProtectionOptions {
|
|
230
|
-
/**
|
|
231
|
-
|
|
241
|
+
/**
|
|
242
|
+
* "off" disables scanning for this request; "manual-only" enforces only the
|
|
243
|
+
* blacklist / whitelist / blocked TLDs (no provider scan, no scan fee);
|
|
244
|
+
* "normal" scans with Google Web Risk; "zscaler" classifies through your
|
|
245
|
+
* organization's Zscaler connection. Enforced teams may raise the mode per
|
|
246
|
+
* request but never lower it.
|
|
247
|
+
*/
|
|
248
|
+
mode?: "off" | "manual-only" | "normal" | "zscaler";
|
|
232
249
|
/** Block verdicts at or above this risk score (integer 0-100). */
|
|
233
250
|
riskScoreThreshold?: number;
|
|
234
251
|
/** Exact domains or globs like "*.example.com" to always block (max 1000). */
|
|
@@ -246,7 +263,7 @@ interface ParseFile {
|
|
|
246
263
|
filename: string;
|
|
247
264
|
contentType?: string;
|
|
248
265
|
}
|
|
249
|
-
type ParseOptions = Omit<ScrapeOptions, "formats" | "waitFor" | "mobile" | "actions" | "location" | "maxAge" | "minAge" | "storeInCache" | "lockdown" | "proxy" | "threatProtection"> & {
|
|
266
|
+
type ParseOptions = Omit<ScrapeOptions, "formats" | "waitFor" | "mobile" | "actions" | "location" | "maxAge" | "minAge" | "storeInCache" | "lockdown" | "proxy" | "threatProtection" | "toolDetail"> & {
|
|
250
267
|
formats?: ParseFormatOption[];
|
|
251
268
|
proxy?: "basic" | "auto";
|
|
252
269
|
};
|
|
@@ -662,14 +679,15 @@ interface SearchData {
|
|
|
662
679
|
}
|
|
663
680
|
/** A complete tool contract returned by semantic or contextual discovery. */
|
|
664
681
|
interface DiscoveredTool {
|
|
682
|
+
next?: AlexandriaCall;
|
|
665
683
|
id?: string;
|
|
666
684
|
provider: string;
|
|
667
685
|
capability: string;
|
|
668
|
-
name
|
|
686
|
+
name?: string;
|
|
669
687
|
description: string;
|
|
670
|
-
creditsCost
|
|
671
|
-
perRecord
|
|
672
|
-
options
|
|
688
|
+
creditsCost?: number;
|
|
689
|
+
perRecord?: boolean;
|
|
690
|
+
options?: Array<{
|
|
673
691
|
name: string;
|
|
674
692
|
type: string;
|
|
675
693
|
required?: boolean;
|
|
@@ -797,8 +815,10 @@ interface CategoryOption {
|
|
|
797
815
|
}
|
|
798
816
|
interface SearchRequest {
|
|
799
817
|
query: string;
|
|
800
|
-
/** Include domain-matched
|
|
818
|
+
/** Include domain-matched tools in tools alongside semantic matches. */
|
|
801
819
|
domainTools?: boolean;
|
|
820
|
+
/** Compact by default; summary adds metadata, full includes contracts. */
|
|
821
|
+
toolDetail?: "compact" | "summary" | "full";
|
|
802
822
|
sources?: Array<"web" | "news" | "images" | "alexandria" | {
|
|
803
823
|
type: "web" | "news" | "images" | "alexandria";
|
|
804
824
|
}>;
|
|
@@ -822,7 +842,7 @@ interface SearchRequest {
|
|
|
822
842
|
timeout?: number;
|
|
823
843
|
/** Generate query-relevant highlights for search results. Defaults to true. */
|
|
824
844
|
highlights?: boolean;
|
|
825
|
-
scrapeOptions?: ScrapeOptions
|
|
845
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
826
846
|
/**
|
|
827
847
|
* Enterprise search options. Use `["zdr"]` for end-to-end Zero Data
|
|
828
848
|
* Retention or `["anon"]` for anonymized search. Must be enabled for
|
|
@@ -850,7 +870,7 @@ interface CrawlOptions {
|
|
|
850
870
|
delay?: number | null;
|
|
851
871
|
maxConcurrency?: number | null;
|
|
852
872
|
webhook?: string | WebhookConfig | null;
|
|
853
|
-
scrapeOptions?: ScrapeOptions | null;
|
|
873
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail"> | null;
|
|
854
874
|
regexOnFullURL?: boolean;
|
|
855
875
|
zeroDataRetention?: boolean;
|
|
856
876
|
integration?: string;
|
|
@@ -871,7 +891,7 @@ interface CrawlJob {
|
|
|
871
891
|
data: Document[];
|
|
872
892
|
}
|
|
873
893
|
interface BatchScrapeOptions {
|
|
874
|
-
options?: ScrapeOptions
|
|
894
|
+
options?: Omit<ScrapeOptions, "toolDetail">;
|
|
875
895
|
webhook?: string | WebhookConfig;
|
|
876
896
|
appendToId?: string;
|
|
877
897
|
ignoreInvalidURLs?: boolean;
|
|
@@ -1005,14 +1025,14 @@ interface MonitorScrapeTarget {
|
|
|
1005
1025
|
id?: string;
|
|
1006
1026
|
type: "scrape";
|
|
1007
1027
|
urls: string[];
|
|
1008
|
-
scrapeOptions?: ScrapeOptions
|
|
1028
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
1009
1029
|
}
|
|
1010
1030
|
interface MonitorCrawlTarget {
|
|
1011
1031
|
id?: string;
|
|
1012
1032
|
type: "crawl";
|
|
1013
1033
|
url: string;
|
|
1014
1034
|
crawlOptions?: CrawlOptions;
|
|
1015
|
-
scrapeOptions?: ScrapeOptions
|
|
1035
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
1016
1036
|
}
|
|
1017
1037
|
interface MonitorSearchTarget {
|
|
1018
1038
|
id?: string;
|
|
@@ -3389,4 +3409,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
3389
3409
|
get v1(): FirecrawlApp;
|
|
3390
3410
|
}
|
|
3391
3411
|
|
|
3392
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentEffort, type AgentExchangeOptions, type AgentExchangeSummary, type AgentListItem, type AgentListOptions, type AgentListResponse, type AgentMode, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentSuggestion, type AgentThread, type AgentThreadResponse, type AgentThreadRun, type AgentTraceActiveBrowserSession, type AgentTraceAgentFinishedEvent, type AgentTraceAgentIdentity, type AgentTraceAgentRole, type AgentTraceAgentStartedEvent, type AgentTraceArtifactChange, type AgentTraceArtifactUpdatedEvent, type AgentTraceBrowserSessionFinishedEvent, type AgentTraceBrowserSessionStartedEvent, type AgentTraceError, type AgentTraceErrorOccurredEvent, type AgentTraceEvent, type AgentTraceProgressReportedEvent, type AgentTraceReasoningSummaryEvent, type AgentTraceResponse, type AgentTraceRunCancelRequestedEvent, type AgentTraceRunFinishedEvent, type AgentTraceRunStartedEvent, type AgentTraceToolCallFinishedEvent, type AgentTraceToolCallStartedEvent, type AgentWebhookConfig, type AgentWebhookEvent, type AlexandriaCall, type AlexandriaOptions, type AlexandriaScrapeData, type AlexandriaScrapeError, type AlexandriaScrapeRequest, type AlexandriaScrapeResult, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type DiscoveredTool, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, type FindToolsData, type FindToolsOptions, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PendingApproval, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, type RequiresAction, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default, parseRequiresAction };
|
|
3412
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentEffort, type AgentExchangeOptions, type AgentExchangeSummary, type AgentListItem, type AgentListOptions, type AgentListResponse, type AgentMode, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentSuggestion, type AgentThread, type AgentThreadResponse, type AgentThreadRun, type AgentTraceActiveBrowserSession, type AgentTraceAgentFinishedEvent, type AgentTraceAgentIdentity, type AgentTraceAgentRole, type AgentTraceAgentStartedEvent, type AgentTraceArtifactChange, type AgentTraceArtifactUpdatedEvent, type AgentTraceBrowserSessionFinishedEvent, type AgentTraceBrowserSessionStartedEvent, type AgentTraceError, type AgentTraceErrorOccurredEvent, type AgentTraceEvent, type AgentTraceProgressReportedEvent, type AgentTraceReasoningSummaryEvent, type AgentTraceResponse, type AgentTraceRunCancelRequestedEvent, type AgentTraceRunFinishedEvent, type AgentTraceRunStartedEvent, type AgentTraceToolCallFinishedEvent, type AgentTraceToolCallStartedEvent, type AgentWebhookConfig, type AgentWebhookEvent, type AlexandriaCall, type AlexandriaOptions, type AlexandriaScrapeData, type AlexandriaScrapeError, type AlexandriaScrapeRequest, type AlexandriaScrapeResult, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type DiscoveredTool, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, type FindToolsData, type FindToolsOptions, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, type ImageParser, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PendingApproval, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, type RequiresAction, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default, parseRequiresAction };
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-I2DFXQYR.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -647,6 +647,7 @@ function prepareSearchPayload(req) {
|
|
|
647
647
|
const payload = {
|
|
648
648
|
query: req.query
|
|
649
649
|
};
|
|
650
|
+
if (req.toolDetail != null) payload.toolDetail = req.toolDetail;
|
|
650
651
|
if (req.domainTools != null) payload.domainTools = req.domainTools;
|
|
651
652
|
if (req.sources) payload.sources = req.sources;
|
|
652
653
|
if (req.categories) payload.categories = req.categories;
|
|
@@ -1315,7 +1316,8 @@ function prepareExtractPayload(args) {
|
|
|
1315
1316
|
body.threatProtection = args.threatProtection;
|
|
1316
1317
|
if (args.scrapeOptions) {
|
|
1317
1318
|
ensureValidScrapeOptions(args.scrapeOptions);
|
|
1318
|
-
|
|
1319
|
+
const { toolDetail, domainTools, ...scrapeOptions } = args.scrapeOptions;
|
|
1320
|
+
body.scrapeOptions = scrapeOptions;
|
|
1319
1321
|
}
|
|
1320
1322
|
return body;
|
|
1321
1323
|
}
|
|
@@ -2665,7 +2667,7 @@ var FirecrawlApp = class {
|
|
|
2665
2667
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2666
2668
|
return process.env.npm_package_version;
|
|
2667
2669
|
}
|
|
2668
|
-
const packageJson = await import("./package-
|
|
2670
|
+
const packageJson = await import("./package-V72LKFNW.js");
|
|
2669
2671
|
return packageJson.default.version;
|
|
2670
2672
|
} catch (error) {
|
|
2671
2673
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "firecrawl",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.41.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -3,6 +3,29 @@ import { startExtract } from "../../../v2/methods/extract";
|
|
|
3
3
|
import type { WebhookConfig } from "../../../v2/types";
|
|
4
4
|
|
|
5
5
|
describe("v2.extract unit", () => {
|
|
6
|
+
test("omits discovery options from extract without mutating shared options", async () => {
|
|
7
|
+
const post = jest
|
|
8
|
+
.fn()
|
|
9
|
+
.mockResolvedValue({ status: 200, data: { id: "extract-job" } });
|
|
10
|
+
const scrapeOptions = {
|
|
11
|
+
toolDetail: "compact" as const,
|
|
12
|
+
domainTools: true,
|
|
13
|
+
onlyMainContent: true,
|
|
14
|
+
};
|
|
15
|
+
await startExtract({ post } as any, {
|
|
16
|
+
urls: ["https://example.com"],
|
|
17
|
+
scrapeOptions,
|
|
18
|
+
});
|
|
19
|
+
expect(post).toHaveBeenCalledWith("/v2/extract", {
|
|
20
|
+
urls: ["https://example.com"],
|
|
21
|
+
scrapeOptions: { onlyMainContent: true },
|
|
22
|
+
});
|
|
23
|
+
expect(scrapeOptions).toEqual({
|
|
24
|
+
toolDetail: "compact",
|
|
25
|
+
domainTools: true,
|
|
26
|
+
onlyMainContent: true,
|
|
27
|
+
});
|
|
28
|
+
});
|
|
6
29
|
test("startExtract forwards string webhook in request payload", async () => {
|
|
7
30
|
const post = jest.fn().mockResolvedValue({
|
|
8
31
|
status: 200,
|
|
@@ -1,4 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import {
|
|
2
|
+
afterAll,
|
|
3
|
+
beforeAll,
|
|
4
|
+
beforeEach,
|
|
5
|
+
describe,
|
|
6
|
+
expect,
|
|
7
|
+
jest,
|
|
8
|
+
test,
|
|
9
|
+
} from "@jest/globals";
|
|
2
10
|
import { createServer, type Server } from "node:http";
|
|
3
11
|
import { FirecrawlClient } from "../../../v2/client";
|
|
4
12
|
|
|
@@ -52,6 +60,13 @@ const next = {
|
|
|
52
60
|
capability: "find-tools",
|
|
53
61
|
options: { providers: ["particle"], level: "tools" },
|
|
54
62
|
};
|
|
63
|
+
const compactTools = [tool, productionTool].map(
|
|
64
|
+
({ provider, capability, description }) => ({
|
|
65
|
+
provider,
|
|
66
|
+
capability,
|
|
67
|
+
description,
|
|
68
|
+
}),
|
|
69
|
+
);
|
|
55
70
|
let server: Server;
|
|
56
71
|
let client: FirecrawlClient;
|
|
57
72
|
const sent: Array<{ body: any; id: string | undefined }> = [];
|
|
@@ -76,10 +91,20 @@ beforeAll(async () => {
|
|
|
76
91
|
warning: "Example warning",
|
|
77
92
|
data: {
|
|
78
93
|
web: [{ url: "https://podcasts.apple.com" }],
|
|
79
|
-
tools:
|
|
94
|
+
tools:
|
|
95
|
+
body.toolDetail === "compact"
|
|
96
|
+
? compactTools
|
|
97
|
+
: [tool, productionTool],
|
|
80
98
|
},
|
|
81
99
|
}),
|
|
82
100
|
);
|
|
101
|
+
if (req.url === "/v2/scrape" && body.url)
|
|
102
|
+
return res.end(
|
|
103
|
+
JSON.stringify({
|
|
104
|
+
success: true,
|
|
105
|
+
data: { markdown: "Example", tools: compactTools },
|
|
106
|
+
}),
|
|
107
|
+
);
|
|
83
108
|
if (body.alexandria[0].provider === "retry" && attempts++ === 0) {
|
|
84
109
|
res.statusCode = 502;
|
|
85
110
|
return res.end("{}");
|
|
@@ -150,6 +175,26 @@ afterAll(async () => {
|
|
|
150
175
|
});
|
|
151
176
|
|
|
152
177
|
describe("Alexandria contracts and execution", () => {
|
|
178
|
+
test.each(["compact", "summary", "full"] as const)(
|
|
179
|
+
"forwards %s discovery detail",
|
|
180
|
+
async (toolDetail) => {
|
|
181
|
+
const result = await client.search("records", { toolDetail });
|
|
182
|
+
if (toolDetail === "compact") expect(result.tools).toEqual(compactTools);
|
|
183
|
+
expect(sent.at(-1)?.body).toMatchObject({ toolDetail });
|
|
184
|
+
},
|
|
185
|
+
);
|
|
186
|
+
test("URL scrape forwards compact detail and preserves tools", async () => {
|
|
187
|
+
const result = await client.scrape("https://example.com", {
|
|
188
|
+
domainTools: true,
|
|
189
|
+
toolDetail: "compact",
|
|
190
|
+
});
|
|
191
|
+
expect(sent.at(-1)?.body).toMatchObject({
|
|
192
|
+
url: "https://example.com",
|
|
193
|
+
domainTools: true,
|
|
194
|
+
toolDetail: "compact",
|
|
195
|
+
});
|
|
196
|
+
expect(result.tools).toEqual(compactTools);
|
|
197
|
+
});
|
|
153
198
|
test("returns complete unified tools and warning beside web results", async () => {
|
|
154
199
|
const result = await client.search("podcasts", {
|
|
155
200
|
sources: ["web", { type: "alexandria" }],
|
|
@@ -226,15 +271,27 @@ describe("Alexandria contracts and execution", () => {
|
|
|
226
271
|
});
|
|
227
272
|
});
|
|
228
273
|
|
|
229
|
-
|
|
230
|
-
|
|
274
|
+
test.each([
|
|
275
|
+
[undefined, 80000],
|
|
276
|
+
[1000, 31000],
|
|
277
|
+
[100000, 80000],
|
|
278
|
+
])(
|
|
231
279
|
"Alexandria timeout %s allows response delivery (%s ms)",
|
|
232
280
|
async (timeout, timeoutMs) => {
|
|
233
|
-
const http = {
|
|
234
|
-
|
|
235
|
-
|
|
281
|
+
const http = {
|
|
282
|
+
post: jest.fn(async () => ({
|
|
283
|
+
status: 200,
|
|
284
|
+
data: {
|
|
285
|
+
success: true,
|
|
286
|
+
data: { alexandria: [], creditsCost: 0 },
|
|
287
|
+
},
|
|
288
|
+
})),
|
|
289
|
+
};
|
|
236
290
|
await scrapeAlexandria(http as any, [next], { timeout });
|
|
237
|
-
expect(http.post).toHaveBeenCalledWith(
|
|
238
|
-
|
|
291
|
+
expect(http.post).toHaveBeenCalledWith(
|
|
292
|
+
"/v2/scrape",
|
|
293
|
+
expect.anything(),
|
|
294
|
+
expect.objectContaining({ timeoutMs }),
|
|
295
|
+
);
|
|
239
296
|
},
|
|
240
297
|
);
|
|
@@ -40,7 +40,8 @@ function prepareExtractPayload(args: {
|
|
|
40
40
|
body.threatProtection = args.threatProtection;
|
|
41
41
|
if (args.scrapeOptions) {
|
|
42
42
|
ensureValidScrapeOptions(args.scrapeOptions);
|
|
43
|
-
|
|
43
|
+
const { toolDetail, domainTools, ...scrapeOptions } = args.scrapeOptions;
|
|
44
|
+
body.scrapeOptions = scrapeOptions;
|
|
44
45
|
}
|
|
45
46
|
return body;
|
|
46
47
|
}
|
package/src/v2/methods/search.ts
CHANGED
|
@@ -27,6 +27,7 @@ function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
|
|
|
27
27
|
const payload: Record<string, unknown> = {
|
|
28
28
|
query: req.query,
|
|
29
29
|
};
|
|
30
|
+
if (req.toolDetail != null) payload.toolDetail = req.toolDetail;
|
|
30
31
|
if (req.domainTools != null) payload.domainTools = req.domainTools;
|
|
31
32
|
if (req.sources) payload.sources = req.sources;
|
|
32
33
|
if (req.categories) payload.categories = req.categories;
|
package/src/v2/types.ts
CHANGED
|
@@ -188,6 +188,16 @@ export interface AuditMetadata {
|
|
|
188
188
|
username: string;
|
|
189
189
|
}
|
|
190
190
|
|
|
191
|
+
/**
|
|
192
|
+
* OCR raster images (PNG, JPEG, JPEG 2000, TIFF, GIF, BMP, WebP, AVIF) as
|
|
193
|
+
* one-page documents. Part of the default parsers list next to "pdf"; takes
|
|
194
|
+
* no options, so the string "image" is equivalent. Omit it from an explicit
|
|
195
|
+
* list to keep image URLs failing as unsupported files.
|
|
196
|
+
*/
|
|
197
|
+
export type ImageParser = {
|
|
198
|
+
type: "image";
|
|
199
|
+
};
|
|
200
|
+
|
|
191
201
|
export type PDFParser = {
|
|
192
202
|
type: "pdf";
|
|
193
203
|
mode?: "fast" | "auto" | "ocr";
|
|
@@ -249,7 +259,7 @@ export interface ScrapeOptions {
|
|
|
249
259
|
timeout?: number;
|
|
250
260
|
waitFor?: number;
|
|
251
261
|
mobile?: boolean;
|
|
252
|
-
parsers?: Array<string | PDFParser>;
|
|
262
|
+
parsers?: Array<string | PDFParser | ImageParser>;
|
|
253
263
|
actions?: ActionOption[];
|
|
254
264
|
location?: LocationConfig;
|
|
255
265
|
skipTlsVerification?: boolean;
|
|
@@ -275,8 +285,10 @@ export interface ScrapeOptions {
|
|
|
275
285
|
};
|
|
276
286
|
integration?: string;
|
|
277
287
|
origin?: string;
|
|
278
|
-
/** Include domain-matched Alexandria
|
|
288
|
+
/** Include domain-matched Alexandria tools for this URL in `tools`. Default off. */
|
|
279
289
|
domainTools?: boolean;
|
|
290
|
+
/** Summary by default; compact returns identity and description, full includes contracts. */
|
|
291
|
+
toolDetail?: "compact" | "summary" | "full";
|
|
280
292
|
}
|
|
281
293
|
|
|
282
294
|
export type RedactPIIEntity =
|
|
@@ -306,8 +318,14 @@ export interface RedactPIIOptions {
|
|
|
306
318
|
* fields you provide replace the team policy's values.
|
|
307
319
|
*/
|
|
308
320
|
export interface ThreatProtectionOptions {
|
|
309
|
-
/**
|
|
310
|
-
|
|
321
|
+
/**
|
|
322
|
+
* "off" disables scanning for this request; "manual-only" enforces only the
|
|
323
|
+
* blacklist / whitelist / blocked TLDs (no provider scan, no scan fee);
|
|
324
|
+
* "normal" scans with Google Web Risk; "zscaler" classifies through your
|
|
325
|
+
* organization's Zscaler connection. Enforced teams may raise the mode per
|
|
326
|
+
* request but never lower it.
|
|
327
|
+
*/
|
|
328
|
+
mode?: "off" | "manual-only" | "normal" | "zscaler";
|
|
311
329
|
/** Block verdicts at or above this risk score (integer 0-100). */
|
|
312
330
|
riskScoreThreshold?: number;
|
|
313
331
|
/** Exact domains or globs like "*.example.com" to always block (max 1000). */
|
|
@@ -342,6 +360,7 @@ export type ParseOptions = Omit<
|
|
|
342
360
|
| "lockdown"
|
|
343
361
|
| "proxy"
|
|
344
362
|
| "threatProtection"
|
|
363
|
+
| "toolDetail"
|
|
345
364
|
> & {
|
|
346
365
|
formats?: ParseFormatOption[];
|
|
347
366
|
proxy?: "basic" | "auto";
|
|
@@ -810,14 +829,15 @@ export interface SearchData {
|
|
|
810
829
|
|
|
811
830
|
/** A complete tool contract returned by semantic or contextual discovery. */
|
|
812
831
|
export interface DiscoveredTool {
|
|
832
|
+
next?: AlexandriaCall;
|
|
813
833
|
id?: string;
|
|
814
834
|
provider: string;
|
|
815
835
|
capability: string;
|
|
816
|
-
name
|
|
836
|
+
name?: string;
|
|
817
837
|
description: string;
|
|
818
|
-
creditsCost
|
|
819
|
-
perRecord
|
|
820
|
-
options
|
|
838
|
+
creditsCost?: number;
|
|
839
|
+
perRecord?: boolean;
|
|
840
|
+
options?: Array<{
|
|
821
841
|
name: string;
|
|
822
842
|
type: string;
|
|
823
843
|
required?: boolean;
|
|
@@ -954,8 +974,10 @@ export interface CategoryOption {
|
|
|
954
974
|
|
|
955
975
|
export interface SearchRequest {
|
|
956
976
|
query: string;
|
|
957
|
-
/** Include domain-matched
|
|
977
|
+
/** Include domain-matched tools in tools alongside semantic matches. */
|
|
958
978
|
domainTools?: boolean;
|
|
979
|
+
/** Compact by default; summary adds metadata, full includes contracts. */
|
|
980
|
+
toolDetail?: "compact" | "summary" | "full";
|
|
959
981
|
sources?: Array<
|
|
960
982
|
"web" | "news" | "images" | "alexandria"
|
|
961
983
|
| { type: "web" | "news" | "images" | "alexandria" }
|
|
@@ -982,7 +1004,7 @@ export interface SearchRequest {
|
|
|
982
1004
|
timeout?: number; // ms
|
|
983
1005
|
/** Generate query-relevant highlights for search results. Defaults to true. */
|
|
984
1006
|
highlights?: boolean;
|
|
985
|
-
scrapeOptions?: ScrapeOptions
|
|
1007
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
986
1008
|
/**
|
|
987
1009
|
* Enterprise search options. Use `["zdr"]` for end-to-end Zero Data
|
|
988
1010
|
* Retention or `["anon"]` for anonymized search. Must be enabled for
|
|
@@ -1011,7 +1033,7 @@ export interface CrawlOptions {
|
|
|
1011
1033
|
delay?: number | null;
|
|
1012
1034
|
maxConcurrency?: number | null;
|
|
1013
1035
|
webhook?: string | WebhookConfig | null;
|
|
1014
|
-
scrapeOptions?: ScrapeOptions | null;
|
|
1036
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail"> | null;
|
|
1015
1037
|
regexOnFullURL?: boolean;
|
|
1016
1038
|
zeroDataRetention?: boolean;
|
|
1017
1039
|
integration?: string;
|
|
@@ -1035,7 +1057,7 @@ export interface CrawlJob {
|
|
|
1035
1057
|
}
|
|
1036
1058
|
|
|
1037
1059
|
export interface BatchScrapeOptions {
|
|
1038
|
-
options?: ScrapeOptions
|
|
1060
|
+
options?: Omit<ScrapeOptions, "toolDetail">;
|
|
1039
1061
|
webhook?: string | WebhookConfig;
|
|
1040
1062
|
appendToId?: string;
|
|
1041
1063
|
ignoreInvalidURLs?: boolean;
|
|
@@ -1185,7 +1207,7 @@ export interface MonitorScrapeTarget {
|
|
|
1185
1207
|
id?: string;
|
|
1186
1208
|
type: "scrape";
|
|
1187
1209
|
urls: string[];
|
|
1188
|
-
scrapeOptions?: ScrapeOptions
|
|
1210
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
1189
1211
|
}
|
|
1190
1212
|
|
|
1191
1213
|
export interface MonitorCrawlTarget {
|
|
@@ -1193,7 +1215,7 @@ export interface MonitorCrawlTarget {
|
|
|
1193
1215
|
type: "crawl";
|
|
1194
1216
|
url: string;
|
|
1195
1217
|
crawlOptions?: CrawlOptions;
|
|
1196
|
-
scrapeOptions?: ScrapeOptions
|
|
1218
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
1197
1219
|
}
|
|
1198
1220
|
|
|
1199
1221
|
export interface MonitorSearchTarget {
|