@mendable/firecrawl-js 4.39.0 → 4.41.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -0
- package/dist/{chunk-4DPCCKWG.js → chunk-I2DFXQYR.js} +1 -1
- package/dist/index.cjs +169 -10
- package/dist/index.d.cts +167 -13
- package/dist/index.d.ts +167 -13
- package/dist/index.js +168 -10
- package/dist/{package-D2QZMRW7.js → package-V72LKFNW.js} +1 -1
- package/package.json +1 -1
- package/pnpm-workspace.yaml +2 -2
- package/src/__tests__/unit/v2/extract.unit.test.ts +23 -0
- package/src/__tests__/unit/v2/threat-protection.unit.test.ts +18 -1
- package/src/__tests__/unit/v2/tools.unit.test.ts +297 -0
- package/src/v2/client.ts +39 -3
- package/src/v2/methods/extract.ts +2 -1
- package/src/v2/methods/scrape.ts +3 -1
- package/src/v2/methods/search.ts +8 -1
- package/src/v2/methods/tools.ts +137 -0
- package/src/v2/types.ts +186 -9
- package/src/v2/utils/errorHandler.ts +19 -3
package/dist/index.d.ts
CHANGED
|
@@ -118,6 +118,15 @@ type ActionOption = WaitAction | ScreenshotAction | ClickAction | WriteAction |
|
|
|
118
118
|
interface AuditMetadata$1 {
|
|
119
119
|
username: string;
|
|
120
120
|
}
|
|
121
|
+
/**
|
|
122
|
+
* OCR raster images (PNG, JPEG, JPEG 2000, TIFF, GIF, BMP, WebP, AVIF) as
|
|
123
|
+
* one-page documents. Part of the default parsers list next to "pdf"; takes
|
|
124
|
+
* no options, so the string "image" is equivalent. Omit it from an explicit
|
|
125
|
+
* list to keep image URLs failing as unsupported files.
|
|
126
|
+
*/
|
|
127
|
+
type ImageParser = {
|
|
128
|
+
type: "image";
|
|
129
|
+
};
|
|
121
130
|
type PDFParser = {
|
|
122
131
|
type: "pdf";
|
|
123
132
|
mode?: "fast" | "auto" | "ocr";
|
|
@@ -174,7 +183,7 @@ interface ScrapeOptions {
|
|
|
174
183
|
timeout?: number;
|
|
175
184
|
waitFor?: number;
|
|
176
185
|
mobile?: boolean;
|
|
177
|
-
parsers?: Array<string | PDFParser>;
|
|
186
|
+
parsers?: Array<string | PDFParser | ImageParser>;
|
|
178
187
|
actions?: ActionOption[];
|
|
179
188
|
location?: LocationConfig$1;
|
|
180
189
|
skipTlsVerification?: boolean;
|
|
@@ -200,6 +209,10 @@ interface ScrapeOptions {
|
|
|
200
209
|
};
|
|
201
210
|
integration?: string;
|
|
202
211
|
origin?: string;
|
|
212
|
+
/** Include domain-matched Alexandria tools for this URL in `tools`. Default off. */
|
|
213
|
+
domainTools?: boolean;
|
|
214
|
+
/** Summary by default; compact returns identity and description, full includes contracts. */
|
|
215
|
+
toolDetail?: "compact" | "summary" | "full";
|
|
203
216
|
}
|
|
204
217
|
type RedactPIIEntity = "PERSON" | "EMAIL" | "PHONE" | "LOCATION" | "FINANCIAL" | "SECRET";
|
|
205
218
|
interface RedactPIIOptions {
|
|
@@ -225,8 +238,14 @@ interface RedactPIIOptions {
|
|
|
225
238
|
* fields you provide replace the team policy's values.
|
|
226
239
|
*/
|
|
227
240
|
interface ThreatProtectionOptions {
|
|
228
|
-
/**
|
|
229
|
-
|
|
241
|
+
/**
|
|
242
|
+
* "off" disables scanning for this request; "manual-only" enforces only the
|
|
243
|
+
* blacklist / whitelist / blocked TLDs (no provider scan, no scan fee);
|
|
244
|
+
* "normal" scans with Google Web Risk; "zscaler" classifies through your
|
|
245
|
+
* organization's Zscaler connection. Enforced teams may raise the mode per
|
|
246
|
+
* request but never lower it.
|
|
247
|
+
*/
|
|
248
|
+
mode?: "off" | "manual-only" | "normal" | "zscaler";
|
|
230
249
|
/** Block verdicts at or above this risk score (integer 0-100). */
|
|
231
250
|
riskScoreThreshold?: number;
|
|
232
251
|
/** Exact domains or globs like "*.example.com" to always block (max 1000). */
|
|
@@ -244,7 +263,7 @@ interface ParseFile {
|
|
|
244
263
|
filename: string;
|
|
245
264
|
contentType?: string;
|
|
246
265
|
}
|
|
247
|
-
type ParseOptions = Omit<ScrapeOptions, "formats" | "waitFor" | "mobile" | "actions" | "location" | "maxAge" | "minAge" | "storeInCache" | "lockdown" | "proxy" | "threatProtection"> & {
|
|
266
|
+
type ParseOptions = Omit<ScrapeOptions, "formats" | "waitFor" | "mobile" | "actions" | "location" | "maxAge" | "minAge" | "storeInCache" | "lockdown" | "proxy" | "threatProtection" | "toolDetail"> & {
|
|
248
267
|
formats?: ParseFormatOption[];
|
|
249
268
|
proxy?: "basic" | "auto";
|
|
250
269
|
};
|
|
@@ -549,6 +568,8 @@ interface Document {
|
|
|
549
568
|
pages?: PdfPage[];
|
|
550
569
|
/** Typed PDF layout blocks, present only when `parsers[].blocks` is true. */
|
|
551
570
|
blocks?: PdfPageBlocks[];
|
|
571
|
+
/** Present when `domainTools` discovered contracts matching this URL. */
|
|
572
|
+
tools?: DiscoveredTool[];
|
|
552
573
|
}
|
|
553
574
|
interface PaginationConfig {
|
|
554
575
|
/** When true (default), automatically follow `next` links and aggregate all documents. */
|
|
@@ -650,9 +671,122 @@ interface SearchResultImages {
|
|
|
650
671
|
position?: number;
|
|
651
672
|
}
|
|
652
673
|
interface SearchData {
|
|
674
|
+
warning?: string;
|
|
653
675
|
web?: Array<SearchResultWeb | Document>;
|
|
654
676
|
news?: Array<SearchResultNews | Document>;
|
|
655
677
|
images?: Array<SearchResultImages | Document>;
|
|
678
|
+
tools?: DiscoveredTool[];
|
|
679
|
+
}
|
|
680
|
+
/** A complete tool contract returned by semantic or contextual discovery. */
|
|
681
|
+
interface DiscoveredTool {
|
|
682
|
+
next?: AlexandriaCall;
|
|
683
|
+
id?: string;
|
|
684
|
+
provider: string;
|
|
685
|
+
capability: string;
|
|
686
|
+
name?: string;
|
|
687
|
+
description: string;
|
|
688
|
+
creditsCost?: number;
|
|
689
|
+
perRecord?: boolean;
|
|
690
|
+
options?: Array<{
|
|
691
|
+
name: string;
|
|
692
|
+
type: string;
|
|
693
|
+
required?: boolean;
|
|
694
|
+
[key: string]: unknown;
|
|
695
|
+
}>;
|
|
696
|
+
requiresOneOf?: string[][];
|
|
697
|
+
response?: {
|
|
698
|
+
about?: string;
|
|
699
|
+
key?: string;
|
|
700
|
+
fields?: Array<{
|
|
701
|
+
name: string;
|
|
702
|
+
type: string;
|
|
703
|
+
[key: string]: unknown;
|
|
704
|
+
}>;
|
|
705
|
+
[key: string]: unknown;
|
|
706
|
+
};
|
|
707
|
+
examples?: Partial<Record<"javascript" | "python" | "curl", string>> & Record<string, string>;
|
|
708
|
+
example?: {
|
|
709
|
+
recordedAt: string;
|
|
710
|
+
request: Record<string, unknown>;
|
|
711
|
+
response: unknown;
|
|
712
|
+
};
|
|
713
|
+
label?: string;
|
|
714
|
+
whenToUse?: string;
|
|
715
|
+
returns?: unknown;
|
|
716
|
+
discovery?: unknown;
|
|
717
|
+
attribution?: unknown;
|
|
718
|
+
matchedBy?: Array<"semantic" | "domain">;
|
|
719
|
+
matchedUrls?: string[];
|
|
720
|
+
concept?: string;
|
|
721
|
+
cohorts?: string[];
|
|
722
|
+
similarity?: number;
|
|
723
|
+
}
|
|
724
|
+
interface FindToolsOptions {
|
|
725
|
+
urls?: string[];
|
|
726
|
+
providers?: string[];
|
|
727
|
+
categories?: string[];
|
|
728
|
+
groups?: string[];
|
|
729
|
+
capabilities?: string[];
|
|
730
|
+
level?: "providers" | "groups" | "tools";
|
|
731
|
+
expand?: Array<"options" | "response" | "examples">;
|
|
732
|
+
limit?: number;
|
|
733
|
+
offset?: number;
|
|
734
|
+
}
|
|
735
|
+
interface FindToolsData {
|
|
736
|
+
level: "providers" | "groups" | "tools";
|
|
737
|
+
items: Array<{
|
|
738
|
+
id: string;
|
|
739
|
+
name: string;
|
|
740
|
+
next?: AlexandriaCall;
|
|
741
|
+
execute?: Pick<AlexandriaCall, "provider" | "capability">;
|
|
742
|
+
[key: string]: unknown;
|
|
743
|
+
}>;
|
|
744
|
+
total: number;
|
|
745
|
+
next: AlexandriaCall | null;
|
|
746
|
+
}
|
|
747
|
+
interface AlexandriaScrapeRequest extends AlexandriaOptions {
|
|
748
|
+
alexandria: AlexandriaCall | AlexandriaCall[];
|
|
749
|
+
}
|
|
750
|
+
interface AlexandriaCall {
|
|
751
|
+
provider: string;
|
|
752
|
+
capability: string;
|
|
753
|
+
options?: Record<string, unknown>;
|
|
754
|
+
}
|
|
755
|
+
interface AlexandriaScrapeError {
|
|
756
|
+
code: string;
|
|
757
|
+
message: string;
|
|
758
|
+
status?: number;
|
|
759
|
+
/** Present when credits were captured before the failure. */
|
|
760
|
+
chargeId?: string;
|
|
761
|
+
}
|
|
762
|
+
type AlexandriaScrapeResult = {
|
|
763
|
+
provider: string;
|
|
764
|
+
capability: string;
|
|
765
|
+
creditsCost: number;
|
|
766
|
+
data: unknown;
|
|
767
|
+
records?: number;
|
|
768
|
+
upstreamStatus?: number;
|
|
769
|
+
recordedAt?: string;
|
|
770
|
+
error?: undefined;
|
|
771
|
+
[key: string]: unknown;
|
|
772
|
+
} | {
|
|
773
|
+
provider?: string;
|
|
774
|
+
capability?: string;
|
|
775
|
+
error: AlexandriaScrapeError;
|
|
776
|
+
[key: string]: unknown;
|
|
777
|
+
};
|
|
778
|
+
interface AlexandriaScrapeData {
|
|
779
|
+
scrapeId: string;
|
|
780
|
+
requestId: string;
|
|
781
|
+
alexandria: AlexandriaScrapeResult[];
|
|
782
|
+
creditsCost: number;
|
|
783
|
+
}
|
|
784
|
+
interface AlexandriaOptions {
|
|
785
|
+
/** Reuse this ID with the identical payload when retrying an execution. */
|
|
786
|
+
requestId?: string;
|
|
787
|
+
timeout?: number;
|
|
788
|
+
integration?: string;
|
|
789
|
+
origin?: string;
|
|
656
790
|
}
|
|
657
791
|
/**
|
|
658
792
|
* Narrows ordinary **web search**. A category does not switch `search()` to a
|
|
@@ -681,8 +815,12 @@ interface CategoryOption {
|
|
|
681
815
|
}
|
|
682
816
|
interface SearchRequest {
|
|
683
817
|
query: string;
|
|
684
|
-
|
|
685
|
-
|
|
818
|
+
/** Include domain-matched tools in tools alongside semantic matches. */
|
|
819
|
+
domainTools?: boolean;
|
|
820
|
+
/** Compact by default; summary adds metadata, full includes contracts. */
|
|
821
|
+
toolDetail?: "compact" | "summary" | "full";
|
|
822
|
+
sources?: Array<"web" | "news" | "images" | "alexandria" | {
|
|
823
|
+
type: "web" | "news" | "images" | "alexandria";
|
|
686
824
|
}>;
|
|
687
825
|
/**
|
|
688
826
|
* Narrow web search by category. See {@link CategoryOption}.
|
|
@@ -704,7 +842,7 @@ interface SearchRequest {
|
|
|
704
842
|
timeout?: number;
|
|
705
843
|
/** Generate query-relevant highlights for search results. Defaults to true. */
|
|
706
844
|
highlights?: boolean;
|
|
707
|
-
scrapeOptions?: ScrapeOptions
|
|
845
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
708
846
|
/**
|
|
709
847
|
* Enterprise search options. Use `["zdr"]` for end-to-end Zero Data
|
|
710
848
|
* Retention or `["anon"]` for anonymized search. Must be enabled for
|
|
@@ -732,7 +870,7 @@ interface CrawlOptions {
|
|
|
732
870
|
delay?: number | null;
|
|
733
871
|
maxConcurrency?: number | null;
|
|
734
872
|
webhook?: string | WebhookConfig | null;
|
|
735
|
-
scrapeOptions?: ScrapeOptions | null;
|
|
873
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail"> | null;
|
|
736
874
|
regexOnFullURL?: boolean;
|
|
737
875
|
zeroDataRetention?: boolean;
|
|
738
876
|
integration?: string;
|
|
@@ -753,7 +891,7 @@ interface CrawlJob {
|
|
|
753
891
|
data: Document[];
|
|
754
892
|
}
|
|
755
893
|
interface BatchScrapeOptions {
|
|
756
|
-
options?: ScrapeOptions
|
|
894
|
+
options?: Omit<ScrapeOptions, "toolDetail">;
|
|
757
895
|
webhook?: string | WebhookConfig;
|
|
758
896
|
appendToId?: string;
|
|
759
897
|
ignoreInvalidURLs?: boolean;
|
|
@@ -887,14 +1025,14 @@ interface MonitorScrapeTarget {
|
|
|
887
1025
|
id?: string;
|
|
888
1026
|
type: "scrape";
|
|
889
1027
|
urls: string[];
|
|
890
|
-
scrapeOptions?: ScrapeOptions
|
|
1028
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
891
1029
|
}
|
|
892
1030
|
interface MonitorCrawlTarget {
|
|
893
1031
|
id?: string;
|
|
894
1032
|
type: "crawl";
|
|
895
1033
|
url: string;
|
|
896
1034
|
crawlOptions?: CrawlOptions;
|
|
897
|
-
scrapeOptions?: ScrapeOptions
|
|
1035
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
898
1036
|
}
|
|
899
1037
|
interface MonitorSearchTarget {
|
|
900
1038
|
id?: string;
|
|
@@ -1441,12 +1579,25 @@ interface ErrorDetails {
|
|
|
1441
1579
|
details?: Record<string, unknown>;
|
|
1442
1580
|
status?: number;
|
|
1443
1581
|
}
|
|
1582
|
+
/** An out-of-band step the API requires before the request can succeed. */
|
|
1583
|
+
interface RequiresAction {
|
|
1584
|
+
type: "accept_terms" | (string & {});
|
|
1585
|
+
terms?: string;
|
|
1586
|
+
version?: string;
|
|
1587
|
+
url?: string;
|
|
1588
|
+
}
|
|
1589
|
+
declare function parseRequiresAction(value: unknown): RequiresAction | undefined;
|
|
1444
1590
|
declare class SdkError extends Error {
|
|
1591
|
+
requestId?: string;
|
|
1445
1592
|
status?: number;
|
|
1446
1593
|
code?: string;
|
|
1447
1594
|
details?: unknown;
|
|
1448
1595
|
jobId?: string;
|
|
1449
|
-
|
|
1596
|
+
/** Present on exchange-mediated scrape failures that already captured credits. */
|
|
1597
|
+
chargeId?: string;
|
|
1598
|
+
/** Present when the API needs an out-of-band step first, such as accepting provider terms. */
|
|
1599
|
+
requiresAction?: RequiresAction;
|
|
1600
|
+
constructor(message: string, status?: number, code?: string, details?: unknown, jobId?: string, chargeId?: string, requiresAction?: RequiresAction);
|
|
1450
1601
|
}
|
|
1451
1602
|
declare class JobTimeoutError extends SdkError {
|
|
1452
1603
|
timeoutSeconds: number;
|
|
@@ -1972,6 +2123,9 @@ declare class FirecrawlClient {
|
|
|
1972
2123
|
json?: InferredJsonFromOptions<Opts>;
|
|
1973
2124
|
}>;
|
|
1974
2125
|
scrape(url: string, options?: ScrapeCallOptions): Promise<Document>;
|
|
2126
|
+
scrape(request: AlexandriaScrapeRequest): Promise<AlexandriaScrapeData>;
|
|
2127
|
+
/** Explore the catalogue without executing the tools it returns. */
|
|
2128
|
+
findTools(options?: FindToolsOptions): Promise<FindToolsData>;
|
|
1975
2129
|
/**
|
|
1976
2130
|
* Interact with the browser session associated with a scrape job.
|
|
1977
2131
|
* @param jobId Scrape job id.
|
|
@@ -3255,4 +3409,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
3255
3409
|
get v1(): FirecrawlApp;
|
|
3256
3410
|
}
|
|
3257
3411
|
|
|
3258
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentEffort, type AgentExchangeOptions, type AgentExchangeSummary, type AgentListItem, type AgentListOptions, type AgentListResponse, type AgentMode, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentSuggestion, type AgentThread, type AgentThreadResponse, type AgentThreadRun, type AgentTraceActiveBrowserSession, type AgentTraceAgentFinishedEvent, type AgentTraceAgentIdentity, type AgentTraceAgentRole, type AgentTraceAgentStartedEvent, type AgentTraceArtifactChange, type AgentTraceArtifactUpdatedEvent, type AgentTraceBrowserSessionFinishedEvent, type AgentTraceBrowserSessionStartedEvent, type AgentTraceError, type AgentTraceErrorOccurredEvent, type AgentTraceEvent, type AgentTraceProgressReportedEvent, type AgentTraceReasoningSummaryEvent, type AgentTraceResponse, type AgentTraceRunCancelRequestedEvent, type AgentTraceRunFinishedEvent, type AgentTraceRunStartedEvent, type AgentTraceToolCallFinishedEvent, type AgentTraceToolCallStartedEvent, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PendingApproval, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
|
3412
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentEffort, type AgentExchangeOptions, type AgentExchangeSummary, type AgentListItem, type AgentListOptions, type AgentListResponse, type AgentMode, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentSuggestion, type AgentThread, type AgentThreadResponse, type AgentThreadRun, type AgentTraceActiveBrowserSession, type AgentTraceAgentFinishedEvent, type AgentTraceAgentIdentity, type AgentTraceAgentRole, type AgentTraceAgentStartedEvent, type AgentTraceArtifactChange, type AgentTraceArtifactUpdatedEvent, type AgentTraceBrowserSessionFinishedEvent, type AgentTraceBrowserSessionStartedEvent, type AgentTraceError, type AgentTraceErrorOccurredEvent, type AgentTraceEvent, type AgentTraceProgressReportedEvent, type AgentTraceReasoningSummaryEvent, type AgentTraceResponse, type AgentTraceRunCancelRequestedEvent, type AgentTraceRunFinishedEvent, type AgentTraceRunStartedEvent, type AgentTraceToolCallFinishedEvent, type AgentTraceToolCallStartedEvent, type AgentWebhookConfig, type AgentWebhookEvent, type AlexandriaCall, type AlexandriaOptions, type AlexandriaScrapeData, type AlexandriaScrapeError, type AlexandriaScrapeRequest, type AlexandriaScrapeResult, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type DiscoveredTool, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, type FindToolsData, type FindToolsOptions, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, type ImageParser, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PendingApproval, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, type RequiresAction, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default, parseRequiresAction };
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-I2DFXQYR.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -138,18 +138,35 @@ var HttpClient = class {
|
|
|
138
138
|
};
|
|
139
139
|
|
|
140
140
|
// src/v2/types.ts
|
|
141
|
+
function parseRequiresAction(value) {
|
|
142
|
+
if (!value || typeof value !== "object") return void 0;
|
|
143
|
+
const record = value;
|
|
144
|
+
if (typeof record.type !== "string") return void 0;
|
|
145
|
+
const action = { type: record.type };
|
|
146
|
+
if (typeof record.terms === "string") action.terms = record.terms;
|
|
147
|
+
if (typeof record.version === "string") action.version = record.version;
|
|
148
|
+
if (typeof record.url === "string") action.url = record.url;
|
|
149
|
+
return action;
|
|
150
|
+
}
|
|
141
151
|
var SdkError = class extends Error {
|
|
152
|
+
requestId;
|
|
142
153
|
status;
|
|
143
154
|
code;
|
|
144
155
|
details;
|
|
145
156
|
jobId;
|
|
146
|
-
|
|
157
|
+
/** Present on exchange-mediated scrape failures that already captured credits. */
|
|
158
|
+
chargeId;
|
|
159
|
+
/** Present when the API needs an out-of-band step first, such as accepting provider terms. */
|
|
160
|
+
requiresAction;
|
|
161
|
+
constructor(message, status, code, details, jobId, chargeId, requiresAction) {
|
|
147
162
|
super(message);
|
|
148
163
|
this.name = "FirecrawlSdkError";
|
|
149
164
|
this.status = status;
|
|
150
165
|
this.code = code;
|
|
151
166
|
this.details = details;
|
|
152
167
|
this.jobId = jobId;
|
|
168
|
+
this.chargeId = chargeId;
|
|
169
|
+
this.requiresAction = requiresAction;
|
|
153
170
|
}
|
|
154
171
|
};
|
|
155
172
|
var JobTimeoutError = class extends SdkError {
|
|
@@ -401,14 +418,30 @@ function throwForBadResponse(resp, action) {
|
|
|
401
418
|
const status = resp.status;
|
|
402
419
|
const body = resp.data || {};
|
|
403
420
|
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
|
|
404
|
-
throw new SdkError(
|
|
421
|
+
throw new SdkError(
|
|
422
|
+
msg,
|
|
423
|
+
status,
|
|
424
|
+
body?.code,
|
|
425
|
+
body?.details,
|
|
426
|
+
void 0,
|
|
427
|
+
body?.chargeId,
|
|
428
|
+
parseRequiresAction(body?.requiresAction)
|
|
429
|
+
);
|
|
405
430
|
}
|
|
406
431
|
function normalizeAxiosError(err, action) {
|
|
407
432
|
const status = err.response?.status;
|
|
408
433
|
const body = err.response?.data;
|
|
409
434
|
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
|
|
410
435
|
const code = body?.code || err.code;
|
|
411
|
-
throw new SdkError(
|
|
436
|
+
throw new SdkError(
|
|
437
|
+
message,
|
|
438
|
+
status,
|
|
439
|
+
code,
|
|
440
|
+
body?.details ?? body,
|
|
441
|
+
void 0,
|
|
442
|
+
body?.chargeId,
|
|
443
|
+
parseRequiresAction(body?.requiresAction)
|
|
444
|
+
);
|
|
412
445
|
}
|
|
413
446
|
function isRetryableError(err) {
|
|
414
447
|
if (err instanceof JobTimeoutError) {
|
|
@@ -454,9 +487,10 @@ async function scrape(http, url, options, deps = {}) {
|
|
|
454
487
|
throw new Error("URL cannot be empty");
|
|
455
488
|
}
|
|
456
489
|
if (options) ensureValidScrapeOptions(options);
|
|
457
|
-
const { autoResume, ...requestOptions } = options ?? {};
|
|
490
|
+
const { autoResume, domainTools, ...requestOptions } = options ?? {};
|
|
458
491
|
const payload = { url: url.trim() };
|
|
459
492
|
Object.assign(payload, requestOptions);
|
|
493
|
+
if (domainTools) payload.domainTools = domainTools;
|
|
460
494
|
const requestTimeout = autoResume === false ? typeof options?.timeout === "number" ? { timeoutMs: options.timeout + 5e3 } : {} : {
|
|
461
495
|
timeoutMs: Math.max(
|
|
462
496
|
typeof options?.timeout === "number" ? options.timeout : 3e5,
|
|
@@ -613,6 +647,8 @@ function prepareSearchPayload(req) {
|
|
|
613
647
|
const payload = {
|
|
614
648
|
query: req.query
|
|
615
649
|
};
|
|
650
|
+
if (req.toolDetail != null) payload.toolDetail = req.toolDetail;
|
|
651
|
+
if (req.domainTools != null) payload.domainTools = req.domainTools;
|
|
616
652
|
if (req.sources) payload.sources = req.sources;
|
|
617
653
|
if (req.categories) payload.categories = req.categories;
|
|
618
654
|
if (req.includeDomains) payload.includeDomains = req.includeDomains;
|
|
@@ -665,10 +701,12 @@ async function search(http, request) {
|
|
|
665
701
|
}
|
|
666
702
|
const data = res.data.data || {};
|
|
667
703
|
const out = {};
|
|
704
|
+
if (res.data.warning) out.warning = res.data.warning;
|
|
668
705
|
if (data.web) out.web = transformArray(data.web);
|
|
669
706
|
if (data.news) out.news = transformArray(data.news);
|
|
670
707
|
if (data.images)
|
|
671
708
|
out.images = transformArray(data.images);
|
|
709
|
+
if (data.tools) out.tools = data.tools;
|
|
672
710
|
Object.defineProperty(out, "data", {
|
|
673
711
|
get() {
|
|
674
712
|
const parts = [];
|
|
@@ -676,7 +714,9 @@ async function search(http, request) {
|
|
|
676
714
|
if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
|
|
677
715
|
if (out.images?.length)
|
|
678
716
|
parts.push(`.images (${out.images.length} results)`);
|
|
679
|
-
|
|
717
|
+
if (out.tools?.length)
|
|
718
|
+
parts.push(`.tools (${out.tools.length} results)`);
|
|
719
|
+
const available = parts.length ? parts.join(", ") : ".web, .news, .images, or .tools";
|
|
680
720
|
throw new Error(
|
|
681
721
|
`SearchData has no '.data'. Results are grouped by source: ${available}`
|
|
682
722
|
);
|
|
@@ -691,6 +731,101 @@ async function search(http, request) {
|
|
|
691
731
|
}
|
|
692
732
|
}
|
|
693
733
|
|
|
734
|
+
// src/v2/methods/tools.ts
|
|
735
|
+
var ALEXANDRIA_MAX_CALLS = 10;
|
|
736
|
+
function prepareAlexandriaPayload(calls, opts) {
|
|
737
|
+
if (!Array.isArray(calls) || calls.length === 0) {
|
|
738
|
+
throw new Error("alexandria requires at least one call");
|
|
739
|
+
}
|
|
740
|
+
if (calls.length > ALEXANDRIA_MAX_CALLS) {
|
|
741
|
+
throw new Error(`alexandria accepts at most ${ALEXANDRIA_MAX_CALLS} calls`);
|
|
742
|
+
}
|
|
743
|
+
const alexandria = calls.map((call, index) => {
|
|
744
|
+
if (!call || typeof call.provider !== "string" || !call.provider.trim()) {
|
|
745
|
+
throw new Error(`alexandria[${index}].provider cannot be empty`);
|
|
746
|
+
}
|
|
747
|
+
if (typeof call.capability !== "string" || !call.capability.trim()) {
|
|
748
|
+
throw new Error(`alexandria[${index}].capability cannot be empty`);
|
|
749
|
+
}
|
|
750
|
+
if (Object.keys(call).some(
|
|
751
|
+
(key) => !["provider", "capability", "options"].includes(key)
|
|
752
|
+
))
|
|
753
|
+
throw new Error("Unknown alexandria call option");
|
|
754
|
+
const item = {
|
|
755
|
+
provider: call.provider.trim(),
|
|
756
|
+
capability: call.capability.trim()
|
|
757
|
+
};
|
|
758
|
+
if (call.options != null) {
|
|
759
|
+
if (typeof call.options !== "object" || Array.isArray(call.options)) {
|
|
760
|
+
throw new Error(`alexandria[${index}].options must be an object`);
|
|
761
|
+
}
|
|
762
|
+
item.options = call.options;
|
|
763
|
+
}
|
|
764
|
+
return item;
|
|
765
|
+
});
|
|
766
|
+
if (opts.timeout != null && (!Number.isInteger(opts.timeout) || opts.timeout <= 0)) {
|
|
767
|
+
throw new Error("timeout must be a positive integer");
|
|
768
|
+
}
|
|
769
|
+
const payload = { alexandria };
|
|
770
|
+
if (opts.timeout != null) payload.timeout = opts.timeout;
|
|
771
|
+
if (opts.integration && opts.integration.trim())
|
|
772
|
+
payload.integration = opts.integration.trim();
|
|
773
|
+
if (opts.origin) payload.origin = opts.origin;
|
|
774
|
+
return payload;
|
|
775
|
+
}
|
|
776
|
+
async function scrapeAlexandria(http, calls, opts = {}) {
|
|
777
|
+
const payload = prepareAlexandriaPayload(calls, opts);
|
|
778
|
+
const requestId = opts.requestId ?? crypto.randomUUID();
|
|
779
|
+
if (!/^[A-Za-z0-9._:-]{1,128}$/.test(requestId))
|
|
780
|
+
throw new Error("Invalid requestId");
|
|
781
|
+
try {
|
|
782
|
+
const res = await http.post("/v2/scrape", payload, {
|
|
783
|
+
headers: { "x-request-id": requestId },
|
|
784
|
+
// Allow response delivery after the API's capped execution deadline.
|
|
785
|
+
timeoutMs: Math.min(opts.timeout ?? 5e4, 5e4) + 3e4
|
|
786
|
+
});
|
|
787
|
+
if (res.status !== 200 || !res.data?.success) {
|
|
788
|
+
throwForBadResponse(res, "alexandria");
|
|
789
|
+
}
|
|
790
|
+
const data = res.data.data;
|
|
791
|
+
if (!data || !Array.isArray(data.alexandria) || typeof data.creditsCost !== "number" || !Number.isInteger(data.creditsCost) || data.creditsCost < 0) {
|
|
792
|
+
throw new SdkError("Invalid alexandria response");
|
|
793
|
+
}
|
|
794
|
+
return {
|
|
795
|
+
scrapeId: res.data.scrape_id ?? "",
|
|
796
|
+
requestId,
|
|
797
|
+
alexandria: data.alexandria,
|
|
798
|
+
creditsCost: data.creditsCost
|
|
799
|
+
};
|
|
800
|
+
} catch (err) {
|
|
801
|
+
try {
|
|
802
|
+
if (err?.isAxiosError) normalizeAxiosError(err, "alexandria");
|
|
803
|
+
throw err;
|
|
804
|
+
} catch (error) {
|
|
805
|
+
if (error && typeof error === "object")
|
|
806
|
+
Object.assign(error, { requestId });
|
|
807
|
+
throw error;
|
|
808
|
+
}
|
|
809
|
+
}
|
|
810
|
+
}
|
|
811
|
+
async function findTools(http, options = {}) {
|
|
812
|
+
const result = await scrapeAlexandria(http, [
|
|
813
|
+
{
|
|
814
|
+
provider: "firecrawl",
|
|
815
|
+
capability: "find-tools",
|
|
816
|
+
options: { ...options }
|
|
817
|
+
}
|
|
818
|
+
]);
|
|
819
|
+
const item = result.alexandria[0];
|
|
820
|
+
if (!item) throw new SdkError("Missing Find Tools result");
|
|
821
|
+
if (item.error)
|
|
822
|
+
throw Object.assign(
|
|
823
|
+
new SdkError(item.error.message, item.error.status, item.error.code),
|
|
824
|
+
{ requestId: result.requestId }
|
|
825
|
+
);
|
|
826
|
+
return item.data;
|
|
827
|
+
}
|
|
828
|
+
|
|
694
829
|
// src/v2/methods/developer.ts
|
|
695
830
|
var ENDPOINT = "/v2/search/developer";
|
|
696
831
|
async function developerSearch(http, query, options = {}) {
|
|
@@ -1181,7 +1316,8 @@ function prepareExtractPayload(args) {
|
|
|
1181
1316
|
body.threatProtection = args.threatProtection;
|
|
1182
1317
|
if (args.scrapeOptions) {
|
|
1183
1318
|
ensureValidScrapeOptions(args.scrapeOptions);
|
|
1184
|
-
|
|
1319
|
+
const { toolDetail, domainTools, ...scrapeOptions } = args.scrapeOptions;
|
|
1320
|
+
body.scrapeOptions = scrapeOptions;
|
|
1185
1321
|
}
|
|
1186
1322
|
return body;
|
|
1187
1323
|
}
|
|
@@ -2018,7 +2154,28 @@ var FirecrawlClient = class {
|
|
|
2018
2154
|
});
|
|
2019
2155
|
}
|
|
2020
2156
|
async scrape(url, options) {
|
|
2021
|
-
return scrape(this.http, url, options);
|
|
2157
|
+
if (typeof url === "string") return scrape(this.http, url, options);
|
|
2158
|
+
if (!url || options !== void 0 || Object.keys(url).some(
|
|
2159
|
+
(key) => ![
|
|
2160
|
+
"alexandria",
|
|
2161
|
+
"requestId",
|
|
2162
|
+
"timeout",
|
|
2163
|
+
"integration",
|
|
2164
|
+
"origin"
|
|
2165
|
+
].includes(key)
|
|
2166
|
+
)) {
|
|
2167
|
+
throw new Error("Provide an alexandria request without URL scrape options");
|
|
2168
|
+
}
|
|
2169
|
+
const { alexandria, ...opts } = url;
|
|
2170
|
+
return scrapeAlexandria(
|
|
2171
|
+
this.http,
|
|
2172
|
+
Array.isArray(alexandria) ? alexandria : [alexandria],
|
|
2173
|
+
opts
|
|
2174
|
+
);
|
|
2175
|
+
}
|
|
2176
|
+
/** Explore the catalogue without executing the tools it returns. */
|
|
2177
|
+
async findTools(options) {
|
|
2178
|
+
return findTools(this.http, options);
|
|
2022
2179
|
}
|
|
2023
2180
|
/**
|
|
2024
2181
|
* Interact with the browser session associated with a scrape job.
|
|
@@ -2510,7 +2667,7 @@ var FirecrawlApp = class {
|
|
|
2510
2667
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2511
2668
|
return process.env.npm_package_version;
|
|
2512
2669
|
}
|
|
2513
|
-
const packageJson = await import("./package-
|
|
2670
|
+
const packageJson = await import("./package-V72LKFNW.js");
|
|
2514
2671
|
return packageJson.default.version;
|
|
2515
2672
|
} catch (error) {
|
|
2516
2673
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
|
@@ -3910,5 +4067,6 @@ export {
|
|
|
3910
4067
|
ResearchClient,
|
|
3911
4068
|
SdkError,
|
|
3912
4069
|
Watcher,
|
|
3913
|
-
index_default as default
|
|
4070
|
+
index_default as default,
|
|
4071
|
+
parseRequiresAction
|
|
3914
4072
|
};
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mendable/firecrawl-js",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.41.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
package/pnpm-workspace.yaml
CHANGED
|
@@ -8,8 +8,8 @@ overrides:
|
|
|
8
8
|
"picomatch@<4.0.4": ">=4.0.4"
|
|
9
9
|
handlebars: ">=4.7.9"
|
|
10
10
|
"brace-expansion@>=5.0.0 <5.0.9": "5.0.9"
|
|
11
|
-
"js-yaml@>=3.0.0 <3.15.
|
|
12
|
-
"js-yaml@>=4.0.0 <4.3.
|
|
11
|
+
"js-yaml@>=3.0.0 <3.15.2": "3.15.2"
|
|
12
|
+
"js-yaml@>=4.0.0 <4.3.2": "4.3.2"
|
|
13
13
|
"axios@>=1.0.0 <1.16.0": "1.16.1"
|
|
14
14
|
"follow-redirects@<1.16.0": ">=1.16.0 <2.0.0"
|
|
15
15
|
"form-data@>=4.0.0 <4.0.6": "4.0.6"
|
|
@@ -3,6 +3,29 @@ import { startExtract } from "../../../v2/methods/extract";
|
|
|
3
3
|
import type { WebhookConfig } from "../../../v2/types";
|
|
4
4
|
|
|
5
5
|
describe("v2.extract unit", () => {
|
|
6
|
+
test("omits discovery options from extract without mutating shared options", async () => {
|
|
7
|
+
const post = jest
|
|
8
|
+
.fn()
|
|
9
|
+
.mockResolvedValue({ status: 200, data: { id: "extract-job" } });
|
|
10
|
+
const scrapeOptions = {
|
|
11
|
+
toolDetail: "compact" as const,
|
|
12
|
+
domainTools: true,
|
|
13
|
+
onlyMainContent: true,
|
|
14
|
+
};
|
|
15
|
+
await startExtract({ post } as any, {
|
|
16
|
+
urls: ["https://example.com"],
|
|
17
|
+
scrapeOptions,
|
|
18
|
+
});
|
|
19
|
+
expect(post).toHaveBeenCalledWith("/v2/extract", {
|
|
20
|
+
urls: ["https://example.com"],
|
|
21
|
+
scrapeOptions: { onlyMainContent: true },
|
|
22
|
+
});
|
|
23
|
+
expect(scrapeOptions).toEqual({
|
|
24
|
+
toolDetail: "compact",
|
|
25
|
+
domainTools: true,
|
|
26
|
+
onlyMainContent: true,
|
|
27
|
+
});
|
|
28
|
+
});
|
|
6
29
|
test("startExtract forwards string webhook in request payload", async () => {
|
|
7
30
|
const post = jest.fn().mockResolvedValue({
|
|
8
31
|
status: 200,
|
|
@@ -32,7 +32,17 @@ describe("v2 threatProtection request serialization", () => {
|
|
|
32
32
|
expect(http.post).toHaveBeenCalledWith(
|
|
33
33
|
"/v2/scrape",
|
|
34
34
|
expect.objectContaining({ url: "https://example.com", threatProtection }),
|
|
35
|
-
{},
|
|
35
|
+
{ timeoutMs: 330000 },
|
|
36
|
+
);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("scrape sends domainTools when true", async () => {
|
|
40
|
+
const http = makeHttp({ success: true, data: {} });
|
|
41
|
+
await scrape(http, "https://example.com", { domainTools: true });
|
|
42
|
+
expect(http.post).toHaveBeenCalledWith(
|
|
43
|
+
"/v2/scrape",
|
|
44
|
+
expect.objectContaining({ url: "https://example.com", domainTools: true }),
|
|
45
|
+
expect.anything(),
|
|
36
46
|
);
|
|
37
47
|
});
|
|
38
48
|
|
|
@@ -121,3 +131,10 @@ describe("v2 threatProtection request serialization", () => {
|
|
|
121
131
|
expect(body).not.toHaveProperty("threatProtection");
|
|
122
132
|
});
|
|
123
133
|
});
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
test.each([false, undefined])("scrape omits domainTools when %s", async (domainTools) => {
|
|
137
|
+
const http = makeHttp({ success: true, data: {} });
|
|
138
|
+
await scrape(http, "https://example.com", { domainTools });
|
|
139
|
+
expect(http.post.mock.calls[0][1]).not.toHaveProperty("domainTools");
|
|
140
|
+
});
|