firecrawl 4.29.2 → 4.30.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-N3GO36OX.js → chunk-2LUWTM44.js} +1 -1
- package/dist/index.cjs +10 -1
- package/dist/index.d.cts +29 -2
- package/dist/index.d.ts +29 -2
- package/dist/index.js +11 -2
- package/dist/{package-VAY7ZMTB.js → package-2VNCXFNY.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/search-highlights.unit.test.ts +18 -0
- package/src/__tests__/unit/v2/threat-protection.unit.test.ts +123 -0
- package/src/v2/methods/agent.ts +4 -1
- package/src/v2/methods/extract.ts +4 -1
- package/src/v2/methods/map.ts +2 -0
- package/src/v2/methods/search.ts +3 -0
- package/src/v2/types.ts +27 -0
|
@@ -12,7 +12,7 @@ var require_package = __commonJS({
|
|
|
12
12
|
"package.json"(exports, module) {
|
|
13
13
|
module.exports = {
|
|
14
14
|
name: "@mendable/firecrawl-js",
|
|
15
|
-
version: "4.
|
|
15
|
+
version: "4.30.1",
|
|
16
16
|
description: "JavaScript SDK for Firecrawl API",
|
|
17
17
|
main: "dist/index.js",
|
|
18
18
|
types: "dist/index.d.ts",
|
package/dist/index.cjs
CHANGED
|
@@ -39,7 +39,7 @@ var require_package = __commonJS({
|
|
|
39
39
|
"package.json"(exports2, module2) {
|
|
40
40
|
module2.exports = {
|
|
41
41
|
name: "@mendable/firecrawl-js",
|
|
42
|
-
version: "4.
|
|
42
|
+
version: "4.30.1",
|
|
43
43
|
description: "JavaScript SDK for Firecrawl API",
|
|
44
44
|
main: "dist/index.js",
|
|
45
45
|
types: "dist/index.d.ts",
|
|
@@ -717,10 +717,13 @@ function prepareSearchPayload(req) {
|
|
|
717
717
|
if (req.ignoreInvalidURLs != null)
|
|
718
718
|
payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
|
|
719
719
|
if (req.timeout != null) payload.timeout = req.timeout;
|
|
720
|
+
if (req.highlights != null) payload.highlights = req.highlights;
|
|
720
721
|
if (req.integration && req.integration.trim())
|
|
721
722
|
payload.integration = req.integration.trim();
|
|
722
723
|
if (req.origin) payload.origin = req.origin;
|
|
723
724
|
if (req.enterprise) payload.enterprise = req.enterprise;
|
|
725
|
+
if (req.threatProtection != null)
|
|
726
|
+
payload.threatProtection = req.threatProtection;
|
|
724
727
|
if (req.scrapeOptions) {
|
|
725
728
|
ensureValidScrapeOptions(req.scrapeOptions);
|
|
726
729
|
payload.scrapeOptions = req.scrapeOptions;
|
|
@@ -797,6 +800,8 @@ function prepareMapPayload(url, options) {
|
|
|
797
800
|
payload.integration = options.integration.trim();
|
|
798
801
|
if (options.origin) payload.origin = options.origin;
|
|
799
802
|
if (options.location != null) payload.location = options.location;
|
|
803
|
+
if (options.threatProtection != null)
|
|
804
|
+
payload.threatProtection = options.threatProtection;
|
|
800
805
|
}
|
|
801
806
|
return payload;
|
|
802
807
|
}
|
|
@@ -1229,6 +1234,8 @@ function prepareExtractPayload(args) {
|
|
|
1229
1234
|
if (args.origin) body.origin = args.origin;
|
|
1230
1235
|
if (args.agent) body.agent = args.agent;
|
|
1231
1236
|
if (args.webhook != null) body.webhook = args.webhook;
|
|
1237
|
+
if (args.threatProtection != null)
|
|
1238
|
+
body.threatProtection = args.threatProtection;
|
|
1232
1239
|
if (args.scrapeOptions) {
|
|
1233
1240
|
ensureValidScrapeOptions(args.scrapeOptions);
|
|
1234
1241
|
body.scrapeOptions = args.scrapeOptions;
|
|
@@ -1286,6 +1293,8 @@ function prepareAgentPayload(args) {
|
|
|
1286
1293
|
if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== void 0) body.strictConstrainToURLs = args.strictConstrainToURLs;
|
|
1287
1294
|
if (args.model !== null && args.model !== void 0) body.model = args.model;
|
|
1288
1295
|
if (args.webhook != null) body.webhook = args.webhook;
|
|
1296
|
+
if (args.threatProtection != null)
|
|
1297
|
+
body.threatProtection = args.threatProtection;
|
|
1289
1298
|
return body;
|
|
1290
1299
|
}
|
|
1291
1300
|
async function startAgent(http, args) {
|
package/dist/index.d.cts
CHANGED
|
@@ -141,6 +141,7 @@ interface ScrapeOptions {
|
|
|
141
141
|
storeInCache?: boolean;
|
|
142
142
|
lockdown?: boolean;
|
|
143
143
|
redactPII?: boolean | RedactPIIOptions;
|
|
144
|
+
threatProtection?: ThreatProtectionOptions;
|
|
144
145
|
profile?: {
|
|
145
146
|
name: string;
|
|
146
147
|
saveChanges?: boolean;
|
|
@@ -165,13 +166,33 @@ interface RedactPIIOptions {
|
|
|
165
166
|
*/
|
|
166
167
|
replaceStyle?: "tag" | "mask" | "remove";
|
|
167
168
|
}
|
|
169
|
+
/**
|
|
170
|
+
* Enterprise: per-request field-level override of your team's threat
|
|
171
|
+
* protection policy. Requires threat protection to be enabled for your team
|
|
172
|
+
* and request overrides to be allowed in the team configuration. Only the
|
|
173
|
+
* fields you provide replace the team policy's values.
|
|
174
|
+
*/
|
|
175
|
+
interface ThreatProtectionOptions {
|
|
176
|
+
/** "off" disables scanning for this request; "normal" applies the policy. */
|
|
177
|
+
mode?: "off" | "normal";
|
|
178
|
+
/** Block verdicts at or above this risk score (integer 0-100). */
|
|
179
|
+
riskScoreThreshold?: number;
|
|
180
|
+
/** Exact domains or globs like "*.example.com" to always block (max 1000). */
|
|
181
|
+
blacklist?: string[];
|
|
182
|
+
/** Exact domains or globs to always allow; wins over everything (max 1000). */
|
|
183
|
+
whitelist?: string[];
|
|
184
|
+
/** Lowercase TLDs without the leading dot, e.g. "zip" (max 1000). */
|
|
185
|
+
blockedTlds?: string[];
|
|
186
|
+
/** Behavior when scanning is unavailable: "closed" blocks, "open" allows. */
|
|
187
|
+
failurePolicy?: "open" | "closed";
|
|
188
|
+
}
|
|
168
189
|
type ParseFileData = Blob | File | Buffer | Uint8Array | ArrayBuffer | string;
|
|
169
190
|
interface ParseFile {
|
|
170
191
|
data: ParseFileData;
|
|
171
192
|
filename: string;
|
|
172
193
|
contentType?: string;
|
|
173
194
|
}
|
|
174
|
-
type ParseOptions = Omit<ScrapeOptions, "formats" | "waitFor" | "mobile" | "actions" | "location" | "maxAge" | "minAge" | "storeInCache" | "lockdown" | "proxy"> & {
|
|
195
|
+
type ParseOptions = Omit<ScrapeOptions, "formats" | "waitFor" | "mobile" | "actions" | "location" | "maxAge" | "minAge" | "storeInCache" | "lockdown" | "proxy" | "threatProtection"> & {
|
|
175
196
|
formats?: ParseFormatOption[];
|
|
176
197
|
proxy?: "basic" | "auto";
|
|
177
198
|
};
|
|
@@ -523,6 +544,8 @@ interface SearchRequest {
|
|
|
523
544
|
location?: string;
|
|
524
545
|
ignoreInvalidURLs?: boolean;
|
|
525
546
|
timeout?: number;
|
|
547
|
+
/** Generate query-relevant highlights for search results. Defaults to true. */
|
|
548
|
+
highlights?: boolean;
|
|
526
549
|
scrapeOptions?: ScrapeOptions;
|
|
527
550
|
/**
|
|
528
551
|
* Enterprise search options. Use `["zdr"]` for end-to-end Zero Data
|
|
@@ -530,6 +553,7 @@ interface SearchRequest {
|
|
|
530
553
|
* your team.
|
|
531
554
|
*/
|
|
532
555
|
enterprise?: Array<"default" | "anon" | "zdr">;
|
|
556
|
+
threatProtection?: ThreatProtectionOptions;
|
|
533
557
|
integration?: string;
|
|
534
558
|
origin?: string;
|
|
535
559
|
}
|
|
@@ -610,6 +634,7 @@ interface MapOptions {
|
|
|
610
634
|
integration?: string;
|
|
611
635
|
origin?: string;
|
|
612
636
|
location?: LocationConfig$1;
|
|
637
|
+
threatProtection?: ThreatProtectionOptions;
|
|
613
638
|
}
|
|
614
639
|
type FeedbackRating = "good" | "partial" | "bad";
|
|
615
640
|
type EndpointFeedbackEndpoint = "search" | "scrape" | "parse" | "map";
|
|
@@ -1250,6 +1275,7 @@ declare function prepareExtractPayload(args: {
|
|
|
1250
1275
|
origin?: string;
|
|
1251
1276
|
agent?: AgentOptions$1;
|
|
1252
1277
|
webhook?: string | WebhookConfig | null;
|
|
1278
|
+
threatProtection?: ThreatProtectionOptions;
|
|
1253
1279
|
}): Record<string, unknown>;
|
|
1254
1280
|
/**
|
|
1255
1281
|
* @deprecated The extract endpoint is in maintenance mode and its use is discouraged.
|
|
@@ -1267,6 +1293,7 @@ declare function prepareAgentPayload(args: {
|
|
|
1267
1293
|
strictConstrainToURLs?: boolean;
|
|
1268
1294
|
model?: "spark-1-pro" | "spark-1-mini";
|
|
1269
1295
|
webhook?: string | AgentWebhookConfig;
|
|
1296
|
+
threatProtection?: ThreatProtectionOptions;
|
|
1270
1297
|
}): Record<string, unknown>;
|
|
1271
1298
|
declare function startAgent(http: HttpClient, args: Parameters<typeof prepareAgentPayload>[0]): Promise<AgentResponse>;
|
|
1272
1299
|
|
|
@@ -2629,4 +2656,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
2629
2656
|
get v1(): FirecrawlApp;
|
|
2630
2657
|
}
|
|
2631
2658
|
|
|
2632
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
|
2659
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
package/dist/index.d.ts
CHANGED
|
@@ -141,6 +141,7 @@ interface ScrapeOptions {
|
|
|
141
141
|
storeInCache?: boolean;
|
|
142
142
|
lockdown?: boolean;
|
|
143
143
|
redactPII?: boolean | RedactPIIOptions;
|
|
144
|
+
threatProtection?: ThreatProtectionOptions;
|
|
144
145
|
profile?: {
|
|
145
146
|
name: string;
|
|
146
147
|
saveChanges?: boolean;
|
|
@@ -165,13 +166,33 @@ interface RedactPIIOptions {
|
|
|
165
166
|
*/
|
|
166
167
|
replaceStyle?: "tag" | "mask" | "remove";
|
|
167
168
|
}
|
|
169
|
+
/**
|
|
170
|
+
* Enterprise: per-request field-level override of your team's threat
|
|
171
|
+
* protection policy. Requires threat protection to be enabled for your team
|
|
172
|
+
* and request overrides to be allowed in the team configuration. Only the
|
|
173
|
+
* fields you provide replace the team policy's values.
|
|
174
|
+
*/
|
|
175
|
+
interface ThreatProtectionOptions {
|
|
176
|
+
/** "off" disables scanning for this request; "normal" applies the policy. */
|
|
177
|
+
mode?: "off" | "normal";
|
|
178
|
+
/** Block verdicts at or above this risk score (integer 0-100). */
|
|
179
|
+
riskScoreThreshold?: number;
|
|
180
|
+
/** Exact domains or globs like "*.example.com" to always block (max 1000). */
|
|
181
|
+
blacklist?: string[];
|
|
182
|
+
/** Exact domains or globs to always allow; wins over everything (max 1000). */
|
|
183
|
+
whitelist?: string[];
|
|
184
|
+
/** Lowercase TLDs without the leading dot, e.g. "zip" (max 1000). */
|
|
185
|
+
blockedTlds?: string[];
|
|
186
|
+
/** Behavior when scanning is unavailable: "closed" blocks, "open" allows. */
|
|
187
|
+
failurePolicy?: "open" | "closed";
|
|
188
|
+
}
|
|
168
189
|
type ParseFileData = Blob | File | Buffer | Uint8Array | ArrayBuffer | string;
|
|
169
190
|
interface ParseFile {
|
|
170
191
|
data: ParseFileData;
|
|
171
192
|
filename: string;
|
|
172
193
|
contentType?: string;
|
|
173
194
|
}
|
|
174
|
-
type ParseOptions = Omit<ScrapeOptions, "formats" | "waitFor" | "mobile" | "actions" | "location" | "maxAge" | "minAge" | "storeInCache" | "lockdown" | "proxy"> & {
|
|
195
|
+
type ParseOptions = Omit<ScrapeOptions, "formats" | "waitFor" | "mobile" | "actions" | "location" | "maxAge" | "minAge" | "storeInCache" | "lockdown" | "proxy" | "threatProtection"> & {
|
|
175
196
|
formats?: ParseFormatOption[];
|
|
176
197
|
proxy?: "basic" | "auto";
|
|
177
198
|
};
|
|
@@ -523,6 +544,8 @@ interface SearchRequest {
|
|
|
523
544
|
location?: string;
|
|
524
545
|
ignoreInvalidURLs?: boolean;
|
|
525
546
|
timeout?: number;
|
|
547
|
+
/** Generate query-relevant highlights for search results. Defaults to true. */
|
|
548
|
+
highlights?: boolean;
|
|
526
549
|
scrapeOptions?: ScrapeOptions;
|
|
527
550
|
/**
|
|
528
551
|
* Enterprise search options. Use `["zdr"]` for end-to-end Zero Data
|
|
@@ -530,6 +553,7 @@ interface SearchRequest {
|
|
|
530
553
|
* your team.
|
|
531
554
|
*/
|
|
532
555
|
enterprise?: Array<"default" | "anon" | "zdr">;
|
|
556
|
+
threatProtection?: ThreatProtectionOptions;
|
|
533
557
|
integration?: string;
|
|
534
558
|
origin?: string;
|
|
535
559
|
}
|
|
@@ -610,6 +634,7 @@ interface MapOptions {
|
|
|
610
634
|
integration?: string;
|
|
611
635
|
origin?: string;
|
|
612
636
|
location?: LocationConfig$1;
|
|
637
|
+
threatProtection?: ThreatProtectionOptions;
|
|
613
638
|
}
|
|
614
639
|
type FeedbackRating = "good" | "partial" | "bad";
|
|
615
640
|
type EndpointFeedbackEndpoint = "search" | "scrape" | "parse" | "map";
|
|
@@ -1250,6 +1275,7 @@ declare function prepareExtractPayload(args: {
|
|
|
1250
1275
|
origin?: string;
|
|
1251
1276
|
agent?: AgentOptions$1;
|
|
1252
1277
|
webhook?: string | WebhookConfig | null;
|
|
1278
|
+
threatProtection?: ThreatProtectionOptions;
|
|
1253
1279
|
}): Record<string, unknown>;
|
|
1254
1280
|
/**
|
|
1255
1281
|
* @deprecated The extract endpoint is in maintenance mode and its use is discouraged.
|
|
@@ -1267,6 +1293,7 @@ declare function prepareAgentPayload(args: {
|
|
|
1267
1293
|
strictConstrainToURLs?: boolean;
|
|
1268
1294
|
model?: "spark-1-pro" | "spark-1-mini";
|
|
1269
1295
|
webhook?: string | AgentWebhookConfig;
|
|
1296
|
+
threatProtection?: ThreatProtectionOptions;
|
|
1270
1297
|
}): Record<string, unknown>;
|
|
1271
1298
|
declare function startAgent(http: HttpClient, args: Parameters<typeof prepareAgentPayload>[0]): Promise<AgentResponse>;
|
|
1272
1299
|
|
|
@@ -2629,4 +2656,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
2629
2656
|
get v1(): FirecrawlApp;
|
|
2630
2657
|
}
|
|
2631
2658
|
|
|
2632
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
|
2659
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type AgentWebhookEvent, type AttributesFormat, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default };
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-2LUWTM44.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -584,10 +584,13 @@ function prepareSearchPayload(req) {
|
|
|
584
584
|
if (req.ignoreInvalidURLs != null)
|
|
585
585
|
payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
|
|
586
586
|
if (req.timeout != null) payload.timeout = req.timeout;
|
|
587
|
+
if (req.highlights != null) payload.highlights = req.highlights;
|
|
587
588
|
if (req.integration && req.integration.trim())
|
|
588
589
|
payload.integration = req.integration.trim();
|
|
589
590
|
if (req.origin) payload.origin = req.origin;
|
|
590
591
|
if (req.enterprise) payload.enterprise = req.enterprise;
|
|
592
|
+
if (req.threatProtection != null)
|
|
593
|
+
payload.threatProtection = req.threatProtection;
|
|
591
594
|
if (req.scrapeOptions) {
|
|
592
595
|
ensureValidScrapeOptions(req.scrapeOptions);
|
|
593
596
|
payload.scrapeOptions = req.scrapeOptions;
|
|
@@ -664,6 +667,8 @@ function prepareMapPayload(url, options) {
|
|
|
664
667
|
payload.integration = options.integration.trim();
|
|
665
668
|
if (options.origin) payload.origin = options.origin;
|
|
666
669
|
if (options.location != null) payload.location = options.location;
|
|
670
|
+
if (options.threatProtection != null)
|
|
671
|
+
payload.threatProtection = options.threatProtection;
|
|
667
672
|
}
|
|
668
673
|
return payload;
|
|
669
674
|
}
|
|
@@ -1096,6 +1101,8 @@ function prepareExtractPayload(args) {
|
|
|
1096
1101
|
if (args.origin) body.origin = args.origin;
|
|
1097
1102
|
if (args.agent) body.agent = args.agent;
|
|
1098
1103
|
if (args.webhook != null) body.webhook = args.webhook;
|
|
1104
|
+
if (args.threatProtection != null)
|
|
1105
|
+
body.threatProtection = args.threatProtection;
|
|
1099
1106
|
if (args.scrapeOptions) {
|
|
1100
1107
|
ensureValidScrapeOptions(args.scrapeOptions);
|
|
1101
1108
|
body.scrapeOptions = args.scrapeOptions;
|
|
@@ -1153,6 +1160,8 @@ function prepareAgentPayload(args) {
|
|
|
1153
1160
|
if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== void 0) body.strictConstrainToURLs = args.strictConstrainToURLs;
|
|
1154
1161
|
if (args.model !== null && args.model !== void 0) body.model = args.model;
|
|
1155
1162
|
if (args.webhook != null) body.webhook = args.webhook;
|
|
1163
|
+
if (args.threatProtection != null)
|
|
1164
|
+
body.threatProtection = args.threatProtection;
|
|
1156
1165
|
return body;
|
|
1157
1166
|
}
|
|
1158
1167
|
async function startAgent(http, args) {
|
|
@@ -2303,7 +2312,7 @@ var FirecrawlApp = class {
|
|
|
2303
2312
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2304
2313
|
return process.env.npm_package_version;
|
|
2305
2314
|
}
|
|
2306
|
-
const packageJson = await import("./package-
|
|
2315
|
+
const packageJson = await import("./package-2VNCXFNY.js");
|
|
2307
2316
|
return packageJson.default.version;
|
|
2308
2317
|
} catch (error) {
|
|
2309
2318
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
package/package.json
CHANGED
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import { describe, expect, jest, test } from "@jest/globals";
|
|
2
|
+
import { search } from "../../../v2/methods/search";
|
|
3
|
+
|
|
4
|
+
describe("v2 search highlights", () => {
|
|
5
|
+
test("forwards the highlights option", async () => {
|
|
6
|
+
const http = {
|
|
7
|
+
post: jest.fn(async () => ({ status: 200, data: { success: true } })),
|
|
8
|
+
} as any;
|
|
9
|
+
|
|
10
|
+
await search(http, { query: "firecrawl", highlights: false });
|
|
11
|
+
|
|
12
|
+
expect(http.post).toHaveBeenCalledWith(
|
|
13
|
+
"/v2/search",
|
|
14
|
+
{ query: "firecrawl", highlights: false },
|
|
15
|
+
{},
|
|
16
|
+
);
|
|
17
|
+
});
|
|
18
|
+
});
|
|
@@ -0,0 +1,123 @@
|
|
|
1
|
+
import { describe, test, expect, jest } from "@jest/globals";
|
|
2
|
+
import { scrape } from "../../../v2/methods/scrape";
|
|
3
|
+
import { startBatchScrape } from "../../../v2/methods/batch";
|
|
4
|
+
import { startCrawl } from "../../../v2/methods/crawl";
|
|
5
|
+
import { search } from "../../../v2/methods/search";
|
|
6
|
+
import { map } from "../../../v2/methods/map";
|
|
7
|
+
import { startExtract } from "../../../v2/methods/extract";
|
|
8
|
+
import { startAgent } from "../../../v2/methods/agent";
|
|
9
|
+
import type { ThreatProtectionOptions } from "../../../v2/types";
|
|
10
|
+
|
|
11
|
+
const threatProtection: ThreatProtectionOptions = {
|
|
12
|
+
mode: "normal",
|
|
13
|
+
riskScoreThreshold: 80,
|
|
14
|
+
blacklist: ["*.blocked.example.com"],
|
|
15
|
+
whitelist: ["allowed.example.com"],
|
|
16
|
+
blockedTlds: ["zip"],
|
|
17
|
+
failurePolicy: "open",
|
|
18
|
+
};
|
|
19
|
+
|
|
20
|
+
function makeHttp(data: Record<string, unknown>) {
|
|
21
|
+
const post = jest.fn(async () => ({ status: 200, data }));
|
|
22
|
+
return {
|
|
23
|
+
post,
|
|
24
|
+
prepareHeaders: jest.fn(() => undefined),
|
|
25
|
+
} as any;
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
describe("v2 threatProtection request serialization", () => {
|
|
29
|
+
test("scrape sends threatProtection at top level", async () => {
|
|
30
|
+
const http = makeHttp({ success: true, data: {} });
|
|
31
|
+
await scrape(http, "https://example.com", { threatProtection });
|
|
32
|
+
expect(http.post).toHaveBeenCalledWith(
|
|
33
|
+
"/v2/scrape",
|
|
34
|
+
expect.objectContaining({ url: "https://example.com", threatProtection }),
|
|
35
|
+
{},
|
|
36
|
+
);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
test("batch scrape sends threatProtection at top level", async () => {
|
|
40
|
+
const http = makeHttp({ success: true, id: "job", url: "u" });
|
|
41
|
+
await startBatchScrape(http, ["https://example.com"], {
|
|
42
|
+
options: { threatProtection },
|
|
43
|
+
});
|
|
44
|
+
expect(http.post).toHaveBeenCalledWith(
|
|
45
|
+
"/v2/batch/scrape",
|
|
46
|
+
expect.objectContaining({
|
|
47
|
+
urls: ["https://example.com"],
|
|
48
|
+
threatProtection,
|
|
49
|
+
}),
|
|
50
|
+
expect.anything(),
|
|
51
|
+
);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("crawl sends threatProtection under scrapeOptions", async () => {
|
|
55
|
+
const http = makeHttp({ success: true, id: "job", url: "u" });
|
|
56
|
+
await startCrawl(http, {
|
|
57
|
+
url: "https://example.com",
|
|
58
|
+
scrapeOptions: { threatProtection },
|
|
59
|
+
});
|
|
60
|
+
expect(http.post).toHaveBeenCalledWith(
|
|
61
|
+
"/v2/crawl",
|
|
62
|
+
expect.objectContaining({
|
|
63
|
+
scrapeOptions: expect.objectContaining({ threatProtection }),
|
|
64
|
+
}),
|
|
65
|
+
);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
test("search sends threatProtection at top level and under scrapeOptions", async () => {
|
|
69
|
+
const http = makeHttp({ success: true, data: {} });
|
|
70
|
+
await search(http, {
|
|
71
|
+
query: "firecrawl",
|
|
72
|
+
threatProtection,
|
|
73
|
+
scrapeOptions: { threatProtection },
|
|
74
|
+
});
|
|
75
|
+
expect(http.post).toHaveBeenCalledWith(
|
|
76
|
+
"/v2/search",
|
|
77
|
+
expect.objectContaining({
|
|
78
|
+
threatProtection,
|
|
79
|
+
scrapeOptions: expect.objectContaining({ threatProtection }),
|
|
80
|
+
}),
|
|
81
|
+
{},
|
|
82
|
+
);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
test("map sends threatProtection at top level", async () => {
|
|
86
|
+
const http = makeHttp({ success: true, links: [] });
|
|
87
|
+
await map(http, "https://example.com", { threatProtection });
|
|
88
|
+
expect(http.post).toHaveBeenCalledWith(
|
|
89
|
+
"/v2/map",
|
|
90
|
+
expect.objectContaining({ threatProtection }),
|
|
91
|
+
{},
|
|
92
|
+
);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("extract sends threatProtection at top level", async () => {
|
|
96
|
+
const http = makeHttp({ success: true, id: "job" });
|
|
97
|
+
await startExtract(http, {
|
|
98
|
+
urls: ["https://example.com"],
|
|
99
|
+
prompt: "extract",
|
|
100
|
+
threatProtection,
|
|
101
|
+
});
|
|
102
|
+
expect(http.post).toHaveBeenCalledWith(
|
|
103
|
+
"/v2/extract",
|
|
104
|
+
expect.objectContaining({ threatProtection }),
|
|
105
|
+
);
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
test("agent sends threatProtection at top level", async () => {
|
|
109
|
+
const http = makeHttp({ success: true, id: "job", status: "processing" });
|
|
110
|
+
await startAgent(http, { prompt: "find pricing", threatProtection });
|
|
111
|
+
expect(http.post).toHaveBeenCalledWith(
|
|
112
|
+
"/v2/agent",
|
|
113
|
+
expect.objectContaining({ threatProtection }),
|
|
114
|
+
);
|
|
115
|
+
});
|
|
116
|
+
|
|
117
|
+
test("threatProtection is omitted when not provided", async () => {
|
|
118
|
+
const http = makeHttp({ success: true, data: {} });
|
|
119
|
+
await scrape(http, "https://example.com", { onlyMainContent: true });
|
|
120
|
+
const body = http.post.mock.calls[0][1] as Record<string, unknown>;
|
|
121
|
+
expect(body).not.toHaveProperty("threatProtection");
|
|
122
|
+
});
|
|
123
|
+
});
|
package/src/v2/methods/agent.ts
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig } from "../types";
|
|
1
|
+
import { type AgentResponse, type AgentStatusResponse, type AgentWebhookConfig, type ThreatProtectionOptions } from "../types";
|
|
2
2
|
import { HttpClient } from "../utils/httpClient";
|
|
3
3
|
import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
|
|
4
4
|
import { isZodSchema, zodSchemaToJsonSchema } from "../../utils/zodSchemaToJson";
|
|
@@ -14,6 +14,7 @@ function prepareAgentPayload(args: {
|
|
|
14
14
|
strictConstrainToURLs?: boolean;
|
|
15
15
|
model?: "spark-1-pro" | "spark-1-mini";
|
|
16
16
|
webhook?: string | AgentWebhookConfig;
|
|
17
|
+
threatProtection?: ThreatProtectionOptions;
|
|
17
18
|
}): Record<string, unknown> {
|
|
18
19
|
const body: Record<string, unknown> = {};
|
|
19
20
|
if (args.urls) body.urls = args.urls;
|
|
@@ -27,6 +28,8 @@ function prepareAgentPayload(args: {
|
|
|
27
28
|
if (args.strictConstrainToURLs !== null && args.strictConstrainToURLs !== undefined) body.strictConstrainToURLs = args.strictConstrainToURLs;
|
|
28
29
|
if (args.model !== null && args.model !== undefined) body.model = args.model;
|
|
29
30
|
if (args.webhook != null) body.webhook = args.webhook;
|
|
31
|
+
if (args.threatProtection != null)
|
|
32
|
+
body.threatProtection = args.threatProtection;
|
|
30
33
|
return body;
|
|
31
34
|
}
|
|
32
35
|
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { type ExtractResponse, type ScrapeOptions, type AgentOptions, type WebhookConfig } from "../types";
|
|
1
|
+
import { type ExtractResponse, type ScrapeOptions, type AgentOptions, type ThreatProtectionOptions, type WebhookConfig } from "../types";
|
|
2
2
|
import { HttpClient } from "../utils/httpClient";
|
|
3
3
|
import { ensureValidScrapeOptions } from "../utils/validation";
|
|
4
4
|
import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
|
|
@@ -19,6 +19,7 @@ function prepareExtractPayload(args: {
|
|
|
19
19
|
origin?: string;
|
|
20
20
|
agent?: AgentOptions;
|
|
21
21
|
webhook?: string | WebhookConfig | null;
|
|
22
|
+
threatProtection?: ThreatProtectionOptions;
|
|
22
23
|
}): Record<string, unknown> {
|
|
23
24
|
const body: Record<string, unknown> = {};
|
|
24
25
|
if (args.urls) body.urls = args.urls;
|
|
@@ -35,6 +36,8 @@ function prepareExtractPayload(args: {
|
|
|
35
36
|
if (args.origin) body.origin = args.origin;
|
|
36
37
|
if (args.agent) body.agent = args.agent;
|
|
37
38
|
if (args.webhook != null) body.webhook = args.webhook;
|
|
39
|
+
if (args.threatProtection != null)
|
|
40
|
+
body.threatProtection = args.threatProtection;
|
|
38
41
|
if (args.scrapeOptions) {
|
|
39
42
|
ensureValidScrapeOptions(args.scrapeOptions);
|
|
40
43
|
body.scrapeOptions = args.scrapeOptions;
|
package/src/v2/methods/map.ts
CHANGED
|
@@ -24,6 +24,8 @@ function prepareMapPayload(
|
|
|
24
24
|
payload.integration = options.integration.trim();
|
|
25
25
|
if (options.origin) payload.origin = options.origin;
|
|
26
26
|
if (options.location != null) payload.location = options.location;
|
|
27
|
+
if (options.threatProtection != null)
|
|
28
|
+
payload.threatProtection = options.threatProtection;
|
|
27
29
|
}
|
|
28
30
|
return payload;
|
|
29
31
|
}
|
package/src/v2/methods/search.ts
CHANGED
|
@@ -37,10 +37,13 @@ function prepareSearchPayload(req: SearchRequest): Record<string, unknown> {
|
|
|
37
37
|
if (req.ignoreInvalidURLs != null)
|
|
38
38
|
payload.ignoreInvalidURLs = req.ignoreInvalidURLs;
|
|
39
39
|
if (req.timeout != null) payload.timeout = req.timeout;
|
|
40
|
+
if (req.highlights != null) payload.highlights = req.highlights;
|
|
40
41
|
if (req.integration && req.integration.trim())
|
|
41
42
|
payload.integration = req.integration.trim();
|
|
42
43
|
if (req.origin) payload.origin = req.origin;
|
|
43
44
|
if (req.enterprise) payload.enterprise = req.enterprise;
|
|
45
|
+
if (req.threatProtection != null)
|
|
46
|
+
payload.threatProtection = req.threatProtection;
|
|
44
47
|
if (req.scrapeOptions) {
|
|
45
48
|
ensureValidScrapeOptions(req.scrapeOptions as ScrapeOptions);
|
|
46
49
|
payload.scrapeOptions = req.scrapeOptions;
|
package/src/v2/types.ts
CHANGED
|
@@ -208,6 +208,7 @@ export interface ScrapeOptions {
|
|
|
208
208
|
storeInCache?: boolean;
|
|
209
209
|
lockdown?: boolean;
|
|
210
210
|
redactPII?: boolean | RedactPIIOptions;
|
|
211
|
+
threatProtection?: ThreatProtectionOptions;
|
|
211
212
|
profile?: {
|
|
212
213
|
name: string;
|
|
213
214
|
saveChanges?: boolean;
|
|
@@ -241,6 +242,27 @@ export interface RedactPIIOptions {
|
|
|
241
242
|
replaceStyle?: "tag" | "mask" | "remove";
|
|
242
243
|
}
|
|
243
244
|
|
|
245
|
+
/**
|
|
246
|
+
* Enterprise: per-request field-level override of your team's threat
|
|
247
|
+
* protection policy. Requires threat protection to be enabled for your team
|
|
248
|
+
* and request overrides to be allowed in the team configuration. Only the
|
|
249
|
+
* fields you provide replace the team policy's values.
|
|
250
|
+
*/
|
|
251
|
+
export interface ThreatProtectionOptions {
|
|
252
|
+
/** "off" disables scanning for this request; "normal" applies the policy. */
|
|
253
|
+
mode?: "off" | "normal";
|
|
254
|
+
/** Block verdicts at or above this risk score (integer 0-100). */
|
|
255
|
+
riskScoreThreshold?: number;
|
|
256
|
+
/** Exact domains or globs like "*.example.com" to always block (max 1000). */
|
|
257
|
+
blacklist?: string[];
|
|
258
|
+
/** Exact domains or globs to always allow; wins over everything (max 1000). */
|
|
259
|
+
whitelist?: string[];
|
|
260
|
+
/** Lowercase TLDs without the leading dot, e.g. "zip" (max 1000). */
|
|
261
|
+
blockedTlds?: string[];
|
|
262
|
+
/** Behavior when scanning is unavailable: "closed" blocks, "open" allows. */
|
|
263
|
+
failurePolicy?: "open" | "closed";
|
|
264
|
+
}
|
|
265
|
+
|
|
244
266
|
export type ParseFileData =
|
|
245
267
|
| Blob
|
|
246
268
|
| File
|
|
@@ -267,6 +289,7 @@ export type ParseOptions = Omit<
|
|
|
267
289
|
| "storeInCache"
|
|
268
290
|
| "lockdown"
|
|
269
291
|
| "proxy"
|
|
292
|
+
| "threatProtection"
|
|
270
293
|
> & {
|
|
271
294
|
formats?: ParseFormatOption[];
|
|
272
295
|
proxy?: "basic" | "auto";
|
|
@@ -674,6 +697,8 @@ export interface SearchRequest {
|
|
|
674
697
|
location?: string;
|
|
675
698
|
ignoreInvalidURLs?: boolean;
|
|
676
699
|
timeout?: number; // ms
|
|
700
|
+
/** Generate query-relevant highlights for search results. Defaults to true. */
|
|
701
|
+
highlights?: boolean;
|
|
677
702
|
scrapeOptions?: ScrapeOptions;
|
|
678
703
|
/**
|
|
679
704
|
* Enterprise search options. Use `["zdr"]` for end-to-end Zero Data
|
|
@@ -681,6 +706,7 @@ export interface SearchRequest {
|
|
|
681
706
|
* your team.
|
|
682
707
|
*/
|
|
683
708
|
enterprise?: Array<"default" | "anon" | "zdr">;
|
|
709
|
+
threatProtection?: ThreatProtectionOptions;
|
|
684
710
|
integration?: string;
|
|
685
711
|
origin?: string;
|
|
686
712
|
}
|
|
@@ -769,6 +795,7 @@ export interface MapOptions {
|
|
|
769
795
|
integration?: string;
|
|
770
796
|
origin?: string;
|
|
771
797
|
location?: LocationConfig;
|
|
798
|
+
threatProtection?: ThreatProtectionOptions;
|
|
772
799
|
}
|
|
773
800
|
|
|
774
801
|
export type FeedbackRating = "good" | "partial" | "bad";
|