@mendable/firecrawl-js 4.40.0 → 4.42.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-OJ4YJQ4J.js → chunk-OCJTYRX5.js} +1 -1
- package/dist/index.cjs +18 -10
- package/dist/index.d.cts +166 -27
- package/dist/index.d.ts +166 -27
- package/dist/index.js +19 -11
- package/dist/{package-F6O7J2XI.js → package-ECWDBUY3.js} +1 -1
- package/package.json +1 -1
- package/src/__tests__/unit/v2/agent-hints.test.ts +116 -0
- package/src/__tests__/unit/v2/agent-thread.unit.test.ts +122 -0
- package/src/__tests__/unit/v2/extract.unit.test.ts +23 -0
- package/src/__tests__/unit/v2/tools.unit.test.ts +66 -9
- package/src/v2/methods/extract.ts +2 -1
- package/src/v2/methods/map.ts +2 -1
- package/src/v2/methods/parse.ts +2 -1
- package/src/v2/methods/scrape.ts +2 -1
- package/src/v2/methods/search.ts +3 -1
- package/src/v2/types.ts +172 -25
- package/src/v2/utils/agentHints.ts +7 -0
- package/src/v2/utils/errorHandler.ts +5 -5
package/dist/index.d.ts
CHANGED
|
@@ -118,6 +118,15 @@ type ActionOption = WaitAction | ScreenshotAction | ClickAction | WriteAction |
|
|
|
118
118
|
interface AuditMetadata$1 {
|
|
119
119
|
username: string;
|
|
120
120
|
}
|
|
121
|
+
/**
|
|
122
|
+
* OCR raster images (PNG, JPEG, JPEG 2000, TIFF, GIF, BMP, WebP, AVIF) as
|
|
123
|
+
* one-page documents. Part of the default parsers list next to "pdf"; takes
|
|
124
|
+
* no options, so the string "image" is equivalent. Omit it from an explicit
|
|
125
|
+
* list to keep image URLs failing as unsupported files.
|
|
126
|
+
*/
|
|
127
|
+
type ImageParser = {
|
|
128
|
+
type: "image";
|
|
129
|
+
};
|
|
121
130
|
type PDFParser = {
|
|
122
131
|
type: "pdf";
|
|
123
132
|
mode?: "fast" | "auto" | "ocr";
|
|
@@ -174,7 +183,7 @@ interface ScrapeOptions {
|
|
|
174
183
|
timeout?: number;
|
|
175
184
|
waitFor?: number;
|
|
176
185
|
mobile?: boolean;
|
|
177
|
-
parsers?: Array<string | PDFParser>;
|
|
186
|
+
parsers?: Array<string | PDFParser | ImageParser>;
|
|
178
187
|
actions?: ActionOption[];
|
|
179
188
|
location?: LocationConfig$1;
|
|
180
189
|
skipTlsVerification?: boolean;
|
|
@@ -200,8 +209,10 @@ interface ScrapeOptions {
|
|
|
200
209
|
};
|
|
201
210
|
integration?: string;
|
|
202
211
|
origin?: string;
|
|
203
|
-
/** Include domain-matched Alexandria
|
|
212
|
+
/** Include domain-matched Alexandria tools for this URL in `tools`. Default off. */
|
|
204
213
|
domainTools?: boolean;
|
|
214
|
+
/** Summary by default; compact returns identity and description, full includes contracts. */
|
|
215
|
+
toolDetail?: "compact" | "summary" | "full";
|
|
205
216
|
}
|
|
206
217
|
type RedactPIIEntity = "PERSON" | "EMAIL" | "PHONE" | "LOCATION" | "FINANCIAL" | "SECRET";
|
|
207
218
|
interface RedactPIIOptions {
|
|
@@ -227,8 +238,14 @@ interface RedactPIIOptions {
|
|
|
227
238
|
* fields you provide replace the team policy's values.
|
|
228
239
|
*/
|
|
229
240
|
interface ThreatProtectionOptions {
|
|
230
|
-
/**
|
|
231
|
-
|
|
241
|
+
/**
|
|
242
|
+
* "off" disables scanning for this request; "manual-only" enforces only the
|
|
243
|
+
* blacklist / whitelist / blocked TLDs (no provider scan, no scan fee);
|
|
244
|
+
* "normal" scans with Google Web Risk; "zscaler" classifies through your
|
|
245
|
+
* organization's Zscaler connection. Enforced teams may raise the mode per
|
|
246
|
+
* request but never lower it.
|
|
247
|
+
*/
|
|
248
|
+
mode?: "off" | "manual-only" | "normal" | "zscaler";
|
|
232
249
|
/** Block verdicts at or above this risk score (integer 0-100). */
|
|
233
250
|
riskScoreThreshold?: number;
|
|
234
251
|
/** Exact domains or globs like "*.example.com" to always block (max 1000). */
|
|
@@ -246,7 +263,7 @@ interface ParseFile {
|
|
|
246
263
|
filename: string;
|
|
247
264
|
contentType?: string;
|
|
248
265
|
}
|
|
249
|
-
type ParseOptions = Omit<ScrapeOptions, "formats" | "waitFor" | "mobile" | "actions" | "location" | "maxAge" | "minAge" | "storeInCache" | "lockdown" | "proxy" | "threatProtection"> & {
|
|
266
|
+
type ParseOptions = Omit<ScrapeOptions, "formats" | "waitFor" | "mobile" | "actions" | "location" | "maxAge" | "minAge" | "storeInCache" | "lockdown" | "proxy" | "threatProtection" | "toolDetail"> & {
|
|
250
267
|
formats?: ParseFormatOption[];
|
|
251
268
|
proxy?: "basic" | "auto";
|
|
252
269
|
};
|
|
@@ -523,6 +540,8 @@ interface DocumentMetadata {
|
|
|
523
540
|
[key: string]: unknown;
|
|
524
541
|
}
|
|
525
542
|
interface Document {
|
|
543
|
+
/** API guidance, separate from extracted page content. */
|
|
544
|
+
agent_hints?: string[];
|
|
526
545
|
markdown?: string;
|
|
527
546
|
html?: string;
|
|
528
547
|
rawHtml?: string;
|
|
@@ -654,6 +673,7 @@ interface SearchResultImages {
|
|
|
654
673
|
position?: number;
|
|
655
674
|
}
|
|
656
675
|
interface SearchData {
|
|
676
|
+
agent_hints?: string[];
|
|
657
677
|
warning?: string;
|
|
658
678
|
web?: Array<SearchResultWeb | Document>;
|
|
659
679
|
news?: Array<SearchResultNews | Document>;
|
|
@@ -662,14 +682,15 @@ interface SearchData {
|
|
|
662
682
|
}
|
|
663
683
|
/** A complete tool contract returned by semantic or contextual discovery. */
|
|
664
684
|
interface DiscoveredTool {
|
|
685
|
+
next?: AlexandriaCall;
|
|
665
686
|
id?: string;
|
|
666
687
|
provider: string;
|
|
667
688
|
capability: string;
|
|
668
|
-
name
|
|
689
|
+
name?: string;
|
|
669
690
|
description: string;
|
|
670
|
-
creditsCost
|
|
671
|
-
perRecord
|
|
672
|
-
options
|
|
691
|
+
creditsCost?: number;
|
|
692
|
+
perRecord?: boolean;
|
|
693
|
+
options?: Array<{
|
|
673
694
|
name: string;
|
|
674
695
|
type: string;
|
|
675
696
|
required?: boolean;
|
|
@@ -797,8 +818,10 @@ interface CategoryOption {
|
|
|
797
818
|
}
|
|
798
819
|
interface SearchRequest {
|
|
799
820
|
query: string;
|
|
800
|
-
/** Include domain-matched
|
|
821
|
+
/** Include domain-matched tools in tools alongside semantic matches. */
|
|
801
822
|
domainTools?: boolean;
|
|
823
|
+
/** Compact by default; summary adds metadata, full includes contracts. */
|
|
824
|
+
toolDetail?: "compact" | "summary" | "full";
|
|
802
825
|
sources?: Array<"web" | "news" | "images" | "alexandria" | {
|
|
803
826
|
type: "web" | "news" | "images" | "alexandria";
|
|
804
827
|
}>;
|
|
@@ -822,7 +845,7 @@ interface SearchRequest {
|
|
|
822
845
|
timeout?: number;
|
|
823
846
|
/** Generate query-relevant highlights for search results. Defaults to true. */
|
|
824
847
|
highlights?: boolean;
|
|
825
|
-
scrapeOptions?: ScrapeOptions
|
|
848
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
826
849
|
/**
|
|
827
850
|
* Enterprise search options. Use `["zdr"]` for end-to-end Zero Data
|
|
828
851
|
* Retention or `["anon"]` for anonymized search. Must be enabled for
|
|
@@ -850,7 +873,7 @@ interface CrawlOptions {
|
|
|
850
873
|
delay?: number | null;
|
|
851
874
|
maxConcurrency?: number | null;
|
|
852
875
|
webhook?: string | WebhookConfig | null;
|
|
853
|
-
scrapeOptions?: ScrapeOptions | null;
|
|
876
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail"> | null;
|
|
854
877
|
regexOnFullURL?: boolean;
|
|
855
878
|
zeroDataRetention?: boolean;
|
|
856
879
|
integration?: string;
|
|
@@ -871,7 +894,7 @@ interface CrawlJob {
|
|
|
871
894
|
data: Document[];
|
|
872
895
|
}
|
|
873
896
|
interface BatchScrapeOptions {
|
|
874
|
-
options?: ScrapeOptions
|
|
897
|
+
options?: Omit<ScrapeOptions, "toolDetail">;
|
|
875
898
|
webhook?: string | WebhookConfig;
|
|
876
899
|
appendToId?: string;
|
|
877
900
|
ignoreInvalidURLs?: boolean;
|
|
@@ -897,6 +920,7 @@ interface BatchScrapeJob {
|
|
|
897
920
|
data: Document[];
|
|
898
921
|
}
|
|
899
922
|
interface MapData {
|
|
923
|
+
agent_hints?: string[];
|
|
900
924
|
id?: string;
|
|
901
925
|
links: SearchResultWeb[];
|
|
902
926
|
}
|
|
@@ -1005,14 +1029,14 @@ interface MonitorScrapeTarget {
|
|
|
1005
1029
|
id?: string;
|
|
1006
1030
|
type: "scrape";
|
|
1007
1031
|
urls: string[];
|
|
1008
|
-
scrapeOptions?: ScrapeOptions
|
|
1032
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
1009
1033
|
}
|
|
1010
1034
|
interface MonitorCrawlTarget {
|
|
1011
1035
|
id?: string;
|
|
1012
1036
|
type: "crawl";
|
|
1013
1037
|
url: string;
|
|
1014
1038
|
crawlOptions?: CrawlOptions;
|
|
1015
|
-
scrapeOptions?: ScrapeOptions
|
|
1039
|
+
scrapeOptions?: Omit<ScrapeOptions, "toolDetail">;
|
|
1016
1040
|
}
|
|
1017
1041
|
interface MonitorSearchTarget {
|
|
1018
1042
|
id?: string;
|
|
@@ -1216,7 +1240,11 @@ interface AgentExchangeOptions {
|
|
|
1216
1240
|
toolkits?: string[];
|
|
1217
1241
|
maxCalls?: number;
|
|
1218
1242
|
requireApproval?: boolean;
|
|
1219
|
-
/**
|
|
1243
|
+
/**
|
|
1244
|
+
* Answers a pendingApproval from the previous turn of the thread. A `terms`
|
|
1245
|
+
* approval is accepted or declined as a whole: `callIds` and `always` are
|
|
1246
|
+
* ignored on it.
|
|
1247
|
+
*/
|
|
1220
1248
|
approve?: {
|
|
1221
1249
|
approvalId: string;
|
|
1222
1250
|
callIds?: string[];
|
|
@@ -1225,6 +1253,75 @@ interface AgentExchangeOptions {
|
|
|
1225
1253
|
decline?: {
|
|
1226
1254
|
approvalId: string;
|
|
1227
1255
|
};
|
|
1256
|
+
/**
|
|
1257
|
+
* What to do when a provider the agent would use needs data terms the team
|
|
1258
|
+
* has not accepted. Gated providers are never called in any mode:
|
|
1259
|
+
* - "skip" (server default): answer with accepted providers only and list
|
|
1260
|
+
* the gated ones in `exchange.skippedProviders`.
|
|
1261
|
+
* - "ask": the same, plus `exchange.requiresAction` and a `terms`
|
|
1262
|
+
* pendingApproval. Get your user's explicit consent, call terms/accept,
|
|
1263
|
+
* then continue the thread with `approve: { approvalId }`.
|
|
1264
|
+
* Terms gating is rolling out: until it is on for a thread, none of the
|
|
1265
|
+
* terms fields appear. There is no auto-accept mode. Omitted on a follow-up turn inherits the
|
|
1266
|
+
* previous turn's value.
|
|
1267
|
+
*/
|
|
1268
|
+
onTermsRequired?: AgentOnTermsRequired;
|
|
1269
|
+
}
|
|
1270
|
+
type AgentOnTermsRequired = "skip" | "ask";
|
|
1271
|
+
/** A gated provider the run would have used but did not. */
|
|
1272
|
+
interface AgentSkippedProvider {
|
|
1273
|
+
provider: string;
|
|
1274
|
+
name: string;
|
|
1275
|
+
capability?: string;
|
|
1276
|
+
/** What it would have added, in the agent's words. */
|
|
1277
|
+
adds?: string;
|
|
1278
|
+
reason: "terms_required";
|
|
1279
|
+
/** The gating terms version. */
|
|
1280
|
+
version: string;
|
|
1281
|
+
/** Where a person accepts the terms in the dashboard. */
|
|
1282
|
+
termsUrl: string;
|
|
1283
|
+
}
|
|
1284
|
+
/**
|
|
1285
|
+
* The exact Exchange calls to view and accept a provider's terms. Nothing here
|
|
1286
|
+
* is executed for you, and terms/accept must only be called after the user
|
|
1287
|
+
* has explicitly agreed to that provider's terms.
|
|
1288
|
+
*/
|
|
1289
|
+
interface AgentTermsRequiredAction {
|
|
1290
|
+
type: "accept_terms";
|
|
1291
|
+
/**
|
|
1292
|
+
* The `terms` pendingApproval that answers this. After the user agrees and
|
|
1293
|
+
* terms/accept succeeds, continue the thread with
|
|
1294
|
+
* `exchange.approve: { approvalId }`, or refuse with `decline`.
|
|
1295
|
+
*/
|
|
1296
|
+
approvalId: string;
|
|
1297
|
+
providers: {
|
|
1298
|
+
provider: string;
|
|
1299
|
+
name: string;
|
|
1300
|
+
capability?: string;
|
|
1301
|
+
adds?: string;
|
|
1302
|
+
version: string;
|
|
1303
|
+
/** null when the catalog published no digest; terms/show returns it. */
|
|
1304
|
+
digest: string | null;
|
|
1305
|
+
url: string;
|
|
1306
|
+
show: {
|
|
1307
|
+
provider: "firecrawl";
|
|
1308
|
+
capability: "terms/show";
|
|
1309
|
+
options: {
|
|
1310
|
+
provider: string;
|
|
1311
|
+
};
|
|
1312
|
+
};
|
|
1313
|
+
accept: {
|
|
1314
|
+
provider: "firecrawl";
|
|
1315
|
+
capability: "terms/accept";
|
|
1316
|
+
options: {
|
|
1317
|
+
provider: string;
|
|
1318
|
+
version: string;
|
|
1319
|
+
/** null when the catalog published no digest; terms/show returns it. */
|
|
1320
|
+
digest: string | null;
|
|
1321
|
+
confirmed: true;
|
|
1322
|
+
};
|
|
1323
|
+
};
|
|
1324
|
+
}[];
|
|
1228
1325
|
}
|
|
1229
1326
|
/** Per-run summary reported on a status response. */
|
|
1230
1327
|
interface AgentExchangeSummary {
|
|
@@ -1232,33 +1329,72 @@ interface AgentExchangeSummary {
|
|
|
1232
1329
|
/** What the run resolved to after thread inheritance, not what it requested. */
|
|
1233
1330
|
toolkits?: string[];
|
|
1234
1331
|
requireApproval?: boolean;
|
|
1332
|
+
onTermsRequired?: AgentOnTermsRequired;
|
|
1235
1333
|
paidCalls: number;
|
|
1236
1334
|
creditsUsed: number | null;
|
|
1335
|
+
/** Gated providers that would have helped and were not used. Any mode. */
|
|
1336
|
+
skippedProviders?: AgentSkippedProvider[];
|
|
1337
|
+
/** "ask" mode, when a terms offer ended the turn. */
|
|
1338
|
+
requiresAction?: AgentTermsRequiredAction;
|
|
1339
|
+
}
|
|
1340
|
+
/** A provider in a `terms` pendingApproval. */
|
|
1341
|
+
interface AgentTermsGate {
|
|
1342
|
+
provider: string;
|
|
1343
|
+
name: string;
|
|
1344
|
+
logo?: string;
|
|
1345
|
+
capability?: string;
|
|
1346
|
+
adds?: string;
|
|
1347
|
+
version: string;
|
|
1348
|
+
/** null when the catalog published no digest; terms/show returns it. */
|
|
1349
|
+
digest: string | null;
|
|
1350
|
+
url: string;
|
|
1237
1351
|
}
|
|
1238
1352
|
/** A follow-up the agent offers for the next turn of the thread. */
|
|
1239
1353
|
interface AgentSuggestion {
|
|
1240
1354
|
label: string;
|
|
1241
1355
|
prompt: string;
|
|
1242
1356
|
}
|
|
1243
|
-
/** A
|
|
1244
|
-
interface
|
|
1357
|
+
/** A paid call a `calls` pendingApproval is holding back. */
|
|
1358
|
+
interface PendingApprovalCall {
|
|
1359
|
+
id: string;
|
|
1360
|
+
provider: string;
|
|
1361
|
+
capability: string;
|
|
1362
|
+
input: Record<string, unknown>;
|
|
1363
|
+
more?: Record<string, unknown>[];
|
|
1364
|
+
creditsEstimate: number | null;
|
|
1365
|
+
}
|
|
1366
|
+
interface PendingApprovalBase {
|
|
1245
1367
|
id: string;
|
|
1246
1368
|
reason: string;
|
|
1247
|
-
calls: {
|
|
1248
|
-
id: string;
|
|
1249
|
-
provider: string;
|
|
1250
|
-
capability: string;
|
|
1251
|
-
input: Record<string, unknown>;
|
|
1252
|
-
more?: Record<string, unknown>[];
|
|
1253
|
-
creditsEstimate: number | null;
|
|
1254
|
-
}[];
|
|
1255
1369
|
resolution: null | {
|
|
1256
1370
|
approved: boolean;
|
|
1371
|
+
/** Calls approved. Ignored on terms offers. */
|
|
1257
1372
|
callIds: string[];
|
|
1258
1373
|
always: boolean;
|
|
1259
1374
|
byRunId: string;
|
|
1260
1375
|
};
|
|
1261
1376
|
}
|
|
1377
|
+
/**
|
|
1378
|
+
* A turn that ended waiting for the caller to allow or refuse paid calls.
|
|
1379
|
+
* `kind` is absent on items written before terms offers existed.
|
|
1380
|
+
*/
|
|
1381
|
+
interface PendingCallsApproval extends PendingApprovalBase {
|
|
1382
|
+
kind?: "calls";
|
|
1383
|
+
calls: PendingApprovalCall[];
|
|
1384
|
+
terms?: never;
|
|
1385
|
+
}
|
|
1386
|
+
/**
|
|
1387
|
+
* A turn that ended waiting for the caller to accept providers' data terms
|
|
1388
|
+
* ("ask" mode). `calls` is always empty (typed `never[]` rather
|
|
1389
|
+
* than `[]` so existing `pendingApproval.calls[0]` code still compiles).
|
|
1390
|
+
*/
|
|
1391
|
+
interface PendingTermsApproval extends PendingApprovalBase {
|
|
1392
|
+
kind: "terms";
|
|
1393
|
+
calls: never[];
|
|
1394
|
+
terms: AgentTermsGate[];
|
|
1395
|
+
}
|
|
1396
|
+
/** Narrow on `kind === "terms"`. */
|
|
1397
|
+
type PendingApproval = PendingCallsApproval | PendingTermsApproval;
|
|
1262
1398
|
interface AgentResponse {
|
|
1263
1399
|
success: boolean;
|
|
1264
1400
|
id: string;
|
|
@@ -1568,6 +1704,7 @@ interface RequiresAction {
|
|
|
1568
1704
|
}
|
|
1569
1705
|
declare function parseRequiresAction(value: unknown): RequiresAction | undefined;
|
|
1570
1706
|
declare class SdkError extends Error {
|
|
1707
|
+
agent_hints?: string[];
|
|
1571
1708
|
requestId?: string;
|
|
1572
1709
|
status?: number;
|
|
1573
1710
|
code?: string;
|
|
@@ -1611,10 +1748,12 @@ interface BrowserExecuteResponse {
|
|
|
1611
1748
|
stderr?: string;
|
|
1612
1749
|
exitCode?: number;
|
|
1613
1750
|
killed?: boolean;
|
|
1751
|
+
truncated?: boolean;
|
|
1614
1752
|
error?: string;
|
|
1615
1753
|
}
|
|
1616
1754
|
interface BrowserDeleteResponse {
|
|
1617
1755
|
success: boolean;
|
|
1756
|
+
status?: string;
|
|
1618
1757
|
sessionDurationMs?: number;
|
|
1619
1758
|
creditsBilled?: number;
|
|
1620
1759
|
error?: string;
|
|
@@ -3389,4 +3528,4 @@ declare class Firecrawl extends FirecrawlClient {
|
|
|
3389
3528
|
get v1(): FirecrawlApp;
|
|
3390
3529
|
}
|
|
3391
3530
|
|
|
3392
|
-
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentEffort, type AgentExchangeOptions, type AgentExchangeSummary, type AgentListItem, type AgentListOptions, type AgentListResponse, type AgentMode, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentSnapshotResponse, type AgentStatusResponse, type AgentSuggestion, type AgentThread, type AgentThreadResponse, type AgentThreadRun, type AgentTraceActiveBrowserSession, type AgentTraceAgentFinishedEvent, type AgentTraceAgentIdentity, type AgentTraceAgentRole, type AgentTraceAgentStartedEvent, type AgentTraceArtifactChange, type AgentTraceArtifactUpdatedEvent, type AgentTraceBrowserSessionFinishedEvent, type AgentTraceBrowserSessionStartedEvent, type AgentTraceError, type AgentTraceErrorOccurredEvent, type AgentTraceEvent, type AgentTraceProgressReportedEvent, type AgentTraceReasoningSummaryEvent, type AgentTraceResponse, type AgentTraceRunCancelRequestedEvent, type AgentTraceRunFinishedEvent, type AgentTraceRunStartedEvent, type AgentTraceToolCallFinishedEvent, type AgentTraceToolCallStartedEvent, type AgentWebhookConfig, type AgentWebhookEvent, type AlexandriaCall, type AlexandriaOptions, type AlexandriaScrapeData, type AlexandriaScrapeError, type AlexandriaScrapeRequest, type AlexandriaScrapeResult, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type DiscoveredTool, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, type FindToolsData, type FindToolsOptions, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PendingApproval, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, type RequiresAction, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default, parseRequiresAction };
|
|
3531
|
+
export { type ActionOption, type ActiveCrawl, type ActiveCrawlsResponse, type AgentEffort, type AgentExchangeOptions, type AgentExchangeSummary, type AgentListItem, type AgentListOptions, type AgentListResponse, type AgentMode, type AgentOnTermsRequired, type AgentOptions$1 as AgentOptions, type AgentResponse, type AgentSkippedProvider, type AgentSnapshotResponse, type AgentStatusResponse, type AgentSuggestion, type AgentTermsGate, type AgentTermsRequiredAction, type AgentThread, type AgentThreadResponse, type AgentThreadRun, type AgentTraceActiveBrowserSession, type AgentTraceAgentFinishedEvent, type AgentTraceAgentIdentity, type AgentTraceAgentRole, type AgentTraceAgentStartedEvent, type AgentTraceArtifactChange, type AgentTraceArtifactUpdatedEvent, type AgentTraceBrowserSessionFinishedEvent, type AgentTraceBrowserSessionStartedEvent, type AgentTraceError, type AgentTraceErrorOccurredEvent, type AgentTraceEvent, type AgentTraceProgressReportedEvent, type AgentTraceReasoningSummaryEvent, type AgentTraceResponse, type AgentTraceRunCancelRequestedEvent, type AgentTraceRunFinishedEvent, type AgentTraceRunStartedEvent, type AgentTraceToolCallFinishedEvent, type AgentTraceToolCallStartedEvent, type AgentWebhookConfig, type AgentWebhookEvent, type AlexandriaCall, type AlexandriaOptions, type AlexandriaScrapeData, type AlexandriaScrapeError, type AlexandriaScrapeRequest, type AlexandriaScrapeResult, type AttributesFormat, type AuditMetadata$1 as AuditMetadata, type BatchScrapeJob, type BatchScrapeOptions, type BatchScrapeResponse$1 as BatchScrapeResponse, type BrandingProfile, type BrowserCreateResponse, type BrowserDeleteResponse, type BrowserExecuteResponse, type BrowserListResponse, type BrowserSession, type CategoryOption, type ChangeTrackingFormat, type ClickAction, type ConcurrencyCheck, type CrawlErrorsResponse$1 as CrawlErrorsResponse, type CrawlJob, type CrawlOptions, type CrawlResponse$1 as CrawlResponse, type CreateMonitorRequest, type CreditUsage, type CreditUsageHistoricalPeriod, type CreditUsageHistoricalResponse, type DeveloperSearchLicenseDisclosure, type DeveloperSearchOptions, type DeveloperSearchPassage, type DeveloperSearchRepoStatus, type DeveloperSearchResponse, type DeveloperSearchResult, type DeveloperSearchSourceStatus, type DeveloperSearchType, type DiscoveredTool, type Document, type DocumentMetadata, type EndpointFeedbackEndpoint, type EndpointFeedbackRequest, type ErrorDetails, type ExecuteJavascriptAction, type ExtractResponse$1 as ExtractResponse, type FeedbackMissingContent, type FeedbackRating, type FeedbackResponse, type FeedbackValuableSource, type FindToolsData, type FindToolsOptions, Firecrawl, FirecrawlApp as FirecrawlAppV1, FirecrawlClient, type FirecrawlClientInput, type FirecrawlClientOptions, type Format, type FormatOption, type FormatString, type GetMonitorCheckOptions, type GetPaperOptions, type GitHubScoreBreakdown, type GitHubSearchItem, type GitHubSearchResponse, type HighlightsFormat, type IdMap, type ImageParser, JobTimeoutError, type JsonFormat, type ListMonitorChecksOptions, type ListMonitorsOptions, type LocationConfig$1 as LocationConfig, type MapData, type MapOptions, type MenuAvailability, type MenuImage, type MenuItem, type MenuItemIdentifiers, type MenuMerchant, type MenuPrice, type MenuProfile, type MenuSection, type Monitor, type MonitorCheck, type MonitorCheckDetail, type MonitorCheckPage, type MonitorCrawlTarget, type MonitorCrawlTargetResult, type MonitorEmailNotification, type MonitorEmailRecipientSubscription, type MonitorJsonFieldDiff, type MonitorNotification, type MonitorPageDiff, type MonitorPageJudgment, type MonitorPageSnapshot, type MonitorSchedule, type MonitorScrapeTarget, type MonitorScrapeTargetResult, type MonitorSearchTarget, type MonitorSearchTargetResult, type MonitorSummary, type MonitorTarget, type MonitorTargetResult, type MonitorWebhookConfig, type PDFAction, type PDFParser, type PaginationConfig, type PaperMetadata, type PaperMetadataResponse, type PaperResult, type PaperSignals, type ParseFile, type ParseFileData, type ParseFormat, type ParseFormatOption, type ParseFormatString, type ParseOptions, type Passage, type PdfBlockConfidence, type PdfBlockItem, type PdfPage, type PdfPageBlocks, type PendingApproval, type PendingApprovalCall, type PendingCallsApproval, type PendingTermsApproval, type PressAction, type ProductAvailability, type ProductImage, type ProductPrice, type ProductProfile, type ProductSale, type ProductVariant, type QueryFormat, type QuestionFormat, type QueueStatusResponse$1 as QueueStatusResponse, type ReadPaperResponse, type RedactPIIEntity, type RedactPIIOptions, type RequiresAction, ResearchClient, type ScrapeAction, type ScrapeBrowserDeleteResponse, type ScrapeExecuteRequest, type ScrapeExecuteResponse, type ScrapeOptions, type ScreenshotAction, type ScreenshotFormat, type ScrollAction, SdkError, type SearchData, type SearchFeedbackRequest, type SearchGithubOptions, type SearchPapersOptions, type SearchPapersResponse, type SearchRequest, type SearchResultImages, type SearchResultNews, type SearchResultWeb, type SimilarPapersOptions, type SimilarPapersResponse, type ThreatProtectionOptions, type TokenUsage, type TokenUsageHistoricalPeriod, type TokenUsageHistoricalResponse, type UpdateMonitorRequest, type Viewport, type WaitAction, Watcher, type WatcherOptions, type WebhookConfig, type WriteAction, Firecrawl as default, parseRequiresAction };
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-OCJTYRX5.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -185,6 +185,12 @@ var JobTimeoutError = class extends SdkError {
|
|
|
185
185
|
}
|
|
186
186
|
};
|
|
187
187
|
|
|
188
|
+
// src/v2/utils/agentHints.ts
|
|
189
|
+
function agentHintMetadata(body) {
|
|
190
|
+
const hints = body?.agent_hints;
|
|
191
|
+
return Array.isArray(hints) && hints.every((hint) => typeof hint === "string") ? { agent_hints: [...hints] } : {};
|
|
192
|
+
}
|
|
193
|
+
|
|
188
194
|
// src/utils/zodSchemaToJson.ts
|
|
189
195
|
import { zodToJsonSchema as zodToJsonSchemaLib } from "zod-to-json-schema";
|
|
190
196
|
function isZodSchema(value) {
|
|
@@ -418,7 +424,7 @@ function throwForBadResponse(resp, action) {
|
|
|
418
424
|
const status = resp.status;
|
|
419
425
|
const body = resp.data || {};
|
|
420
426
|
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
|
|
421
|
-
throw new SdkError(
|
|
427
|
+
throw Object.assign(new SdkError(
|
|
422
428
|
msg,
|
|
423
429
|
status,
|
|
424
430
|
body?.code,
|
|
@@ -426,14 +432,14 @@ function throwForBadResponse(resp, action) {
|
|
|
426
432
|
void 0,
|
|
427
433
|
body?.chargeId,
|
|
428
434
|
parseRequiresAction(body?.requiresAction)
|
|
429
|
-
);
|
|
435
|
+
), agentHintMetadata(body));
|
|
430
436
|
}
|
|
431
437
|
function normalizeAxiosError(err, action) {
|
|
432
438
|
const status = err.response?.status;
|
|
433
439
|
const body = err.response?.data;
|
|
434
440
|
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
|
|
435
441
|
const code = body?.code || err.code;
|
|
436
|
-
throw new SdkError(
|
|
442
|
+
throw Object.assign(new SdkError(
|
|
437
443
|
message,
|
|
438
444
|
status,
|
|
439
445
|
code,
|
|
@@ -441,7 +447,7 @@ function normalizeAxiosError(err, action) {
|
|
|
441
447
|
void 0,
|
|
442
448
|
body?.chargeId,
|
|
443
449
|
parseRequiresAction(body?.requiresAction)
|
|
444
|
-
);
|
|
450
|
+
), agentHintMetadata(body));
|
|
445
451
|
}
|
|
446
452
|
function isRetryableError(err) {
|
|
447
453
|
if (err instanceof JobTimeoutError) {
|
|
@@ -509,7 +515,7 @@ async function scrape(http, url, options, deps = {}) {
|
|
|
509
515
|
if (res.status !== 200 || !res.data?.success) {
|
|
510
516
|
throwForBadResponse(res, "scrape");
|
|
511
517
|
}
|
|
512
|
-
return res.data.data
|
|
518
|
+
return { ...res.data.data, ...agentHintMetadata(res.data) };
|
|
513
519
|
} catch (err) {
|
|
514
520
|
const delayMs = autoResume !== false ? processingContinuesDelayMs(err) : void 0;
|
|
515
521
|
if (delayMs !== void 0 && resumes < RESUME_MAX_ATTEMPTS && resumeWaitedMs + delayMs <= RESUME_MAX_TOTAL_WAIT_MS) {
|
|
@@ -626,7 +632,7 @@ async function parse(http, file, options) {
|
|
|
626
632
|
if (res.status !== 200 || !res.data?.success) {
|
|
627
633
|
throwForBadResponse(res, "parse");
|
|
628
634
|
}
|
|
629
|
-
return res.data.data
|
|
635
|
+
return { ...res.data.data, ...agentHintMetadata(res.data) };
|
|
630
636
|
} catch (err) {
|
|
631
637
|
if (err?.isAxiosError) return normalizeAxiosError(err, "parse");
|
|
632
638
|
throw err;
|
|
@@ -647,6 +653,7 @@ function prepareSearchPayload(req) {
|
|
|
647
653
|
const payload = {
|
|
648
654
|
query: req.query
|
|
649
655
|
};
|
|
656
|
+
if (req.toolDetail != null) payload.toolDetail = req.toolDetail;
|
|
650
657
|
if (req.domainTools != null) payload.domainTools = req.domainTools;
|
|
651
658
|
if (req.sources) payload.sources = req.sources;
|
|
652
659
|
if (req.categories) payload.categories = req.categories;
|
|
@@ -699,7 +706,7 @@ async function search(http, request) {
|
|
|
699
706
|
throwForBadResponse(res, "search");
|
|
700
707
|
}
|
|
701
708
|
const data = res.data.data || {};
|
|
702
|
-
const out = {};
|
|
709
|
+
const out = { ...agentHintMetadata(res.data) };
|
|
703
710
|
if (res.data.warning) out.warning = res.data.warning;
|
|
704
711
|
if (data.web) out.web = transformArray(data.web);
|
|
705
712
|
if (data.news) out.news = transformArray(data.news);
|
|
@@ -904,7 +911,7 @@ async function map(http, url, options) {
|
|
|
904
911
|
description: item.description
|
|
905
912
|
});
|
|
906
913
|
}
|
|
907
|
-
return { id: res.data.id, links };
|
|
914
|
+
return { id: res.data.id, links, ...agentHintMetadata(res.data) };
|
|
908
915
|
} catch (err) {
|
|
909
916
|
if (err?.isAxiosError) return normalizeAxiosError(err, "map");
|
|
910
917
|
throw err;
|
|
@@ -1315,7 +1322,8 @@ function prepareExtractPayload(args) {
|
|
|
1315
1322
|
body.threatProtection = args.threatProtection;
|
|
1316
1323
|
if (args.scrapeOptions) {
|
|
1317
1324
|
ensureValidScrapeOptions(args.scrapeOptions);
|
|
1318
|
-
|
|
1325
|
+
const { toolDetail, domainTools, ...scrapeOptions } = args.scrapeOptions;
|
|
1326
|
+
body.scrapeOptions = scrapeOptions;
|
|
1319
1327
|
}
|
|
1320
1328
|
return body;
|
|
1321
1329
|
}
|
|
@@ -2665,7 +2673,7 @@ var FirecrawlApp = class {
|
|
|
2665
2673
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
2666
2674
|
return process.env.npm_package_version;
|
|
2667
2675
|
}
|
|
2668
|
-
const packageJson = await import("./package-
|
|
2676
|
+
const packageJson = await import("./package-ECWDBUY3.js");
|
|
2669
2677
|
return packageJson.default.version;
|
|
2670
2678
|
} catch (error) {
|
|
2671
2679
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@mendable/firecrawl-js",
|
|
3
|
-
"version": "4.
|
|
3
|
+
"version": "4.42.0",
|
|
4
4
|
"description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
|
|
5
5
|
"main": "dist/index.js",
|
|
6
6
|
"types": "dist/index.d.ts",
|
|
@@ -0,0 +1,116 @@
|
|
|
1
|
+
import { describe, expect, it, jest } from "@jest/globals";
|
|
2
|
+
import { search } from "../../../v2/methods/search";
|
|
3
|
+
import { scrape } from "../../../v2/methods/scrape";
|
|
4
|
+
import { parse } from "../../../v2/methods/parse";
|
|
5
|
+
import { map } from "../../../v2/methods/map";
|
|
6
|
+
import {
|
|
7
|
+
normalizeAxiosError,
|
|
8
|
+
throwForBadResponse,
|
|
9
|
+
} from "../../../v2/utils/errorHandler";
|
|
10
|
+
|
|
11
|
+
const hints = [
|
|
12
|
+
"Scrape a selected search result if you need full page content.",
|
|
13
|
+
"Evaluate this result before submitting feedback.",
|
|
14
|
+
];
|
|
15
|
+
function httpFor(body: object) {
|
|
16
|
+
return {
|
|
17
|
+
post: jest.fn<any>().mockResolvedValue({ status: 200, data: body }),
|
|
18
|
+
postMultipart: jest
|
|
19
|
+
.fn<any>()
|
|
20
|
+
.mockResolvedValue({ status: 200, data: body }),
|
|
21
|
+
} as any;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
describe("agent hints survive SDK response unwrapping", () => {
|
|
25
|
+
it("preserves search hints without adding them to records", async () => {
|
|
26
|
+
const web = [{ url: "https://example.com" }];
|
|
27
|
+
const result = await search(
|
|
28
|
+
httpFor({ success: true, data: { web }, agent_hints: hints }),
|
|
29
|
+
{ query: "test" },
|
|
30
|
+
);
|
|
31
|
+
expect(result.web).toEqual(web);
|
|
32
|
+
expect(result.web[0]).not.toHaveProperty("agent_hints");
|
|
33
|
+
expect(result.agent_hints).toEqual(hints);
|
|
34
|
+
expect(JSON.parse(JSON.stringify(result)).agent_hints).toEqual(hints);
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
it("keeps existing search output unchanged when hints are absent", async () => {
|
|
38
|
+
const result = await search(httpFor({ success: true, data: { web: [] } }), {
|
|
39
|
+
query: "test",
|
|
40
|
+
});
|
|
41
|
+
expect(Object.keys(result)).toEqual(["web"]);
|
|
42
|
+
});
|
|
43
|
+
|
|
44
|
+
it.each(["scrape", "parse"])(
|
|
45
|
+
"keeps %s hints separate from document content",
|
|
46
|
+
async (method) => {
|
|
47
|
+
const http = httpFor({
|
|
48
|
+
success: true,
|
|
49
|
+
data: { markdown: "# Page" },
|
|
50
|
+
agent_hints: hints,
|
|
51
|
+
});
|
|
52
|
+
const result =
|
|
53
|
+
method === "scrape"
|
|
54
|
+
? await scrape(http, "https://example.com")
|
|
55
|
+
: await parse(http, { filename: "page.txt", data: "Page" });
|
|
56
|
+
expect(result.markdown).toBe("# Page");
|
|
57
|
+
expect(result.agent_hints).toEqual(hints);
|
|
58
|
+
},
|
|
59
|
+
);
|
|
60
|
+
|
|
61
|
+
it("preserves map metadata alongside normalized links", async () => {
|
|
62
|
+
const result = await map(
|
|
63
|
+
httpFor({
|
|
64
|
+
success: true,
|
|
65
|
+
id: "map-id",
|
|
66
|
+
links: ["https://example.com"],
|
|
67
|
+
agent_hints: hints,
|
|
68
|
+
}),
|
|
69
|
+
"https://example.com",
|
|
70
|
+
);
|
|
71
|
+
expect(result).toEqual({
|
|
72
|
+
id: "map-id",
|
|
73
|
+
links: [{ url: "https://example.com" }],
|
|
74
|
+
agent_hints: hints,
|
|
75
|
+
});
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
it.each([null, "unexpected", [123]])(
|
|
79
|
+
"ignores malformed optional hints: %j",
|
|
80
|
+
async (value) => {
|
|
81
|
+
const result = await search(
|
|
82
|
+
httpFor({ success: true, data: {}, agent_hints: value }),
|
|
83
|
+
{ query: "test" },
|
|
84
|
+
);
|
|
85
|
+
expect(result).not.toHaveProperty("agent_hints");
|
|
86
|
+
},
|
|
87
|
+
);
|
|
88
|
+
|
|
89
|
+
it("preserves hints on HTTP and Axios errors without changing their status", () => {
|
|
90
|
+
const response = {
|
|
91
|
+
status: 400,
|
|
92
|
+
data: {
|
|
93
|
+
success: false,
|
|
94
|
+
code: "INVALID",
|
|
95
|
+
error: "Invalid request",
|
|
96
|
+
agent_hints: hints,
|
|
97
|
+
},
|
|
98
|
+
} as any;
|
|
99
|
+
for (const raise of [
|
|
100
|
+
() => throwForBadResponse(response, "search"),
|
|
101
|
+
() =>
|
|
102
|
+
normalizeAxiosError({ response, isAxiosError: true } as any, "search"),
|
|
103
|
+
]) {
|
|
104
|
+
try {
|
|
105
|
+
raise();
|
|
106
|
+
throw new Error("Expected SDK error");
|
|
107
|
+
} catch (error) {
|
|
108
|
+
expect(error).toMatchObject({
|
|
109
|
+
status: 400,
|
|
110
|
+
code: "INVALID",
|
|
111
|
+
agent_hints: hints,
|
|
112
|
+
});
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
});
|
|
116
|
+
});
|