@mendable/firecrawl-js 1.18.0-beta.8 → 1.18.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +22 -286
- package/dist/index.d.cts +5 -148
- package/dist/index.d.ts +5 -148
- package/dist/index.js +22 -286
- package/package.json +1 -1
- package/src/index.ts +3 -1
- package/dump.rdb +0 -0
package/dist/index.cjs
CHANGED
|
@@ -42,11 +42,9 @@ var import_isows = require("isows");
|
|
|
42
42
|
var import_typescript_event_target = require("typescript-event-target");
|
|
43
43
|
var FirecrawlError = class extends Error {
|
|
44
44
|
statusCode;
|
|
45
|
-
|
|
46
|
-
constructor(message, statusCode, details) {
|
|
45
|
+
constructor(message, statusCode) {
|
|
47
46
|
super(message);
|
|
48
47
|
this.statusCode = statusCode;
|
|
49
|
-
this.details = details;
|
|
50
48
|
}
|
|
51
49
|
};
|
|
52
50
|
var FirecrawlApp = class {
|
|
@@ -93,20 +91,6 @@ var FirecrawlApp = class {
|
|
|
93
91
|
}
|
|
94
92
|
};
|
|
95
93
|
}
|
|
96
|
-
if (jsonData?.jsonOptions?.schema) {
|
|
97
|
-
let schema = jsonData.jsonOptions.schema;
|
|
98
|
-
try {
|
|
99
|
-
schema = (0, import_zod_to_json_schema.zodToJsonSchema)(schema);
|
|
100
|
-
} catch (error) {
|
|
101
|
-
}
|
|
102
|
-
jsonData = {
|
|
103
|
-
...jsonData,
|
|
104
|
-
jsonOptions: {
|
|
105
|
-
...jsonData.jsonOptions,
|
|
106
|
-
schema
|
|
107
|
-
}
|
|
108
|
-
};
|
|
109
|
-
}
|
|
110
94
|
try {
|
|
111
95
|
const response = await import_axios.default.post(
|
|
112
96
|
this.apiUrl + `/v1/scrape`,
|
|
@@ -261,26 +245,16 @@ var FirecrawlApp = class {
|
|
|
261
245
|
* Checks the status of a crawl job using the Firecrawl API.
|
|
262
246
|
* @param id - The ID of the crawl operation.
|
|
263
247
|
* @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
|
|
264
|
-
* @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
|
|
265
|
-
* @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
|
|
266
|
-
* @param limit - How many entries to return. Only used when `getAllData = false`.
|
|
267
248
|
* @returns The response containing the job status.
|
|
268
249
|
*/
|
|
269
|
-
async checkCrawlStatus(id, getAllData = false
|
|
250
|
+
async checkCrawlStatus(id, getAllData = false) {
|
|
270
251
|
if (!id) {
|
|
271
252
|
throw new FirecrawlError("No crawl ID provided", 400);
|
|
272
253
|
}
|
|
273
254
|
const headers = this.prepareHeaders();
|
|
274
|
-
const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/crawl/${id}`);
|
|
275
|
-
if (skip !== void 0) {
|
|
276
|
-
targetURL.searchParams.set("skip", skip.toString());
|
|
277
|
-
}
|
|
278
|
-
if (limit !== void 0) {
|
|
279
|
-
targetURL.searchParams.set("limit", limit.toString());
|
|
280
|
-
}
|
|
281
255
|
try {
|
|
282
256
|
const response = await this.getRequest(
|
|
283
|
-
|
|
257
|
+
`${this.apiUrl}/v1/crawl/${id}`,
|
|
284
258
|
headers
|
|
285
259
|
);
|
|
286
260
|
if (response.status === 200) {
|
|
@@ -305,7 +279,6 @@ var FirecrawlApp = class {
|
|
|
305
279
|
total: response.data.total,
|
|
306
280
|
completed: response.data.completed,
|
|
307
281
|
creditsUsed: response.data.creditsUsed,
|
|
308
|
-
next: getAllData ? void 0 : response.data.next,
|
|
309
282
|
expiresAt: new Date(response.data.expiresAt),
|
|
310
283
|
data: allData
|
|
311
284
|
};
|
|
@@ -328,28 +301,6 @@ var FirecrawlApp = class {
|
|
|
328
301
|
}
|
|
329
302
|
return { success: false, error: "Internal server error." };
|
|
330
303
|
}
|
|
331
|
-
/**
|
|
332
|
-
* Returns information about crawl errors.
|
|
333
|
-
* @param id - The ID of the crawl operation.
|
|
334
|
-
* @returns Information about crawl errors.
|
|
335
|
-
*/
|
|
336
|
-
async checkCrawlErrors(id) {
|
|
337
|
-
const headers = this.prepareHeaders();
|
|
338
|
-
try {
|
|
339
|
-
const response = await this.deleteRequest(
|
|
340
|
-
`${this.apiUrl}/v1/crawl/${id}/errors`,
|
|
341
|
-
headers
|
|
342
|
-
);
|
|
343
|
-
if (response.status === 200) {
|
|
344
|
-
return response.data;
|
|
345
|
-
} else {
|
|
346
|
-
this.handleError(response, "check crawl errors");
|
|
347
|
-
}
|
|
348
|
-
} catch (error) {
|
|
349
|
-
throw new FirecrawlError(error.message, 500);
|
|
350
|
-
}
|
|
351
|
-
return { success: false, error: "Internal server error." };
|
|
352
|
-
}
|
|
353
304
|
/**
|
|
354
305
|
* Cancels a crawl job using the Firecrawl API.
|
|
355
306
|
* @param id - The ID of the crawl operation.
|
|
@@ -438,20 +389,6 @@ var FirecrawlApp = class {
|
|
|
438
389
|
}
|
|
439
390
|
};
|
|
440
391
|
}
|
|
441
|
-
if (jsonData?.jsonOptions?.schema) {
|
|
442
|
-
let schema = jsonData.jsonOptions.schema;
|
|
443
|
-
try {
|
|
444
|
-
schema = (0, import_zod_to_json_schema.zodToJsonSchema)(schema);
|
|
445
|
-
} catch (error) {
|
|
446
|
-
}
|
|
447
|
-
jsonData = {
|
|
448
|
-
...jsonData,
|
|
449
|
-
jsonOptions: {
|
|
450
|
-
...jsonData.jsonOptions,
|
|
451
|
-
schema
|
|
452
|
-
}
|
|
453
|
-
};
|
|
454
|
-
}
|
|
455
392
|
try {
|
|
456
393
|
const response = await this.postRequest(
|
|
457
394
|
this.apiUrl + `/v1/batch/scrape`,
|
|
@@ -515,26 +452,16 @@ var FirecrawlApp = class {
|
|
|
515
452
|
* Checks the status of a batch scrape job using the Firecrawl API.
|
|
516
453
|
* @param id - The ID of the batch scrape operation.
|
|
517
454
|
* @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
|
|
518
|
-
* @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
|
|
519
|
-
* @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
|
|
520
|
-
* @param limit - How many entries to return. Only used when `getAllData = false`.
|
|
521
455
|
* @returns The response containing the job status.
|
|
522
456
|
*/
|
|
523
|
-
async checkBatchScrapeStatus(id, getAllData = false
|
|
457
|
+
async checkBatchScrapeStatus(id, getAllData = false) {
|
|
524
458
|
if (!id) {
|
|
525
459
|
throw new FirecrawlError("No batch scrape ID provided", 400);
|
|
526
460
|
}
|
|
527
461
|
const headers = this.prepareHeaders();
|
|
528
|
-
const targetURL = new URL(nextURL ?? `${this.apiUrl}/v1/batch/scrape/${id}`);
|
|
529
|
-
if (skip !== void 0) {
|
|
530
|
-
targetURL.searchParams.set("skip", skip.toString());
|
|
531
|
-
}
|
|
532
|
-
if (limit !== void 0) {
|
|
533
|
-
targetURL.searchParams.set("limit", limit.toString());
|
|
534
|
-
}
|
|
535
462
|
try {
|
|
536
463
|
const response = await this.getRequest(
|
|
537
|
-
|
|
464
|
+
`${this.apiUrl}/v1/batch/scrape/${id}`,
|
|
538
465
|
headers
|
|
539
466
|
);
|
|
540
467
|
if (response.status === 200) {
|
|
@@ -559,7 +486,6 @@ var FirecrawlApp = class {
|
|
|
559
486
|
total: response.data.total,
|
|
560
487
|
completed: response.data.completed,
|
|
561
488
|
creditsUsed: response.data.creditsUsed,
|
|
562
|
-
next: getAllData ? void 0 : response.data.next,
|
|
563
489
|
expiresAt: new Date(response.data.expiresAt),
|
|
564
490
|
data: allData
|
|
565
491
|
};
|
|
@@ -582,28 +508,6 @@ var FirecrawlApp = class {
|
|
|
582
508
|
}
|
|
583
509
|
return { success: false, error: "Internal server error." };
|
|
584
510
|
}
|
|
585
|
-
/**
|
|
586
|
-
* Returns information about batch scrape errors.
|
|
587
|
-
* @param id - The ID of the batch scrape operation.
|
|
588
|
-
* @returns Information about batch scrape errors.
|
|
589
|
-
*/
|
|
590
|
-
async checkBatchScrapeErrors(id) {
|
|
591
|
-
const headers = this.prepareHeaders();
|
|
592
|
-
try {
|
|
593
|
-
const response = await this.deleteRequest(
|
|
594
|
-
`${this.apiUrl}/v1/batch/scrape/${id}/errors`,
|
|
595
|
-
headers
|
|
596
|
-
);
|
|
597
|
-
if (response.status === 200) {
|
|
598
|
-
return response.data;
|
|
599
|
-
} else {
|
|
600
|
-
this.handleError(response, "check batch scrape errors");
|
|
601
|
-
}
|
|
602
|
-
} catch (error) {
|
|
603
|
-
throw new FirecrawlError(error.message, 500);
|
|
604
|
-
}
|
|
605
|
-
return { success: false, error: "Internal server error." };
|
|
606
|
-
}
|
|
607
511
|
/**
|
|
608
512
|
* Extracts information from URLs using the Firecrawl API.
|
|
609
513
|
* Currently in Beta. Expect breaking changes on future minor versions.
|
|
@@ -626,66 +530,6 @@ var FirecrawlApp = class {
|
|
|
626
530
|
} catch (error) {
|
|
627
531
|
throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
|
|
628
532
|
}
|
|
629
|
-
try {
|
|
630
|
-
const response = await this.postRequest(
|
|
631
|
-
this.apiUrl + `/v1/extract`,
|
|
632
|
-
{ ...jsonData, schema: jsonSchema, origin: params?.origin || "api-sdk" },
|
|
633
|
-
headers
|
|
634
|
-
);
|
|
635
|
-
if (response.status === 200) {
|
|
636
|
-
const jobId = response.data.id;
|
|
637
|
-
let extractStatus;
|
|
638
|
-
do {
|
|
639
|
-
const statusResponse = await this.getRequest(
|
|
640
|
-
`${this.apiUrl}/v1/extract/${jobId}`,
|
|
641
|
-
headers
|
|
642
|
-
);
|
|
643
|
-
extractStatus = statusResponse.data;
|
|
644
|
-
if (extractStatus.status === "completed") {
|
|
645
|
-
if (extractStatus.success) {
|
|
646
|
-
return {
|
|
647
|
-
success: true,
|
|
648
|
-
data: extractStatus.data,
|
|
649
|
-
warning: extractStatus.warning,
|
|
650
|
-
error: extractStatus.error,
|
|
651
|
-
sources: extractStatus?.sources || void 0
|
|
652
|
-
};
|
|
653
|
-
} else {
|
|
654
|
-
throw new FirecrawlError(`Failed to extract data. Error: ${extractStatus.error}`, statusResponse.status);
|
|
655
|
-
}
|
|
656
|
-
} else if (extractStatus.status === "failed" || extractStatus.status === "cancelled") {
|
|
657
|
-
throw new FirecrawlError(`Extract job ${extractStatus.status}. Error: ${extractStatus.error}`, statusResponse.status);
|
|
658
|
-
}
|
|
659
|
-
await new Promise((resolve) => setTimeout(resolve, 1e3));
|
|
660
|
-
} while (extractStatus.status !== "completed");
|
|
661
|
-
} else {
|
|
662
|
-
this.handleError(response, "extract");
|
|
663
|
-
}
|
|
664
|
-
} catch (error) {
|
|
665
|
-
throw new FirecrawlError(error.message, 500, error.response?.data?.details);
|
|
666
|
-
}
|
|
667
|
-
return { success: false, error: "Internal server error." };
|
|
668
|
-
}
|
|
669
|
-
/**
|
|
670
|
-
* Initiates an asynchronous extract job for a URL using the Firecrawl API.
|
|
671
|
-
* @param url - The URL to extract data from.
|
|
672
|
-
* @param params - Additional parameters for the extract request.
|
|
673
|
-
* @param idempotencyKey - Optional idempotency key for the request.
|
|
674
|
-
* @returns The response from the extract operation.
|
|
675
|
-
*/
|
|
676
|
-
async asyncExtract(urls, params, idempotencyKey) {
|
|
677
|
-
const headers = this.prepareHeaders(idempotencyKey);
|
|
678
|
-
let jsonData = { urls, ...params };
|
|
679
|
-
let jsonSchema;
|
|
680
|
-
try {
|
|
681
|
-
if (params?.schema instanceof zt.ZodType) {
|
|
682
|
-
jsonSchema = (0, import_zod_to_json_schema.zodToJsonSchema)(params.schema);
|
|
683
|
-
} else {
|
|
684
|
-
jsonSchema = params?.schema;
|
|
685
|
-
}
|
|
686
|
-
} catch (error) {
|
|
687
|
-
throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
|
|
688
|
-
}
|
|
689
533
|
try {
|
|
690
534
|
const response = await this.postRequest(
|
|
691
535
|
this.apiUrl + `/v1/extract`,
|
|
@@ -693,34 +537,24 @@ var FirecrawlApp = class {
|
|
|
693
537
|
headers
|
|
694
538
|
);
|
|
695
539
|
if (response.status === 200) {
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
* @param jobId - The ID of the extract job.
|
|
708
|
-
* @returns The status of the extract job.
|
|
709
|
-
*/
|
|
710
|
-
async getExtractStatus(jobId) {
|
|
711
|
-
try {
|
|
712
|
-
const response = await this.getRequest(
|
|
713
|
-
`${this.apiUrl}/v1/extract/${jobId}`,
|
|
714
|
-
this.prepareHeaders()
|
|
715
|
-
);
|
|
716
|
-
if (response.status === 200) {
|
|
717
|
-
return response.data;
|
|
540
|
+
const responseData = response.data;
|
|
541
|
+
if (responseData.success) {
|
|
542
|
+
return {
|
|
543
|
+
success: true,
|
|
544
|
+
data: responseData.data,
|
|
545
|
+
warning: responseData.warning,
|
|
546
|
+
error: responseData.error
|
|
547
|
+
};
|
|
548
|
+
} else {
|
|
549
|
+
throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);
|
|
550
|
+
}
|
|
718
551
|
} else {
|
|
719
|
-
this.handleError(response, "
|
|
552
|
+
this.handleError(response, "extract");
|
|
720
553
|
}
|
|
721
554
|
} catch (error) {
|
|
722
555
|
throw new FirecrawlError(error.message, 500);
|
|
723
556
|
}
|
|
557
|
+
return { success: false, error: "Internal server error." };
|
|
724
558
|
}
|
|
725
559
|
/**
|
|
726
560
|
* Prepares the headers for an API request.
|
|
@@ -836,13 +670,11 @@ var FirecrawlApp = class {
|
|
|
836
670
|
* @param {string} action - The action being performed when the error occurred.
|
|
837
671
|
*/
|
|
838
672
|
handleError(response, action) {
|
|
839
|
-
if ([
|
|
673
|
+
if ([402, 408, 409, 500].includes(response.status)) {
|
|
840
674
|
const errorMessage = response.data.error || "Unknown error occurred";
|
|
841
|
-
const details = response.data.details ? ` - ${JSON.stringify(response.data.details)}` : "";
|
|
842
675
|
throw new FirecrawlError(
|
|
843
|
-
`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}
|
|
844
|
-
response.status
|
|
845
|
-
response?.data?.details
|
|
676
|
+
`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`,
|
|
677
|
+
response.status
|
|
846
678
|
);
|
|
847
679
|
} else {
|
|
848
680
|
throw new FirecrawlError(
|
|
@@ -851,101 +683,6 @@ var FirecrawlApp = class {
|
|
|
851
683
|
);
|
|
852
684
|
}
|
|
853
685
|
}
|
|
854
|
-
/**
|
|
855
|
-
* Initiates a deep research operation on a given topic and polls until completion.
|
|
856
|
-
* @param params - Parameters for the deep research operation.
|
|
857
|
-
* @returns The final research results.
|
|
858
|
-
*/
|
|
859
|
-
async __deepResearch(topic, params) {
|
|
860
|
-
try {
|
|
861
|
-
const response = await this.__asyncDeepResearch(topic, params);
|
|
862
|
-
if (!response.success || "error" in response) {
|
|
863
|
-
return { success: false, error: "error" in response ? response.error : "Unknown error" };
|
|
864
|
-
}
|
|
865
|
-
if (!response.id) {
|
|
866
|
-
throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);
|
|
867
|
-
}
|
|
868
|
-
const jobId = response.id;
|
|
869
|
-
let researchStatus;
|
|
870
|
-
while (true) {
|
|
871
|
-
researchStatus = await this.__checkDeepResearchStatus(jobId);
|
|
872
|
-
if ("error" in researchStatus && !researchStatus.success) {
|
|
873
|
-
return researchStatus;
|
|
874
|
-
}
|
|
875
|
-
if (researchStatus.status === "completed") {
|
|
876
|
-
return researchStatus;
|
|
877
|
-
}
|
|
878
|
-
if (researchStatus.status === "failed") {
|
|
879
|
-
throw new FirecrawlError(
|
|
880
|
-
`Research job ${researchStatus.status}. Error: ${researchStatus.error}`,
|
|
881
|
-
500
|
|
882
|
-
);
|
|
883
|
-
}
|
|
884
|
-
if (researchStatus.status !== "processing") {
|
|
885
|
-
break;
|
|
886
|
-
}
|
|
887
|
-
await new Promise((resolve) => setTimeout(resolve, 2e3));
|
|
888
|
-
}
|
|
889
|
-
return { success: false, error: "Research job terminated unexpectedly" };
|
|
890
|
-
} catch (error) {
|
|
891
|
-
throw new FirecrawlError(error.message, 500, error.response?.data?.details);
|
|
892
|
-
}
|
|
893
|
-
}
|
|
894
|
-
/**
|
|
895
|
-
* Initiates a deep research operation on a given topic without polling.
|
|
896
|
-
* @param params - Parameters for the deep research operation.
|
|
897
|
-
* @returns The response containing the research job ID.
|
|
898
|
-
*/
|
|
899
|
-
async __asyncDeepResearch(topic, params) {
|
|
900
|
-
const headers = this.prepareHeaders();
|
|
901
|
-
try {
|
|
902
|
-
const response = await this.postRequest(
|
|
903
|
-
`${this.apiUrl}/v1/deep-research`,
|
|
904
|
-
{ topic, ...params },
|
|
905
|
-
headers
|
|
906
|
-
);
|
|
907
|
-
if (response.status === 200) {
|
|
908
|
-
return response.data;
|
|
909
|
-
} else {
|
|
910
|
-
this.handleError(response, "start deep research");
|
|
911
|
-
}
|
|
912
|
-
} catch (error) {
|
|
913
|
-
if (error.response?.data?.error) {
|
|
914
|
-
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
|
|
915
|
-
} else {
|
|
916
|
-
throw new FirecrawlError(error.message, 500);
|
|
917
|
-
}
|
|
918
|
-
}
|
|
919
|
-
return { success: false, error: "Internal server error." };
|
|
920
|
-
}
|
|
921
|
-
/**
|
|
922
|
-
* Checks the status of a deep research operation.
|
|
923
|
-
* @param id - The ID of the deep research operation.
|
|
924
|
-
* @returns The current status and results of the research operation.
|
|
925
|
-
*/
|
|
926
|
-
async __checkDeepResearchStatus(id) {
|
|
927
|
-
const headers = this.prepareHeaders();
|
|
928
|
-
try {
|
|
929
|
-
const response = await this.getRequest(
|
|
930
|
-
`${this.apiUrl}/v1/deep-research/${id}`,
|
|
931
|
-
headers
|
|
932
|
-
);
|
|
933
|
-
if (response.status === 200) {
|
|
934
|
-
return response.data;
|
|
935
|
-
} else if (response.status === 404) {
|
|
936
|
-
throw new FirecrawlError("Deep research job not found", 404);
|
|
937
|
-
} else {
|
|
938
|
-
this.handleError(response, "check deep research status");
|
|
939
|
-
}
|
|
940
|
-
} catch (error) {
|
|
941
|
-
if (error.response?.data?.error) {
|
|
942
|
-
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
|
|
943
|
-
} else {
|
|
944
|
-
throw new FirecrawlError(error.message, 500);
|
|
945
|
-
}
|
|
946
|
-
}
|
|
947
|
-
return { success: false, error: "Internal server error." };
|
|
948
|
-
}
|
|
949
686
|
};
|
|
950
687
|
var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget {
|
|
951
688
|
ws;
|
|
@@ -955,8 +692,7 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
|
|
|
955
692
|
constructor(id, app) {
|
|
956
693
|
super();
|
|
957
694
|
this.id = id;
|
|
958
|
-
|
|
959
|
-
this.ws = new import_isows.WebSocket(`${wsUrl}/v1/crawl/${id}`, app.apiKey);
|
|
695
|
+
this.ws = new import_isows.WebSocket(`${app.apiUrl}/v1/crawl/${id}`, app.apiKey);
|
|
960
696
|
this.status = "scraping";
|
|
961
697
|
this.data = [];
|
|
962
698
|
const messageHandler = (msg) => {
|
package/dist/index.d.cts
CHANGED
|
@@ -61,7 +61,6 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
|
|
|
61
61
|
rawHtml?: string;
|
|
62
62
|
links?: string[];
|
|
63
63
|
extract?: T;
|
|
64
|
-
json?: T;
|
|
65
64
|
screenshot?: string;
|
|
66
65
|
metadata?: FirecrawlDocumentMetadata;
|
|
67
66
|
actions: ActionsSchema;
|
|
@@ -73,7 +72,7 @@ interface FirecrawlDocument<T = any, ActionsSchema extends (ActionsResult | neve
|
|
|
73
72
|
* Defines the options and configurations available for scraping web content.
|
|
74
73
|
*/
|
|
75
74
|
interface CrawlScrapeOptions {
|
|
76
|
-
formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract"
|
|
75
|
+
formats: ("markdown" | "html" | "rawHtml" | "content" | "links" | "screenshot" | "screenshot@fullPage" | "extract")[];
|
|
77
76
|
headers?: Record<string, string>;
|
|
78
77
|
includeTags?: string[];
|
|
79
78
|
excludeTags?: string[];
|
|
@@ -87,7 +86,6 @@ interface CrawlScrapeOptions {
|
|
|
87
86
|
mobile?: boolean;
|
|
88
87
|
skipTlsVerification?: boolean;
|
|
89
88
|
removeBase64Images?: boolean;
|
|
90
|
-
blockAds?: boolean;
|
|
91
89
|
}
|
|
92
90
|
type Action = {
|
|
93
91
|
type: "wait";
|
|
@@ -121,11 +119,6 @@ interface ScrapeParams<LLMSchema extends zt.ZodSchema = any, ActionsSchema exten
|
|
|
121
119
|
schema?: LLMSchema;
|
|
122
120
|
systemPrompt?: string;
|
|
123
121
|
};
|
|
124
|
-
jsonOptions?: {
|
|
125
|
-
prompt?: string;
|
|
126
|
-
schema?: LLMSchema;
|
|
127
|
-
systemPrompt?: string;
|
|
128
|
-
};
|
|
129
122
|
actions?: ActionsSchema;
|
|
130
123
|
}
|
|
131
124
|
interface ActionsResult {
|
|
@@ -157,7 +150,6 @@ interface CrawlParams {
|
|
|
157
150
|
url: string;
|
|
158
151
|
headers?: Record<string, string>;
|
|
159
152
|
metadata?: Record<string, string>;
|
|
160
|
-
events?: ["completed", "failed", "page", "started"][number][];
|
|
161
153
|
};
|
|
162
154
|
deduplicateSimilarURLs?: boolean;
|
|
163
155
|
ignoreQueryParameters?: boolean;
|
|
@@ -221,7 +213,6 @@ interface MapParams {
|
|
|
221
213
|
includeSubdomains?: boolean;
|
|
222
214
|
sitemapOnly?: boolean;
|
|
223
215
|
limit?: number;
|
|
224
|
-
timeout?: number;
|
|
225
216
|
}
|
|
226
217
|
/**
|
|
227
218
|
* Response interface for mapping operations.
|
|
@@ -241,10 +232,7 @@ interface ExtractParams<LLMSchema extends zt.ZodSchema = any> {
|
|
|
241
232
|
schema?: LLMSchema | object;
|
|
242
233
|
systemPrompt?: string;
|
|
243
234
|
allowExternalLinks?: boolean;
|
|
244
|
-
enableWebSearch?: boolean;
|
|
245
235
|
includeSubdomains?: boolean;
|
|
246
|
-
origin?: string;
|
|
247
|
-
showSources?: boolean;
|
|
248
236
|
}
|
|
249
237
|
/**
|
|
250
238
|
* Response interface for extracting information from URLs.
|
|
@@ -255,7 +243,6 @@ interface ExtractResponse<LLMSchema extends zt.ZodSchema = any> {
|
|
|
255
243
|
data: LLMSchema;
|
|
256
244
|
error?: string;
|
|
257
245
|
warning?: string;
|
|
258
|
-
sources?: string[];
|
|
259
246
|
}
|
|
260
247
|
/**
|
|
261
248
|
* Error response interface.
|
|
@@ -271,8 +258,7 @@ interface ErrorResponse {
|
|
|
271
258
|
*/
|
|
272
259
|
declare class FirecrawlError extends Error {
|
|
273
260
|
statusCode: number;
|
|
274
|
-
|
|
275
|
-
constructor(message: string, statusCode: number, details?: any);
|
|
261
|
+
constructor(message: string, statusCode: number);
|
|
276
262
|
}
|
|
277
263
|
/**
|
|
278
264
|
* Parameters for search operations.
|
|
@@ -299,85 +285,6 @@ interface SearchResponse {
|
|
|
299
285
|
warning?: string;
|
|
300
286
|
error?: string;
|
|
301
287
|
}
|
|
302
|
-
/**
|
|
303
|
-
* Response interface for crawl/batch scrape error monitoring.
|
|
304
|
-
*/
|
|
305
|
-
interface CrawlErrorsResponse {
|
|
306
|
-
/**
|
|
307
|
-
* Scrapes that errored out + error details
|
|
308
|
-
*/
|
|
309
|
-
errors: {
|
|
310
|
-
id: string;
|
|
311
|
-
timestamp?: string;
|
|
312
|
-
url: string;
|
|
313
|
-
error: string;
|
|
314
|
-
}[];
|
|
315
|
-
/**
|
|
316
|
-
* URLs blocked by robots.txt
|
|
317
|
-
*/
|
|
318
|
-
robotsBlocked: string[];
|
|
319
|
-
}
|
|
320
|
-
/**
|
|
321
|
-
* Parameters for deep research operations.
|
|
322
|
-
* Defines options for conducting deep research on a topic.
|
|
323
|
-
*/
|
|
324
|
-
interface DeepResearchParams {
|
|
325
|
-
/**
|
|
326
|
-
* Maximum depth of research iterations (1-10)
|
|
327
|
-
* @default 7
|
|
328
|
-
*/
|
|
329
|
-
maxDepth?: number;
|
|
330
|
-
/**
|
|
331
|
-
* Time limit in seconds (30-300)
|
|
332
|
-
* @default 270
|
|
333
|
-
*/
|
|
334
|
-
timeLimit?: number;
|
|
335
|
-
/**
|
|
336
|
-
* Experimental flag for streaming steps
|
|
337
|
-
*/
|
|
338
|
-
__experimental_streamSteps?: boolean;
|
|
339
|
-
}
|
|
340
|
-
/**
|
|
341
|
-
* Response interface for deep research operations.
|
|
342
|
-
*/
|
|
343
|
-
interface DeepResearchResponse {
|
|
344
|
-
success: boolean;
|
|
345
|
-
id: string;
|
|
346
|
-
}
|
|
347
|
-
/**
|
|
348
|
-
* Status response interface for deep research operations.
|
|
349
|
-
*/
|
|
350
|
-
interface DeepResearchStatusResponse {
|
|
351
|
-
success: boolean;
|
|
352
|
-
data: {
|
|
353
|
-
findings: Array<{
|
|
354
|
-
text: string;
|
|
355
|
-
source: string;
|
|
356
|
-
}>;
|
|
357
|
-
finalAnalysis: string;
|
|
358
|
-
analysis: string;
|
|
359
|
-
completedSteps: number;
|
|
360
|
-
totalSteps: number;
|
|
361
|
-
};
|
|
362
|
-
status: "processing" | "completed" | "failed";
|
|
363
|
-
error?: string;
|
|
364
|
-
expiresAt: string;
|
|
365
|
-
currentDepth: number;
|
|
366
|
-
maxDepth: number;
|
|
367
|
-
activities: Array<{
|
|
368
|
-
type: string;
|
|
369
|
-
status: string;
|
|
370
|
-
message: string;
|
|
371
|
-
timestamp: string;
|
|
372
|
-
depth: number;
|
|
373
|
-
}>;
|
|
374
|
-
sources: Array<{
|
|
375
|
-
url: string;
|
|
376
|
-
title: string;
|
|
377
|
-
description: string;
|
|
378
|
-
}>;
|
|
379
|
-
summaries: string[];
|
|
380
|
-
}
|
|
381
288
|
/**
|
|
382
289
|
* Main class for interacting with the Firecrawl API.
|
|
383
290
|
* Provides methods for scraping, searching, crawling, and mapping web content.
|
|
@@ -419,18 +326,9 @@ declare class FirecrawlApp {
|
|
|
419
326
|
* Checks the status of a crawl job using the Firecrawl API.
|
|
420
327
|
* @param id - The ID of the crawl operation.
|
|
421
328
|
* @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
|
|
422
|
-
* @param nextURL - The `next` URL from the previous crawl status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
|
|
423
|
-
* @param skip - How many entries to skip to paginate. Only required if you're not providing `nextURL`. Only used when `getAllData = false`.
|
|
424
|
-
* @param limit - How many entries to return. Only used when `getAllData = false`.
|
|
425
329
|
* @returns The response containing the job status.
|
|
426
330
|
*/
|
|
427
|
-
checkCrawlStatus(id?: string, getAllData?: boolean
|
|
428
|
-
/**
|
|
429
|
-
* Returns information about crawl errors.
|
|
430
|
-
* @param id - The ID of the crawl operation.
|
|
431
|
-
* @returns Information about crawl errors.
|
|
432
|
-
*/
|
|
433
|
-
checkCrawlErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse>;
|
|
331
|
+
checkCrawlStatus(id?: string, getAllData?: boolean): Promise<CrawlStatusResponse | ErrorResponse>;
|
|
434
332
|
/**
|
|
435
333
|
* Cancels a crawl job using the Firecrawl API.
|
|
436
334
|
* @param id - The ID of the crawl operation.
|
|
@@ -475,18 +373,9 @@ declare class FirecrawlApp {
|
|
|
475
373
|
* Checks the status of a batch scrape job using the Firecrawl API.
|
|
476
374
|
* @param id - The ID of the batch scrape operation.
|
|
477
375
|
* @param getAllData - Paginate through all the pages of documents, returning the full list of all documents. (default: `false`)
|
|
478
|
-
* @param nextURL - The `next` URL from the previous batch scrape status. Only required if you're not manually increasing `skip`. Only used when `getAllData = false`.
|
|
479
|
-
* @param skip - How many entries to skip to paginate. Only used when `getAllData = false`.
|
|
480
|
-
* @param limit - How many entries to return. Only used when `getAllData = false`.
|
|
481
376
|
* @returns The response containing the job status.
|
|
482
377
|
*/
|
|
483
|
-
checkBatchScrapeStatus(id?: string, getAllData?: boolean
|
|
484
|
-
/**
|
|
485
|
-
* Returns information about batch scrape errors.
|
|
486
|
-
* @param id - The ID of the batch scrape operation.
|
|
487
|
-
* @returns Information about batch scrape errors.
|
|
488
|
-
*/
|
|
489
|
-
checkBatchScrapeErrors(id: string): Promise<CrawlErrorsResponse | ErrorResponse>;
|
|
378
|
+
checkBatchScrapeStatus(id?: string, getAllData?: boolean): Promise<BatchScrapeStatusResponse | ErrorResponse>;
|
|
490
379
|
/**
|
|
491
380
|
* Extracts information from URLs using the Firecrawl API.
|
|
492
381
|
* Currently in Beta. Expect breaking changes on future minor versions.
|
|
@@ -495,20 +384,6 @@ declare class FirecrawlApp {
|
|
|
495
384
|
* @returns The response from the extract operation.
|
|
496
385
|
*/
|
|
497
386
|
extract<T extends zt.ZodSchema = any>(urls: string[], params?: ExtractParams<T>): Promise<ExtractResponse<zt.infer<T>> | ErrorResponse>;
|
|
498
|
-
/**
|
|
499
|
-
* Initiates an asynchronous extract job for a URL using the Firecrawl API.
|
|
500
|
-
* @param url - The URL to extract data from.
|
|
501
|
-
* @param params - Additional parameters for the extract request.
|
|
502
|
-
* @param idempotencyKey - Optional idempotency key for the request.
|
|
503
|
-
* @returns The response from the extract operation.
|
|
504
|
-
*/
|
|
505
|
-
asyncExtract(urls: string[], params?: ExtractParams, idempotencyKey?: string): Promise<ExtractResponse | ErrorResponse>;
|
|
506
|
-
/**
|
|
507
|
-
* Retrieves the status of an extract job.
|
|
508
|
-
* @param jobId - The ID of the extract job.
|
|
509
|
-
* @returns The status of the extract job.
|
|
510
|
-
*/
|
|
511
|
-
getExtractStatus(jobId: string): Promise<any>;
|
|
512
387
|
/**
|
|
513
388
|
* Prepares the headers for an API request.
|
|
514
389
|
* @param idempotencyKey - Optional key to ensure idempotency.
|
|
@@ -552,24 +427,6 @@ declare class FirecrawlApp {
|
|
|
552
427
|
* @param {string} action - The action being performed when the error occurred.
|
|
553
428
|
*/
|
|
554
429
|
handleError(response: AxiosResponse, action: string): void;
|
|
555
|
-
/**
|
|
556
|
-
* Initiates a deep research operation on a given topic and polls until completion.
|
|
557
|
-
* @param params - Parameters for the deep research operation.
|
|
558
|
-
* @returns The final research results.
|
|
559
|
-
*/
|
|
560
|
-
__deepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchStatusResponse | ErrorResponse>;
|
|
561
|
-
/**
|
|
562
|
-
* Initiates a deep research operation on a given topic without polling.
|
|
563
|
-
* @param params - Parameters for the deep research operation.
|
|
564
|
-
* @returns The response containing the research job ID.
|
|
565
|
-
*/
|
|
566
|
-
__asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse>;
|
|
567
|
-
/**
|
|
568
|
-
* Checks the status of a deep research operation.
|
|
569
|
-
* @param id - The ID of the deep research operation.
|
|
570
|
-
* @returns The current status and results of the research operation.
|
|
571
|
-
*/
|
|
572
|
-
__checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse>;
|
|
573
430
|
}
|
|
574
431
|
interface CrawlWatcherEvents {
|
|
575
432
|
document: CustomEvent<FirecrawlDocument<undefined>>;
|
|
@@ -592,4 +449,4 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
|
|
|
592
449
|
close(): void;
|
|
593
450
|
}
|
|
594
451
|
|
|
595
|
-
export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type
|
|
452
|
+
export { type Action, type ActionsResult, type BatchScrapeResponse, type BatchScrapeStatusResponse, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type ExtractParams, type ExtractResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, type SearchParams, type SearchResponse, FirecrawlApp as default };
|