@mendable/firecrawl 1.18.6 → 1.19.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +118 -1
- package/dist/index.d.cts +49 -8
- package/dist/index.d.ts +49 -8
- package/dist/index.js +118 -1
- package/package.json +1 -1
- package/src/__tests__/e2e_withAuth/index.test.ts +1 -1
- package/src/__tests__/v1/e2e_withAuth/index.test.ts +2 -2
- package/src/index.ts +168 -11
package/dist/index.cjs
CHANGED
|
@@ -618,7 +618,7 @@ var FirecrawlApp = class {
|
|
|
618
618
|
try {
|
|
619
619
|
if (!params?.schema) {
|
|
620
620
|
jsonSchema = void 0;
|
|
621
|
-
} else if (params.schema
|
|
621
|
+
} else if (typeof params.schema === "object" && params.schema !== null && Object.getPrototypeOf(params.schema)?.constructor?.name?.startsWith("Zod")) {
|
|
622
622
|
jsonSchema = (0, import_zod_to_json_schema.zodToJsonSchema)(params.schema);
|
|
623
623
|
} else {
|
|
624
624
|
jsonSchema = params.schema;
|
|
@@ -852,6 +852,121 @@ var FirecrawlApp = class {
|
|
|
852
852
|
}
|
|
853
853
|
}
|
|
854
854
|
/**
|
|
855
|
+
* Initiates a deep research operation on a given query and polls until completion.
|
|
856
|
+
* @param query - The query to research.
|
|
857
|
+
* @param params - Parameters for the deep research operation.
|
|
858
|
+
* @param onActivity - Optional callback to receive activity updates in real-time.
|
|
859
|
+
* @param onSource - Optional callback to receive source updates in real-time.
|
|
860
|
+
* @returns The final research results.
|
|
861
|
+
*/
|
|
862
|
+
async deepResearch(query, params, onActivity, onSource) {
|
|
863
|
+
try {
|
|
864
|
+
const response = await this.asyncDeepResearch(query, params);
|
|
865
|
+
if (!response.success || "error" in response) {
|
|
866
|
+
return { success: false, error: "error" in response ? response.error : "Unknown error" };
|
|
867
|
+
}
|
|
868
|
+
if (!response.id) {
|
|
869
|
+
throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);
|
|
870
|
+
}
|
|
871
|
+
const jobId = response.id;
|
|
872
|
+
let researchStatus;
|
|
873
|
+
let lastActivityCount = 0;
|
|
874
|
+
let lastSourceCount = 0;
|
|
875
|
+
while (true) {
|
|
876
|
+
researchStatus = await this.checkDeepResearchStatus(jobId);
|
|
877
|
+
if ("error" in researchStatus && !researchStatus.success) {
|
|
878
|
+
return researchStatus;
|
|
879
|
+
}
|
|
880
|
+
if (onActivity && researchStatus.activities) {
|
|
881
|
+
const newActivities = researchStatus.activities.slice(lastActivityCount);
|
|
882
|
+
for (const activity of newActivities) {
|
|
883
|
+
onActivity(activity);
|
|
884
|
+
}
|
|
885
|
+
lastActivityCount = researchStatus.activities.length;
|
|
886
|
+
}
|
|
887
|
+
if (onSource && researchStatus.sources) {
|
|
888
|
+
const newSources = researchStatus.sources.slice(lastSourceCount);
|
|
889
|
+
for (const source of newSources) {
|
|
890
|
+
onSource(source);
|
|
891
|
+
}
|
|
892
|
+
lastSourceCount = researchStatus.sources.length;
|
|
893
|
+
}
|
|
894
|
+
if (researchStatus.status === "completed") {
|
|
895
|
+
return researchStatus;
|
|
896
|
+
}
|
|
897
|
+
if (researchStatus.status === "failed") {
|
|
898
|
+
throw new FirecrawlError(
|
|
899
|
+
`Research job ${researchStatus.status}. Error: ${researchStatus.error}`,
|
|
900
|
+
500
|
|
901
|
+
);
|
|
902
|
+
}
|
|
903
|
+
if (researchStatus.status !== "processing") {
|
|
904
|
+
break;
|
|
905
|
+
}
|
|
906
|
+
await new Promise((resolve) => setTimeout(resolve, 2e3));
|
|
907
|
+
}
|
|
908
|
+
return { success: false, error: "Research job terminated unexpectedly" };
|
|
909
|
+
} catch (error) {
|
|
910
|
+
throw new FirecrawlError(error.message, 500, error.response?.data?.details);
|
|
911
|
+
}
|
|
912
|
+
}
|
|
913
|
+
/**
|
|
914
|
+
* Initiates a deep research operation on a given query without polling.
|
|
915
|
+
* @param params - Parameters for the deep research operation.
|
|
916
|
+
* @returns The response containing the research job ID.
|
|
917
|
+
*/
|
|
918
|
+
async asyncDeepResearch(query, params) {
|
|
919
|
+
const headers = this.prepareHeaders();
|
|
920
|
+
try {
|
|
921
|
+
const response = await this.postRequest(
|
|
922
|
+
`${this.apiUrl}/v1/deep-research`,
|
|
923
|
+
{ query, ...params },
|
|
924
|
+
headers
|
|
925
|
+
);
|
|
926
|
+
if (response.status === 200) {
|
|
927
|
+
return response.data;
|
|
928
|
+
} else {
|
|
929
|
+
this.handleError(response, "start deep research");
|
|
930
|
+
}
|
|
931
|
+
} catch (error) {
|
|
932
|
+
if (error.response?.data?.error) {
|
|
933
|
+
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
|
|
934
|
+
} else {
|
|
935
|
+
throw new FirecrawlError(error.message, 500);
|
|
936
|
+
}
|
|
937
|
+
}
|
|
938
|
+
return { success: false, error: "Internal server error." };
|
|
939
|
+
}
|
|
940
|
+
/**
|
|
941
|
+
* Checks the status of a deep research operation.
|
|
942
|
+
* @param id - The ID of the deep research operation.
|
|
943
|
+
* @returns The current status and results of the research operation.
|
|
944
|
+
*/
|
|
945
|
+
async checkDeepResearchStatus(id) {
|
|
946
|
+
const headers = this.prepareHeaders();
|
|
947
|
+
try {
|
|
948
|
+
const response = await this.getRequest(
|
|
949
|
+
`${this.apiUrl}/v1/deep-research/${id}`,
|
|
950
|
+
headers
|
|
951
|
+
);
|
|
952
|
+
if (response.status === 200) {
|
|
953
|
+
return response.data;
|
|
954
|
+
} else if (response.status === 404) {
|
|
955
|
+
throw new FirecrawlError("Deep research job not found", 404);
|
|
956
|
+
} else {
|
|
957
|
+
this.handleError(response, "check deep research status");
|
|
958
|
+
}
|
|
959
|
+
} catch (error) {
|
|
960
|
+
if (error.response?.data?.error) {
|
|
961
|
+
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
|
|
962
|
+
} else {
|
|
963
|
+
throw new FirecrawlError(error.message, 500);
|
|
964
|
+
}
|
|
965
|
+
}
|
|
966
|
+
return { success: false, error: "Internal server error." };
|
|
967
|
+
}
|
|
968
|
+
/**
|
|
969
|
+
* @deprecated Use deepResearch() instead
|
|
855
970
|
* Initiates a deep research operation on a given topic and polls until completion.
|
|
856
971
|
* @param topic - The topic to research.
|
|
857
972
|
* @param params - Parameters for the deep research operation.
|
|
@@ -902,6 +1017,7 @@ var FirecrawlApp = class {
|
|
|
902
1017
|
}
|
|
903
1018
|
}
|
|
904
1019
|
/**
|
|
1020
|
+
* @deprecated Use asyncDeepResearch() instead
|
|
905
1021
|
* Initiates a deep research operation on a given topic without polling.
|
|
906
1022
|
* @param params - Parameters for the deep research operation.
|
|
907
1023
|
* @returns The response containing the research job ID.
|
|
@@ -929,6 +1045,7 @@ var FirecrawlApp = class {
|
|
|
929
1045
|
return { success: false, error: "Internal server error." };
|
|
930
1046
|
}
|
|
931
1047
|
/**
|
|
1048
|
+
* @deprecated Use checkDeepResearchStatus() instead
|
|
932
1049
|
* Checks the status of a deep research operation.
|
|
933
1050
|
* @param id - The ID of the deep research operation.
|
|
934
1051
|
* @returns The current status and results of the research operation.
|
package/dist/index.d.cts
CHANGED
|
@@ -162,6 +162,7 @@ interface CrawlParams {
|
|
|
162
162
|
};
|
|
163
163
|
deduplicateSimilarURLs?: boolean;
|
|
164
164
|
ignoreQueryParameters?: boolean;
|
|
165
|
+
regexOnFullURL?: boolean;
|
|
165
166
|
}
|
|
166
167
|
/**
|
|
167
168
|
* Response interface for crawling operations.
|
|
@@ -321,7 +322,7 @@ interface CrawlErrorsResponse {
|
|
|
321
322
|
}
|
|
322
323
|
/**
|
|
323
324
|
* Parameters for deep research operations.
|
|
324
|
-
* Defines options for conducting deep research on a
|
|
325
|
+
* Defines options for conducting deep research on a query.
|
|
325
326
|
*/
|
|
326
327
|
interface DeepResearchParams {
|
|
327
328
|
/**
|
|
@@ -357,14 +358,19 @@ interface DeepResearchResponse {
|
|
|
357
358
|
interface DeepResearchStatusResponse {
|
|
358
359
|
success: boolean;
|
|
359
360
|
data: {
|
|
360
|
-
findings: Array<{
|
|
361
|
-
text: string;
|
|
362
|
-
source: string;
|
|
363
|
-
}>;
|
|
364
361
|
finalAnalysis: string;
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
362
|
+
activities: Array<{
|
|
363
|
+
type: string;
|
|
364
|
+
status: string;
|
|
365
|
+
message: string;
|
|
366
|
+
timestamp: string;
|
|
367
|
+
depth: number;
|
|
368
|
+
}>;
|
|
369
|
+
sources: Array<{
|
|
370
|
+
url: string;
|
|
371
|
+
title: string;
|
|
372
|
+
description: string;
|
|
373
|
+
}>;
|
|
368
374
|
};
|
|
369
375
|
status: "processing" | "completed" | "failed";
|
|
370
376
|
error?: string;
|
|
@@ -599,6 +605,39 @@ declare class FirecrawlApp {
|
|
|
599
605
|
*/
|
|
600
606
|
handleError(response: AxiosResponse, action: string): void;
|
|
601
607
|
/**
|
|
608
|
+
* Initiates a deep research operation on a given query and polls until completion.
|
|
609
|
+
* @param query - The query to research.
|
|
610
|
+
* @param params - Parameters for the deep research operation.
|
|
611
|
+
* @param onActivity - Optional callback to receive activity updates in real-time.
|
|
612
|
+
* @param onSource - Optional callback to receive source updates in real-time.
|
|
613
|
+
* @returns The final research results.
|
|
614
|
+
*/
|
|
615
|
+
deepResearch(query: string, params: DeepResearchParams, onActivity?: (activity: {
|
|
616
|
+
type: string;
|
|
617
|
+
status: string;
|
|
618
|
+
message: string;
|
|
619
|
+
timestamp: string;
|
|
620
|
+
depth: number;
|
|
621
|
+
}) => void, onSource?: (source: {
|
|
622
|
+
url: string;
|
|
623
|
+
title?: string;
|
|
624
|
+
description?: string;
|
|
625
|
+
icon?: string;
|
|
626
|
+
}) => void): Promise<DeepResearchStatusResponse | ErrorResponse>;
|
|
627
|
+
/**
|
|
628
|
+
* Initiates a deep research operation on a given query without polling.
|
|
629
|
+
* @param params - Parameters for the deep research operation.
|
|
630
|
+
* @returns The response containing the research job ID.
|
|
631
|
+
*/
|
|
632
|
+
asyncDeepResearch(query: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse>;
|
|
633
|
+
/**
|
|
634
|
+
* Checks the status of a deep research operation.
|
|
635
|
+
* @param id - The ID of the deep research operation.
|
|
636
|
+
* @returns The current status and results of the research operation.
|
|
637
|
+
*/
|
|
638
|
+
checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse>;
|
|
639
|
+
/**
|
|
640
|
+
* @deprecated Use deepResearch() instead
|
|
602
641
|
* Initiates a deep research operation on a given topic and polls until completion.
|
|
603
642
|
* @param topic - The topic to research.
|
|
604
643
|
* @param params - Parameters for the deep research operation.
|
|
@@ -613,12 +652,14 @@ declare class FirecrawlApp {
|
|
|
613
652
|
depth: number;
|
|
614
653
|
}) => void): Promise<DeepResearchStatusResponse | ErrorResponse>;
|
|
615
654
|
/**
|
|
655
|
+
* @deprecated Use asyncDeepResearch() instead
|
|
616
656
|
* Initiates a deep research operation on a given topic without polling.
|
|
617
657
|
* @param params - Parameters for the deep research operation.
|
|
618
658
|
* @returns The response containing the research job ID.
|
|
619
659
|
*/
|
|
620
660
|
__asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse>;
|
|
621
661
|
/**
|
|
662
|
+
* @deprecated Use checkDeepResearchStatus() instead
|
|
622
663
|
* Checks the status of a deep research operation.
|
|
623
664
|
* @param id - The ID of the deep research operation.
|
|
624
665
|
* @returns The current status and results of the research operation.
|
package/dist/index.d.ts
CHANGED
|
@@ -162,6 +162,7 @@ interface CrawlParams {
|
|
|
162
162
|
};
|
|
163
163
|
deduplicateSimilarURLs?: boolean;
|
|
164
164
|
ignoreQueryParameters?: boolean;
|
|
165
|
+
regexOnFullURL?: boolean;
|
|
165
166
|
}
|
|
166
167
|
/**
|
|
167
168
|
* Response interface for crawling operations.
|
|
@@ -321,7 +322,7 @@ interface CrawlErrorsResponse {
|
|
|
321
322
|
}
|
|
322
323
|
/**
|
|
323
324
|
* Parameters for deep research operations.
|
|
324
|
-
* Defines options for conducting deep research on a
|
|
325
|
+
* Defines options for conducting deep research on a query.
|
|
325
326
|
*/
|
|
326
327
|
interface DeepResearchParams {
|
|
327
328
|
/**
|
|
@@ -357,14 +358,19 @@ interface DeepResearchResponse {
|
|
|
357
358
|
interface DeepResearchStatusResponse {
|
|
358
359
|
success: boolean;
|
|
359
360
|
data: {
|
|
360
|
-
findings: Array<{
|
|
361
|
-
text: string;
|
|
362
|
-
source: string;
|
|
363
|
-
}>;
|
|
364
361
|
finalAnalysis: string;
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
362
|
+
activities: Array<{
|
|
363
|
+
type: string;
|
|
364
|
+
status: string;
|
|
365
|
+
message: string;
|
|
366
|
+
timestamp: string;
|
|
367
|
+
depth: number;
|
|
368
|
+
}>;
|
|
369
|
+
sources: Array<{
|
|
370
|
+
url: string;
|
|
371
|
+
title: string;
|
|
372
|
+
description: string;
|
|
373
|
+
}>;
|
|
368
374
|
};
|
|
369
375
|
status: "processing" | "completed" | "failed";
|
|
370
376
|
error?: string;
|
|
@@ -599,6 +605,39 @@ declare class FirecrawlApp {
|
|
|
599
605
|
*/
|
|
600
606
|
handleError(response: AxiosResponse, action: string): void;
|
|
601
607
|
/**
|
|
608
|
+
* Initiates a deep research operation on a given query and polls until completion.
|
|
609
|
+
* @param query - The query to research.
|
|
610
|
+
* @param params - Parameters for the deep research operation.
|
|
611
|
+
* @param onActivity - Optional callback to receive activity updates in real-time.
|
|
612
|
+
* @param onSource - Optional callback to receive source updates in real-time.
|
|
613
|
+
* @returns The final research results.
|
|
614
|
+
*/
|
|
615
|
+
deepResearch(query: string, params: DeepResearchParams, onActivity?: (activity: {
|
|
616
|
+
type: string;
|
|
617
|
+
status: string;
|
|
618
|
+
message: string;
|
|
619
|
+
timestamp: string;
|
|
620
|
+
depth: number;
|
|
621
|
+
}) => void, onSource?: (source: {
|
|
622
|
+
url: string;
|
|
623
|
+
title?: string;
|
|
624
|
+
description?: string;
|
|
625
|
+
icon?: string;
|
|
626
|
+
}) => void): Promise<DeepResearchStatusResponse | ErrorResponse>;
|
|
627
|
+
/**
|
|
628
|
+
* Initiates a deep research operation on a given query without polling.
|
|
629
|
+
* @param params - Parameters for the deep research operation.
|
|
630
|
+
* @returns The response containing the research job ID.
|
|
631
|
+
*/
|
|
632
|
+
asyncDeepResearch(query: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse>;
|
|
633
|
+
/**
|
|
634
|
+
* Checks the status of a deep research operation.
|
|
635
|
+
* @param id - The ID of the deep research operation.
|
|
636
|
+
* @returns The current status and results of the research operation.
|
|
637
|
+
*/
|
|
638
|
+
checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse>;
|
|
639
|
+
/**
|
|
640
|
+
* @deprecated Use deepResearch() instead
|
|
602
641
|
* Initiates a deep research operation on a given topic and polls until completion.
|
|
603
642
|
* @param topic - The topic to research.
|
|
604
643
|
* @param params - Parameters for the deep research operation.
|
|
@@ -613,12 +652,14 @@ declare class FirecrawlApp {
|
|
|
613
652
|
depth: number;
|
|
614
653
|
}) => void): Promise<DeepResearchStatusResponse | ErrorResponse>;
|
|
615
654
|
/**
|
|
655
|
+
* @deprecated Use asyncDeepResearch() instead
|
|
616
656
|
* Initiates a deep research operation on a given topic without polling.
|
|
617
657
|
* @param params - Parameters for the deep research operation.
|
|
618
658
|
* @returns The response containing the research job ID.
|
|
619
659
|
*/
|
|
620
660
|
__asyncDeepResearch(topic: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse>;
|
|
621
661
|
/**
|
|
662
|
+
* @deprecated Use checkDeepResearchStatus() instead
|
|
622
663
|
* Checks the status of a deep research operation.
|
|
623
664
|
* @param id - The ID of the deep research operation.
|
|
624
665
|
* @returns The current status and results of the research operation.
|
package/dist/index.js
CHANGED
|
@@ -582,7 +582,7 @@ var FirecrawlApp = class {
|
|
|
582
582
|
try {
|
|
583
583
|
if (!params?.schema) {
|
|
584
584
|
jsonSchema = void 0;
|
|
585
|
-
} else if (params.schema
|
|
585
|
+
} else if (typeof params.schema === "object" && params.schema !== null && Object.getPrototypeOf(params.schema)?.constructor?.name?.startsWith("Zod")) {
|
|
586
586
|
jsonSchema = zodToJsonSchema(params.schema);
|
|
587
587
|
} else {
|
|
588
588
|
jsonSchema = params.schema;
|
|
@@ -816,6 +816,121 @@ var FirecrawlApp = class {
|
|
|
816
816
|
}
|
|
817
817
|
}
|
|
818
818
|
/**
|
|
819
|
+
* Initiates a deep research operation on a given query and polls until completion.
|
|
820
|
+
* @param query - The query to research.
|
|
821
|
+
* @param params - Parameters for the deep research operation.
|
|
822
|
+
* @param onActivity - Optional callback to receive activity updates in real-time.
|
|
823
|
+
* @param onSource - Optional callback to receive source updates in real-time.
|
|
824
|
+
* @returns The final research results.
|
|
825
|
+
*/
|
|
826
|
+
async deepResearch(query, params, onActivity, onSource) {
|
|
827
|
+
try {
|
|
828
|
+
const response = await this.asyncDeepResearch(query, params);
|
|
829
|
+
if (!response.success || "error" in response) {
|
|
830
|
+
return { success: false, error: "error" in response ? response.error : "Unknown error" };
|
|
831
|
+
}
|
|
832
|
+
if (!response.id) {
|
|
833
|
+
throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);
|
|
834
|
+
}
|
|
835
|
+
const jobId = response.id;
|
|
836
|
+
let researchStatus;
|
|
837
|
+
let lastActivityCount = 0;
|
|
838
|
+
let lastSourceCount = 0;
|
|
839
|
+
while (true) {
|
|
840
|
+
researchStatus = await this.checkDeepResearchStatus(jobId);
|
|
841
|
+
if ("error" in researchStatus && !researchStatus.success) {
|
|
842
|
+
return researchStatus;
|
|
843
|
+
}
|
|
844
|
+
if (onActivity && researchStatus.activities) {
|
|
845
|
+
const newActivities = researchStatus.activities.slice(lastActivityCount);
|
|
846
|
+
for (const activity of newActivities) {
|
|
847
|
+
onActivity(activity);
|
|
848
|
+
}
|
|
849
|
+
lastActivityCount = researchStatus.activities.length;
|
|
850
|
+
}
|
|
851
|
+
if (onSource && researchStatus.sources) {
|
|
852
|
+
const newSources = researchStatus.sources.slice(lastSourceCount);
|
|
853
|
+
for (const source of newSources) {
|
|
854
|
+
onSource(source);
|
|
855
|
+
}
|
|
856
|
+
lastSourceCount = researchStatus.sources.length;
|
|
857
|
+
}
|
|
858
|
+
if (researchStatus.status === "completed") {
|
|
859
|
+
return researchStatus;
|
|
860
|
+
}
|
|
861
|
+
if (researchStatus.status === "failed") {
|
|
862
|
+
throw new FirecrawlError(
|
|
863
|
+
`Research job ${researchStatus.status}. Error: ${researchStatus.error}`,
|
|
864
|
+
500
|
|
865
|
+
);
|
|
866
|
+
}
|
|
867
|
+
if (researchStatus.status !== "processing") {
|
|
868
|
+
break;
|
|
869
|
+
}
|
|
870
|
+
await new Promise((resolve) => setTimeout(resolve, 2e3));
|
|
871
|
+
}
|
|
872
|
+
return { success: false, error: "Research job terminated unexpectedly" };
|
|
873
|
+
} catch (error) {
|
|
874
|
+
throw new FirecrawlError(error.message, 500, error.response?.data?.details);
|
|
875
|
+
}
|
|
876
|
+
}
|
|
877
|
+
/**
|
|
878
|
+
* Initiates a deep research operation on a given query without polling.
|
|
879
|
+
* @param params - Parameters for the deep research operation.
|
|
880
|
+
* @returns The response containing the research job ID.
|
|
881
|
+
*/
|
|
882
|
+
async asyncDeepResearch(query, params) {
|
|
883
|
+
const headers = this.prepareHeaders();
|
|
884
|
+
try {
|
|
885
|
+
const response = await this.postRequest(
|
|
886
|
+
`${this.apiUrl}/v1/deep-research`,
|
|
887
|
+
{ query, ...params },
|
|
888
|
+
headers
|
|
889
|
+
);
|
|
890
|
+
if (response.status === 200) {
|
|
891
|
+
return response.data;
|
|
892
|
+
} else {
|
|
893
|
+
this.handleError(response, "start deep research");
|
|
894
|
+
}
|
|
895
|
+
} catch (error) {
|
|
896
|
+
if (error.response?.data?.error) {
|
|
897
|
+
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
|
|
898
|
+
} else {
|
|
899
|
+
throw new FirecrawlError(error.message, 500);
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
return { success: false, error: "Internal server error." };
|
|
903
|
+
}
|
|
904
|
+
/**
|
|
905
|
+
* Checks the status of a deep research operation.
|
|
906
|
+
* @param id - The ID of the deep research operation.
|
|
907
|
+
* @returns The current status and results of the research operation.
|
|
908
|
+
*/
|
|
909
|
+
async checkDeepResearchStatus(id) {
|
|
910
|
+
const headers = this.prepareHeaders();
|
|
911
|
+
try {
|
|
912
|
+
const response = await this.getRequest(
|
|
913
|
+
`${this.apiUrl}/v1/deep-research/${id}`,
|
|
914
|
+
headers
|
|
915
|
+
);
|
|
916
|
+
if (response.status === 200) {
|
|
917
|
+
return response.data;
|
|
918
|
+
} else if (response.status === 404) {
|
|
919
|
+
throw new FirecrawlError("Deep research job not found", 404);
|
|
920
|
+
} else {
|
|
921
|
+
this.handleError(response, "check deep research status");
|
|
922
|
+
}
|
|
923
|
+
} catch (error) {
|
|
924
|
+
if (error.response?.data?.error) {
|
|
925
|
+
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
|
|
926
|
+
} else {
|
|
927
|
+
throw new FirecrawlError(error.message, 500);
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
return { success: false, error: "Internal server error." };
|
|
931
|
+
}
|
|
932
|
+
/**
|
|
933
|
+
* @deprecated Use deepResearch() instead
|
|
819
934
|
* Initiates a deep research operation on a given topic and polls until completion.
|
|
820
935
|
* @param topic - The topic to research.
|
|
821
936
|
* @param params - Parameters for the deep research operation.
|
|
@@ -866,6 +981,7 @@ var FirecrawlApp = class {
|
|
|
866
981
|
}
|
|
867
982
|
}
|
|
868
983
|
/**
|
|
984
|
+
* @deprecated Use asyncDeepResearch() instead
|
|
869
985
|
* Initiates a deep research operation on a given topic without polling.
|
|
870
986
|
* @param params - Parameters for the deep research operation.
|
|
871
987
|
* @returns The response containing the research job ID.
|
|
@@ -893,6 +1009,7 @@ var FirecrawlApp = class {
|
|
|
893
1009
|
return { success: false, error: "Internal server error." };
|
|
894
1010
|
}
|
|
895
1011
|
/**
|
|
1012
|
+
* @deprecated Use checkDeepResearchStatus() instead
|
|
896
1013
|
* Checks the status of a deep research operation.
|
|
897
1014
|
* @param id - The ID of the deep research operation.
|
|
898
1015
|
* @returns The current status and results of the research operation.
|
package/package.json
CHANGED
|
@@ -55,7 +55,7 @@ describe('FirecrawlApp<"v0"> E2E Tests', () => {
|
|
|
55
55
|
"should return successful response with valid preview token",
|
|
56
56
|
async () => {
|
|
57
57
|
const app = new FirecrawlApp<"v0">({
|
|
58
|
-
apiKey:
|
|
58
|
+
apiKey: process.env.PREVIEW_TOKEN,
|
|
59
59
|
apiUrl: API_URL,
|
|
60
60
|
version: "v0",
|
|
61
61
|
});
|
|
@@ -40,7 +40,7 @@ describe('FirecrawlApp E2E Tests', () => {
|
|
|
40
40
|
});
|
|
41
41
|
|
|
42
42
|
test.concurrent('should return successful response with valid preview token', async () => {
|
|
43
|
-
const app = new FirecrawlApp({ apiKey:
|
|
43
|
+
const app = new FirecrawlApp({ apiKey: process.env.PREVIEW_TOKEN, apiUrl: API_URL });
|
|
44
44
|
const response = await app.scrapeUrl('https://roastmywebsite.ai');
|
|
45
45
|
if (!response.success) {
|
|
46
46
|
throw new Error(response.error);
|
|
@@ -365,7 +365,7 @@ describe('FirecrawlApp E2E Tests', () => {
|
|
|
365
365
|
});
|
|
366
366
|
|
|
367
367
|
test.concurrent('should return successful response with valid preview token', async () => {
|
|
368
|
-
const app = new FirecrawlApp({ apiKey:
|
|
368
|
+
const app = new FirecrawlApp({ apiKey: process.env.PREVIEW_TOKEN, apiUrl: API_URL });
|
|
369
369
|
const response = await app.mapUrl('https://roastmywebsite.ai') as MapResponse;
|
|
370
370
|
expect(response).not.toBeNull();
|
|
371
371
|
expect(response.links?.length).toBeGreaterThan(0);
|
package/src/index.ts
CHANGED
|
@@ -173,6 +173,7 @@ export interface CrawlParams {
|
|
|
173
173
|
};
|
|
174
174
|
deduplicateSimilarURLs?: boolean;
|
|
175
175
|
ignoreQueryParameters?: boolean;
|
|
176
|
+
regexOnFullURL?: boolean;
|
|
176
177
|
}
|
|
177
178
|
|
|
178
179
|
/**
|
|
@@ -351,7 +352,7 @@ export interface CrawlErrorsResponse {
|
|
|
351
352
|
|
|
352
353
|
/**
|
|
353
354
|
* Parameters for deep research operations.
|
|
354
|
-
* Defines options for conducting deep research on a
|
|
355
|
+
* Defines options for conducting deep research on a query.
|
|
355
356
|
*/
|
|
356
357
|
export interface DeepResearchParams {
|
|
357
358
|
/**
|
|
@@ -389,14 +390,19 @@ export interface DeepResearchResponse {
|
|
|
389
390
|
export interface DeepResearchStatusResponse {
|
|
390
391
|
success: boolean;
|
|
391
392
|
data: {
|
|
392
|
-
findings: Array<{
|
|
393
|
-
text: string;
|
|
394
|
-
source: string;
|
|
395
|
-
}>;
|
|
396
393
|
finalAnalysis: string;
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
394
|
+
activities: Array<{
|
|
395
|
+
type: string;
|
|
396
|
+
status: string;
|
|
397
|
+
message: string;
|
|
398
|
+
timestamp: string;
|
|
399
|
+
depth: number;
|
|
400
|
+
}>;
|
|
401
|
+
sources: Array<{
|
|
402
|
+
url: string;
|
|
403
|
+
title: string;
|
|
404
|
+
description: string;
|
|
405
|
+
}>;
|
|
400
406
|
};
|
|
401
407
|
status: "processing" | "completed" | "failed";
|
|
402
408
|
error?: string;
|
|
@@ -1124,15 +1130,14 @@ export default class FirecrawlApp {
|
|
|
1124
1130
|
try {
|
|
1125
1131
|
if (!params?.schema) {
|
|
1126
1132
|
jsonSchema = undefined;
|
|
1127
|
-
} else if (params.schema
|
|
1128
|
-
jsonSchema = zodToJsonSchema(params.schema);
|
|
1133
|
+
} else if (typeof params.schema === "object" && params.schema !== null && Object.getPrototypeOf(params.schema)?.constructor?.name?.startsWith("Zod")) {
|
|
1134
|
+
jsonSchema = zodToJsonSchema(params.schema as zt.ZodType);
|
|
1129
1135
|
} else {
|
|
1130
1136
|
jsonSchema = params.schema;
|
|
1131
1137
|
}
|
|
1132
1138
|
} catch (error: any) {
|
|
1133
1139
|
throw new FirecrawlError("Invalid schema. Schema must be either a valid Zod schema or JSON schema object.", 400);
|
|
1134
1140
|
}
|
|
1135
|
-
|
|
1136
1141
|
|
|
1137
1142
|
try {
|
|
1138
1143
|
const response: AxiosResponse = await this.postRequest(
|
|
@@ -1395,6 +1400,156 @@ export default class FirecrawlApp {
|
|
|
1395
1400
|
}
|
|
1396
1401
|
|
|
1397
1402
|
/**
|
|
1403
|
+
* Initiates a deep research operation on a given query and polls until completion.
|
|
1404
|
+
* @param query - The query to research.
|
|
1405
|
+
* @param params - Parameters for the deep research operation.
|
|
1406
|
+
* @param onActivity - Optional callback to receive activity updates in real-time.
|
|
1407
|
+
* @param onSource - Optional callback to receive source updates in real-time.
|
|
1408
|
+
* @returns The final research results.
|
|
1409
|
+
*/
|
|
1410
|
+
async deepResearch(
|
|
1411
|
+
query: string,
|
|
1412
|
+
params: DeepResearchParams,
|
|
1413
|
+
onActivity?: (activity: {
|
|
1414
|
+
type: string;
|
|
1415
|
+
status: string;
|
|
1416
|
+
message: string;
|
|
1417
|
+
timestamp: string;
|
|
1418
|
+
depth: number;
|
|
1419
|
+
}) => void,
|
|
1420
|
+
onSource?: (source: {
|
|
1421
|
+
url: string;
|
|
1422
|
+
title?: string;
|
|
1423
|
+
description?: string;
|
|
1424
|
+
icon?: string;
|
|
1425
|
+
}) => void
|
|
1426
|
+
): Promise<DeepResearchStatusResponse | ErrorResponse> {
|
|
1427
|
+
try {
|
|
1428
|
+
const response = await this.asyncDeepResearch(query, params);
|
|
1429
|
+
|
|
1430
|
+
if (!response.success || 'error' in response) {
|
|
1431
|
+
return { success: false, error: 'error' in response ? response.error : 'Unknown error' };
|
|
1432
|
+
}
|
|
1433
|
+
|
|
1434
|
+
if (!response.id) {
|
|
1435
|
+
throw new FirecrawlError(`Failed to start research. No job ID returned.`, 500);
|
|
1436
|
+
}
|
|
1437
|
+
|
|
1438
|
+
const jobId = response.id;
|
|
1439
|
+
let researchStatus;
|
|
1440
|
+
let lastActivityCount = 0;
|
|
1441
|
+
let lastSourceCount = 0;
|
|
1442
|
+
|
|
1443
|
+
while (true) {
|
|
1444
|
+
researchStatus = await this.checkDeepResearchStatus(jobId);
|
|
1445
|
+
|
|
1446
|
+
if ('error' in researchStatus && !researchStatus.success) {
|
|
1447
|
+
return researchStatus;
|
|
1448
|
+
}
|
|
1449
|
+
|
|
1450
|
+
// Stream new activities through the callback if provided
|
|
1451
|
+
if (onActivity && researchStatus.activities) {
|
|
1452
|
+
const newActivities = researchStatus.activities.slice(lastActivityCount);
|
|
1453
|
+
for (const activity of newActivities) {
|
|
1454
|
+
onActivity(activity);
|
|
1455
|
+
}
|
|
1456
|
+
lastActivityCount = researchStatus.activities.length;
|
|
1457
|
+
}
|
|
1458
|
+
|
|
1459
|
+
// Stream new sources through the callback if provided
|
|
1460
|
+
if (onSource && researchStatus.sources) {
|
|
1461
|
+
const newSources = researchStatus.sources.slice(lastSourceCount);
|
|
1462
|
+
for (const source of newSources) {
|
|
1463
|
+
onSource(source);
|
|
1464
|
+
}
|
|
1465
|
+
lastSourceCount = researchStatus.sources.length;
|
|
1466
|
+
}
|
|
1467
|
+
|
|
1468
|
+
if (researchStatus.status === "completed") {
|
|
1469
|
+
return researchStatus;
|
|
1470
|
+
}
|
|
1471
|
+
|
|
1472
|
+
if (researchStatus.status === "failed") {
|
|
1473
|
+
throw new FirecrawlError(
|
|
1474
|
+
`Research job ${researchStatus.status}. Error: ${researchStatus.error}`,
|
|
1475
|
+
500
|
|
1476
|
+
);
|
|
1477
|
+
}
|
|
1478
|
+
|
|
1479
|
+
if (researchStatus.status !== "processing") {
|
|
1480
|
+
break;
|
|
1481
|
+
}
|
|
1482
|
+
|
|
1483
|
+
await new Promise(resolve => setTimeout(resolve, 2000));
|
|
1484
|
+
}
|
|
1485
|
+
|
|
1486
|
+
return { success: false, error: "Research job terminated unexpectedly" };
|
|
1487
|
+
} catch (error: any) {
|
|
1488
|
+
throw new FirecrawlError(error.message, 500, error.response?.data?.details);
|
|
1489
|
+
}
|
|
1490
|
+
}
|
|
1491
|
+
|
|
1492
|
+
/**
|
|
1493
|
+
* Initiates a deep research operation on a given query without polling.
|
|
1494
|
+
* @param params - Parameters for the deep research operation.
|
|
1495
|
+
* @returns The response containing the research job ID.
|
|
1496
|
+
*/
|
|
1497
|
+
async asyncDeepResearch(query: string, params: DeepResearchParams): Promise<DeepResearchResponse | ErrorResponse> {
|
|
1498
|
+
const headers = this.prepareHeaders();
|
|
1499
|
+
try {
|
|
1500
|
+
const response: AxiosResponse = await this.postRequest(
|
|
1501
|
+
`${this.apiUrl}/v1/deep-research`,
|
|
1502
|
+
{ query, ...params },
|
|
1503
|
+
headers
|
|
1504
|
+
);
|
|
1505
|
+
|
|
1506
|
+
if (response.status === 200) {
|
|
1507
|
+
return response.data;
|
|
1508
|
+
} else {
|
|
1509
|
+
this.handleError(response, "start deep research");
|
|
1510
|
+
}
|
|
1511
|
+
} catch (error: any) {
|
|
1512
|
+
if (error.response?.data?.error) {
|
|
1513
|
+
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
|
|
1514
|
+
} else {
|
|
1515
|
+
throw new FirecrawlError(error.message, 500);
|
|
1516
|
+
}
|
|
1517
|
+
}
|
|
1518
|
+
return { success: false, error: "Internal server error." };
|
|
1519
|
+
}
|
|
1520
|
+
|
|
1521
|
+
/**
|
|
1522
|
+
* Checks the status of a deep research operation.
|
|
1523
|
+
* @param id - The ID of the deep research operation.
|
|
1524
|
+
* @returns The current status and results of the research operation.
|
|
1525
|
+
*/
|
|
1526
|
+
async checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {
|
|
1527
|
+
const headers = this.prepareHeaders();
|
|
1528
|
+
try {
|
|
1529
|
+
const response: AxiosResponse = await this.getRequest(
|
|
1530
|
+
`${this.apiUrl}/v1/deep-research/${id}`,
|
|
1531
|
+
headers
|
|
1532
|
+
);
|
|
1533
|
+
|
|
1534
|
+
if (response.status === 200) {
|
|
1535
|
+
return response.data;
|
|
1536
|
+
} else if (response.status === 404) {
|
|
1537
|
+
throw new FirecrawlError("Deep research job not found", 404);
|
|
1538
|
+
} else {
|
|
1539
|
+
this.handleError(response, "check deep research status");
|
|
1540
|
+
}
|
|
1541
|
+
} catch (error: any) {
|
|
1542
|
+
if (error.response?.data?.error) {
|
|
1543
|
+
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
|
|
1544
|
+
} else {
|
|
1545
|
+
throw new FirecrawlError(error.message, 500);
|
|
1546
|
+
}
|
|
1547
|
+
}
|
|
1548
|
+
return { success: false, error: "Internal server error." };
|
|
1549
|
+
}
|
|
1550
|
+
|
|
1551
|
+
/**
|
|
1552
|
+
* @deprecated Use deepResearch() instead
|
|
1398
1553
|
* Initiates a deep research operation on a given topic and polls until completion.
|
|
1399
1554
|
* @param topic - The topic to research.
|
|
1400
1555
|
* @param params - Parameters for the deep research operation.
|
|
@@ -1468,6 +1623,7 @@ export default class FirecrawlApp {
|
|
|
1468
1623
|
}
|
|
1469
1624
|
|
|
1470
1625
|
/**
|
|
1626
|
+
* @deprecated Use asyncDeepResearch() instead
|
|
1471
1627
|
* Initiates a deep research operation on a given topic without polling.
|
|
1472
1628
|
* @param params - Parameters for the deep research operation.
|
|
1473
1629
|
* @returns The response containing the research job ID.
|
|
@@ -1497,6 +1653,7 @@ export default class FirecrawlApp {
|
|
|
1497
1653
|
}
|
|
1498
1654
|
|
|
1499
1655
|
/**
|
|
1656
|
+
* @deprecated Use checkDeepResearchStatus() instead
|
|
1500
1657
|
* Checks the status of a deep research operation.
|
|
1501
1658
|
* @param id - The ID of the deep research operation.
|
|
1502
1659
|
* @returns The current status and results of the research operation.
|