firecrawl 4.39.0 → 4.40.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +46 -0
- package/dist/{chunk-4DPCCKWG.js → chunk-OJ4YJQ4J.js} +1 -1
- package/dist/index.cjs +166 -9
- package/dist/index.d.cts +138 -4
- package/dist/index.d.ts +138 -4
- package/dist/index.js +165 -9
- package/dist/{package-D2QZMRW7.js → package-F6O7J2XI.js} +1 -1
- package/package.json +1 -1
- package/pnpm-workspace.yaml +2 -2
- package/src/__tests__/unit/v2/threat-protection.unit.test.ts +18 -1
- package/src/__tests__/unit/v2/tools.unit.test.ts +240 -0
- package/src/v2/client.ts +39 -3
- package/src/v2/methods/scrape.ts +3 -1
- package/src/v2/methods/search.ts +7 -1
- package/src/v2/methods/tools.ts +137 -0
- package/src/v2/types.ts +156 -1
- package/src/v2/utils/errorHandler.ts +19 -3
package/src/v2/types.ts
CHANGED
|
@@ -275,6 +275,8 @@ export interface ScrapeOptions {
|
|
|
275
275
|
};
|
|
276
276
|
integration?: string;
|
|
277
277
|
origin?: string;
|
|
278
|
+
/** Include domain-matched Alexandria contracts for this URL in `tools`. Default off. */
|
|
279
|
+
domainTools?: boolean;
|
|
278
280
|
}
|
|
279
281
|
|
|
280
282
|
export type RedactPIIEntity =
|
|
@@ -685,6 +687,8 @@ export interface Document {
|
|
|
685
687
|
pages?: PdfPage[];
|
|
686
688
|
/** Typed PDF layout blocks, present only when `parsers[].blocks` is true. */
|
|
687
689
|
blocks?: PdfPageBlocks[];
|
|
690
|
+
/** Present when `domainTools` discovered contracts matching this URL. */
|
|
691
|
+
tools?: DiscoveredTool[];
|
|
688
692
|
}
|
|
689
693
|
|
|
690
694
|
// Pagination configuration for auto-fetching pages from v2 endpoints that return a `next` URL
|
|
@@ -797,9 +801,129 @@ export interface SearchResultImages {
|
|
|
797
801
|
}
|
|
798
802
|
|
|
799
803
|
export interface SearchData {
|
|
804
|
+
warning?: string;
|
|
800
805
|
web?: Array<SearchResultWeb | Document>;
|
|
801
806
|
news?: Array<SearchResultNews | Document>;
|
|
802
807
|
images?: Array<SearchResultImages | Document>;
|
|
808
|
+
tools?: DiscoveredTool[];
|
|
809
|
+
}
|
|
810
|
+
|
|
811
|
+
/** A complete tool contract returned by semantic or contextual discovery. */
|
|
812
|
+
export interface DiscoveredTool {
|
|
813
|
+
id?: string;
|
|
814
|
+
provider: string;
|
|
815
|
+
capability: string;
|
|
816
|
+
name: string;
|
|
817
|
+
description: string;
|
|
818
|
+
creditsCost: number;
|
|
819
|
+
perRecord: boolean;
|
|
820
|
+
options: Array<{
|
|
821
|
+
name: string;
|
|
822
|
+
type: string;
|
|
823
|
+
required?: boolean;
|
|
824
|
+
[key: string]: unknown;
|
|
825
|
+
}>;
|
|
826
|
+
requiresOneOf?: string[][];
|
|
827
|
+
response?: {
|
|
828
|
+
about?: string;
|
|
829
|
+
key?: string;
|
|
830
|
+
fields?: Array<{ name: string; type: string; [key: string]: unknown }>;
|
|
831
|
+
[key: string]: unknown;
|
|
832
|
+
};
|
|
833
|
+
examples?: Partial<Record<"javascript" | "python" | "curl", string>> &
|
|
834
|
+
Record<string, string>;
|
|
835
|
+
example?: {
|
|
836
|
+
recordedAt: string;
|
|
837
|
+
request: Record<string, unknown>;
|
|
838
|
+
response: unknown;
|
|
839
|
+
};
|
|
840
|
+
label?: string;
|
|
841
|
+
whenToUse?: string;
|
|
842
|
+
returns?: unknown;
|
|
843
|
+
discovery?: unknown;
|
|
844
|
+
attribution?: unknown;
|
|
845
|
+
matchedBy?: Array<"semantic" | "domain">;
|
|
846
|
+
matchedUrls?: string[];
|
|
847
|
+
concept?: string;
|
|
848
|
+
cohorts?: string[];
|
|
849
|
+
similarity?: number;
|
|
850
|
+
}
|
|
851
|
+
|
|
852
|
+
export interface FindToolsOptions {
|
|
853
|
+
urls?: string[];
|
|
854
|
+
providers?: string[];
|
|
855
|
+
categories?: string[];
|
|
856
|
+
groups?: string[];
|
|
857
|
+
capabilities?: string[];
|
|
858
|
+
level?: "providers" | "groups" | "tools";
|
|
859
|
+
expand?: Array<"options" | "response" | "examples">;
|
|
860
|
+
limit?: number;
|
|
861
|
+
offset?: number;
|
|
862
|
+
}
|
|
863
|
+
|
|
864
|
+
export interface FindToolsData {
|
|
865
|
+
level: "providers" | "groups" | "tools";
|
|
866
|
+
items: Array<{
|
|
867
|
+
id: string;
|
|
868
|
+
name: string;
|
|
869
|
+
next?: AlexandriaCall;
|
|
870
|
+
execute?: Pick<AlexandriaCall, "provider" | "capability">;
|
|
871
|
+
[key: string]: unknown;
|
|
872
|
+
}>;
|
|
873
|
+
total: number;
|
|
874
|
+
next: AlexandriaCall | null;
|
|
875
|
+
}
|
|
876
|
+
|
|
877
|
+
export interface AlexandriaScrapeRequest extends AlexandriaOptions {
|
|
878
|
+
alexandria: AlexandriaCall | AlexandriaCall[];
|
|
879
|
+
}
|
|
880
|
+
|
|
881
|
+
export interface AlexandriaCall {
|
|
882
|
+
provider: string;
|
|
883
|
+
capability: string;
|
|
884
|
+
options?: Record<string, unknown>;
|
|
885
|
+
}
|
|
886
|
+
|
|
887
|
+
export interface AlexandriaScrapeError {
|
|
888
|
+
code: string;
|
|
889
|
+
message: string;
|
|
890
|
+
status?: number;
|
|
891
|
+
/** Present when credits were captured before the failure. */
|
|
892
|
+
chargeId?: string;
|
|
893
|
+
}
|
|
894
|
+
|
|
895
|
+
export type AlexandriaScrapeResult =
|
|
896
|
+
| {
|
|
897
|
+
provider: string;
|
|
898
|
+
capability: string;
|
|
899
|
+
creditsCost: number;
|
|
900
|
+
data: unknown;
|
|
901
|
+
records?: number;
|
|
902
|
+
upstreamStatus?: number;
|
|
903
|
+
recordedAt?: string;
|
|
904
|
+
error?: undefined;
|
|
905
|
+
[key: string]: unknown;
|
|
906
|
+
}
|
|
907
|
+
| {
|
|
908
|
+
provider?: string;
|
|
909
|
+
capability?: string;
|
|
910
|
+
error: AlexandriaScrapeError;
|
|
911
|
+
[key: string]: unknown;
|
|
912
|
+
};
|
|
913
|
+
|
|
914
|
+
export interface AlexandriaScrapeData {
|
|
915
|
+
scrapeId: string;
|
|
916
|
+
requestId: string;
|
|
917
|
+
alexandria: AlexandriaScrapeResult[];
|
|
918
|
+
creditsCost: number;
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
export interface AlexandriaOptions {
|
|
922
|
+
/** Reuse this ID with the identical payload when retrying an execution. */
|
|
923
|
+
requestId?: string;
|
|
924
|
+
timeout?: number;
|
|
925
|
+
integration?: string;
|
|
926
|
+
origin?: string;
|
|
803
927
|
}
|
|
804
928
|
|
|
805
929
|
/**
|
|
@@ -830,8 +954,11 @@ export interface CategoryOption {
|
|
|
830
954
|
|
|
831
955
|
export interface SearchRequest {
|
|
832
956
|
query: string;
|
|
957
|
+
/** Include domain-matched contracts in tools alongside semantic matches. */
|
|
958
|
+
domainTools?: boolean;
|
|
833
959
|
sources?: Array<
|
|
834
|
-
"web" | "news" | "images" |
|
|
960
|
+
"web" | "news" | "images" | "alexandria"
|
|
961
|
+
| { type: "web" | "news" | "images" | "alexandria" }
|
|
835
962
|
>;
|
|
836
963
|
/**
|
|
837
964
|
* Narrow web search by category. See {@link CategoryOption}.
|
|
@@ -1707,17 +1834,43 @@ export interface ErrorDetails {
|
|
|
1707
1834
|
status?: number;
|
|
1708
1835
|
}
|
|
1709
1836
|
|
|
1837
|
+
/** An out-of-band step the API requires before the request can succeed. */
|
|
1838
|
+
export interface RequiresAction {
|
|
1839
|
+
type: "accept_terms" | (string & {});
|
|
1840
|
+
terms?: string;
|
|
1841
|
+
version?: string;
|
|
1842
|
+
url?: string;
|
|
1843
|
+
}
|
|
1844
|
+
|
|
1845
|
+
export function parseRequiresAction(value: unknown): RequiresAction | undefined {
|
|
1846
|
+
if (!value || typeof value !== "object") return undefined;
|
|
1847
|
+
const record = value as Record<string, unknown>;
|
|
1848
|
+
if (typeof record.type !== "string") return undefined;
|
|
1849
|
+
const action: RequiresAction = { type: record.type };
|
|
1850
|
+
if (typeof record.terms === "string") action.terms = record.terms;
|
|
1851
|
+
if (typeof record.version === "string") action.version = record.version;
|
|
1852
|
+
if (typeof record.url === "string") action.url = record.url;
|
|
1853
|
+
return action;
|
|
1854
|
+
}
|
|
1855
|
+
|
|
1710
1856
|
export class SdkError extends Error {
|
|
1857
|
+
requestId?: string;
|
|
1711
1858
|
status?: number;
|
|
1712
1859
|
code?: string;
|
|
1713
1860
|
details?: unknown;
|
|
1714
1861
|
jobId?: string;
|
|
1862
|
+
/** Present on exchange-mediated scrape failures that already captured credits. */
|
|
1863
|
+
chargeId?: string;
|
|
1864
|
+
/** Present when the API needs an out-of-band step first, such as accepting provider terms. */
|
|
1865
|
+
requiresAction?: RequiresAction;
|
|
1715
1866
|
constructor(
|
|
1716
1867
|
message: string,
|
|
1717
1868
|
status?: number,
|
|
1718
1869
|
code?: string,
|
|
1719
1870
|
details?: unknown,
|
|
1720
1871
|
jobId?: string,
|
|
1872
|
+
chargeId?: string,
|
|
1873
|
+
requiresAction?: RequiresAction,
|
|
1721
1874
|
) {
|
|
1722
1875
|
super(message);
|
|
1723
1876
|
this.name = "FirecrawlSdkError";
|
|
@@ -1725,6 +1878,8 @@ export class SdkError extends Error {
|
|
|
1725
1878
|
this.code = code;
|
|
1726
1879
|
this.details = details;
|
|
1727
1880
|
this.jobId = jobId;
|
|
1881
|
+
this.chargeId = chargeId;
|
|
1882
|
+
this.requiresAction = requiresAction;
|
|
1728
1883
|
}
|
|
1729
1884
|
}
|
|
1730
1885
|
|
|
@@ -1,11 +1,19 @@
|
|
|
1
1
|
import { type AxiosError, type AxiosResponse } from "axios";
|
|
2
|
-
import { SdkError, JobTimeoutError } from "../types";
|
|
2
|
+
import { SdkError, JobTimeoutError, parseRequiresAction } from "../types";
|
|
3
3
|
|
|
4
4
|
export function throwForBadResponse(resp: AxiosResponse, action: string): never {
|
|
5
5
|
const status = resp.status;
|
|
6
6
|
const body = resp.data || {};
|
|
7
7
|
const msg = body?.error || body?.message || `Request failed (${status}) while trying to ${action}`;
|
|
8
|
-
throw new SdkError(
|
|
8
|
+
throw new SdkError(
|
|
9
|
+
msg,
|
|
10
|
+
status,
|
|
11
|
+
body?.code,
|
|
12
|
+
body?.details,
|
|
13
|
+
undefined,
|
|
14
|
+
body?.chargeId,
|
|
15
|
+
parseRequiresAction(body?.requiresAction),
|
|
16
|
+
);
|
|
9
17
|
}
|
|
10
18
|
|
|
11
19
|
export function normalizeAxiosError(err: AxiosError, action: string): never {
|
|
@@ -13,7 +21,15 @@ export function normalizeAxiosError(err: AxiosError, action: string): never {
|
|
|
13
21
|
const body: any = err.response?.data;
|
|
14
22
|
const message = body?.error || err.message || `Request failed${status ? ` (${status})` : ""} while trying to ${action}`;
|
|
15
23
|
const code = (body?.code as string) || err.code;
|
|
16
|
-
throw new SdkError(
|
|
24
|
+
throw new SdkError(
|
|
25
|
+
message,
|
|
26
|
+
status,
|
|
27
|
+
code,
|
|
28
|
+
body?.details ?? body,
|
|
29
|
+
undefined,
|
|
30
|
+
body?.chargeId,
|
|
31
|
+
parseRequiresAction(body?.requiresAction),
|
|
32
|
+
);
|
|
17
33
|
}
|
|
18
34
|
|
|
19
35
|
export function isRetryableError(err: any): boolean {
|