@mendable/firecrawl 4.3.6 → 4.3.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/{chunk-R5TAYLVH.js → chunk-I2FDKS37.js} +1 -1
- package/dist/index.cjs +4 -1
- package/dist/index.d.cts +59 -30
- package/dist/index.d.ts +59 -30
- package/dist/index.js +5 -2
- package/dist/{package-HNRON2YA.js → package-O3Y6ICGR.js} +1 -1
- package/package.json +1 -1
- package/src/v2/types.ts +99 -41
- package/src/v2/utils/httpClient.ts +5 -0
|
@@ -8,7 +8,7 @@ var require_package = __commonJS({
|
|
|
8
8
|
"package.json"(exports, module) {
|
|
9
9
|
module.exports = {
|
|
10
10
|
name: "@mendable/firecrawl-js",
|
|
11
|
-
version: "4.3.
|
|
11
|
+
version: "4.3.8",
|
|
12
12
|
description: "JavaScript SDK for Firecrawl API",
|
|
13
13
|
main: "dist/index.js",
|
|
14
14
|
types: "dist/index.d.ts",
|
package/dist/index.cjs
CHANGED
|
@@ -35,7 +35,7 @@ var require_package = __commonJS({
|
|
|
35
35
|
"package.json"(exports2, module2) {
|
|
36
36
|
module2.exports = {
|
|
37
37
|
name: "@mendable/firecrawl-js",
|
|
38
|
-
version: "4.3.
|
|
38
|
+
version: "4.3.8",
|
|
39
39
|
description: "JavaScript SDK for Firecrawl API",
|
|
40
40
|
main: "dist/index.js",
|
|
41
41
|
types: "dist/index.d.ts",
|
|
@@ -174,6 +174,9 @@ var HttpClient = class {
|
|
|
174
174
|
if (cfg.method && ["post", "put", "patch"].includes(cfg.method.toLowerCase())) {
|
|
175
175
|
const data = cfg.data ?? {};
|
|
176
176
|
cfg.data = { ...data, origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${version}` };
|
|
177
|
+
if (typeof data.timeout === "number") {
|
|
178
|
+
cfg.timeout = data.timeout + 5e3;
|
|
179
|
+
}
|
|
177
180
|
}
|
|
178
181
|
const res = await this.instance.request(cfg);
|
|
179
182
|
if (res.status === 502 && attempt < this.maxRetries - 1) {
|
package/dist/index.d.cts
CHANGED
|
@@ -4,7 +4,7 @@ import { AxiosResponse, AxiosRequestHeaders } from 'axios';
|
|
|
4
4
|
import { EventEmitter } from 'events';
|
|
5
5
|
import { TypedEventTarget } from 'typescript-event-target';
|
|
6
6
|
|
|
7
|
-
type FormatString =
|
|
7
|
+
type FormatString = 'markdown' | 'html' | 'rawHtml' | 'links' | 'images' | 'screenshot' | 'summary' | 'changeTracking' | 'json' | 'attributes';
|
|
8
8
|
interface Viewport {
|
|
9
9
|
width: number;
|
|
10
10
|
height: number;
|
|
@@ -13,12 +13,12 @@ interface Format {
|
|
|
13
13
|
type: FormatString;
|
|
14
14
|
}
|
|
15
15
|
interface JsonFormat extends Format {
|
|
16
|
-
type:
|
|
16
|
+
type: 'json';
|
|
17
17
|
prompt?: string;
|
|
18
18
|
schema?: Record<string, unknown> | ZodTypeAny;
|
|
19
19
|
}
|
|
20
20
|
interface ScreenshotFormat {
|
|
21
|
-
type:
|
|
21
|
+
type: 'screenshot';
|
|
22
22
|
fullPage?: boolean;
|
|
23
23
|
quality?: number;
|
|
24
24
|
viewport?: Viewport | {
|
|
@@ -27,14 +27,14 @@ interface ScreenshotFormat {
|
|
|
27
27
|
};
|
|
28
28
|
}
|
|
29
29
|
interface ChangeTrackingFormat extends Format {
|
|
30
|
-
type:
|
|
31
|
-
modes: (
|
|
30
|
+
type: 'changeTracking';
|
|
31
|
+
modes: ('git-diff' | 'json')[];
|
|
32
32
|
schema?: Record<string, unknown>;
|
|
33
33
|
prompt?: string;
|
|
34
34
|
tag?: string;
|
|
35
35
|
}
|
|
36
36
|
interface AttributesFormat extends Format {
|
|
37
|
-
type:
|
|
37
|
+
type: 'attributes';
|
|
38
38
|
selectors: Array<{
|
|
39
39
|
selector: string;
|
|
40
40
|
attribute: string;
|
|
@@ -46,12 +46,12 @@ interface LocationConfig$1 {
|
|
|
46
46
|
languages?: string[];
|
|
47
47
|
}
|
|
48
48
|
interface WaitAction {
|
|
49
|
-
type:
|
|
49
|
+
type: 'wait';
|
|
50
50
|
milliseconds?: number;
|
|
51
51
|
selector?: string;
|
|
52
52
|
}
|
|
53
53
|
interface ScreenshotAction {
|
|
54
|
-
type:
|
|
54
|
+
type: 'screenshot';
|
|
55
55
|
fullPage?: boolean;
|
|
56
56
|
quality?: number;
|
|
57
57
|
viewport?: Viewport | {
|
|
@@ -60,32 +60,32 @@ interface ScreenshotAction {
|
|
|
60
60
|
};
|
|
61
61
|
}
|
|
62
62
|
interface ClickAction {
|
|
63
|
-
type:
|
|
63
|
+
type: 'click';
|
|
64
64
|
selector: string;
|
|
65
65
|
}
|
|
66
66
|
interface WriteAction {
|
|
67
|
-
type:
|
|
67
|
+
type: 'write';
|
|
68
68
|
text: string;
|
|
69
69
|
}
|
|
70
70
|
interface PressAction {
|
|
71
|
-
type:
|
|
71
|
+
type: 'press';
|
|
72
72
|
key: string;
|
|
73
73
|
}
|
|
74
74
|
interface ScrollAction {
|
|
75
|
-
type:
|
|
76
|
-
direction:
|
|
75
|
+
type: 'scroll';
|
|
76
|
+
direction: 'up' | 'down';
|
|
77
77
|
selector?: string;
|
|
78
78
|
}
|
|
79
79
|
interface ScrapeAction {
|
|
80
|
-
type:
|
|
80
|
+
type: 'scrape';
|
|
81
81
|
}
|
|
82
82
|
interface ExecuteJavascriptAction {
|
|
83
|
-
type:
|
|
83
|
+
type: 'executeJavascript';
|
|
84
84
|
script: string;
|
|
85
85
|
}
|
|
86
86
|
interface PDFAction {
|
|
87
|
-
type:
|
|
88
|
-
format?:
|
|
87
|
+
type: 'pdf';
|
|
88
|
+
format?: 'A0' | 'A1' | 'A2' | 'A3' | 'A4' | 'A5' | 'A6' | 'Letter' | 'Legal' | 'Tabloid' | 'Ledger';
|
|
89
89
|
landscape?: boolean;
|
|
90
90
|
scale?: number;
|
|
91
91
|
}
|
|
@@ -100,7 +100,7 @@ interface ScrapeOptions {
|
|
|
100
100
|
waitFor?: number;
|
|
101
101
|
mobile?: boolean;
|
|
102
102
|
parsers?: Array<string | {
|
|
103
|
-
type:
|
|
103
|
+
type: 'pdf';
|
|
104
104
|
maxPages?: number;
|
|
105
105
|
}>;
|
|
106
106
|
actions?: ActionOption[];
|
|
@@ -110,7 +110,7 @@ interface ScrapeOptions {
|
|
|
110
110
|
fastMode?: boolean;
|
|
111
111
|
useMock?: string;
|
|
112
112
|
blockAds?: boolean;
|
|
113
|
-
proxy?:
|
|
113
|
+
proxy?: 'basic' | 'stealth' | 'auto' | string;
|
|
114
114
|
maxAge?: number;
|
|
115
115
|
storeInCache?: boolean;
|
|
116
116
|
integration?: string;
|
|
@@ -119,11 +119,12 @@ interface WebhookConfig {
|
|
|
119
119
|
url: string;
|
|
120
120
|
headers?: Record<string, string>;
|
|
121
121
|
metadata?: Record<string, string>;
|
|
122
|
-
events?: Array<
|
|
122
|
+
events?: Array<'completed' | 'failed' | 'page' | 'started'>;
|
|
123
123
|
}
|
|
124
124
|
interface DocumentMetadata {
|
|
125
125
|
title?: string;
|
|
126
126
|
description?: string;
|
|
127
|
+
url?: string;
|
|
127
128
|
language?: string;
|
|
128
129
|
keywords?: string | string[];
|
|
129
130
|
robots?: string;
|
|
@@ -131,8 +132,36 @@ interface DocumentMetadata {
|
|
|
131
132
|
ogDescription?: string;
|
|
132
133
|
ogUrl?: string;
|
|
133
134
|
ogImage?: string;
|
|
135
|
+
ogAudio?: string;
|
|
136
|
+
ogDeterminer?: string;
|
|
137
|
+
ogLocale?: string;
|
|
138
|
+
ogLocaleAlternate?: string[];
|
|
139
|
+
ogSiteName?: string;
|
|
140
|
+
ogVideo?: string;
|
|
141
|
+
favicon?: string;
|
|
142
|
+
dcTermsCreated?: string;
|
|
143
|
+
dcDateCreated?: string;
|
|
144
|
+
dcDate?: string;
|
|
145
|
+
dcTermsType?: string;
|
|
146
|
+
dcType?: string;
|
|
147
|
+
dcTermsAudience?: string;
|
|
148
|
+
dcTermsSubject?: string;
|
|
149
|
+
dcSubject?: string;
|
|
150
|
+
dcDescription?: string;
|
|
151
|
+
dcTermsKeywords?: string;
|
|
152
|
+
modifiedTime?: string;
|
|
153
|
+
publishedTime?: string;
|
|
154
|
+
articleTag?: string;
|
|
155
|
+
articleSection?: string;
|
|
134
156
|
sourceURL?: string;
|
|
135
157
|
statusCode?: number;
|
|
158
|
+
scrapeId?: string;
|
|
159
|
+
numPages?: number;
|
|
160
|
+
contentType?: string;
|
|
161
|
+
proxyUsed?: 'basic' | 'stealth';
|
|
162
|
+
cacheState?: 'hit' | 'miss';
|
|
163
|
+
cachedAt?: string;
|
|
164
|
+
creditsUsed?: number;
|
|
136
165
|
error?: string;
|
|
137
166
|
[key: string]: unknown;
|
|
138
167
|
}
|
|
@@ -194,14 +223,14 @@ interface SearchData {
|
|
|
194
223
|
images?: Array<SearchResultImages | Document>;
|
|
195
224
|
}
|
|
196
225
|
interface CategoryOption {
|
|
197
|
-
type:
|
|
226
|
+
type: 'github' | 'research';
|
|
198
227
|
}
|
|
199
228
|
interface SearchRequest {
|
|
200
229
|
query: string;
|
|
201
|
-
sources?: Array<
|
|
202
|
-
type:
|
|
230
|
+
sources?: Array<'web' | 'news' | 'images' | {
|
|
231
|
+
type: 'web' | 'news' | 'images';
|
|
203
232
|
}>;
|
|
204
|
-
categories?: Array<
|
|
233
|
+
categories?: Array<'github' | 'research' | CategoryOption>;
|
|
205
234
|
limit?: number;
|
|
206
235
|
tbs?: string;
|
|
207
236
|
location?: string;
|
|
@@ -215,7 +244,7 @@ interface CrawlOptions {
|
|
|
215
244
|
excludePaths?: string[] | null;
|
|
216
245
|
includePaths?: string[] | null;
|
|
217
246
|
maxDiscoveryDepth?: number | null;
|
|
218
|
-
sitemap?:
|
|
247
|
+
sitemap?: 'skip' | 'include';
|
|
219
248
|
ignoreQueryParameters?: boolean;
|
|
220
249
|
limit?: number | null;
|
|
221
250
|
crawlEntireDomain?: boolean;
|
|
@@ -233,7 +262,7 @@ interface CrawlResponse$1 {
|
|
|
233
262
|
url: string;
|
|
234
263
|
}
|
|
235
264
|
interface CrawlJob {
|
|
236
|
-
status:
|
|
265
|
+
status: 'scraping' | 'completed' | 'failed' | 'cancelled';
|
|
237
266
|
total: number;
|
|
238
267
|
completed: number;
|
|
239
268
|
creditsUsed?: number;
|
|
@@ -257,7 +286,7 @@ interface BatchScrapeResponse$1 {
|
|
|
257
286
|
invalidURLs?: string[];
|
|
258
287
|
}
|
|
259
288
|
interface BatchScrapeJob {
|
|
260
|
-
status:
|
|
289
|
+
status: 'scraping' | 'completed' | 'failed' | 'cancelled';
|
|
261
290
|
completed: number;
|
|
262
291
|
total: number;
|
|
263
292
|
creditsUsed?: number;
|
|
@@ -270,7 +299,7 @@ interface MapData {
|
|
|
270
299
|
}
|
|
271
300
|
interface MapOptions {
|
|
272
301
|
search?: string;
|
|
273
|
-
sitemap?:
|
|
302
|
+
sitemap?: 'only' | 'include' | 'skip';
|
|
274
303
|
includeSubdomains?: boolean;
|
|
275
304
|
limit?: number;
|
|
276
305
|
timeout?: number;
|
|
@@ -280,7 +309,7 @@ interface MapOptions {
|
|
|
280
309
|
interface ExtractResponse$1 {
|
|
281
310
|
success?: boolean;
|
|
282
311
|
id?: string;
|
|
283
|
-
status?:
|
|
312
|
+
status?: 'processing' | 'completed' | 'failed' | 'cancelled';
|
|
284
313
|
data?: unknown;
|
|
285
314
|
error?: string;
|
|
286
315
|
warning?: string;
|
|
@@ -288,7 +317,7 @@ interface ExtractResponse$1 {
|
|
|
288
317
|
expiresAt?: string;
|
|
289
318
|
}
|
|
290
319
|
interface AgentOptions$1 {
|
|
291
|
-
model:
|
|
320
|
+
model: 'FIRE-1';
|
|
292
321
|
}
|
|
293
322
|
interface ConcurrencyCheck {
|
|
294
323
|
concurrency: number;
|
package/dist/index.d.ts
CHANGED
|
@@ -4,7 +4,7 @@ import { AxiosResponse, AxiosRequestHeaders } from 'axios';
|
|
|
4
4
|
import { EventEmitter } from 'events';
|
|
5
5
|
import { TypedEventTarget } from 'typescript-event-target';
|
|
6
6
|
|
|
7
|
-
type FormatString =
|
|
7
|
+
type FormatString = 'markdown' | 'html' | 'rawHtml' | 'links' | 'images' | 'screenshot' | 'summary' | 'changeTracking' | 'json' | 'attributes';
|
|
8
8
|
interface Viewport {
|
|
9
9
|
width: number;
|
|
10
10
|
height: number;
|
|
@@ -13,12 +13,12 @@ interface Format {
|
|
|
13
13
|
type: FormatString;
|
|
14
14
|
}
|
|
15
15
|
interface JsonFormat extends Format {
|
|
16
|
-
type:
|
|
16
|
+
type: 'json';
|
|
17
17
|
prompt?: string;
|
|
18
18
|
schema?: Record<string, unknown> | ZodTypeAny;
|
|
19
19
|
}
|
|
20
20
|
interface ScreenshotFormat {
|
|
21
|
-
type:
|
|
21
|
+
type: 'screenshot';
|
|
22
22
|
fullPage?: boolean;
|
|
23
23
|
quality?: number;
|
|
24
24
|
viewport?: Viewport | {
|
|
@@ -27,14 +27,14 @@ interface ScreenshotFormat {
|
|
|
27
27
|
};
|
|
28
28
|
}
|
|
29
29
|
interface ChangeTrackingFormat extends Format {
|
|
30
|
-
type:
|
|
31
|
-
modes: (
|
|
30
|
+
type: 'changeTracking';
|
|
31
|
+
modes: ('git-diff' | 'json')[];
|
|
32
32
|
schema?: Record<string, unknown>;
|
|
33
33
|
prompt?: string;
|
|
34
34
|
tag?: string;
|
|
35
35
|
}
|
|
36
36
|
interface AttributesFormat extends Format {
|
|
37
|
-
type:
|
|
37
|
+
type: 'attributes';
|
|
38
38
|
selectors: Array<{
|
|
39
39
|
selector: string;
|
|
40
40
|
attribute: string;
|
|
@@ -46,12 +46,12 @@ interface LocationConfig$1 {
|
|
|
46
46
|
languages?: string[];
|
|
47
47
|
}
|
|
48
48
|
interface WaitAction {
|
|
49
|
-
type:
|
|
49
|
+
type: 'wait';
|
|
50
50
|
milliseconds?: number;
|
|
51
51
|
selector?: string;
|
|
52
52
|
}
|
|
53
53
|
interface ScreenshotAction {
|
|
54
|
-
type:
|
|
54
|
+
type: 'screenshot';
|
|
55
55
|
fullPage?: boolean;
|
|
56
56
|
quality?: number;
|
|
57
57
|
viewport?: Viewport | {
|
|
@@ -60,32 +60,32 @@ interface ScreenshotAction {
|
|
|
60
60
|
};
|
|
61
61
|
}
|
|
62
62
|
interface ClickAction {
|
|
63
|
-
type:
|
|
63
|
+
type: 'click';
|
|
64
64
|
selector: string;
|
|
65
65
|
}
|
|
66
66
|
interface WriteAction {
|
|
67
|
-
type:
|
|
67
|
+
type: 'write';
|
|
68
68
|
text: string;
|
|
69
69
|
}
|
|
70
70
|
interface PressAction {
|
|
71
|
-
type:
|
|
71
|
+
type: 'press';
|
|
72
72
|
key: string;
|
|
73
73
|
}
|
|
74
74
|
interface ScrollAction {
|
|
75
|
-
type:
|
|
76
|
-
direction:
|
|
75
|
+
type: 'scroll';
|
|
76
|
+
direction: 'up' | 'down';
|
|
77
77
|
selector?: string;
|
|
78
78
|
}
|
|
79
79
|
interface ScrapeAction {
|
|
80
|
-
type:
|
|
80
|
+
type: 'scrape';
|
|
81
81
|
}
|
|
82
82
|
interface ExecuteJavascriptAction {
|
|
83
|
-
type:
|
|
83
|
+
type: 'executeJavascript';
|
|
84
84
|
script: string;
|
|
85
85
|
}
|
|
86
86
|
interface PDFAction {
|
|
87
|
-
type:
|
|
88
|
-
format?:
|
|
87
|
+
type: 'pdf';
|
|
88
|
+
format?: 'A0' | 'A1' | 'A2' | 'A3' | 'A4' | 'A5' | 'A6' | 'Letter' | 'Legal' | 'Tabloid' | 'Ledger';
|
|
89
89
|
landscape?: boolean;
|
|
90
90
|
scale?: number;
|
|
91
91
|
}
|
|
@@ -100,7 +100,7 @@ interface ScrapeOptions {
|
|
|
100
100
|
waitFor?: number;
|
|
101
101
|
mobile?: boolean;
|
|
102
102
|
parsers?: Array<string | {
|
|
103
|
-
type:
|
|
103
|
+
type: 'pdf';
|
|
104
104
|
maxPages?: number;
|
|
105
105
|
}>;
|
|
106
106
|
actions?: ActionOption[];
|
|
@@ -110,7 +110,7 @@ interface ScrapeOptions {
|
|
|
110
110
|
fastMode?: boolean;
|
|
111
111
|
useMock?: string;
|
|
112
112
|
blockAds?: boolean;
|
|
113
|
-
proxy?:
|
|
113
|
+
proxy?: 'basic' | 'stealth' | 'auto' | string;
|
|
114
114
|
maxAge?: number;
|
|
115
115
|
storeInCache?: boolean;
|
|
116
116
|
integration?: string;
|
|
@@ -119,11 +119,12 @@ interface WebhookConfig {
|
|
|
119
119
|
url: string;
|
|
120
120
|
headers?: Record<string, string>;
|
|
121
121
|
metadata?: Record<string, string>;
|
|
122
|
-
events?: Array<
|
|
122
|
+
events?: Array<'completed' | 'failed' | 'page' | 'started'>;
|
|
123
123
|
}
|
|
124
124
|
interface DocumentMetadata {
|
|
125
125
|
title?: string;
|
|
126
126
|
description?: string;
|
|
127
|
+
url?: string;
|
|
127
128
|
language?: string;
|
|
128
129
|
keywords?: string | string[];
|
|
129
130
|
robots?: string;
|
|
@@ -131,8 +132,36 @@ interface DocumentMetadata {
|
|
|
131
132
|
ogDescription?: string;
|
|
132
133
|
ogUrl?: string;
|
|
133
134
|
ogImage?: string;
|
|
135
|
+
ogAudio?: string;
|
|
136
|
+
ogDeterminer?: string;
|
|
137
|
+
ogLocale?: string;
|
|
138
|
+
ogLocaleAlternate?: string[];
|
|
139
|
+
ogSiteName?: string;
|
|
140
|
+
ogVideo?: string;
|
|
141
|
+
favicon?: string;
|
|
142
|
+
dcTermsCreated?: string;
|
|
143
|
+
dcDateCreated?: string;
|
|
144
|
+
dcDate?: string;
|
|
145
|
+
dcTermsType?: string;
|
|
146
|
+
dcType?: string;
|
|
147
|
+
dcTermsAudience?: string;
|
|
148
|
+
dcTermsSubject?: string;
|
|
149
|
+
dcSubject?: string;
|
|
150
|
+
dcDescription?: string;
|
|
151
|
+
dcTermsKeywords?: string;
|
|
152
|
+
modifiedTime?: string;
|
|
153
|
+
publishedTime?: string;
|
|
154
|
+
articleTag?: string;
|
|
155
|
+
articleSection?: string;
|
|
134
156
|
sourceURL?: string;
|
|
135
157
|
statusCode?: number;
|
|
158
|
+
scrapeId?: string;
|
|
159
|
+
numPages?: number;
|
|
160
|
+
contentType?: string;
|
|
161
|
+
proxyUsed?: 'basic' | 'stealth';
|
|
162
|
+
cacheState?: 'hit' | 'miss';
|
|
163
|
+
cachedAt?: string;
|
|
164
|
+
creditsUsed?: number;
|
|
136
165
|
error?: string;
|
|
137
166
|
[key: string]: unknown;
|
|
138
167
|
}
|
|
@@ -194,14 +223,14 @@ interface SearchData {
|
|
|
194
223
|
images?: Array<SearchResultImages | Document>;
|
|
195
224
|
}
|
|
196
225
|
interface CategoryOption {
|
|
197
|
-
type:
|
|
226
|
+
type: 'github' | 'research';
|
|
198
227
|
}
|
|
199
228
|
interface SearchRequest {
|
|
200
229
|
query: string;
|
|
201
|
-
sources?: Array<
|
|
202
|
-
type:
|
|
230
|
+
sources?: Array<'web' | 'news' | 'images' | {
|
|
231
|
+
type: 'web' | 'news' | 'images';
|
|
203
232
|
}>;
|
|
204
|
-
categories?: Array<
|
|
233
|
+
categories?: Array<'github' | 'research' | CategoryOption>;
|
|
205
234
|
limit?: number;
|
|
206
235
|
tbs?: string;
|
|
207
236
|
location?: string;
|
|
@@ -215,7 +244,7 @@ interface CrawlOptions {
|
|
|
215
244
|
excludePaths?: string[] | null;
|
|
216
245
|
includePaths?: string[] | null;
|
|
217
246
|
maxDiscoveryDepth?: number | null;
|
|
218
|
-
sitemap?:
|
|
247
|
+
sitemap?: 'skip' | 'include';
|
|
219
248
|
ignoreQueryParameters?: boolean;
|
|
220
249
|
limit?: number | null;
|
|
221
250
|
crawlEntireDomain?: boolean;
|
|
@@ -233,7 +262,7 @@ interface CrawlResponse$1 {
|
|
|
233
262
|
url: string;
|
|
234
263
|
}
|
|
235
264
|
interface CrawlJob {
|
|
236
|
-
status:
|
|
265
|
+
status: 'scraping' | 'completed' | 'failed' | 'cancelled';
|
|
237
266
|
total: number;
|
|
238
267
|
completed: number;
|
|
239
268
|
creditsUsed?: number;
|
|
@@ -257,7 +286,7 @@ interface BatchScrapeResponse$1 {
|
|
|
257
286
|
invalidURLs?: string[];
|
|
258
287
|
}
|
|
259
288
|
interface BatchScrapeJob {
|
|
260
|
-
status:
|
|
289
|
+
status: 'scraping' | 'completed' | 'failed' | 'cancelled';
|
|
261
290
|
completed: number;
|
|
262
291
|
total: number;
|
|
263
292
|
creditsUsed?: number;
|
|
@@ -270,7 +299,7 @@ interface MapData {
|
|
|
270
299
|
}
|
|
271
300
|
interface MapOptions {
|
|
272
301
|
search?: string;
|
|
273
|
-
sitemap?:
|
|
302
|
+
sitemap?: 'only' | 'include' | 'skip';
|
|
274
303
|
includeSubdomains?: boolean;
|
|
275
304
|
limit?: number;
|
|
276
305
|
timeout?: number;
|
|
@@ -280,7 +309,7 @@ interface MapOptions {
|
|
|
280
309
|
interface ExtractResponse$1 {
|
|
281
310
|
success?: boolean;
|
|
282
311
|
id?: string;
|
|
283
|
-
status?:
|
|
312
|
+
status?: 'processing' | 'completed' | 'failed' | 'cancelled';
|
|
284
313
|
data?: unknown;
|
|
285
314
|
error?: string;
|
|
286
315
|
warning?: string;
|
|
@@ -288,7 +317,7 @@ interface ExtractResponse$1 {
|
|
|
288
317
|
expiresAt?: string;
|
|
289
318
|
}
|
|
290
319
|
interface AgentOptions$1 {
|
|
291
|
-
model:
|
|
320
|
+
model: 'FIRE-1';
|
|
292
321
|
}
|
|
293
322
|
interface ConcurrencyCheck {
|
|
294
323
|
concurrency: number;
|
package/dist/index.js
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import {
|
|
2
2
|
require_package
|
|
3
|
-
} from "./chunk-
|
|
3
|
+
} from "./chunk-I2FDKS37.js";
|
|
4
4
|
|
|
5
5
|
// src/v2/utils/httpClient.ts
|
|
6
6
|
import axios from "axios";
|
|
@@ -58,6 +58,9 @@ var HttpClient = class {
|
|
|
58
58
|
if (cfg.method && ["post", "put", "patch"].includes(cfg.method.toLowerCase())) {
|
|
59
59
|
const data = cfg.data ?? {};
|
|
60
60
|
cfg.data = { ...data, origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${version}` };
|
|
61
|
+
if (typeof data.timeout === "number") {
|
|
62
|
+
cfg.timeout = data.timeout + 5e3;
|
|
63
|
+
}
|
|
61
64
|
}
|
|
62
65
|
const res = await this.instance.request(cfg);
|
|
63
66
|
if (res.status === 502 && attempt < this.maxRetries - 1) {
|
|
@@ -1065,7 +1068,7 @@ var FirecrawlApp = class {
|
|
|
1065
1068
|
if (typeof process !== "undefined" && process.env && process.env.npm_package_version) {
|
|
1066
1069
|
return process.env.npm_package_version;
|
|
1067
1070
|
}
|
|
1068
|
-
const packageJson = await import("./package-
|
|
1071
|
+
const packageJson = await import("./package-O3Y6ICGR.js");
|
|
1069
1072
|
return packageJson.default.version;
|
|
1070
1073
|
} catch (error) {
|
|
1071
1074
|
const isTest = typeof process !== "undefined" && (process.env.JEST_WORKER_ID != null || false);
|
package/package.json
CHANGED
package/src/v2/types.ts
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import type { ZodTypeAny } from
|
|
1
|
+
import type { ZodTypeAny } from 'zod';
|
|
2
2
|
// Public types for Firecrawl JS/TS SDK v2 (camelCase only)
|
|
3
3
|
|
|
4
4
|
export type FormatString =
|
|
5
|
-
|
|
|
6
|
-
|
|
|
7
|
-
|
|
|
8
|
-
|
|
|
9
|
-
|
|
|
10
|
-
|
|
|
11
|
-
|
|
|
12
|
-
|
|
|
13
|
-
|
|
|
14
|
-
|
|
|
5
|
+
| 'markdown'
|
|
6
|
+
| 'html'
|
|
7
|
+
| 'rawHtml'
|
|
8
|
+
| 'links'
|
|
9
|
+
| 'images'
|
|
10
|
+
| 'screenshot'
|
|
11
|
+
| 'summary'
|
|
12
|
+
| 'changeTracking'
|
|
13
|
+
| 'json'
|
|
14
|
+
| 'attributes';
|
|
15
15
|
|
|
16
16
|
export interface Viewport {
|
|
17
17
|
width: number;
|
|
@@ -23,27 +23,27 @@ export interface Format {
|
|
|
23
23
|
}
|
|
24
24
|
|
|
25
25
|
export interface JsonFormat extends Format {
|
|
26
|
-
type:
|
|
26
|
+
type: 'json';
|
|
27
27
|
prompt?: string;
|
|
28
28
|
schema?: Record<string, unknown> | ZodTypeAny;
|
|
29
29
|
}
|
|
30
30
|
|
|
31
31
|
export interface ScreenshotFormat {
|
|
32
|
-
type:
|
|
32
|
+
type: 'screenshot';
|
|
33
33
|
fullPage?: boolean;
|
|
34
34
|
quality?: number;
|
|
35
35
|
viewport?: Viewport | { width: number; height: number };
|
|
36
36
|
}
|
|
37
37
|
|
|
38
38
|
export interface ChangeTrackingFormat extends Format {
|
|
39
|
-
type:
|
|
40
|
-
modes: (
|
|
39
|
+
type: 'changeTracking';
|
|
40
|
+
modes: ('git-diff' | 'json')[];
|
|
41
41
|
schema?: Record<string, unknown>;
|
|
42
42
|
prompt?: string;
|
|
43
43
|
tag?: string;
|
|
44
44
|
}
|
|
45
45
|
export interface AttributesFormat extends Format {
|
|
46
|
-
type:
|
|
46
|
+
type: 'attributes';
|
|
47
47
|
selectors: Array<{
|
|
48
48
|
selector: string;
|
|
49
49
|
attribute: string;
|
|
@@ -64,51 +64,62 @@ export interface LocationConfig {
|
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
export interface WaitAction {
|
|
67
|
-
type:
|
|
67
|
+
type: 'wait';
|
|
68
68
|
milliseconds?: number;
|
|
69
69
|
selector?: string;
|
|
70
70
|
}
|
|
71
71
|
|
|
72
72
|
export interface ScreenshotAction {
|
|
73
|
-
type:
|
|
73
|
+
type: 'screenshot';
|
|
74
74
|
fullPage?: boolean;
|
|
75
75
|
quality?: number;
|
|
76
76
|
viewport?: Viewport | { width: number; height: number };
|
|
77
77
|
}
|
|
78
78
|
|
|
79
79
|
export interface ClickAction {
|
|
80
|
-
type:
|
|
80
|
+
type: 'click';
|
|
81
81
|
selector: string;
|
|
82
82
|
}
|
|
83
83
|
|
|
84
84
|
export interface WriteAction {
|
|
85
|
-
type:
|
|
85
|
+
type: 'write';
|
|
86
86
|
text: string;
|
|
87
87
|
}
|
|
88
88
|
|
|
89
89
|
export interface PressAction {
|
|
90
|
-
type:
|
|
90
|
+
type: 'press';
|
|
91
91
|
key: string;
|
|
92
92
|
}
|
|
93
93
|
|
|
94
94
|
export interface ScrollAction {
|
|
95
|
-
type:
|
|
96
|
-
direction:
|
|
95
|
+
type: 'scroll';
|
|
96
|
+
direction: 'up' | 'down';
|
|
97
97
|
selector?: string;
|
|
98
98
|
}
|
|
99
99
|
|
|
100
100
|
export interface ScrapeAction {
|
|
101
|
-
type:
|
|
101
|
+
type: 'scrape';
|
|
102
102
|
}
|
|
103
103
|
|
|
104
104
|
export interface ExecuteJavascriptAction {
|
|
105
|
-
type:
|
|
105
|
+
type: 'executeJavascript';
|
|
106
106
|
script: string;
|
|
107
107
|
}
|
|
108
108
|
|
|
109
109
|
export interface PDFAction {
|
|
110
|
-
type:
|
|
111
|
-
format?:
|
|
110
|
+
type: 'pdf';
|
|
111
|
+
format?:
|
|
112
|
+
| 'A0'
|
|
113
|
+
| 'A1'
|
|
114
|
+
| 'A2'
|
|
115
|
+
| 'A3'
|
|
116
|
+
| 'A4'
|
|
117
|
+
| 'A5'
|
|
118
|
+
| 'A6'
|
|
119
|
+
| 'Letter'
|
|
120
|
+
| 'Legal'
|
|
121
|
+
| 'Tabloid'
|
|
122
|
+
| 'Ledger';
|
|
112
123
|
landscape?: boolean;
|
|
113
124
|
scale?: number;
|
|
114
125
|
}
|
|
@@ -133,7 +144,7 @@ export interface ScrapeOptions {
|
|
|
133
144
|
timeout?: number;
|
|
134
145
|
waitFor?: number;
|
|
135
146
|
mobile?: boolean;
|
|
136
|
-
parsers?: Array<string | { type:
|
|
147
|
+
parsers?: Array<string | { type: 'pdf'; maxPages?: number }>;
|
|
137
148
|
actions?: ActionOption[];
|
|
138
149
|
location?: LocationConfig;
|
|
139
150
|
skipTlsVerification?: boolean;
|
|
@@ -141,7 +152,7 @@ export interface ScrapeOptions {
|
|
|
141
152
|
fastMode?: boolean;
|
|
142
153
|
useMock?: string;
|
|
143
154
|
blockAds?: boolean;
|
|
144
|
-
proxy?:
|
|
155
|
+
proxy?: 'basic' | 'stealth' | 'auto' | string;
|
|
145
156
|
maxAge?: number;
|
|
146
157
|
storeInCache?: boolean;
|
|
147
158
|
integration?: string;
|
|
@@ -151,22 +162,62 @@ export interface WebhookConfig {
|
|
|
151
162
|
url: string;
|
|
152
163
|
headers?: Record<string, string>;
|
|
153
164
|
metadata?: Record<string, string>;
|
|
154
|
-
events?: Array<
|
|
165
|
+
events?: Array<'completed' | 'failed' | 'page' | 'started'>;
|
|
155
166
|
}
|
|
156
167
|
|
|
157
168
|
export interface DocumentMetadata {
|
|
169
|
+
// Common metadata fields
|
|
158
170
|
title?: string;
|
|
159
171
|
description?: string;
|
|
172
|
+
url?: string;
|
|
160
173
|
language?: string;
|
|
161
174
|
keywords?: string | string[];
|
|
162
175
|
robots?: string;
|
|
176
|
+
|
|
177
|
+
// OpenGraph and social metadata
|
|
163
178
|
ogTitle?: string;
|
|
164
179
|
ogDescription?: string;
|
|
165
180
|
ogUrl?: string;
|
|
166
181
|
ogImage?: string;
|
|
182
|
+
ogAudio?: string;
|
|
183
|
+
ogDeterminer?: string;
|
|
184
|
+
ogLocale?: string;
|
|
185
|
+
ogLocaleAlternate?: string[];
|
|
186
|
+
ogSiteName?: string;
|
|
187
|
+
ogVideo?: string;
|
|
188
|
+
|
|
189
|
+
// Dublin Core and other site metadata
|
|
190
|
+
favicon?: string;
|
|
191
|
+
dcTermsCreated?: string;
|
|
192
|
+
dcDateCreated?: string;
|
|
193
|
+
dcDate?: string;
|
|
194
|
+
dcTermsType?: string;
|
|
195
|
+
dcType?: string;
|
|
196
|
+
dcTermsAudience?: string;
|
|
197
|
+
dcTermsSubject?: string;
|
|
198
|
+
dcSubject?: string;
|
|
199
|
+
dcDescription?: string;
|
|
200
|
+
dcTermsKeywords?: string;
|
|
201
|
+
|
|
202
|
+
modifiedTime?: string;
|
|
203
|
+
publishedTime?: string;
|
|
204
|
+
articleTag?: string;
|
|
205
|
+
articleSection?: string;
|
|
206
|
+
|
|
207
|
+
// Response-level metadata
|
|
167
208
|
sourceURL?: string;
|
|
168
209
|
statusCode?: number;
|
|
210
|
+
scrapeId?: string;
|
|
211
|
+
numPages?: number;
|
|
212
|
+
contentType?: string;
|
|
213
|
+
proxyUsed?: 'basic' | 'stealth';
|
|
214
|
+
cacheState?: 'hit' | 'miss';
|
|
215
|
+
cachedAt?: string;
|
|
216
|
+
creditsUsed?: number;
|
|
217
|
+
|
|
218
|
+
// Error information
|
|
169
219
|
error?: string;
|
|
220
|
+
|
|
170
221
|
[key: string]: unknown;
|
|
171
222
|
}
|
|
172
223
|
|
|
@@ -235,13 +286,15 @@ export interface SearchData {
|
|
|
235
286
|
}
|
|
236
287
|
|
|
237
288
|
export interface CategoryOption {
|
|
238
|
-
type:
|
|
289
|
+
type: 'github' | 'research';
|
|
239
290
|
}
|
|
240
291
|
|
|
241
292
|
export interface SearchRequest {
|
|
242
293
|
query: string;
|
|
243
|
-
sources?: Array<
|
|
244
|
-
|
|
294
|
+
sources?: Array<
|
|
295
|
+
'web' | 'news' | 'images' | { type: 'web' | 'news' | 'images' }
|
|
296
|
+
>;
|
|
297
|
+
categories?: Array<'github' | 'research' | CategoryOption>;
|
|
245
298
|
limit?: number;
|
|
246
299
|
tbs?: string;
|
|
247
300
|
location?: string;
|
|
@@ -256,7 +309,7 @@ export interface CrawlOptions {
|
|
|
256
309
|
excludePaths?: string[] | null;
|
|
257
310
|
includePaths?: string[] | null;
|
|
258
311
|
maxDiscoveryDepth?: number | null;
|
|
259
|
-
sitemap?:
|
|
312
|
+
sitemap?: 'skip' | 'include';
|
|
260
313
|
ignoreQueryParameters?: boolean;
|
|
261
314
|
limit?: number | null;
|
|
262
315
|
crawlEntireDomain?: boolean;
|
|
@@ -276,7 +329,7 @@ export interface CrawlResponse {
|
|
|
276
329
|
}
|
|
277
330
|
|
|
278
331
|
export interface CrawlJob {
|
|
279
|
-
status:
|
|
332
|
+
status: 'scraping' | 'completed' | 'failed' | 'cancelled';
|
|
280
333
|
total: number;
|
|
281
334
|
completed: number;
|
|
282
335
|
creditsUsed?: number;
|
|
@@ -303,7 +356,7 @@ export interface BatchScrapeResponse {
|
|
|
303
356
|
}
|
|
304
357
|
|
|
305
358
|
export interface BatchScrapeJob {
|
|
306
|
-
status:
|
|
359
|
+
status: 'scraping' | 'completed' | 'failed' | 'cancelled';
|
|
307
360
|
completed: number;
|
|
308
361
|
total: number;
|
|
309
362
|
creditsUsed?: number;
|
|
@@ -318,7 +371,7 @@ export interface MapData {
|
|
|
318
371
|
|
|
319
372
|
export interface MapOptions {
|
|
320
373
|
search?: string;
|
|
321
|
-
sitemap?:
|
|
374
|
+
sitemap?: 'only' | 'include' | 'skip';
|
|
322
375
|
includeSubdomains?: boolean;
|
|
323
376
|
limit?: number;
|
|
324
377
|
timeout?: number;
|
|
@@ -329,7 +382,7 @@ export interface MapOptions {
|
|
|
329
382
|
export interface ExtractResponse {
|
|
330
383
|
success?: boolean;
|
|
331
384
|
id?: string;
|
|
332
|
-
status?:
|
|
385
|
+
status?: 'processing' | 'completed' | 'failed' | 'cancelled';
|
|
333
386
|
data?: unknown;
|
|
334
387
|
error?: string;
|
|
335
388
|
warning?: string;
|
|
@@ -338,7 +391,7 @@ export interface ExtractResponse {
|
|
|
338
391
|
}
|
|
339
392
|
|
|
340
393
|
export interface AgentOptions {
|
|
341
|
-
model:
|
|
394
|
+
model: 'FIRE-1';
|
|
342
395
|
}
|
|
343
396
|
|
|
344
397
|
export interface ConcurrencyCheck {
|
|
@@ -418,9 +471,14 @@ export class SdkError extends Error {
|
|
|
418
471
|
status?: number;
|
|
419
472
|
code?: string;
|
|
420
473
|
details?: unknown;
|
|
421
|
-
constructor(
|
|
474
|
+
constructor(
|
|
475
|
+
message: string,
|
|
476
|
+
status?: number,
|
|
477
|
+
code?: string,
|
|
478
|
+
details?: unknown
|
|
479
|
+
) {
|
|
422
480
|
super(message);
|
|
423
|
-
this.name =
|
|
481
|
+
this.name = 'FirecrawlSdkError';
|
|
424
482
|
this.status = status;
|
|
425
483
|
this.code = code;
|
|
426
484
|
this.details = details;
|
|
@@ -54,6 +54,11 @@ export class HttpClient {
|
|
|
54
54
|
if (cfg.method && ["post", "put", "patch"].includes(cfg.method.toLowerCase())) {
|
|
55
55
|
const data = (cfg.data ?? {}) as Record<string, unknown>;
|
|
56
56
|
cfg.data = { ...data, origin: typeof data.origin === "string" && data.origin.includes("mcp") ? data.origin : `js-sdk@${version}` };
|
|
57
|
+
|
|
58
|
+
// If timeout is specified in the body, use it to override the request timeout
|
|
59
|
+
if (typeof data.timeout === "number") {
|
|
60
|
+
cfg.timeout = data.timeout + 5000;
|
|
61
|
+
}
|
|
57
62
|
}
|
|
58
63
|
const res = await this.instance.request<T>(cfg);
|
|
59
64
|
if (res.status === 502 && attempt < this.maxRetries - 1) {
|