firecrawl 1.5.2 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +84 -20
- package/dist/index.d.cts +35 -1
- package/dist/index.d.ts +35 -1
- package/dist/index.js +82 -19
- package/dump.rdb +0 -0
- package/package.json +1 -1
- package/src/index.ts +92 -19
package/dist/index.cjs
CHANGED
|
@@ -31,6 +31,7 @@ var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: tru
|
|
|
31
31
|
var src_exports = {};
|
|
32
32
|
__export(src_exports, {
|
|
33
33
|
CrawlWatcher: () => CrawlWatcher,
|
|
34
|
+
FirecrawlError: () => FirecrawlError,
|
|
34
35
|
default: () => FirecrawlApp
|
|
35
36
|
});
|
|
36
37
|
module.exports = __toCommonJS(src_exports);
|
|
@@ -38,6 +39,13 @@ var import_axios = __toESM(require("axios"), 1);
|
|
|
38
39
|
var import_zod_to_json_schema = require("zod-to-json-schema");
|
|
39
40
|
var import_isows = require("isows");
|
|
40
41
|
var import_typescript_event_target = require("typescript-event-target");
|
|
42
|
+
var FirecrawlError = class extends Error {
|
|
43
|
+
statusCode;
|
|
44
|
+
constructor(message, statusCode) {
|
|
45
|
+
super(message);
|
|
46
|
+
this.statusCode = statusCode;
|
|
47
|
+
}
|
|
48
|
+
};
|
|
41
49
|
var FirecrawlApp = class {
|
|
42
50
|
apiKey;
|
|
43
51
|
apiUrl;
|
|
@@ -47,7 +55,7 @@ var FirecrawlApp = class {
|
|
|
47
55
|
*/
|
|
48
56
|
constructor({ apiKey = null, apiUrl = null }) {
|
|
49
57
|
if (typeof apiKey !== "string") {
|
|
50
|
-
throw new
|
|
58
|
+
throw new FirecrawlError("No API key provided", 401);
|
|
51
59
|
}
|
|
52
60
|
this.apiKey = apiKey;
|
|
53
61
|
this.apiUrl = apiUrl || "https://api.firecrawl.dev";
|
|
@@ -94,13 +102,13 @@ var FirecrawlApp = class {
|
|
|
94
102
|
...responseData.data
|
|
95
103
|
};
|
|
96
104
|
} else {
|
|
97
|
-
throw new
|
|
105
|
+
throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);
|
|
98
106
|
}
|
|
99
107
|
} else {
|
|
100
108
|
this.handleError(response, "scrape URL");
|
|
101
109
|
}
|
|
102
110
|
} catch (error) {
|
|
103
|
-
|
|
111
|
+
this.handleError(error.response, "scrape URL");
|
|
104
112
|
}
|
|
105
113
|
return { success: false, error: "Internal server error." };
|
|
106
114
|
}
|
|
@@ -111,7 +119,7 @@ var FirecrawlApp = class {
|
|
|
111
119
|
* @returns Throws an error advising to use version 0 of the API.
|
|
112
120
|
*/
|
|
113
121
|
async search(query, params) {
|
|
114
|
-
throw new
|
|
122
|
+
throw new FirecrawlError("Search is not supported in v1, please downgrade Firecrawl to 0.0.36.", 400);
|
|
115
123
|
}
|
|
116
124
|
/**
|
|
117
125
|
* Initiates a crawl job for a URL using the Firecrawl API.
|
|
@@ -138,9 +146,9 @@ var FirecrawlApp = class {
|
|
|
138
146
|
}
|
|
139
147
|
} catch (error) {
|
|
140
148
|
if (error.response?.data?.error) {
|
|
141
|
-
throw new
|
|
149
|
+
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
|
|
142
150
|
} else {
|
|
143
|
-
throw new
|
|
151
|
+
throw new FirecrawlError(error.message, 500);
|
|
144
152
|
}
|
|
145
153
|
}
|
|
146
154
|
return { success: false, error: "Internal server error." };
|
|
@@ -161,9 +169,9 @@ var FirecrawlApp = class {
|
|
|
161
169
|
}
|
|
162
170
|
} catch (error) {
|
|
163
171
|
if (error.response?.data?.error) {
|
|
164
|
-
throw new
|
|
172
|
+
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
|
|
165
173
|
} else {
|
|
166
|
-
throw new
|
|
174
|
+
throw new FirecrawlError(error.message, 500);
|
|
167
175
|
}
|
|
168
176
|
}
|
|
169
177
|
return { success: false, error: "Internal server error." };
|
|
@@ -176,7 +184,7 @@ var FirecrawlApp = class {
|
|
|
176
184
|
*/
|
|
177
185
|
async checkCrawlStatus(id, getAllData = false) {
|
|
178
186
|
if (!id) {
|
|
179
|
-
throw new
|
|
187
|
+
throw new FirecrawlError("No crawl ID provided", 400);
|
|
180
188
|
}
|
|
181
189
|
const headers = this.prepareHeaders();
|
|
182
190
|
try {
|
|
@@ -212,18 +220,53 @@ var FirecrawlApp = class {
|
|
|
212
220
|
this.handleError(response, "check crawl status");
|
|
213
221
|
}
|
|
214
222
|
} catch (error) {
|
|
215
|
-
throw new
|
|
223
|
+
throw new FirecrawlError(error.message, 500);
|
|
216
224
|
}
|
|
217
225
|
return { success: false, error: "Internal server error." };
|
|
218
226
|
}
|
|
227
|
+
/**
|
|
228
|
+
* Cancels a crawl job using the Firecrawl API.
|
|
229
|
+
* @param id - The ID of the crawl operation.
|
|
230
|
+
* @returns The response from the cancel crawl operation.
|
|
231
|
+
*/
|
|
232
|
+
async cancelCrawl(id) {
|
|
233
|
+
const headers = this.prepareHeaders();
|
|
234
|
+
try {
|
|
235
|
+
const response = await this.deleteRequest(
|
|
236
|
+
`${this.apiUrl}/v1/crawl/${id}`,
|
|
237
|
+
headers
|
|
238
|
+
);
|
|
239
|
+
if (response.status === 200) {
|
|
240
|
+
return response.data;
|
|
241
|
+
} else {
|
|
242
|
+
this.handleError(response, "cancel crawl job");
|
|
243
|
+
}
|
|
244
|
+
} catch (error) {
|
|
245
|
+
throw new FirecrawlError(error.message, 500);
|
|
246
|
+
}
|
|
247
|
+
return { success: false, error: "Internal server error." };
|
|
248
|
+
}
|
|
249
|
+
/**
|
|
250
|
+
* Initiates a crawl job and returns a CrawlWatcher to monitor the job via WebSocket.
|
|
251
|
+
* @param url - The URL to crawl.
|
|
252
|
+
* @param params - Additional parameters for the crawl request.
|
|
253
|
+
* @param idempotencyKey - Optional idempotency key for the request.
|
|
254
|
+
* @returns A CrawlWatcher instance to monitor the crawl job.
|
|
255
|
+
*/
|
|
219
256
|
async crawlUrlAndWatch(url, params, idempotencyKey) {
|
|
220
257
|
const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
|
|
221
258
|
if (crawl.success && crawl.id) {
|
|
222
259
|
const id = crawl.id;
|
|
223
260
|
return new CrawlWatcher(id, this);
|
|
224
261
|
}
|
|
225
|
-
throw new
|
|
262
|
+
throw new FirecrawlError("Crawl job failed to start", 400);
|
|
226
263
|
}
|
|
264
|
+
/**
|
|
265
|
+
* Maps a URL using the Firecrawl API.
|
|
266
|
+
* @param url - The URL to map.
|
|
267
|
+
* @param params - Additional parameters for the map request.
|
|
268
|
+
* @returns The response from the map operation.
|
|
269
|
+
*/
|
|
227
270
|
async mapUrl(url, params) {
|
|
228
271
|
const headers = this.prepareHeaders();
|
|
229
272
|
let jsonData = { url, ...params };
|
|
@@ -239,7 +282,7 @@ var FirecrawlApp = class {
|
|
|
239
282
|
this.handleError(response, "map");
|
|
240
283
|
}
|
|
241
284
|
} catch (error) {
|
|
242
|
-
throw new
|
|
285
|
+
throw new FirecrawlError(error.message, 500);
|
|
243
286
|
}
|
|
244
287
|
return { success: false, error: "Internal server error." };
|
|
245
288
|
}
|
|
@@ -282,6 +325,23 @@ var FirecrawlApp = class {
|
|
|
282
325
|
}
|
|
283
326
|
}
|
|
284
327
|
}
|
|
328
|
+
/**
|
|
329
|
+
* Sends a DELETE request to the specified URL.
|
|
330
|
+
* @param url - The URL to send the request to.
|
|
331
|
+
* @param headers - The headers for the request.
|
|
332
|
+
* @returns The response from the DELETE request.
|
|
333
|
+
*/
|
|
334
|
+
async deleteRequest(url, headers) {
|
|
335
|
+
try {
|
|
336
|
+
return await import_axios.default.delete(url, { headers });
|
|
337
|
+
} catch (error) {
|
|
338
|
+
if (error instanceof import_axios.AxiosError && error.response) {
|
|
339
|
+
return error.response;
|
|
340
|
+
} else {
|
|
341
|
+
throw error;
|
|
342
|
+
}
|
|
343
|
+
}
|
|
344
|
+
}
|
|
285
345
|
/**
|
|
286
346
|
* Monitors the status of a crawl job until completion or failure.
|
|
287
347
|
* @param id - The ID of the crawl operation.
|
|
@@ -309,7 +369,7 @@ var FirecrawlApp = class {
|
|
|
309
369
|
statusData.data = data;
|
|
310
370
|
return statusData;
|
|
311
371
|
} else {
|
|
312
|
-
throw new
|
|
372
|
+
throw new FirecrawlError("Crawl job completed but no data was returned", 500);
|
|
313
373
|
}
|
|
314
374
|
} else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
|
|
315
375
|
checkInterval = Math.max(checkInterval, 2);
|
|
@@ -317,8 +377,9 @@ var FirecrawlApp = class {
|
|
|
317
377
|
(resolve) => setTimeout(resolve, checkInterval * 1e3)
|
|
318
378
|
);
|
|
319
379
|
} else {
|
|
320
|
-
throw new
|
|
321
|
-
`Crawl job failed or was stopped. Status: ${statusData.status}
|
|
380
|
+
throw new FirecrawlError(
|
|
381
|
+
`Crawl job failed or was stopped. Status: ${statusData.status}`,
|
|
382
|
+
500
|
|
322
383
|
);
|
|
323
384
|
}
|
|
324
385
|
} else {
|
|
@@ -334,12 +395,14 @@ var FirecrawlApp = class {
|
|
|
334
395
|
handleError(response, action) {
|
|
335
396
|
if ([402, 408, 409, 500].includes(response.status)) {
|
|
336
397
|
const errorMessage = response.data.error || "Unknown error occurred";
|
|
337
|
-
throw new
|
|
338
|
-
`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}
|
|
398
|
+
throw new FirecrawlError(
|
|
399
|
+
`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`,
|
|
400
|
+
response.status
|
|
339
401
|
);
|
|
340
402
|
} else {
|
|
341
|
-
throw new
|
|
342
|
-
`Unexpected error occurred while trying to ${action}. Status code: ${response.status}
|
|
403
|
+
throw new FirecrawlError(
|
|
404
|
+
`Unexpected error occurred while trying to ${action}. Status code: ${response.status}`,
|
|
405
|
+
response.status
|
|
343
406
|
);
|
|
344
407
|
}
|
|
345
408
|
}
|
|
@@ -414,5 +477,6 @@ var CrawlWatcher = class extends import_typescript_event_target.TypedEventTarget
|
|
|
414
477
|
};
|
|
415
478
|
// Annotate the CommonJS export names for ESM import in node:
|
|
416
479
|
0 && (module.exports = {
|
|
417
|
-
CrawlWatcher
|
|
480
|
+
CrawlWatcher,
|
|
481
|
+
FirecrawlError
|
|
418
482
|
});
|
package/dist/index.d.cts
CHANGED
|
@@ -183,6 +183,14 @@ interface ErrorResponse {
|
|
|
183
183
|
success: false;
|
|
184
184
|
error: string;
|
|
185
185
|
}
|
|
186
|
+
/**
|
|
187
|
+
* Custom error class for Firecrawl.
|
|
188
|
+
* Extends the built-in Error class to include a status code.
|
|
189
|
+
*/
|
|
190
|
+
declare class FirecrawlError extends Error {
|
|
191
|
+
statusCode: number;
|
|
192
|
+
constructor(message: string, statusCode: number);
|
|
193
|
+
}
|
|
186
194
|
/**
|
|
187
195
|
* Main class for interacting with the Firecrawl API.
|
|
188
196
|
* Provides methods for scraping, searching, crawling, and mapping web content.
|
|
@@ -226,7 +234,26 @@ declare class FirecrawlApp {
|
|
|
226
234
|
* @returns The response containing the job status.
|
|
227
235
|
*/
|
|
228
236
|
checkCrawlStatus(id?: string, getAllData?: boolean): Promise<CrawlStatusResponse | ErrorResponse>;
|
|
237
|
+
/**
|
|
238
|
+
* Cancels a crawl job using the Firecrawl API.
|
|
239
|
+
* @param id - The ID of the crawl operation.
|
|
240
|
+
* @returns The response from the cancel crawl operation.
|
|
241
|
+
*/
|
|
242
|
+
cancelCrawl(id: string): Promise<ErrorResponse>;
|
|
243
|
+
/**
|
|
244
|
+
* Initiates a crawl job and returns a CrawlWatcher to monitor the job via WebSocket.
|
|
245
|
+
* @param url - The URL to crawl.
|
|
246
|
+
* @param params - Additional parameters for the crawl request.
|
|
247
|
+
* @param idempotencyKey - Optional idempotency key for the request.
|
|
248
|
+
* @returns A CrawlWatcher instance to monitor the crawl job.
|
|
249
|
+
*/
|
|
229
250
|
crawlUrlAndWatch(url: string, params?: CrawlParams, idempotencyKey?: string): Promise<CrawlWatcher>;
|
|
251
|
+
/**
|
|
252
|
+
* Maps a URL using the Firecrawl API.
|
|
253
|
+
* @param url - The URL to map.
|
|
254
|
+
* @param params - Additional parameters for the map request.
|
|
255
|
+
* @returns The response from the map operation.
|
|
256
|
+
*/
|
|
230
257
|
mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse>;
|
|
231
258
|
/**
|
|
232
259
|
* Prepares the headers for an API request.
|
|
@@ -249,6 +276,13 @@ declare class FirecrawlApp {
|
|
|
249
276
|
* @returns The response from the GET request.
|
|
250
277
|
*/
|
|
251
278
|
getRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
|
|
279
|
+
/**
|
|
280
|
+
* Sends a DELETE request to the specified URL.
|
|
281
|
+
* @param url - The URL to send the request to.
|
|
282
|
+
* @param headers - The headers for the request.
|
|
283
|
+
* @returns The response from the DELETE request.
|
|
284
|
+
*/
|
|
285
|
+
deleteRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
|
|
252
286
|
/**
|
|
253
287
|
* Monitors the status of a crawl job until completion or failure.
|
|
254
288
|
* @param id - The ID of the crawl operation.
|
|
@@ -285,4 +319,4 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
|
|
|
285
319
|
close(): void;
|
|
286
320
|
}
|
|
287
321
|
|
|
288
|
-
export { type Action, type ActionsResult, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, FirecrawlApp as default };
|
|
322
|
+
export { type Action, type ActionsResult, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, FirecrawlApp as default };
|
package/dist/index.d.ts
CHANGED
|
@@ -183,6 +183,14 @@ interface ErrorResponse {
|
|
|
183
183
|
success: false;
|
|
184
184
|
error: string;
|
|
185
185
|
}
|
|
186
|
+
/**
|
|
187
|
+
* Custom error class for Firecrawl.
|
|
188
|
+
* Extends the built-in Error class to include a status code.
|
|
189
|
+
*/
|
|
190
|
+
declare class FirecrawlError extends Error {
|
|
191
|
+
statusCode: number;
|
|
192
|
+
constructor(message: string, statusCode: number);
|
|
193
|
+
}
|
|
186
194
|
/**
|
|
187
195
|
* Main class for interacting with the Firecrawl API.
|
|
188
196
|
* Provides methods for scraping, searching, crawling, and mapping web content.
|
|
@@ -226,7 +234,26 @@ declare class FirecrawlApp {
|
|
|
226
234
|
* @returns The response containing the job status.
|
|
227
235
|
*/
|
|
228
236
|
checkCrawlStatus(id?: string, getAllData?: boolean): Promise<CrawlStatusResponse | ErrorResponse>;
|
|
237
|
+
/**
|
|
238
|
+
* Cancels a crawl job using the Firecrawl API.
|
|
239
|
+
* @param id - The ID of the crawl operation.
|
|
240
|
+
* @returns The response from the cancel crawl operation.
|
|
241
|
+
*/
|
|
242
|
+
cancelCrawl(id: string): Promise<ErrorResponse>;
|
|
243
|
+
/**
|
|
244
|
+
* Initiates a crawl job and returns a CrawlWatcher to monitor the job via WebSocket.
|
|
245
|
+
* @param url - The URL to crawl.
|
|
246
|
+
* @param params - Additional parameters for the crawl request.
|
|
247
|
+
* @param idempotencyKey - Optional idempotency key for the request.
|
|
248
|
+
* @returns A CrawlWatcher instance to monitor the crawl job.
|
|
249
|
+
*/
|
|
229
250
|
crawlUrlAndWatch(url: string, params?: CrawlParams, idempotencyKey?: string): Promise<CrawlWatcher>;
|
|
251
|
+
/**
|
|
252
|
+
* Maps a URL using the Firecrawl API.
|
|
253
|
+
* @param url - The URL to map.
|
|
254
|
+
* @param params - Additional parameters for the map request.
|
|
255
|
+
* @returns The response from the map operation.
|
|
256
|
+
*/
|
|
230
257
|
mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse>;
|
|
231
258
|
/**
|
|
232
259
|
* Prepares the headers for an API request.
|
|
@@ -249,6 +276,13 @@ declare class FirecrawlApp {
|
|
|
249
276
|
* @returns The response from the GET request.
|
|
250
277
|
*/
|
|
251
278
|
getRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
|
|
279
|
+
/**
|
|
280
|
+
* Sends a DELETE request to the specified URL.
|
|
281
|
+
* @param url - The URL to send the request to.
|
|
282
|
+
* @param headers - The headers for the request.
|
|
283
|
+
* @returns The response from the DELETE request.
|
|
284
|
+
*/
|
|
285
|
+
deleteRequest(url: string, headers: AxiosRequestHeaders): Promise<AxiosResponse>;
|
|
252
286
|
/**
|
|
253
287
|
* Monitors the status of a crawl job until completion or failure.
|
|
254
288
|
* @param id - The ID of the crawl operation.
|
|
@@ -285,4 +319,4 @@ declare class CrawlWatcher extends TypedEventTarget<CrawlWatcherEvents> {
|
|
|
285
319
|
close(): void;
|
|
286
320
|
}
|
|
287
321
|
|
|
288
|
-
export { type Action, type ActionsResult, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, FirecrawlApp as default };
|
|
322
|
+
export { type Action, type ActionsResult, type CrawlParams, type CrawlResponse, type CrawlScrapeOptions, type CrawlStatusResponse, CrawlWatcher, type ErrorResponse, type FirecrawlAppConfig, type FirecrawlDocument, type FirecrawlDocumentMetadata, FirecrawlError, type MapParams, type MapResponse, type ScrapeParams, type ScrapeResponse, FirecrawlApp as default };
|
package/dist/index.js
CHANGED
|
@@ -3,6 +3,13 @@ import axios, { AxiosError } from "axios";
|
|
|
3
3
|
import { zodToJsonSchema } from "zod-to-json-schema";
|
|
4
4
|
import { WebSocket } from "isows";
|
|
5
5
|
import { TypedEventTarget } from "typescript-event-target";
|
|
6
|
+
var FirecrawlError = class extends Error {
|
|
7
|
+
statusCode;
|
|
8
|
+
constructor(message, statusCode) {
|
|
9
|
+
super(message);
|
|
10
|
+
this.statusCode = statusCode;
|
|
11
|
+
}
|
|
12
|
+
};
|
|
6
13
|
var FirecrawlApp = class {
|
|
7
14
|
apiKey;
|
|
8
15
|
apiUrl;
|
|
@@ -12,7 +19,7 @@ var FirecrawlApp = class {
|
|
|
12
19
|
*/
|
|
13
20
|
constructor({ apiKey = null, apiUrl = null }) {
|
|
14
21
|
if (typeof apiKey !== "string") {
|
|
15
|
-
throw new
|
|
22
|
+
throw new FirecrawlError("No API key provided", 401);
|
|
16
23
|
}
|
|
17
24
|
this.apiKey = apiKey;
|
|
18
25
|
this.apiUrl = apiUrl || "https://api.firecrawl.dev";
|
|
@@ -59,13 +66,13 @@ var FirecrawlApp = class {
|
|
|
59
66
|
...responseData.data
|
|
60
67
|
};
|
|
61
68
|
} else {
|
|
62
|
-
throw new
|
|
69
|
+
throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);
|
|
63
70
|
}
|
|
64
71
|
} else {
|
|
65
72
|
this.handleError(response, "scrape URL");
|
|
66
73
|
}
|
|
67
74
|
} catch (error) {
|
|
68
|
-
|
|
75
|
+
this.handleError(error.response, "scrape URL");
|
|
69
76
|
}
|
|
70
77
|
return { success: false, error: "Internal server error." };
|
|
71
78
|
}
|
|
@@ -76,7 +83,7 @@ var FirecrawlApp = class {
|
|
|
76
83
|
* @returns Throws an error advising to use version 0 of the API.
|
|
77
84
|
*/
|
|
78
85
|
async search(query, params) {
|
|
79
|
-
throw new
|
|
86
|
+
throw new FirecrawlError("Search is not supported in v1, please downgrade Firecrawl to 0.0.36.", 400);
|
|
80
87
|
}
|
|
81
88
|
/**
|
|
82
89
|
* Initiates a crawl job for a URL using the Firecrawl API.
|
|
@@ -103,9 +110,9 @@ var FirecrawlApp = class {
|
|
|
103
110
|
}
|
|
104
111
|
} catch (error) {
|
|
105
112
|
if (error.response?.data?.error) {
|
|
106
|
-
throw new
|
|
113
|
+
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
|
|
107
114
|
} else {
|
|
108
|
-
throw new
|
|
115
|
+
throw new FirecrawlError(error.message, 500);
|
|
109
116
|
}
|
|
110
117
|
}
|
|
111
118
|
return { success: false, error: "Internal server error." };
|
|
@@ -126,9 +133,9 @@ var FirecrawlApp = class {
|
|
|
126
133
|
}
|
|
127
134
|
} catch (error) {
|
|
128
135
|
if (error.response?.data?.error) {
|
|
129
|
-
throw new
|
|
136
|
+
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ""}`, error.response.status);
|
|
130
137
|
} else {
|
|
131
|
-
throw new
|
|
138
|
+
throw new FirecrawlError(error.message, 500);
|
|
132
139
|
}
|
|
133
140
|
}
|
|
134
141
|
return { success: false, error: "Internal server error." };
|
|
@@ -141,7 +148,7 @@ var FirecrawlApp = class {
|
|
|
141
148
|
*/
|
|
142
149
|
async checkCrawlStatus(id, getAllData = false) {
|
|
143
150
|
if (!id) {
|
|
144
|
-
throw new
|
|
151
|
+
throw new FirecrawlError("No crawl ID provided", 400);
|
|
145
152
|
}
|
|
146
153
|
const headers = this.prepareHeaders();
|
|
147
154
|
try {
|
|
@@ -177,18 +184,53 @@ var FirecrawlApp = class {
|
|
|
177
184
|
this.handleError(response, "check crawl status");
|
|
178
185
|
}
|
|
179
186
|
} catch (error) {
|
|
180
|
-
throw new
|
|
187
|
+
throw new FirecrawlError(error.message, 500);
|
|
181
188
|
}
|
|
182
189
|
return { success: false, error: "Internal server error." };
|
|
183
190
|
}
|
|
191
|
+
/**
|
|
192
|
+
* Cancels a crawl job using the Firecrawl API.
|
|
193
|
+
* @param id - The ID of the crawl operation.
|
|
194
|
+
* @returns The response from the cancel crawl operation.
|
|
195
|
+
*/
|
|
196
|
+
async cancelCrawl(id) {
|
|
197
|
+
const headers = this.prepareHeaders();
|
|
198
|
+
try {
|
|
199
|
+
const response = await this.deleteRequest(
|
|
200
|
+
`${this.apiUrl}/v1/crawl/${id}`,
|
|
201
|
+
headers
|
|
202
|
+
);
|
|
203
|
+
if (response.status === 200) {
|
|
204
|
+
return response.data;
|
|
205
|
+
} else {
|
|
206
|
+
this.handleError(response, "cancel crawl job");
|
|
207
|
+
}
|
|
208
|
+
} catch (error) {
|
|
209
|
+
throw new FirecrawlError(error.message, 500);
|
|
210
|
+
}
|
|
211
|
+
return { success: false, error: "Internal server error." };
|
|
212
|
+
}
|
|
213
|
+
/**
|
|
214
|
+
* Initiates a crawl job and returns a CrawlWatcher to monitor the job via WebSocket.
|
|
215
|
+
* @param url - The URL to crawl.
|
|
216
|
+
* @param params - Additional parameters for the crawl request.
|
|
217
|
+
* @param idempotencyKey - Optional idempotency key for the request.
|
|
218
|
+
* @returns A CrawlWatcher instance to monitor the crawl job.
|
|
219
|
+
*/
|
|
184
220
|
async crawlUrlAndWatch(url, params, idempotencyKey) {
|
|
185
221
|
const crawl = await this.asyncCrawlUrl(url, params, idempotencyKey);
|
|
186
222
|
if (crawl.success && crawl.id) {
|
|
187
223
|
const id = crawl.id;
|
|
188
224
|
return new CrawlWatcher(id, this);
|
|
189
225
|
}
|
|
190
|
-
throw new
|
|
226
|
+
throw new FirecrawlError("Crawl job failed to start", 400);
|
|
191
227
|
}
|
|
228
|
+
/**
|
|
229
|
+
* Maps a URL using the Firecrawl API.
|
|
230
|
+
* @param url - The URL to map.
|
|
231
|
+
* @param params - Additional parameters for the map request.
|
|
232
|
+
* @returns The response from the map operation.
|
|
233
|
+
*/
|
|
192
234
|
async mapUrl(url, params) {
|
|
193
235
|
const headers = this.prepareHeaders();
|
|
194
236
|
let jsonData = { url, ...params };
|
|
@@ -204,7 +246,7 @@ var FirecrawlApp = class {
|
|
|
204
246
|
this.handleError(response, "map");
|
|
205
247
|
}
|
|
206
248
|
} catch (error) {
|
|
207
|
-
throw new
|
|
249
|
+
throw new FirecrawlError(error.message, 500);
|
|
208
250
|
}
|
|
209
251
|
return { success: false, error: "Internal server error." };
|
|
210
252
|
}
|
|
@@ -247,6 +289,23 @@ var FirecrawlApp = class {
|
|
|
247
289
|
}
|
|
248
290
|
}
|
|
249
291
|
}
|
|
292
|
+
/**
|
|
293
|
+
* Sends a DELETE request to the specified URL.
|
|
294
|
+
* @param url - The URL to send the request to.
|
|
295
|
+
* @param headers - The headers for the request.
|
|
296
|
+
* @returns The response from the DELETE request.
|
|
297
|
+
*/
|
|
298
|
+
async deleteRequest(url, headers) {
|
|
299
|
+
try {
|
|
300
|
+
return await axios.delete(url, { headers });
|
|
301
|
+
} catch (error) {
|
|
302
|
+
if (error instanceof AxiosError && error.response) {
|
|
303
|
+
return error.response;
|
|
304
|
+
} else {
|
|
305
|
+
throw error;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
}
|
|
250
309
|
/**
|
|
251
310
|
* Monitors the status of a crawl job until completion or failure.
|
|
252
311
|
* @param id - The ID of the crawl operation.
|
|
@@ -274,7 +333,7 @@ var FirecrawlApp = class {
|
|
|
274
333
|
statusData.data = data;
|
|
275
334
|
return statusData;
|
|
276
335
|
} else {
|
|
277
|
-
throw new
|
|
336
|
+
throw new FirecrawlError("Crawl job completed but no data was returned", 500);
|
|
278
337
|
}
|
|
279
338
|
} else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
|
|
280
339
|
checkInterval = Math.max(checkInterval, 2);
|
|
@@ -282,8 +341,9 @@ var FirecrawlApp = class {
|
|
|
282
341
|
(resolve) => setTimeout(resolve, checkInterval * 1e3)
|
|
283
342
|
);
|
|
284
343
|
} else {
|
|
285
|
-
throw new
|
|
286
|
-
`Crawl job failed or was stopped. Status: ${statusData.status}
|
|
344
|
+
throw new FirecrawlError(
|
|
345
|
+
`Crawl job failed or was stopped. Status: ${statusData.status}`,
|
|
346
|
+
500
|
|
287
347
|
);
|
|
288
348
|
}
|
|
289
349
|
} else {
|
|
@@ -299,12 +359,14 @@ var FirecrawlApp = class {
|
|
|
299
359
|
handleError(response, action) {
|
|
300
360
|
if ([402, 408, 409, 500].includes(response.status)) {
|
|
301
361
|
const errorMessage = response.data.error || "Unknown error occurred";
|
|
302
|
-
throw new
|
|
303
|
-
`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}
|
|
362
|
+
throw new FirecrawlError(
|
|
363
|
+
`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`,
|
|
364
|
+
response.status
|
|
304
365
|
);
|
|
305
366
|
} else {
|
|
306
|
-
throw new
|
|
307
|
-
`Unexpected error occurred while trying to ${action}. Status code: ${response.status}
|
|
367
|
+
throw new FirecrawlError(
|
|
368
|
+
`Unexpected error occurred while trying to ${action}. Status code: ${response.status}`,
|
|
369
|
+
response.status
|
|
308
370
|
);
|
|
309
371
|
}
|
|
310
372
|
}
|
|
@@ -379,5 +441,6 @@ var CrawlWatcher = class extends TypedEventTarget {
|
|
|
379
441
|
};
|
|
380
442
|
export {
|
|
381
443
|
CrawlWatcher,
|
|
444
|
+
FirecrawlError,
|
|
382
445
|
FirecrawlApp as default
|
|
383
446
|
};
|
package/dump.rdb
CHANGED
|
Binary file
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -199,6 +199,19 @@ export interface ErrorResponse {
|
|
|
199
199
|
error: string;
|
|
200
200
|
}
|
|
201
201
|
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Custom error class for Firecrawl.
|
|
205
|
+
* Extends the built-in Error class to include a status code.
|
|
206
|
+
*/
|
|
207
|
+
export class FirecrawlError extends Error {
|
|
208
|
+
statusCode: number;
|
|
209
|
+
constructor(message: string, statusCode: number) {
|
|
210
|
+
super(message);
|
|
211
|
+
this.statusCode = statusCode;
|
|
212
|
+
}
|
|
213
|
+
}
|
|
214
|
+
|
|
202
215
|
/**
|
|
203
216
|
* Main class for interacting with the Firecrawl API.
|
|
204
217
|
* Provides methods for scraping, searching, crawling, and mapping web content.
|
|
@@ -213,7 +226,7 @@ export default class FirecrawlApp {
|
|
|
213
226
|
*/
|
|
214
227
|
constructor({ apiKey = null, apiUrl = null }: FirecrawlAppConfig) {
|
|
215
228
|
if (typeof apiKey !== "string") {
|
|
216
|
-
throw new
|
|
229
|
+
throw new FirecrawlError("No API key provided", 401);
|
|
217
230
|
}
|
|
218
231
|
|
|
219
232
|
this.apiKey = apiKey;
|
|
@@ -268,13 +281,13 @@ export default class FirecrawlApp {
|
|
|
268
281
|
...responseData.data
|
|
269
282
|
};
|
|
270
283
|
} else {
|
|
271
|
-
throw new
|
|
284
|
+
throw new FirecrawlError(`Failed to scrape URL. Error: ${responseData.error}`, response.status);
|
|
272
285
|
}
|
|
273
286
|
} else {
|
|
274
287
|
this.handleError(response, "scrape URL");
|
|
275
288
|
}
|
|
276
289
|
} catch (error: any) {
|
|
277
|
-
|
|
290
|
+
this.handleError(error.response, "scrape URL");
|
|
278
291
|
}
|
|
279
292
|
return { success: false, error: "Internal server error." };
|
|
280
293
|
}
|
|
@@ -289,7 +302,7 @@ export default class FirecrawlApp {
|
|
|
289
302
|
query: string,
|
|
290
303
|
params?: any
|
|
291
304
|
): Promise<any> {
|
|
292
|
-
throw new
|
|
305
|
+
throw new FirecrawlError("Search is not supported in v1, please downgrade Firecrawl to 0.0.36.", 400);
|
|
293
306
|
}
|
|
294
307
|
|
|
295
308
|
/**
|
|
@@ -322,9 +335,9 @@ export default class FirecrawlApp {
|
|
|
322
335
|
}
|
|
323
336
|
} catch (error: any) {
|
|
324
337
|
if (error.response?.data?.error) {
|
|
325
|
-
throw new
|
|
338
|
+
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
|
|
326
339
|
} else {
|
|
327
|
-
throw new
|
|
340
|
+
throw new FirecrawlError(error.message, 500);
|
|
328
341
|
}
|
|
329
342
|
}
|
|
330
343
|
return { success: false, error: "Internal server error." };
|
|
@@ -350,9 +363,9 @@ export default class FirecrawlApp {
|
|
|
350
363
|
}
|
|
351
364
|
} catch (error: any) {
|
|
352
365
|
if (error.response?.data?.error) {
|
|
353
|
-
throw new
|
|
366
|
+
throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
|
|
354
367
|
} else {
|
|
355
|
-
throw new
|
|
368
|
+
throw new FirecrawlError(error.message, 500);
|
|
356
369
|
}
|
|
357
370
|
}
|
|
358
371
|
return { success: false, error: "Internal server error." };
|
|
@@ -366,7 +379,7 @@ export default class FirecrawlApp {
|
|
|
366
379
|
*/
|
|
367
380
|
async checkCrawlStatus(id?: string, getAllData = false): Promise<CrawlStatusResponse | ErrorResponse> {
|
|
368
381
|
if (!id) {
|
|
369
|
-
throw new
|
|
382
|
+
throw new FirecrawlError("No crawl ID provided", 400);
|
|
370
383
|
}
|
|
371
384
|
|
|
372
385
|
const headers: AxiosRequestHeaders = this.prepareHeaders();
|
|
@@ -403,11 +416,41 @@ export default class FirecrawlApp {
|
|
|
403
416
|
this.handleError(response, "check crawl status");
|
|
404
417
|
}
|
|
405
418
|
} catch (error: any) {
|
|
406
|
-
throw new
|
|
419
|
+
throw new FirecrawlError(error.message, 500);
|
|
420
|
+
}
|
|
421
|
+
return { success: false, error: "Internal server error." };
|
|
422
|
+
}
|
|
423
|
+
|
|
424
|
+
/**
|
|
425
|
+
* Cancels a crawl job using the Firecrawl API.
|
|
426
|
+
* @param id - The ID of the crawl operation.
|
|
427
|
+
* @returns The response from the cancel crawl operation.
|
|
428
|
+
*/
|
|
429
|
+
async cancelCrawl(id: string): Promise<ErrorResponse> {
|
|
430
|
+
const headers = this.prepareHeaders();
|
|
431
|
+
try {
|
|
432
|
+
const response: AxiosResponse = await this.deleteRequest(
|
|
433
|
+
`${this.apiUrl}/v1/crawl/${id}`,
|
|
434
|
+
headers
|
|
435
|
+
);
|
|
436
|
+
if (response.status === 200) {
|
|
437
|
+
return response.data;
|
|
438
|
+
} else {
|
|
439
|
+
this.handleError(response, "cancel crawl job");
|
|
440
|
+
}
|
|
441
|
+
} catch (error: any) {
|
|
442
|
+
throw new FirecrawlError(error.message, 500);
|
|
407
443
|
}
|
|
408
444
|
return { success: false, error: "Internal server error." };
|
|
409
445
|
}
|
|
410
446
|
|
|
447
|
+
/**
|
|
448
|
+
* Initiates a crawl job and returns a CrawlWatcher to monitor the job via WebSocket.
|
|
449
|
+
* @param url - The URL to crawl.
|
|
450
|
+
* @param params - Additional parameters for the crawl request.
|
|
451
|
+
* @param idempotencyKey - Optional idempotency key for the request.
|
|
452
|
+
* @returns A CrawlWatcher instance to monitor the crawl job.
|
|
453
|
+
*/
|
|
411
454
|
async crawlUrlAndWatch(
|
|
412
455
|
url: string,
|
|
413
456
|
params?: CrawlParams,
|
|
@@ -420,9 +463,15 @@ export default class FirecrawlApp {
|
|
|
420
463
|
return new CrawlWatcher(id, this);
|
|
421
464
|
}
|
|
422
465
|
|
|
423
|
-
throw new
|
|
466
|
+
throw new FirecrawlError("Crawl job failed to start", 400);
|
|
424
467
|
}
|
|
425
468
|
|
|
469
|
+
/**
|
|
470
|
+
* Maps a URL using the Firecrawl API.
|
|
471
|
+
* @param url - The URL to map.
|
|
472
|
+
* @param params - Additional parameters for the map request.
|
|
473
|
+
* @returns The response from the map operation.
|
|
474
|
+
*/
|
|
426
475
|
async mapUrl(url: string, params?: MapParams): Promise<MapResponse | ErrorResponse> {
|
|
427
476
|
const headers = this.prepareHeaders();
|
|
428
477
|
let jsonData: { url: string } & MapParams = { url, ...params };
|
|
@@ -439,7 +488,7 @@ export default class FirecrawlApp {
|
|
|
439
488
|
this.handleError(response, "map");
|
|
440
489
|
}
|
|
441
490
|
} catch (error: any) {
|
|
442
|
-
throw new
|
|
491
|
+
throw new FirecrawlError(error.message, 500);
|
|
443
492
|
}
|
|
444
493
|
return { success: false, error: "Internal server error." };
|
|
445
494
|
}
|
|
@@ -493,6 +542,27 @@ export default class FirecrawlApp {
|
|
|
493
542
|
}
|
|
494
543
|
}
|
|
495
544
|
|
|
545
|
+
/**
|
|
546
|
+
* Sends a DELETE request to the specified URL.
|
|
547
|
+
* @param url - The URL to send the request to.
|
|
548
|
+
* @param headers - The headers for the request.
|
|
549
|
+
* @returns The response from the DELETE request.
|
|
550
|
+
*/
|
|
551
|
+
async deleteRequest(
|
|
552
|
+
url: string,
|
|
553
|
+
headers: AxiosRequestHeaders
|
|
554
|
+
): Promise<AxiosResponse> {
|
|
555
|
+
try {
|
|
556
|
+
return await axios.delete(url, { headers });
|
|
557
|
+
} catch (error) {
|
|
558
|
+
if (error instanceof AxiosError && error.response) {
|
|
559
|
+
return error.response as AxiosResponse;
|
|
560
|
+
} else {
|
|
561
|
+
throw error;
|
|
562
|
+
}
|
|
563
|
+
}
|
|
564
|
+
}
|
|
565
|
+
|
|
496
566
|
/**
|
|
497
567
|
* Monitors the status of a crawl job until completion or failure.
|
|
498
568
|
* @param id - The ID of the crawl operation.
|
|
@@ -524,7 +594,7 @@ export default class FirecrawlApp {
|
|
|
524
594
|
statusData.data = data;
|
|
525
595
|
return statusData;
|
|
526
596
|
} else {
|
|
527
|
-
throw new
|
|
597
|
+
throw new FirecrawlError("Crawl job completed but no data was returned", 500);
|
|
528
598
|
}
|
|
529
599
|
} else if (
|
|
530
600
|
["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
|
|
@@ -534,8 +604,9 @@ export default class FirecrawlApp {
|
|
|
534
604
|
setTimeout(resolve, checkInterval * 1000)
|
|
535
605
|
);
|
|
536
606
|
} else {
|
|
537
|
-
throw new
|
|
538
|
-
`Crawl job failed or was stopped. Status: ${statusData.status}
|
|
607
|
+
throw new FirecrawlError(
|
|
608
|
+
`Crawl job failed or was stopped. Status: ${statusData.status}`,
|
|
609
|
+
500
|
|
539
610
|
);
|
|
540
611
|
}
|
|
541
612
|
} else {
|
|
@@ -553,12 +624,14 @@ export default class FirecrawlApp {
|
|
|
553
624
|
if ([402, 408, 409, 500].includes(response.status)) {
|
|
554
625
|
const errorMessage: string =
|
|
555
626
|
response.data.error || "Unknown error occurred";
|
|
556
|
-
throw new
|
|
557
|
-
`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}
|
|
627
|
+
throw new FirecrawlError(
|
|
628
|
+
`Failed to ${action}. Status code: ${response.status}. Error: ${errorMessage}`,
|
|
629
|
+
response.status
|
|
558
630
|
);
|
|
559
631
|
} else {
|
|
560
|
-
throw new
|
|
561
|
-
`Unexpected error occurred while trying to ${action}. Status code: ${response.status}
|
|
632
|
+
throw new FirecrawlError(
|
|
633
|
+
`Unexpected error occurred while trying to ${action}. Status code: ${response.status}`,
|
|
634
|
+
response.status
|
|
562
635
|
);
|
|
563
636
|
}
|
|
564
637
|
}
|