@mendable/firecrawl-js 0.0.23 → 0.0.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/build/index.js +7 -7
- package/package.json +1 -1
- package/src/index.ts +7 -7
- package/types/index.d.ts +3 -3
package/build/index.js
CHANGED
|
@@ -110,12 +110,12 @@ export default class FirecrawlApp {
|
|
|
110
110
|
* @param {string} url - The URL to crawl.
|
|
111
111
|
* @param {Params | null} params - Additional parameters for the crawl request.
|
|
112
112
|
* @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
|
|
113
|
-
* @param {number}
|
|
113
|
+
* @param {number} pollInterval - Time in seconds for job status checks.
|
|
114
114
|
* @param {string} idempotencyKey - Optional idempotency key for the request.
|
|
115
115
|
* @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
|
|
116
116
|
*/
|
|
117
117
|
crawlUrl(url_1) {
|
|
118
|
-
return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true,
|
|
118
|
+
return __awaiter(this, arguments, void 0, function* (url, params = null, waitUntilDone = true, pollInterval = 2, idempotencyKey) {
|
|
119
119
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
120
120
|
let jsonData = { url };
|
|
121
121
|
if (params) {
|
|
@@ -126,7 +126,7 @@ export default class FirecrawlApp {
|
|
|
126
126
|
if (response.status === 200) {
|
|
127
127
|
const jobId = response.data.jobId;
|
|
128
128
|
if (waitUntilDone) {
|
|
129
|
-
return this.monitorJobStatus(jobId, headers,
|
|
129
|
+
return this.monitorJobStatus(jobId, headers, pollInterval);
|
|
130
130
|
}
|
|
131
131
|
else {
|
|
132
132
|
return { success: true, jobId };
|
|
@@ -208,7 +208,7 @@ export default class FirecrawlApp {
|
|
|
208
208
|
* @param {number} timeout - Timeout in seconds for job status checks.
|
|
209
209
|
* @returns {Promise<any>} The final job status or data.
|
|
210
210
|
*/
|
|
211
|
-
monitorJobStatus(jobId, headers,
|
|
211
|
+
monitorJobStatus(jobId, headers, checkInterval) {
|
|
212
212
|
return __awaiter(this, void 0, void 0, function* () {
|
|
213
213
|
while (true) {
|
|
214
214
|
const statusResponse = yield this.getRequest(this.apiUrl + `/v0/crawl/status/${jobId}`, headers);
|
|
@@ -223,10 +223,10 @@ export default class FirecrawlApp {
|
|
|
223
223
|
}
|
|
224
224
|
}
|
|
225
225
|
else if (["active", "paused", "pending", "queued"].includes(statusData.status)) {
|
|
226
|
-
if (
|
|
227
|
-
|
|
226
|
+
if (checkInterval < 2) {
|
|
227
|
+
checkInterval = 2;
|
|
228
228
|
}
|
|
229
|
-
yield new Promise((resolve) => setTimeout(resolve,
|
|
229
|
+
yield new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
|
|
230
230
|
}
|
|
231
231
|
else {
|
|
232
232
|
throw new Error(`Crawl job failed or was stopped. Status: ${statusData.status}`);
|
package/package.json
CHANGED
package/src/index.ts
CHANGED
|
@@ -175,7 +175,7 @@ export default class FirecrawlApp {
|
|
|
175
175
|
* @param {string} url - The URL to crawl.
|
|
176
176
|
* @param {Params | null} params - Additional parameters for the crawl request.
|
|
177
177
|
* @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
|
|
178
|
-
* @param {number}
|
|
178
|
+
* @param {number} pollInterval - Time in seconds for job status checks.
|
|
179
179
|
* @param {string} idempotencyKey - Optional idempotency key for the request.
|
|
180
180
|
* @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
|
|
181
181
|
*/
|
|
@@ -183,7 +183,7 @@ export default class FirecrawlApp {
|
|
|
183
183
|
url: string,
|
|
184
184
|
params: Params | null = null,
|
|
185
185
|
waitUntilDone: boolean = true,
|
|
186
|
-
|
|
186
|
+
pollInterval: number = 2,
|
|
187
187
|
idempotencyKey?: string
|
|
188
188
|
): Promise<CrawlResponse | any> {
|
|
189
189
|
const headers = this.prepareHeaders(idempotencyKey);
|
|
@@ -200,7 +200,7 @@ export default class FirecrawlApp {
|
|
|
200
200
|
if (response.status === 200) {
|
|
201
201
|
const jobId: string = response.data.jobId;
|
|
202
202
|
if (waitUntilDone) {
|
|
203
|
-
return this.monitorJobStatus(jobId, headers,
|
|
203
|
+
return this.monitorJobStatus(jobId, headers, pollInterval);
|
|
204
204
|
} else {
|
|
205
205
|
return { success: true, jobId };
|
|
206
206
|
}
|
|
@@ -296,7 +296,7 @@ export default class FirecrawlApp {
|
|
|
296
296
|
async monitorJobStatus(
|
|
297
297
|
jobId: string,
|
|
298
298
|
headers: AxiosRequestHeaders,
|
|
299
|
-
|
|
299
|
+
checkInterval: number
|
|
300
300
|
): Promise<any> {
|
|
301
301
|
while (true) {
|
|
302
302
|
const statusResponse: AxiosResponse = await this.getRequest(
|
|
@@ -314,10 +314,10 @@ export default class FirecrawlApp {
|
|
|
314
314
|
} else if (
|
|
315
315
|
["active", "paused", "pending", "queued"].includes(statusData.status)
|
|
316
316
|
) {
|
|
317
|
-
if (
|
|
318
|
-
|
|
317
|
+
if (checkInterval < 2) {
|
|
318
|
+
checkInterval = 2;
|
|
319
319
|
}
|
|
320
|
-
await new Promise((resolve) => setTimeout(resolve,
|
|
320
|
+
await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000)); // Wait for the specified timeout before checking again
|
|
321
321
|
} else {
|
|
322
322
|
throw new Error(
|
|
323
323
|
`Crawl job failed or was stopped. Status: ${statusData.status}`
|
package/types/index.d.ts
CHANGED
|
@@ -84,11 +84,11 @@ export default class FirecrawlApp {
|
|
|
84
84
|
* @param {string} url - The URL to crawl.
|
|
85
85
|
* @param {Params | null} params - Additional parameters for the crawl request.
|
|
86
86
|
* @param {boolean} waitUntilDone - Whether to wait for the crawl job to complete.
|
|
87
|
-
* @param {number}
|
|
87
|
+
* @param {number} pollInterval - Time in seconds for job status checks.
|
|
88
88
|
* @param {string} idempotencyKey - Optional idempotency key for the request.
|
|
89
89
|
* @returns {Promise<CrawlResponse | any>} The response from the crawl operation.
|
|
90
90
|
*/
|
|
91
|
-
crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean,
|
|
91
|
+
crawlUrl(url: string, params?: Params | null, waitUntilDone?: boolean, pollInterval?: number, idempotencyKey?: string): Promise<CrawlResponse | any>;
|
|
92
92
|
/**
|
|
93
93
|
* Checks the status of a crawl job using the Firecrawl API.
|
|
94
94
|
* @param {string} jobId - The job ID of the crawl operation.
|
|
@@ -122,7 +122,7 @@ export default class FirecrawlApp {
|
|
|
122
122
|
* @param {number} timeout - Timeout in seconds for job status checks.
|
|
123
123
|
* @returns {Promise<any>} The final job status or data.
|
|
124
124
|
*/
|
|
125
|
-
monitorJobStatus(jobId: string, headers: AxiosRequestHeaders,
|
|
125
|
+
monitorJobStatus(jobId: string, headers: AxiosRequestHeaders, checkInterval: number): Promise<any>;
|
|
126
126
|
/**
|
|
127
127
|
* Handles errors from API responses.
|
|
128
128
|
* @param {AxiosResponse} response - The response from the API.
|