@mendable/firecrawl-js 1.2.1 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (4) hide show
  1. package/build/cjs/index.js +10 -3
  2. package/build/esm/index.js +10 -3
  3. package/package.json +1 -1
  4. package/src/index.ts +10 -3
package/build/cjs/index.js CHANGED
@@ -236,18 +236,25 @@ class FirecrawlApp {
236
236
  */
237
237
  async monitorJobStatus(id, headers, checkInterval) {
238
238
  while (true) {
239
- const statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
239
+ let statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
240
240
  if (statusResponse.status === 200) {
241
- const statusData = statusResponse.data;
241
+ let statusData = statusResponse.data;
242
242
  if (statusData.status === "completed") {
243
243
  if ("data" in statusData) {
244
+ let data = statusData.data;
245
+ while ('next' in statusData) {
246
+ statusResponse = await this.getRequest(statusData.next, headers);
247
+ statusData = statusResponse.data;
248
+ data = data.concat(statusData.data);
249
+ }
250
+ statusData.data = data;
244
251
  return statusData;
245
252
  }
246
253
  else {
247
254
  throw new Error("Crawl job completed but no data was returned");
248
255
  }
249
256
  }
250
- else if (["active", "paused", "pending", "queued", "scraping"].includes(statusData.status)) {
257
+ else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
251
258
  checkInterval = Math.max(checkInterval, 2);
252
259
  await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000));
253
260
  }
package/build/esm/index.js CHANGED
@@ -230,18 +230,25 @@ export default class FirecrawlApp {
230
230
  */
231
231
  async monitorJobStatus(id, headers, checkInterval) {
232
232
  while (true) {
233
- const statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
233
+ let statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
234
234
  if (statusResponse.status === 200) {
235
- const statusData = statusResponse.data;
235
+ let statusData = statusResponse.data;
236
236
  if (statusData.status === "completed") {
237
237
  if ("data" in statusData) {
238
+ let data = statusData.data;
239
+ while ('next' in statusData) {
240
+ statusResponse = await this.getRequest(statusData.next, headers);
241
+ statusData = statusResponse.data;
242
+ data = data.concat(statusData.data);
243
+ }
244
+ statusData.data = data;
238
245
  return statusData;
239
246
  }
240
247
  else {
241
248
  throw new Error("Crawl job completed but no data was returned");
242
249
  }
243
250
  }
244
- else if (["active", "paused", "pending", "queued", "scraping"].includes(statusData.status)) {
251
+ else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
245
252
  checkInterval = Math.max(checkInterval, 2);
246
253
  await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000));
247
254
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mendable/firecrawl-js",
3
- "version": "1.2.1",
3
+ "version": "1.2.2",
4
4
  "description": "JavaScript SDK for Firecrawl API",
5
5
  "main": "build/cjs/index.js",
6
6
  "types": "types/index.d.ts",
package/src/index.ts CHANGED
@@ -454,20 +454,27 @@ export default class FirecrawlApp {
454
454
  checkInterval: number
455
455
  ): Promise<CrawlStatusResponse> {
456
456
  while (true) {
457
- const statusResponse: AxiosResponse = await this.getRequest(
457
+ let statusResponse: AxiosResponse = await this.getRequest(
458
458
  `${this.apiUrl}/v1/crawl/${id}`,
459
459
  headers
460
460
  );
461
461
  if (statusResponse.status === 200) {
462
- const statusData = statusResponse.data;
462
+ let statusData = statusResponse.data;
463
463
  if (statusData.status === "completed") {
464
464
  if ("data" in statusData) {
465
+ let data = statusData.data;
466
+ while ('next' in statusData) {
467
+ statusResponse = await this.getRequest(statusData.next, headers);
468
+ statusData = statusResponse.data;
469
+ data = data.concat(statusData.data);
470
+ }
471
+ statusData.data = data;
465
472
  return statusData;
466
473
  } else {
467
474
  throw new Error("Crawl job completed but no data was returned");
468
475
  }
469
476
  } else if (
470
- ["active", "paused", "pending", "queued", "scraping"].includes(statusData.status)
477
+ ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
471
478
  ) {
472
479
  checkInterval = Math.max(checkInterval, 2);
473
480
  await new Promise((resolve) =>