@mendable/firecrawl-js 1.2.0 → 1.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. package/build/cjs/index.js +10 -3
  2. package/build/esm/index.js +10 -3
  3. package/package.json +1 -1
  4. package/src/index.ts +11 -3
  5. package/types/index.d.ts +1 -0
package/build/cjs/index.js CHANGED
@@ -236,18 +236,25 @@ class FirecrawlApp {
236
236
  */
237
237
  async monitorJobStatus(id, headers, checkInterval) {
238
238
  while (true) {
239
- const statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
239
+ let statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
240
240
  if (statusResponse.status === 200) {
241
- const statusData = statusResponse.data;
241
+ let statusData = statusResponse.data;
242
242
  if (statusData.status === "completed") {
243
243
  if ("data" in statusData) {
244
+ let data = statusData.data;
245
+ while ('next' in statusData) {
246
+ statusResponse = await this.getRequest(statusData.next, headers);
247
+ statusData = statusResponse.data;
248
+ data = data.concat(statusData.data);
249
+ }
250
+ statusData.data = data;
244
251
  return statusData;
245
252
  }
246
253
  else {
247
254
  throw new Error("Crawl job completed but no data was returned");
248
255
  }
249
256
  }
250
- else if (["active", "paused", "pending", "queued", "scraping"].includes(statusData.status)) {
257
+ else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
251
258
  checkInterval = Math.max(checkInterval, 2);
252
259
  await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000));
253
260
  }
package/build/esm/index.js CHANGED
@@ -230,18 +230,25 @@ export default class FirecrawlApp {
230
230
  */
231
231
  async monitorJobStatus(id, headers, checkInterval) {
232
232
  while (true) {
233
- const statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
233
+ let statusResponse = await this.getRequest(`${this.apiUrl}/v1/crawl/${id}`, headers);
234
234
  if (statusResponse.status === 200) {
235
- const statusData = statusResponse.data;
235
+ let statusData = statusResponse.data;
236
236
  if (statusData.status === "completed") {
237
237
  if ("data" in statusData) {
238
+ let data = statusData.data;
239
+ while ('next' in statusData) {
240
+ statusResponse = await this.getRequest(statusData.next, headers);
241
+ statusData = statusResponse.data;
242
+ data = data.concat(statusData.data);
243
+ }
244
+ statusData.data = data;
238
245
  return statusData;
239
246
  }
240
247
  else {
241
248
  throw new Error("Crawl job completed but no data was returned");
242
249
  }
243
250
  }
244
- else if (["active", "paused", "pending", "queued", "scraping"].includes(statusData.status)) {
251
+ else if (["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)) {
245
252
  checkInterval = Math.max(checkInterval, 2);
246
253
  await new Promise((resolve) => setTimeout(resolve, checkInterval * 1000));
247
254
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@mendable/firecrawl-js",
3
- "version": "1.2.0",
3
+ "version": "1.2.2",
4
4
  "description": "JavaScript SDK for Firecrawl API",
5
5
  "main": "build/cjs/index.js",
6
6
  "types": "types/index.d.ts",
package/src/index.ts CHANGED
@@ -111,6 +111,7 @@ export interface CrawlParams {
111
111
  allowExternalLinks?: boolean;
112
112
  ignoreSitemap?: boolean;
113
113
  scrapeOptions?: ScrapeParams;
114
+ webhook?: string;
114
115
  }
115
116
 
116
117
  /**
@@ -453,20 +454,27 @@ export default class FirecrawlApp {
453
454
  checkInterval: number
454
455
  ): Promise<CrawlStatusResponse> {
455
456
  while (true) {
456
- const statusResponse: AxiosResponse = await this.getRequest(
457
+ let statusResponse: AxiosResponse = await this.getRequest(
457
458
  `${this.apiUrl}/v1/crawl/${id}`,
458
459
  headers
459
460
  );
460
461
  if (statusResponse.status === 200) {
461
- const statusData = statusResponse.data;
462
+ let statusData = statusResponse.data;
462
463
  if (statusData.status === "completed") {
463
464
  if ("data" in statusData) {
465
+ let data = statusData.data;
466
+ while ('next' in statusData) {
467
+ statusResponse = await this.getRequest(statusData.next, headers);
468
+ statusData = statusResponse.data;
469
+ data = data.concat(statusData.data);
470
+ }
471
+ statusData.data = data;
464
472
  return statusData;
465
473
  } else {
466
474
  throw new Error("Crawl job completed but no data was returned");
467
475
  }
468
476
  } else if (
469
- ["active", "paused", "pending", "queued", "scraping"].includes(statusData.status)
477
+ ["active", "paused", "pending", "queued", "waiting", "scraping"].includes(statusData.status)
470
478
  ) {
471
479
  checkInterval = Math.max(checkInterval, 2);
472
480
  await new Promise((resolve) =>
package/types/index.d.ts CHANGED
@@ -103,6 +103,7 @@ export interface CrawlParams {
103
103
  allowExternalLinks?: boolean;
104
104
  ignoreSitemap?: boolean;
105
105
  scrapeOptions?: ScrapeParams;
106
+ webhook?: string;
106
107
  }
107
108
  /**
108
109
  * Response interface for crawling operations.