@adobe/spacecat-shared-scrape-client 1.0.1 → 1.0.3

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [@adobe/spacecat-shared-scrape-client-v1.0.3](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-scrape-client-v1.0.2...@adobe/spacecat-shared-scrape-client-v1.0.3) (2025-07-21)
+
+
+### Bug Fixes
+
+* ScrapeClient handle large url lists ([#854](https://github.com/adobe/spacecat-shared/issues/854)) ([d0768db](https://github.com/adobe/spacecat-shared/commit/d0768db101d65bc604c64473648cba0344612025))
+
+# [@adobe/spacecat-shared-scrape-client-v1.0.2](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-scrape-client-v1.0.1...@adobe/spacecat-shared-scrape-client-v1.0.2) (2025-07-19)
+
+
+### Bug Fixes
+
+* **deps:** update external fixes ([#859](https://github.com/adobe/spacecat-shared/issues/859)) ([7ca9099](https://github.com/adobe/spacecat-shared/commit/7ca90994d61d07f71e580301365447b94ad07a52))
+
 # [@adobe/spacecat-shared-scrape-client-v1.0.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-scrape-client-v1.0.0...@adobe/spacecat-shared-scrape-client-v1.0.1) (2025-07-12)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-scrape-client",
-  "version": "1.0.1",
+  "version": "1.0.3",
   "description": "Shared modules of the Spacecat Services - Scrape Client",
   "type": "module",
   "engines": {
@@ -42,7 +42,7 @@
   "devDependencies": {
     "chai": "5.2.1",
     "chai-as-promised": "8.0.1",
-    "nock": "14.0.5",
+    "nock": "14.0.6",
     "sinon": "20.0.0",
     "sinon-chai": "4.0.0",
     "typescript": "5.8.3"

package/src/clients/scrape-job-supervisor.js

@@ -35,6 +35,7 @@ function ScrapeJobSupervisor(services, config) {
   const {
     queues = [], // Array of scrape queues
     scrapeWorkerQueue, // URL of the scrape worker queue
+    maxUrlsPerMessage,
   } = config;
 
   /**
@@ -129,9 +130,24 @@ function ScrapeJobSupervisor(services, config) {
   }
 
   /**
-   * Queue all URLs as a single message for processing by another function. This will enable
-   * the controller to respond with a new job ID ASAP, while the individual URLs are queued up
-   * asynchronously.
+   * Split an array of URLs into batches of a specified size.
+   * @param urls
+   * @param batchSize
+   * @returns {*[]}
+   */
+  function splitUrlsIntoBatches(urls, batchSize = 1000) {
+    const batches = [];
+    for (let i = 0; i < urls.length; i += batchSize) {
+      batches.push(urls.slice(i, i + batchSize));
+    }
+    log.info(`Split ${urls.length} URLs into ${batches.length} batches of size ${batchSize}.`);
+    return batches;
+  }
+
+  /**
+   * Queue all URLs for processing by another function. Splits URL-Arrays > 1000 into multiple
+   * messages. This will enable the controller to respond with a new job ID ASAP, while the
+   * individual URLs are queued up asynchronously.
    * @param {Array<string>} urls - Array of URL records to queue.
    * @param {object} scrapeJob - The scrape job record.
    * @param {object} customHeaders - Optional custom headers to be sent with each request.
@@ -143,17 +159,32 @@ function ScrapeJobSupervisor(services, config) {
 
     const options = scrapeJob.getOptions();
     const processingType = scrapeJob.getProcessingType();
+    const totalUrlCount = urls.length;
+    const baseUrl = scrapeJob.getBaseURL();
+    let urlBatches = [];
 
-    // Send a single message containing all URLs and the new job ID
-    const message = {
-      processingType,
-      jobId: scrapeJob.getId(),
-      urls,
-      customHeaders,
-      options,
-    };
+    // If there are more than 1000 URLs, split them into multiple messages
+    if (totalUrlCount > maxUrlsPerMessage) {
+      urlBatches = splitUrlsIntoBatches(urls, maxUrlsPerMessage);
+      log.info(`Queuing ${totalUrlCount} URLs for scrape in ${urlBatches.length} messages.`);
+    } else {
+      // If there are 1000 or fewer URLs, we can send them all in a single message
+      log.info(`Queuing ${totalUrlCount} URLs for scrape in a single message.`);
+      urlBatches = [urls]; // Wrap in an array to maintain consistent structure
+    }
+
+    for (const batch of urlBatches) {
+      const message = {
+        processingType,
+        jobId: scrapeJob.getId(),
+        batch,
+        customHeaders,
+        options,
+      };
 
-    await sqs.sendMessage(scrapeWorkerQueue, message);
+      // eslint-disable-next-line no-await-in-loop
+      await sqs.sendMessage(scrapeWorkerQueue, message, baseUrl);
+    }
   }
 
   /**