@adobe/spacecat-shared-rum-api-client 2.34.1 → 2.35.0

package/CHANGELOG.md

@@ -1,3 +1,10 @@
+# [@adobe/spacecat-shared-rum-api-client-v2.35.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.34.1...@adobe/spacecat-shared-rum-api-client-v2.35.0) (2025-07-24)
+
+
+### Features
+
+* **rum-api-client:** add query stream ([#872](https://github.com/adobe/spacecat-shared/issues/872)) ([7702138](https://github.com/adobe/spacecat-shared/commit/77021388a298843f3ec2571c9a6f148184fa2814))
+
 # [@adobe/spacecat-shared-rum-api-client-v2.34.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.34.0...@adobe/spacecat-shared-rum-api-client-v2.34.1) (2025-07-24)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-rum-api-client",
-  "version": "2.34.1",
+  "version": "2.35.0",
   "description": "Shared modules of the Spacecat Services - Rum API client",
   "type": "module",
   "engines": {

package/src/common/rum-bundler-client.js

@@ -207,22 +207,7 @@ async function mergeBundlesWithSameId(bundles) {
 }
 /* c8 ignore end */
 
-async function fetchBundles(opts, log) {
-  const {
-    domain,
-    domainkey,
-    interval = 7,
-    granularity = GRANULARITY.DAILY,
-    checkpoints = [],
-    filterBotTraffic = true,
-    startTime,
-    endTime,
-  } = opts;
-
-  if (!hasText(domain) || !hasText(domainkey)) {
-    throw new Error('Missing required parameters');
-  }
-
+function validateDateRange(startTime, endTime) {
   // Validate startTime and endTime if provided
   if (startTime && endTime) {
     const start = parseDate(startTime);
@@ -236,27 +221,48 @@ async function fetchBundles(opts, log) {
       throw new Error('startTime must be before endTime');
     }
   }
+}
 
-  let urls = [];
-
+function generateURLs(domain, granularity, domainkey, startTime, endTime, interval) {
   if (startTime && endTime) {
+    validateDateRange(startTime, endTime);
     // Use custom date range
-    urls = generateUrlsForDateRange(startTime, endTime, domain, granularity, domainkey);
-  } else {
-    // Use existing interval-based logic
-    const multiplier = granularity.toUpperCase() === GRANULARITY.HOURLY ? ONE_HOUR : ONE_DAY;
-    const range = granularity.toUpperCase() === GRANULARITY.HOURLY
-      ? interval * HOURS_IN_DAY
-      : interval + 1;
-
-    const currentDate = new Date();
-
-    for (let i = 0; i < range; i += 1) {
-      const date = new Date(currentDate.getTime() - i * multiplier);
-      urls.push(constructUrl(domain, date, granularity, domainkey));
-    }
+    return generateUrlsForDateRange(startTime, endTime, domain, granularity, domainkey);
+  }
+
+  // Use existing interval-based logic
+  const multiplier = granularity.toUpperCase() === GRANULARITY.HOURLY ? ONE_HOUR : ONE_DAY;
+  const range = granularity.toUpperCase() === GRANULARITY.HOURLY
+    ? interval * HOURS_IN_DAY
+    : interval + 1;
+
+  const currentDate = new Date();
+  const urls = [];
+
+  for (let i = 0; i < range; i += 1) {
+    const date = new Date(currentDate.getTime() - i * multiplier);
+    urls.push(constructUrl(domain, date, granularity, domainkey));
+  }
+  return urls;
+}
+
+async function fetchBundles(opts, log) {
+  const {
+    domain,
+    domainkey,
+    interval = 7,
+    granularity = GRANULARITY.DAILY,
+    checkpoints = [],
+    filterBotTraffic = true,
+    startTime,
+    endTime,
+  } = opts;
+
+  if (!hasText(domain) || !hasText(domainkey)) {
+    throw new Error('Missing required parameters');
   }
 
+  const urls = generateURLs(domain, granularity, domainkey, startTime, endTime, interval);
   const chunks = getUrlChunks(urls, CHUNK_SIZE);
 
   let totalTransferSize = 0;
@@ -311,6 +317,88 @@ async function fetchBundles(opts, log) {
   return mergeBundlesWithSameId(result);
 }
 
+function createBundleStream(opts, log) {
+  const {
+    domain,
+    domainkey,
+    interval = 7,
+    granularity = GRANULARITY.DAILY,
+    checkpoints = [],
+    filterBotTraffic = true,
+    startTime,
+    endTime,
+    handler,
+  } = opts;
+
+  if (!hasText(domain) || !hasText(domainkey)) {
+    throw new Error('Missing required parameters');
+  }
+
+  const urls = generateURLs(domain, granularity, domainkey, startTime, endTime, interval);
+
+  return new ReadableStream({
+    async start(controller) {
+      const failedUrls = [];
+      let totalTransferSize = 0;
+      let bundlesCount = 0;
+      let lastCheckpoint = 0;
+
+      async function streamBundle(url) {
+        const response = await fetch(url);
+        totalTransferSize += parseInt(response.headers.get('content-length'), 10);
+
+        if (!response.ok) {
+          log.warn(`Failed to fetch URL: ${sanitizeURL(url)} - status: ${response.status}`);
+          failedUrls.push(url);
+          return;
+        }
+
+        const bundles = await response.json();
+
+        const filtered = bundles?.rumBundles?.filter(
+          (bundle) => !filterBotTraffic || !isBotTraffic(bundle),
+        ).map(filterEvents(checkpoints));
+
+        bundlesCount += filtered.length;
+        const currentCheckpoint = Math.floor(bundlesCount / 50000);
+
+        if (currentCheckpoint > lastCheckpoint) {
+          log.info(`Checkpoint: Fetched ${bundlesCount} bundles; resuming...`);
+          lastCheckpoint = currentCheckpoint;
+        }
+
+        const crunchedBundle = handler(filtered || []);
+        controller.enqueue(crunchedBundle);
+      }
+
+      async function worker() {
+        while (urls.length > 0) {
+          const url = urls.shift();
+          // eslint-disable-next-line no-await-in-loop
+          await streamBundle(url);
+        }
+      }
+
+      const workers = Array(CHUNK_SIZE)
+        .fill()
+        .map(() => worker());
+
+      await Promise.all(workers);
+
+      log.info(`Retrieved all RUM bundles. Total transfer size (in KB): ${(totalTransferSize / 1024).toFixed(2)}`);
+
+      // Add failedUrls to opts object for access by callers
+      if (failedUrls.length > 0) {
+        // eslint-disable-next-line no-param-reassign
+        opts.failedUrls = failedUrls;
+      }
+
+      controller.close();
+    },
+  });
+}
+
 export {
   fetchBundles,
+  createBundleStream,
 };

package/src/index.js

@@ -10,7 +10,7 @@
  * governing permissions and limitations under the License.
  */
 import { hasText, fetch } from '@adobe/spacecat-shared-utils';
-import { fetchBundles } from './common/rum-bundler-client.js';
+import { fetchBundles, createBundleStream } from './common/rum-bundler-client.js';
 import notfound from './functions/404.js';
 import notfoundInternalLinks from './functions/404-internal-links.js';
 import cwv from './functions/cwv.js';
@@ -177,4 +177,21 @@ export default class RUMAPIClient {
       throw new Error(`Multi query failed. Queries: ${JSON.stringify(queries)}, Opts: ${JSON.stringify(sanitize(opts))}. Reason: ${e.message}`);
     }
   }
+
+  async queryStream(query, opts) {
+    const { handler, checkpoints } = HANDLERS[query] || {};
+    if (!handler) throw new Error(`Unknown query ${query}`);
+
+    try {
+      const domainkey = await this._getDomainkey(opts);
+      return createBundleStream({
+        ...opts,
+        domainkey,
+        checkpoints,
+        handler,
+      }, this.log);
+    } catch (e) {
+      throw new Error(`Query stream '${query}' failed. Opts: ${JSON.stringify(sanitize(opts))}. Reason: ${e.message}`);
+    }
+  }
 }