@adobe/spacecat-shared-rum-api-client 2.30.0 → 2.31.1

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [@adobe/spacecat-shared-rum-api-client-v2.31.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.31.0...@adobe/spacecat-shared-rum-api-client-v2.31.1) (2025-06-30)
+
+
+### Bug Fixes
+
+* check status before parsing rum bundles ([#807](https://github.com/adobe/spacecat-shared/issues/807)) ([36566e4](https://github.com/adobe/spacecat-shared/commit/36566e4ab1cb8e72e3e5e0673df3171c7793ab82))
+
+# [@adobe/spacecat-shared-rum-api-client-v2.31.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.30.0...@adobe/spacecat-shared-rum-api-client-v2.31.0) (2025-06-25)
+
+
+### Features
+
+* expose urls with most paid traffic  ([#825](https://github.com/adobe/spacecat-shared/issues/825)) ([88dde01](https://github.com/adobe/spacecat-shared/commit/88dde01843ae32643a54d93868b7ef7eec10a62a))
+
 # [@adobe/spacecat-shared-rum-api-client-v2.30.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.29.0...@adobe/spacecat-shared-rum-api-client-v2.30.0) (2025-06-25)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-rum-api-client",
-  "version": "2.30.0",
+  "version": "2.31.1",
   "description": "Shared modules of the Spacecat Services - Rum API client",
   "type": "module",
   "engines": {

package/src/common/rum-bundler-client.js

@@ -57,6 +57,20 @@ function filterEvents(checkpoints = []) {
   };
 }
 
+function sanitizeURL(url) {
+  try {
+    const parsedUrl = new URL(url);
+    if (parsedUrl.searchParams.has('domainkey')) {
+      parsedUrl.searchParams.set('domainkey', 'redacted');
+    }
+    return parsedUrl.toString();
+    /* c8 ignore next 4 */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  } catch (e) {
+    return url;
+  }
+}
+
 function constructUrl(domain, date, granularity, domainkey) {
   const year = date.getUTCFullYear();
   const month = (date.getUTCMonth() + 1).toString().padStart(2, '0');
@@ -246,6 +260,7 @@ async function fetchBundles(opts, log) {
   const chunks = getUrlChunks(urls, CHUNK_SIZE);
 
   let totalTransferSize = 0;
+  const failedUrls = [];
 
   const result = [];
   for (const chunk of chunks) {
@@ -255,7 +270,21 @@ async function fetchBundles(opts, log) {
       totalTransferSize += parseInt(response.headers.get('content-length'), 10);
       return response;
     }));
-    const bundles = await Promise.all(responses.map((response) => response.json()));
+
+    const bundlesRaw = await Promise.all(
+      responses.map(async (response, index) => {
+        if (response.ok) {
+          return response.json();
+        } else {
+          const failedUrl = response.url || chunk[index];
+          log.warn(`Skipping response at index ${index}: status ${response.status} - url: ${sanitizeURL(failedUrl)}`);
+          failedUrls.push(failedUrl);
+          return null;
+        }
+      }),
+    );
+
+    const bundles = bundlesRaw.filter(Boolean);
     bundles.forEach((b) => {
       b.rumBundles
         .filter((bundle) => !filterBotTraffic || !isBotTraffic(bundle))
@@ -264,6 +293,13 @@ async function fetchBundles(opts, log) {
     });
   }
   log.info(`Retrieved RUM bundles. Total transfer size (in KB): ${(totalTransferSize / 1024).toFixed(2)}`);
+
+  // Add failedUrls to opts object for access by callers
+  if (failedUrls.length > 0) {
+    // eslint-disable-next-line no-param-reassign
+    opts.failedUrls = failedUrls;
+  }
+
   return mergeBundlesWithSameId(result);
 }
 

package/src/functions/traffic-metrics.js

@@ -15,8 +15,21 @@ import { DELIMITER, generateKey, loadBundles } from '../utils.js';
 import { classifyTraffic } from '../common/traffic.js';
 import { getPageType, isConsentClick } from '../common/page.js';
 
-function getTrafficSource(bundle) {
-  return classifyTraffic(bundle).type;
+function getTrafficSource(bundle, memo) {
+  const id = `${bundle.id}-${bundle.url}-${bundle.time}`;
+  if (id in memo) {
+    return memo[id];
+  }
+  const result = classifyTraffic(bundle);
+  // eslint-disable-next-line no-param-reassign
+  memo[id] = result;
+  return result;
+}
+
+function getTrafficSourceKey(bundle, memo) {
+  const classifyResult = getTrafficSource(bundle, memo);
+  const { type, category, vendor } = classifyResult;
+  return `${type}:${category}:${vendor}`;
 }
 
 function getDeviceType(bundle) {
@@ -35,36 +48,54 @@ function addPageTypeDeviceTypeFacet(dataChunks, pageTypes) {
   });
 }
 
-function addPageTypeTrafficSourceDeviceTypes(dataChunks, pageTypes) {
+function addPageTypeTrafficSourceDeviceTypes(dataChunks, pageTypes, memo) {
   dataChunks.addFacet('pageTrafficDeviceTypes', (bundle) => {
     const deviceType = getDeviceType(bundle);
     const pageType = getPageType(bundle, pageTypes);
-    return generateKey(pageType, getTrafficSource(bundle), deviceType);
+    return generateKey(pageType, getTrafficSourceKey(bundle, memo), deviceType);
   });
 }
 
-function addPageTypeTrafficSourceFacet(dataChunks, pageTypes) {
+function addPageTypeTrafficSourceFacet(dataChunks, pageTypes, memo) {
   dataChunks.addFacet('pageTypeTrafficSources', (bundle) => {
     const pageType = getPageType(bundle, pageTypes);
-    return generateKey(pageType, getTrafficSource(bundle));
+    return generateKey(pageType, getTrafficSourceKey(bundle, memo));
   });
 }
 
-function handler(bundles, options = { pageTypes: null }) {
+/**
+ * Handler for traffic metrics.
+ * @param {Array} bundles - The RUM bundles.
+ * @param {Object} options - Options object.
+ * @param {Object} [options.pageTypes] - Page type regex or mapping.
+ * @param {string} [options.trafficType] - Eg, 'paid', 'earned', 'owned', 'all'. Defaults to 'all'.
+ */
+function handler(bundles, options = { pageTypes: null, trafficType: 'all' }) {
   const dataChunks = new DataChunks();
-  const { pageTypes: pageTypeOpt } = options;
+  const trafficSourceMemo = {};
+  const { pageTypes: pageTypeOpt, trafficType = 'all' } = options;
+
+  let filteredBundles = bundles;
+  if (trafficType && trafficType !== 'all') {
+    filteredBundles = bundles
+      .filter((bundle) => getTrafficSource(bundle, trafficSourceMemo).type === trafficType);
+  }
+
+  const getTS = (bundle) => getTrafficSourceKey(bundle, trafficSourceMemo);
 
-  loadBundles(bundles, dataChunks);
+  loadBundles(filteredBundles, dataChunks);
 
   const metricFilter = (metrics) => {
-    const { ctr, enters, sumOfAllClicks } = metrics;
+    const {
+      ctr, enters, sumOfAllClicks, facet,
+    } = metrics;
     return {
-      ctr: ctr.sum / ctr.weight,
+      ctr: ctr.weight !== 0 ? ctr.sum / ctr.weight : 0,
       clickedSessions: ctr.sum,
-      totalSessions: ctr.weight,
+      pageViews: facet.weight,
       sessionsWithEnter: enters.sum,
-      clicksOverViews: ctr.weight ? ctr.sum / ctr.weight : 0,
-      bounceRate: ctr.weight ? (1 - (ctr.sum / ctr.weight)) : 0,
+      clicksOverViews: ctr.weight !== 0 ? ctr.sum / ctr.weight : 0,
+      bounceRate: ctr.weight !== 0 ? (1 - (ctr.sum / ctr.weight)) : 1,
       totalNumClicks: sumOfAllClicks.sum,
       avgClicksPerSession: ctr.sum ? sumOfAllClicks.sum / ctr.sum : 0,
     };
@@ -72,25 +103,25 @@ function handler(bundles, options = { pageTypes: null }) {
 
   dataChunks.addFacet('urls', (bundle) => bundle.url);
 
-  dataChunks.addFacet('trafficSources', (bundle) => getTrafficSource(bundle));
+  dataChunks.addFacet('trafficSources', (bundle) => getTS(bundle));
 
-  dataChunks.addFacet('urlTrafficSources', (bundle) => generateKey(bundle.url, getTrafficSource(bundle)));
+  dataChunks.addFacet('urlTrafficSources', (bundle) => generateKey(bundle.url, getTS(bundle)));
 
   dataChunks.addFacet('urlDeviceTypes', (bundle) => generateKey(bundle.url, getDeviceType(bundle)));
 
   dataChunks.addFacet('deviceTypes', (bundle) => getDeviceType(bundle));
 
-  dataChunks.addFacet('urlTrafficSourceDeviceTypes', (bundle) => generateKey(bundle.url, getTrafficSource(bundle), getDeviceType(bundle)));
+  dataChunks.addFacet('urlTrafficSourceDeviceTypes', (bundle) => generateKey(bundle.url, getTS(bundle), getDeviceType(bundle)));
 
-  dataChunks.addFacet('deviceTypeTrafficSources', (bundle) => generateKey(getDeviceType(bundle), getTrafficSource(bundle)));
+  dataChunks.addFacet('deviceTypeTrafficSources', (bundle) => generateKey(getDeviceType(bundle), getTS(bundle)));
 
   addPageTypeFacet(dataChunks, pageTypeOpt);
 
-  addPageTypeTrafficSourceFacet(dataChunks, pageTypeOpt);
+  addPageTypeTrafficSourceFacet(dataChunks, pageTypeOpt, trafficSourceMemo);
 
   addPageTypeDeviceTypeFacet(dataChunks, pageTypeOpt);
 
-  addPageTypeTrafficSourceDeviceTypes(dataChunks, pageTypeOpt);
+  addPageTypeTrafficSourceDeviceTypes(dataChunks, pageTypeOpt, trafficSourceMemo);
 
   dataChunks.addSeries('ctr', (bundle) => {
     const isClicked = bundle.events.some((e) => e.checkpoint === 'click');
@@ -111,100 +142,108 @@ function handler(bundles, options = { pageTypes: null }) {
     return containsEnter ? bundle.weight : 0;
   });
 
-  const urls = dataChunks.facets.urls.map((facet) => {
-    const url = facet.value;
-    return {
-      ...metricFilter(facet.metrics),
-      url,
-    };
-  });
+  const urls = dataChunks.facets.urls.map((facet) => ({
+    ...metricFilter({ ...facet.metrics, facet }),
+    url: facet.value,
+    urls: [facet.value],
+  }));
 
   const pageType = dataChunks.facets.pageType.map((facet) => {
     const type = facet.value;
     return {
-      ...metricFilter(facet.metrics),
+      ...metricFilter({ ...facet.metrics, facet }),
       type,
+      urls: [...new Set(facet.entries.map((b) => b.url))],
     };
   });
 
   const deviceTypes = dataChunks.facets.deviceTypes.map((facet) => {
     const deviceType = facet.value;
     return {
-      ...metricFilter(facet.metrics),
+      ...metricFilter({ ...facet.metrics, facet }),
       deviceType,
+      urls: [...new Set(facet.entries.map((b) => b.url))],
     };
   });
 
   const urlDeviceTypes = dataChunks.facets.urlDeviceTypes.map((facet) => {
     const [url, deviceType] = facet.value.split(DELIMITER);
     return {
-      ...metricFilter(facet.metrics),
+      ...metricFilter({ ...facet.metrics, facet }),
       url,
       deviceType,
+      urls: [...new Set(facet.entries.map((b) => b.url))],
     };
   });
 
   const trafficSources = dataChunks.facets.trafficSources.map((facet) => {
     const source = facet.value;
     return {
-      ...metricFilter(facet.metrics),
+      ...metricFilter({ ...facet.metrics, facet }),
       source,
+      urls: [...new Set(facet.entries.map((b) => b.url))],
     };
   });
 
   const urlTrafficSources = dataChunks.facets.urlTrafficSources.map((facet) => {
     const [url, source] = facet.value.split(DELIMITER);
     return {
-      ...metricFilter(facet.metrics),
+      ...metricFilter({ ...facet.metrics, facet }),
       url,
       source,
+      urls: [...new Set(facet.entries.map((b) => b.url))],
     };
   });
 
   const urlTrafficSourceDeviceTypes = dataChunks.facets.urlTrafficSourceDeviceTypes.map((facet) => {
     const [url, source, deviceType] = facet.value.split(DELIMITER);
     return {
-      ...metricFilter(facet.metrics),
+      ...metricFilter({ ...facet.metrics, facet }),
       url,
       source,
       deviceType,
+      urls: [...new Set(facet.entries.map((b) => b.url))],
     };
   });
 
   const pageTypeTrafficSources = dataChunks.facets.pageTypeTrafficSources.map((facet) => {
     const [type, source] = facet.value.split(DELIMITER);
     return {
-      ...metricFilter(facet.metrics),
+      ...metricFilter({ ...facet.metrics, facet }),
       type,
       source,
+      urls: [...new Set(facet.entries.map((b) => b.url))],
     };
   });
 
   const pageTypeDeviceTypes = dataChunks.facets.pageTypeDeviceTypes.map((facet) => {
     const [type, deviceType] = facet.value.split(DELIMITER);
     return {
-      ...metricFilter(facet.metrics),
+      ...metricFilter({ ...facet.metrics, facet }),
       type,
       deviceType,
+      urls: [...new Set(facet.entries.map((b) => b.url))],
     };
   });
 
   const deviceTypeTrafficSources = dataChunks.facets.deviceTypeTrafficSources.map((facet) => {
     const [deviceType, source] = facet.value.split(DELIMITER);
     return {
-      ...metricFilter(facet.metrics),
+      ...metricFilter({ ...facet.metrics, facet }),
       deviceType,
       source,
+      urls: [...new Set(facet.entries.map((b) => b.url))],
     };
   });
 
   const pageTrafficDeviceTypes = dataChunks.facets.pageTrafficDeviceTypes.map((facet) => {
     const [type, source, deviceType] = facet.value.split(DELIMITER);
     return {
-      ...metricFilter(facet.metrics),
+      ...metricFilter({ ...facet.metrics, facet }),
       type,
       source,
       deviceType,
+      urls: [...new Set(facet.entries.map((b) => b.url))],
     };
   });
 

package/src/index.js

@@ -122,7 +122,6 @@ export default class RUMAPIClient {
 
     try {
       const domainkey = await this._getDomainkey(opts);
-
       const bundles = await fetchBundles({
         ...opts,
         domainkey,
@@ -130,7 +129,6 @@ export default class RUMAPIClient {
       }, this.log);
 
       this.log.info(`Query "${query}" fetched ${bundles.length} bundles`);
-
       return handler(bundles, opts);
     } catch (e) {
       throw new Error(`Query '${query}' failed. Opts: ${JSON.stringify(sanitize(opts))}. Reason: ${e.message}`);
@@ -164,7 +162,6 @@ export default class RUMAPIClient {
       }, this.log);
 
       const results = {};
-
       this.log.info(`Multi query ${JSON.stringify(queries.join(', '))} fetched ${bundles.length} bundles`);
 
       // Execute each query handler sequentially