@adobe/spacecat-shared-rum-api-client 2.31.0 → 2.32.0

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [@adobe/spacecat-shared-rum-api-client-v2.32.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.31.1...@adobe/spacecat-shared-rum-api-client-v2.32.0) (2025-07-11)
+
+
+### Features
+
+* add RUM API Client handler for paid traffic analysis import ([#841](https://github.com/adobe/spacecat-shared/issues/841)) ([6e6656e](https://github.com/adobe/spacecat-shared/commit/6e6656efb7e9741b52d26c21ea910988b219e2cc))
+
+# [@adobe/spacecat-shared-rum-api-client-v2.31.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.31.0...@adobe/spacecat-shared-rum-api-client-v2.31.1) (2025-06-30)
+
+
+### Bug Fixes
+
+* check status before parsing rum bundles ([#807](https://github.com/adobe/spacecat-shared/issues/807)) ([36566e4](https://github.com/adobe/spacecat-shared/commit/36566e4ab1cb8e72e3e5e0673df3171c7793ab82))
+
 # [@adobe/spacecat-shared-rum-api-client-v2.31.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.30.0...@adobe/spacecat-shared-rum-api-client-v2.31.0) (2025-06-25)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-rum-api-client",
-  "version": "2.31.0",
+  "version": "2.32.0",
   "description": "Shared modules of the Spacecat Services - Rum API client",
   "type": "module",
   "engines": {

package/src/common/rum-bundler-client.js

@@ -57,6 +57,20 @@ function filterEvents(checkpoints = []) {
   };
 }
 
+function sanitizeURL(url) {
+  try {
+    const parsedUrl = new URL(url);
+    if (parsedUrl.searchParams.has('domainkey')) {
+      parsedUrl.searchParams.set('domainkey', 'redacted');
+    }
+    return parsedUrl.toString();
+    /* c8 ignore next 4 */
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+  } catch (e) {
+    return url;
+  }
+}
+
 function constructUrl(domain, date, granularity, domainkey) {
   const year = date.getUTCFullYear();
   const month = (date.getUTCMonth() + 1).toString().padStart(2, '0');
@@ -246,6 +260,7 @@ async function fetchBundles(opts, log) {
   const chunks = getUrlChunks(urls, CHUNK_SIZE);
 
   let totalTransferSize = 0;
+  const failedUrls = [];
 
   const result = [];
   for (const chunk of chunks) {
@@ -255,7 +270,21 @@ async function fetchBundles(opts, log) {
       totalTransferSize += parseInt(response.headers.get('content-length'), 10);
       return response;
     }));
-    const bundles = await Promise.all(responses.map((response) => response.json()));
+
+    const bundlesRaw = await Promise.all(
+      responses.map(async (response, index) => {
+        if (response.ok) {
+          return response.json();
+        } else {
+          const failedUrl = response.url || chunk[index];
+          log.warn(`Skipping response at index ${index}: status ${response.status} - url: ${sanitizeURL(failedUrl)}`);
+          failedUrls.push(failedUrl);
+          return null;
+        }
+      }),
+    );
+
+    const bundles = bundlesRaw.filter(Boolean);
     bundles.forEach((b) => {
       b.rumBundles
         .filter((bundle) => !filterBotTraffic || !isBotTraffic(bundle))
@@ -264,6 +293,13 @@ async function fetchBundles(opts, log) {
     });
   }
   log.info(`Retrieved RUM bundles. Total transfer size (in KB): ${(totalTransferSize / 1024).toFixed(2)}`);
+
+  // Add failedUrls to opts object for access by callers
+  if (failedUrls.length > 0) {
+    // eslint-disable-next-line no-param-reassign
+    opts.failedUrls = failedUrls;
+  }
+
   return mergeBundlesWithSameId(result);
 }
 

package/src/functions/traffic-analysis.js

@@ -0,0 +1,129 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { utils } from '@adobe/rum-distiller';
+import { classifyTraffic } from '../common/traffic.js';
+
+function getUTM(bundle, type) {
+  return bundle.events
+    .find((e) => e.checkpoint === 'utm' && e.source === `utm_${type}`)
+    ?.target || null;
+}
+
+function getCWV(bundle, metric) {
+  const measurements = bundle.events
+    .filter((e) => e.checkpoint === `cwv-${metric}`)
+    .map((e) => e.value);
+
+  return measurements.length > 0 ? Math.max(...measurements) : null;
+}
+
+function containsEngagedScroll(bundle) {
+  return bundle.events
+    .some((e) => (e.checkpoint === 'viewmedia' || e.checkpoint === 'viewblock') && e.timeDelta >= 10000)
+    ? 1 : 0;
+}
+
+function getNotFound(bundle) {
+  return bundle.events
+    .find((e) => e.checkpoint === '404')
+    ?.source || null;
+}
+
+function getReferrer(bundle) {
+  const enterCheckpoint = bundle.events
+    .find((e) => e.checkpoint === 'enter')
+    ?.source;
+
+  const navigateCheckpoint = bundle.events
+    .find((e) => e.checkpoint === 'navigate')
+    ?.source;
+
+  return navigateCheckpoint || enterCheckpoint || null;
+}
+
+function getClicked(bundle) {
+  const latestClickEvent = bundle.events
+    .filter((e) => e.checkpoint === 'click')
+    .reduce((latest, current) => {
+      if (!latest || !latest.timeDelta) return current;
+      if (current?.timeDelta > latest.timeDelta) return current;
+      return latest;
+    }, null);
+
+  if (!latestClickEvent) return 0;
+
+  const isConsentClick = !!utils.reclassifyConsent(latestClickEvent).vendor;
+
+  if (isConsentClick) return 0;
+
+  return 1;
+}
+
+function getConsent(bundle) {
+  const consentBannerStatus = bundle.events
+    .find((e) => e.checkpoint === 'consent')
+    ?.target;
+
+  const consentClick = bundle.events.find((e) => e.checkpoint === 'click' && utils.reclassifyConsent(e).vendor);
+
+  if (!consentClick) return consentBannerStatus || null;
+
+  return utils.reclassifyConsent(consentClick).target;
+}
+
+function trafficType(bundle, memo) {
+  const key = `${bundle.id}${bundle.url}${bundle.time}`;
+  if (memo[key]) return memo[key];
+
+  const type = classifyTraffic(bundle);
+  // eslint-disable-next-line no-param-reassign
+  memo[key] = type;
+  return type;
+}
+
+async function handler(bundles) {
+  const memo = {};
+
+  const result = bundles.map((bundle) => {
+    /* eslint-disable camelcase */
+    const trafficData = trafficType(bundle, memo);
+    const clicked = getClicked(bundle);
+
+    return {
+      path: new URL(bundle.url).pathname,
+      trf_type: trafficData.type,
+      trf_channel: trafficData.category,
+      trf_platform: trafficData.vendor || null,
+      device: bundle.userAgent.split(':')[0],
+      utm_source: getUTM(bundle, 'source'),
+      utm_medium: getUTM(bundle, 'medium'),
+      utm_campaign: getUTM(bundle, 'campaign'),
+      referrer: getReferrer(bundle),
+      consent: getConsent(bundle),
+      notfound: getNotFound(bundle),
+      pageviews: bundle.weight,
+      clicked,
+      engaged: containsEngagedScroll(bundle) || clicked,
+      lcp: getCWV(bundle, 'lcp'),
+      inp: getCWV(bundle, 'inp'),
+      cls: getCWV(bundle, 'cls'),
+    };
+    /* eslint-enable camelcase */
+  });
+
+  return result;
+}
+
+export default {
+  handler,
+};

package/src/index.js

@@ -24,6 +24,7 @@ import trafficMetrics from './functions/traffic-metrics.js';
 import rageclick from './functions/opportunities/rageclick.js';
 import highInorganicHighBounceRate from './functions/opportunities/high-inorganic-high-bounce-rate.js';
 import highOrganicLowCtr from './functions/opportunities/high-organic-low-ctr.js';
+import trafficAnalysis from './functions/traffic-analysis.js';
 
 // exported for tests
 export const RUM_BUNDLER_API_HOST = 'https://bundles.aem.page';
@@ -42,6 +43,7 @@ const HANDLERS = {
   'high-organic-low-ctr': highOrganicLowCtr,
   pageviews,
   trafficMetrics,
+  'traffic-analysis': trafficAnalysis,
 };
 
 function sanitize(opts) {
@@ -122,7 +124,6 @@ export default class RUMAPIClient {
 
     try {
       const domainkey = await this._getDomainkey(opts);
-
       const bundles = await fetchBundles({
         ...opts,
         domainkey,
@@ -130,7 +131,6 @@ export default class RUMAPIClient {
       }, this.log);
 
       this.log.info(`Query "${query}" fetched ${bundles.length} bundles`);
-
       return handler(bundles, opts);
     } catch (e) {
       throw new Error(`Query '${query}' failed. Opts: ${JSON.stringify(sanitize(opts))}. Reason: ${e.message}`);
@@ -164,7 +164,6 @@ export default class RUMAPIClient {
       }, this.log);
 
       const results = {};
-
       this.log.info(`Multi query ${JSON.stringify(queries.join(', '))} fetched ${bundles.length} bundles`);
 
       // Execute each query handler sequentially