@adobe/spacecat-shared-rum-api-client 2.11.0 → 2.12.1

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [@adobe/spacecat-shared-rum-api-client-v2.12.1](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.12.0...@adobe/spacecat-shared-rum-api-client-v2.12.1) (2024-11-23)
+
+
+### Bug Fixes
+
+* **deps:** update external fixes ([#454](https://github.com/adobe/spacecat-shared/issues/454)) ([325cf8d](https://github.com/adobe/spacecat-shared/commit/325cf8dded5fcabadaf7d8fdd510d33aeafd08a7))
+
+# [@adobe/spacecat-shared-rum-api-client-v2.12.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.11.0...@adobe/spacecat-shared-rum-api-client-v2.12.0) (2024-11-21)
+
+
+### Features
+
+* **data-cruncher:** port queries to the data cruncher from RUM Explorer ([#375](https://github.com/adobe/spacecat-shared/issues/375)) ([599ebfc](https://github.com/adobe/spacecat-shared/commit/599ebfc7a7e14580ee1af9e42eac0d208acd5d94))
+
 # [@adobe/spacecat-shared-rum-api-client-v2.11.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.10.0...@adobe/spacecat-shared-rum-api-client-v2.11.0) (2024-11-20)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-rum-api-client",
-  "version": "2.11.0",
+  "version": "2.12.1",
   "description": "Shared modules of the Spacecat Services - Rum API client",
   "type": "module",
   "engines": {
@@ -39,8 +39,8 @@
     "@adobe/helix-shared-wrap": "2.0.2",
     "@adobe/helix-universal": "5.0.6",
     "@adobe/spacecat-shared-utils": "1.22.4",
+    "@adobe/rum-distiller": "1.9.0",
     "aws4": "1.13.2",
-    "d3-array": "3.2.4",
     "urijs": "^1.19.11"
   },
   "devDependencies": {
@@ -49,6 +49,6 @@
     "nock": "13.5.6",
     "sinon": "19.0.2",
     "sinon-chai": "4.0.0",
-    "typescript": "5.6.3"
+    "typescript": "5.7.2"
   }
 }

package/src/common/aggregateFns.js

@@ -12,48 +12,6 @@
 
 import { extractTrafficHints, classifyVendor, getSecondLevelDomain } from './traffic.js';
 
-/**
- * Calculates the total page views by URL from an array of bundles.
- * @param {Array<Object>} bundles - An array of RUM bundles (NOT Flat bundles).
- * @returns {Object} An object where keys are URLs and values are the total page views for each URL.
- */
-function pageviewsByUrl(bundles) {
-  return bundles.reduce((acc, cur) => {
-    if (!acc[cur.url]) acc[cur.url] = 0;
-    acc[cur.url] += cur.weight;
-    return acc;
-  }, {});
-}
-
-/**
- * Calculates the Click-Through Rate (CTR) by URL.
- * CTR is defined as the total number of sessions with at least one click event
- * divided by the total number of pageviews for each URL.
- *
- * @param {Array<Object>} bundles - An array of RUM bundles (NOT Flat bundles).
- * @returns {Object} - An object where the key is the URL and the value is the CTR value.
- */
-function getCTRByUrl(bundles) {
-  const aggregated = bundles.reduce((acc, bundle) => {
-    const { url } = bundle;
-    if (!acc[url]) {
-      acc[url] = { sessionsWithClick: 0, totalPageviews: 0 };
-    }
-    const hasClick = bundle.events.some((event) => event.checkpoint === 'click');
-
-    acc[url].totalPageviews += bundle.weight;
-    if (hasClick) {
-      acc[url].sessionsWithClick += bundle.weight;
-    }
-    return acc;
-  }, {});
-  return Object.entries(aggregated)
-    .reduce((acc, [url, { sessionsWithClick, totalPageviews }]) => {
-      acc[url] = (sessionsWithClick / totalPageviews);
-      return acc;
-    }, {});
-}
-
 /**
  * Calculates the Click-Through Rate (CTR) by URL and Referrer.
  * CTR is defined as the total number of sessions with at least one click event per referrer.
@@ -132,7 +90,5 @@ function getSiteAvgCTR(bundles) {
 
 export {
   getSiteAvgCTR,
-  getCTRByUrl,
   getCTRByUrlAndVendor,
-  pageviewsByUrl,
 };

package/src/common/traffic.js

@@ -122,6 +122,7 @@ const anyOf = (truth) => (text) => {
   return truth === text;
 };
 
+/* c8 ignore next 1 */
 const none = (input) => (Array.isArray(input) ? input.length === 0 : !hasText(input));
 
 const not = (truth) => (text) => {
@@ -198,6 +199,7 @@ const RULES = (domain) => ([
 ]);
 
 export function extractTrafficHints(bundle) {
+  /* c8 ignore next 1 */
   const findEvent = (checkpoint, source = '') => bundle.events.find((e) => e.checkpoint === checkpoint && (!source || e.source === source)) || {};
 
   const referrer = findEvent('enter').source || '';
@@ -252,3 +254,20 @@ export function classifyTrafficSource(url, referrer, utmSource, utmMedium, track
     vendor,
   };
 }
+
+export function classifyTraffic(bundle) {
+  const {
+    url,
+    weight,
+    referrer,
+    utmSource,
+    utmMedium,
+    tracking,
+  } = extractTrafficHints(bundle);
+
+  return {
+    url,
+    weight,
+    ...classifyTrafficSource(url, referrer, utmSource, utmMedium, tracking),
+  };
+}

package/src/functions/cwv.js

@@ -10,62 +10,36 @@
  * governing permissions and limitations under the License.
  */
 
-import { quantile } from 'd3-array';
-import { pageviewsByUrl } from '../common/aggregateFns.js';
-import { FlatBundle } from '../common/flat-bundle.js';
-
-const CWV_METRICS = ['lcp', 'cls', 'inp', 'ttfb'].map((metric) => `cwv-${metric}`);
-
-function collectCWVs(groupedByUrlIdTime) {
-  const { url, items: itemsByUrl } = groupedByUrlIdTime;
-
-  // first level: grouped by url
-  const CWVs = itemsByUrl.reduce((acc, { items: itemsById }) => {
-    // second level: grouped by id
-    const itemsByTime = itemsById.flatMap((itemById) => itemById.items);
-    // third level: grouped by time
-    const maximums = itemsByTime.reduce((values, item) => {
-      // each session (id-time) can contain multiple measurement for the same metric
-      // we need to find the maximum per metric type
-      // eslint-disable-next-line no-param-reassign
-      values[item.checkpoint] = Math.max(values[item.checkpoint] || 0, item.value);
-      return values;
-    }, {});
-
-    // max values per id for each metric type are collected into an array
-    CWV_METRICS.forEach((metric) => {
-      if (!acc[metric]) acc[metric] = [];
-      if (maximums[metric]) {
-        acc[metric].push(maximums[metric]);
-      }
-    });
-    return acc;
-  }, {});
-
-  return {
-    url,
-    lcp: quantile(CWVs['cwv-lcp'], 0.75) || null,
-    lcpCount: CWVs['cwv-lcp'].length,
-    cls: quantile(CWVs['cwv-cls'], 0.75) || null,
-    clsCount: CWVs['cwv-cls'].length,
-    inp: quantile(CWVs['cwv-inp'], 0.75) || null,
-    inpCount: CWVs['cwv-inp'].length,
-    ttfb: quantile(CWVs['cwv-ttfb'], 0.75) || null,
-    ttfbCount: CWVs['cwv-ttfb'].length,
-  };
-}
+import {
+  DataChunks, series, facets,
+} from '@adobe/rum-distiller';
+import { loadBundles } from '../utils.js';
 
 function handler(bundles) {
-  const pageviews = pageviewsByUrl(bundles);
-
-  return FlatBundle.fromArray(bundles)
-    .groupBy('url', 'id', 'time')
-    .map(collectCWVs)
+  const dataChunks = new DataChunks();
+
+  loadBundles(bundles, dataChunks);
+
+  dataChunks.addFacet('urls', facets.url);
+
+  dataChunks.addSeries('lcp', series.lcp);
+  dataChunks.addSeries('cls', series.cls);
+  dataChunks.addSeries('inp', series.inp);
+  dataChunks.addSeries('ttfb', series.ttfb);
+
+  return dataChunks.facets.urls.map((urlFacet) => ({
+    url: urlFacet.value,
+    pageviews: urlFacet.weight,
+    lcp: urlFacet.metrics.lcp.percentile(75) || null,
+    lcpCount: urlFacet.metrics.lcp.count,
+    cls: urlFacet.metrics.cls.percentile(75) || null,
+    clsCount: urlFacet.metrics.cls.count,
+    inp: urlFacet.metrics.inp.percentile(75) || null,
+    inpCount: urlFacet.metrics.inp.count,
+    ttfb: urlFacet.metrics.ttfb.percentile(75) || null,
+    ttfbCount: urlFacet.metrics.ttfb.count,
+  }))
     .filter((row) => row.lcp || row.cls || row.inp || row.ttfb) // filter out pages with no cwv data
-    .map((acc) => {
-      acc.pageviews = pageviews[acc.url];
-      return acc;
-    })
     .sort((a, b) => b.pageviews - a.pageviews); // sort desc by pageviews
 }
 

package/src/functions/experiment.js

@@ -10,159 +10,82 @@
  * governing permissions and limitations under the License.
  */
 
-const EXPERIMENT_CHECKPOINT = ['experiment'];
-const METRIC_CHECKPOINTS = ['click', 'convert', 'formsubmit'];
-const CHECKPOINTS = [...EXPERIMENT_CHECKPOINT, ...METRIC_CHECKPOINTS];
+import { DataChunks } from '@adobe/rum-distiller';
+import { generateKey, DELIMITER, loadBundles } from '../utils.js';
 
-function toClassName(name) {
-  return typeof name === 'string'
-    ? name.toLowerCase().replace(/[^0-9a-z]/gi, '-').replace(/-+/g, '-').replace(/^-|-$/g, '')
-    : '';
-}
+const METRICS = ['click', 'convert', 'formsubmit'];
 
-function getOrCreateExperimentObject(urlInsights, experimentName) {
-  let experimentObject = urlInsights.find((e) => e.experiment === toClassName(experimentName));
-  if (!experimentObject) {
-    experimentObject = {
-      experiment: toClassName(experimentName),
-      variants: [],
-    };
-    urlInsights.push(experimentObject);
-  }
-  return experimentObject;
-}
+const experimentsFacetFn = (bundle) => bundle.events.filter((e) => e.checkpoint === 'experiment').map((e) => generateKey(bundle.url, e.source));
+const variantsFacetFn = (bundle) => bundle.events.filter((e) => e.checkpoint === 'experiment').map((e) => generateKey(bundle.url, e.source, e.target));
 
-function getOrCreateVariantObject(variants, variantName) {
-  let variantObject = variants.find((v) => v.name === variantName);
-  if (!variantObject) {
-    variantObject = {
-      name: variantName,
-      views: 0,
-      samples: 0,
-      click: {},
-      convert: {},
-      formsubmit: {},
-    };
-    variants.push(variantObject);
-  }
-  return variantObject;
-}
+const checkpointsFacetFn = (bundle) => {
+  const experiments = bundle.events.filter((e) => e.checkpoint === 'experiment');
+  const metrics = bundle.events.filter((c) => METRICS.includes(c.checkpoint) && c.source);
 
-function updateInferredStartAndEndDate(experimentObject, time) {
-  const bundleTime = new Date(time);
-  const yesterday = new Date();
-  yesterday.setDate(yesterday.getDate() - 1);
-  yesterday.setHours(0, 0, 0, 0);
-  const bundleDate = new Date(bundleTime);
-  bundleDate.setHours(0, 0, 0, 0);
-  if (!experimentObject.inferredStartDate && !experimentObject.inferredEndDate) {
-    // eslint-disable-next-line no-param-reassign
-    experimentObject.inferredStartDate = time;
-    // eslint-disable-next-line no-param-reassign
-    experimentObject.inferredEndDate = time;
-  } else {
-    const inferredStartDateObj = new Date(experimentObject.inferredStartDate);
-    const inferredEndDateObj = new Date(experimentObject.inferredEndDate);
-    if (bundleTime < inferredStartDateObj) {
-      // eslint-disable-next-line no-param-reassign
-      experimentObject.inferredStartDate = time;
-    }
-    if (bundleTime > inferredEndDateObj) {
-      // eslint-disable-next-line no-param-reassign
-      experimentObject.inferredEndDate = time;
-    }
-  }
-}
+  const keys = experiments.flatMap(
+    (exp) => metrics.flatMap((metric) => [
+      generateKey(bundle.url, exp.source, exp.target, metric.checkpoint, metric.source),
+      generateKey(bundle.url, exp.source, exp.target, metric.checkpoint, '*'),
+    ]),
+  );
 
-function calculateMetrics(bundle) {
-  const metrics = {};
-  for (const checkpoint of METRIC_CHECKPOINTS) {
-    metrics[checkpoint] = {};
-  }
-  for (const event of bundle.events) {
-    if (METRIC_CHECKPOINTS.includes(event.checkpoint)) {
-      const { source, checkpoint } = event;
-      if (!metrics[checkpoint][source]) {
-        metrics[checkpoint][source] = {
-          value: bundle.weight,
-          samples: 1,
-        };
-      } else {
-        metrics[checkpoint][source].value += bundle.weight;
-        metrics[checkpoint][source].samples += 1;
-      }
-    }
-  }
-  return metrics;
-}
+  return [...new Set(keys)];
+};
 
 function handler(bundles) {
-  const experimentInsights = {};
-  for (const bundle of bundles) {
-    const experimentEvents = bundle.events?.filter(
-      (e) => (EXPERIMENT_CHECKPOINT.includes(e.checkpoint) && e.source),
-    );
-    const { url, weight, time } = bundle;
-    const metrics = calculateMetrics(bundle);
-    for (const experimentEvent of experimentEvents) {
-      if (!experimentInsights[url]) {
-        experimentInsights[url] = [];
-      }
-      const experimentName = experimentEvent.source;
-      const variantName = experimentEvent.target;
-      const experimentObject = getOrCreateExperimentObject(
-        experimentInsights[url],
-        experimentName,
-      );
-      const variantObject = getOrCreateVariantObject(experimentObject.variants, variantName);
-      updateInferredStartAndEndDate(experimentObject, time);
-      variantObject.views += weight;
-      variantObject.samples += 1;
-      // combine metrics and variantObject, considering the interaction events
-      // only once during the session
-      for (const checkpoint of METRIC_CHECKPOINTS) {
-        // eslint-disable-next-line no-restricted-syntax
-        for (const source in metrics?.[checkpoint]) {
-          if (!variantObject[checkpoint][source]) {
-            variantObject[checkpoint][source] = {
-              value: weight,
-              samples: 1,
-            };
-          } else {
-            variantObject[checkpoint][source].value += weight;
-            variantObject[checkpoint][source].samples += 1;
-          }
-        }
-      }
-      // add each metric to the variantObject's * count by weight
-      for (const checkpoint of Object.keys(metrics)) {
-        if (Object.keys(metrics[checkpoint]).length > 0) {
-          if (!variantObject[checkpoint]['*']) {
-            variantObject[checkpoint]['*'] = {
-              value: weight,
-              samples: 1,
-            };
-          } else {
-            variantObject[checkpoint]['*'].value += weight;
-            variantObject[checkpoint]['*'].samples += 1;
-          }
-        }
-      }
-      // add global interactionsCount if there's any interaction
-      const hasInteraction = Object.values(metrics).some((m) => Object.keys(m).length > 0);
-      if (hasInteraction) {
-        if (!variantObject.interactionsCount) {
-          variantObject.interactionsCount = weight;
-        } else {
-          variantObject.interactionsCount += weight;
-        }
-      }
-    }
-  }
-  return experimentInsights;
+  const dataChunks = new DataChunks();
+
+  loadBundles(bundles.filter((bundle) => bundle.events.some((event) => event.checkpoint === 'experiment')), dataChunks);
+
+  dataChunks.addFacet('experiments', experimentsFacetFn);
+  dataChunks.addFacet('variants', variantsFacetFn);
+  dataChunks.addFacet('checkpoints', checkpointsFacetFn);
+
+  dataChunks.addSeries('experimenttime', (bundle) => new Date(bundle.time).getTime());
+  dataChunks.addSeries('views', (bundle) => bundle.weight);
+  dataChunks.addSeries('interaction', (bundle) => (bundle.events.find((e) => e.checkpoint === 'click' || e.checkpoint === 'formsubmit' || e.checkpoint === 'convert') ? bundle.weight : undefined));
+
+  const { experiments, variants, checkpoints } = dataChunks.facets;
+
+  const result = {};
+  experiments.forEach((uev) => {
+    const [url, experiment] = uev.value.split(DELIMITER);
+    if (!result[url]) result[url] = [];
+    result[url].push({
+      experiment,
+      variants: [],
+      inferredStartDate: new Date(uev.metrics.experimenttime.min).toISOString(),
+      inferredEndDate: new Date(uev.metrics.experimenttime.max).toISOString(),
+    });
+  });
+  variants.forEach((uev) => {
+    const [url, experiment, variant] = uev.value.split(DELIMITER);
+    const eIdx = result[url].findIndex((e) => e.experiment === experiment);
+    result[url][eIdx].variants.push({
+      name: variant,
+      click: {},
+      formsubmit: {},
+      convert: {},
+      interactionsCount: uev.metrics.interaction.sum,
+      samples: uev.metrics.views.count,
+      views: uev.metrics.views.sum,
+    });
+  });
+  checkpoints.forEach((uev) => {
+    const [url, experiment, variant, checkpoint, source] = uev.value.split(DELIMITER);
+    const eIdx = result[url].findIndex((e) => e.experiment === experiment);
+    const vIdx = result[url][eIdx].variants.findIndex((v) => v.name === variant);
+
+    result[url][eIdx].variants[vIdx][checkpoint][source] = {
+      value: uev.metrics.views.sum,
+      samples: uev.metrics.views.count,
+    };
+  });
+
+  return result;
 }
 
 export default {
   handler,
-  checkpoints: CHECKPOINTS,
+  checkpoints: [...METRICS, 'experiment'],
 };

package/src/functions/opportunities/high-inorganic-high-bounce-rate.js

@@ -10,8 +10,8 @@
  * governing permissions and limitations under the License.
  */
 
-import trafficAcquisition from '../traffic-acquisition.js';
-import { getCTRByUrl } from '../../common/aggregateFns.js';
+import { DataChunks, facets } from '@adobe/rum-distiller';
+import { loadBundles, trafficSeriesFn } from '../../utils.js';
 
 const HOMEPAGE_PAID_TRAFFIC_THRESHOLD = 0.8;
 const NON_HOMEPAGE_PAID_TRAFFIC_THRESHOLD = 0.5;
@@ -44,8 +44,7 @@ function convertToOpportunity(traffic) {
   };
 }
 
-function hasHighInorganicTraffic(traffic) {
-  const { url, paid, total } = traffic;
+function hasHighInorganicTraffic(url, paid, total) {
   const isHomepage = new URL(url).pathname === '/';
   const threshold = isHomepage
     ? HOMEPAGE_PAID_TRAFFIC_THRESHOLD
@@ -53,19 +52,39 @@ function hasHighInorganicTraffic(traffic) {
   return paid / total > threshold;
 }
 
-function hasHighBounceRate(ctr) {
-  return ctr < BOUNCE_RATE_THRESHOLD;
-}
-
 function handler(bundles, opts = {}) {
   const { interval = 7 } = opts;
-  const trafficByUrl = trafficAcquisition.handler(bundles);
-  const ctrByUrl = getCTRByUrl(bundles);
 
-  return trafficByUrl.filter((traffic) => traffic.total > interval * DAILY_PAGEVIEW_THRESHOLD)
-    .filter(hasHighInorganicTraffic)
-    .filter((traffic) => hasHighBounceRate(ctrByUrl[traffic.url]))
-    .map((traffic) => ({ ...traffic, bounceRate: 1 - ctrByUrl[traffic.url] }))
+  const dataChunks = new DataChunks();
+
+  loadBundles(bundles, dataChunks);
+
+  dataChunks.addFacet('urls', facets.url);
+
+  dataChunks.addSeries('views', (bundle) => bundle.weight);
+  dataChunks.addSeries('clicks', (bundle) => (bundle.events.some((e) => e.checkpoint === 'click') ? bundle.weight : 0));
+
+  const memo = {};
+  dataChunks.addSeries('earned', trafficSeriesFn(memo, 'earned'));
+  dataChunks.addSeries('owned', trafficSeriesFn(memo, 'owned'));
+  dataChunks.addSeries('paid', trafficSeriesFn(memo, 'paid'));
+
+  return dataChunks.facets.urls
+    .filter((url) => url.metrics.views.sum > interval * DAILY_PAGEVIEW_THRESHOLD)
+    .filter((url) => hasHighInorganicTraffic(
+      url.value,
+      url.metrics.paid.sum,
+      url.metrics.views.sum,
+    ))
+    .filter((url) => url.metrics.clicks.sum / url.metrics.views.sum < BOUNCE_RATE_THRESHOLD)
+    .map((url) => ({
+      url: url.value,
+      total: url.metrics.views.sum,
+      earned: url.metrics.earned.sum,
+      owned: url.metrics.owned.sum,
+      paid: url.metrics.paid.sum,
+      bounceRate: 1 - url.metrics.clicks.sum / url.metrics.views.sum,
+    }))
     .map(convertToOpportunity);
 }
 

package/src/functions/traffic-acquisition.js

@@ -10,7 +10,7 @@
  * governing permissions and limitations under the License.
  */
 
-import { classifyTrafficSource, extractTrafficHints } from '../common/traffic.js';
+import { classifyTraffic } from '../common/traffic.js';
 
 const MAIN_TYPES = ['total', 'paid', 'earned', 'owned'];
 
@@ -40,21 +40,21 @@ function transformFormat(trafficSources) {
   }));
 }
 
+function formatTraffic(row) {
+  const {
+    url, weight, type, category, vendor,
+  } = row;
+  return {
+    url,
+    weight,
+    trafficSource: vendor ? `${type}:${category}:${vendor}` : `${type}:${category}`,
+  };
+}
+
 function handler(bundles) {
   const trafficSources = bundles
-    .map(extractTrafficHints)
-    .map((row) => {
-      const {
-        type,
-        category,
-        vendor,
-      } = classifyTrafficSource(row.url, row.referrer, row.utmSource, row.utmMedium, row.tracking);
-      return {
-        url: row.url,
-        weight: row.weight,
-        trafficSource: vendor ? `${type}:${category}:${vendor}` : `${type}:${category}`,
-      };
-    })
+    .map(classifyTraffic)
+    .map(formatTraffic)
     .reduce(collectByUrlAndTrafficSource, {});
 
   return transformFormat(trafficSources)

package/src/utils.js

@@ -11,6 +11,26 @@
  */
 
 import { context as h2, h1 } from '@adobe/fetch';
+import { utils } from '@adobe/rum-distiller';
+import { classifyTraffic } from './common/traffic.js';
+
+export const DELIMITER = '≡';
+
+export const generateKey = (...keys) => keys.join(DELIMITER);
+
+export const trafficSeriesFn = (memo, type) => (bundle) => {
+  const key = generateKey(bundle.url, bundle.id, bundle.time);
+  if (!memo[key]) {
+    // eslint-disable-next-line no-param-reassign
+    memo[key] = classifyTraffic(bundle).type;
+  }
+
+  return type === memo[key] ? bundle.weight : 0;
+};
+
+export const loadBundles = (bundles, dataChunks) => {
+  dataChunks.load([{ rumBundles: bundles.map(utils.addCalculatedProps) }]);
+};
 
 /* c8 ignore next 3 */
 export const { fetch } = process.env.HELIX_FETCH_FORCE_HTTP1