@adobe/spacecat-shared-rum-api-client 2.3.0 → 2.5.0

package/CHANGELOG.md

@@ -1,3 +1,17 @@
+# [@adobe/spacecat-shared-rum-api-client-v2.5.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.4.0...@adobe/spacecat-shared-rum-api-client-v2.5.0) (2024-07-23)
+
+
+### Features
+
+* Experimentation entity ([#288](https://github.com/adobe/spacecat-shared/issues/288)) ([774e2c7](https://github.com/adobe/spacecat-shared/commit/774e2c7013c9e617c745c494e20e1cdd8cce71e7))
+
+# [@adobe/spacecat-shared-rum-api-client-v2.4.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.3.0...@adobe/spacecat-shared-rum-api-client-v2.4.0) (2024-07-19)
+
+
+### Features
+
+* traffic acquisition detection ([#286](https://github.com/adobe/spacecat-shared/issues/286)) ([b3d1f1c](https://github.com/adobe/spacecat-shared/commit/b3d1f1caa288ed5cdda367b64dea24886cf87afb))
+
 # [@adobe/spacecat-shared-rum-api-client-v2.3.0](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.2.1...@adobe/spacecat-shared-rum-api-client-v2.3.0) (2024-07-12)
 
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@adobe/spacecat-shared-rum-api-client",
-  "version": "2.3.0",
+  "version": "2.5.0",
   "description": "Shared modules of the Spacecat Services - Rum API client",
   "type": "module",
   "main": "src/index.js",

package/src/common/traffic.js

@@ -0,0 +1,128 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+/* eslint-disable object-curly-newline */
+
+import { hasText } from '@adobe/spacecat-shared-utils';
+
+/*
+ * --------- DEFINITIONS ----------------
+ */
+
+// Referrer related
+// matches second level domains 1:1 ignoring subdomains and top-level domains
+// for example: https://l.instagram.com matches, whereas https://wwww.linstagram.com does not
+const searchEngines = /^(https?:\/\/)?(.*\.)?(google|yahoo|bing|yandex|baidu|duckduckgo|brave|ecosia|aol|startpage|ask)\.(.*)(\/|$)/;
+const socialMedias = /^(https?:\/\/)?(.*\.)?(facebook|tiktok|snapchat|x|twitter|pinterest|reddit|linkedin|threads|quora|discord|tumblr|mastodon|bluesky|instagram)\.(.*)(\/|$)/;
+const adNetworks = /googlesyndication|2mdn/;
+const videoPlatforms = /^(https?:\/\/)?(.*\.)?(youtube|vimeo|twitch|dailymotion|wistia)\.(.*)(\/|$)/;
+
+// UTM Source related
+const paidDisplaySources = ['gdn'];
+
+// UTM Medium related
+// matches 'pp', *cp[acmuv]*, *ppc*, *paid*
+const paidUTMMediums = /^\bpp\b|(.*(cp[acmuv]|ppc|paid|display|banner|poster|placement).*)$/;
+const searchEngineUTMMediums = ['google', 'paidsearch', 'paidsearchnb', 'sea', 'sem'];
+const socialMediaUTMMediums = ['facebook', 'gnews', 'instagramfeed', 'instagramreels', 'instagramstories', 'line', 'linkedin', 'metasearch', 'organicsocialown', 'paidsocial', 'social', 'sociallinkedin', 'socialpaid'];
+const affiliateUTMMediums = ['aff', 'affiliate', 'affiliatemarketing'];
+const organicUTMMediums = ['organicsocial'];
+const emailUTMMediums = ['em', 'email', 'mail', 'newsletter'];
+const smsUTMMediums = ['sms', 'mms'];
+const qrUTMMediums = ['qr', 'qrcode'];
+const pushUTMMediums = ['push', 'pushnotification'];
+
+// Tracking params - based on the checkpoints we have in rum-enhancer now
+// const organicTrackingParams = ['srsltid']; WE DO NOT HAVE THIS AS OF NOW
+const paidTrackingParams = ['paid'];
+const emailTrackingParams = ['email'];
+
+/*
+ * --------- HELPERS ----------------
+ */
+
+const any = () => true;
+
+const anyOf = (truth) => (text) => {
+  if (Array.isArray(truth)) return truth.includes(text);
+  if (truth instanceof RegExp) return truth.test(text);
+  return truth === text;
+};
+
+const none = (input) => (Array.isArray(input) ? input.length === 0 : !hasText(input));
+
+const not = (truth) => (text) => {
+  if (!hasText(text)) return false;
+  if (Array.isArray(truth)) return !truth.includes(text);
+  if (truth instanceof RegExp) return !truth.test(text);
+  return truth !== text;
+};
+
+const notEmpty = (text) => hasText(text);
+
+/*
+ * --------- RULES ----------------
+ */
+
+// ORDER IS IMPORTANT
+const RULES = (origin) => ([
+  // PAID
+  { type: 'paid', category: 'search', referrer: anyOf(searchEngines), utmSource: any, utmMedium: anyOf(searchEngineUTMMediums), tracking: none },
+  { type: 'paid', category: 'search', referrer: anyOf(searchEngines), utmSource: any, utmMedium: any, tracking: anyOf(paidTrackingParams) },
+  { type: 'paid', category: 'social', referrer: anyOf(socialMedias), utmSource: any, utmMedium: anyOf(socialMediaUTMMediums), tracking: none },
+  { type: 'paid', category: 'social', referrer: anyOf(socialMedias), utmSource: any, utmMedium: any, tracking: anyOf(paidTrackingParams) },
+  { type: 'paid', category: 'video', referrer: anyOf(videoPlatforms), utmSource: any, utmMedium: anyOf(paidUTMMediums), tracking: any },
+  { type: 'paid', category: 'video', referrer: anyOf(videoPlatforms), utmSource: any, utmMedium: any, tracking: anyOf(paidTrackingParams) },
+  { type: 'paid', category: 'display', referrer: notEmpty, utmSource: any, utmMedium: anyOf(paidUTMMediums), tracking: any },
+  { type: 'paid', category: 'display', referrer: anyOf(adNetworks), utmSource: any, utmMedium: any, tracking: any },
+  { type: 'paid', category: 'display', referrer: notEmpty, utmSource: anyOf(paidDisplaySources), utmMedium: any, tracking: any },
+  { type: 'paid', category: 'affiliate', referrer: notEmpty, utmSource: any, utmMedium: anyOf(affiliateUTMMediums), tracking: any },
+  { type: 'paid', category: 'uncategorized', referrer: not(origin), utmSource: any, utmMedium: anyOf(paidUTMMediums), tracking: any },
+  { type: 'paid', category: 'uncategorized', referrer: not(origin), utmSource: any, utmMedium: any, tracking: anyOf(paidTrackingParams) },
+
+  // EARNED
+  { type: 'earned', category: 'search', referrer: anyOf(searchEngines), utmSource: none, utmMedium: none, tracking: none },
+  { type: 'earned', category: 'search', referrer: anyOf(searchEngines), utmSource: any, utmMedium: not(paidUTMMediums), tracking: not(paidTrackingParams) },
+  { type: 'earned', category: 'social', referrer: anyOf(socialMedias), utmSource: none, utmMedium: none, tracking: none },
+  { type: 'earned', category: 'social', referrer: not(origin), utmSource: any, utmMedium: anyOf(organicUTMMediums), tracking: none },
+  { type: 'earned', category: 'video', referrer: anyOf(videoPlatforms), utmSource: none, utmMedium: none, tracking: none },
+  { type: 'earned', category: 'video', referrer: anyOf(videoPlatforms), utmSource: any, utmMedium: not(paidUTMMediums), tracking: none },
+  { type: 'earned', category: 'referral', referrer: not(origin), utmSource: none, utmMedium: none, tracking: none },
+
+  // OWNED
+  { type: 'owned', category: 'direct', referrer: none, utmSource: none, utmMedium: none, tracking: none },
+  { type: 'owned', category: 'internal', referrer: anyOf(origin), utmSource: none, utmMedium: none, tracking: none },
+  { type: 'owned', category: 'email', referrer: any, utmSource: any, utmMedium: any, tracking: anyOf(emailTrackingParams) },
+  { type: 'owned', category: 'email', referrer: any, utmSource: any, utmMedium: anyOf(emailUTMMediums), tracking: any },
+  { type: 'owned', category: 'sms', referrer: none, utmSource: any, utmMedium: anyOf(smsUTMMediums), tracking: none },
+  { type: 'owned', category: 'qr', referrer: none, utmSource: any, utmMedium: anyOf(qrUTMMediums), tracking: none },
+  { type: 'owned', category: 'push', referrer: none, utmSource: any, utmMedium: anyOf(pushUTMMediums), tracking: none },
+
+  // FALLBACK
+  { type: 'owned', category: 'uncategorized', referrer: any, utmSource: any, utmMedium: any, tracking: any },
+]);
+
+export function classifyTrafficSource(url, referrer, utmSource, utmMedium, trackingParams) {
+  const { origin } = new URL(url);
+  const rules = RULES(origin);
+
+  const { type, category } = rules.find((rule) => (
+    rule.referrer(referrer)
+    && rule.utmSource(utmSource)
+    && rule.utmMedium(utmMedium)
+    && rule.tracking(trackingParams)
+  ));
+
+  return {
+    type,
+    category,
+  };
+}

package/src/functions/experiment.js

@@ -47,12 +47,49 @@ function getOrCreateVariantObject(variants, variantName) {
   return variantObject;
 }
 
+function updateInferredStartAndEndDate(experimentObject, time) {
+  const bundleTime = new Date(time);
+  const yesterday = new Date();
+  yesterday.setDate(yesterday.getDate() - 1);
+  yesterday.setHours(0, 0, 0, 0);
+  const bundleDate = new Date(bundleTime);
+  bundleDate.setHours(0, 0, 0, 0);
+  if (!experimentObject.inferredStartDate && !experimentObject.inferredEndDate) {
+    // adding the inferredStartDate and inferredEndDate properties for the first time
+    // eslint-disable-next-line no-param-reassign
+    experimentObject.inferredStartDate = time;
+    // check if bundleTime is before yesterday
+    if (bundleDate < yesterday) {
+      // RUM data is delayed by a day, so if we don't have
+      // any RUM data for yesterday, so we can infer the endDate
+      // eslint-disable-next-line no-param-reassign
+      experimentObject.inferredEndDate = time;
+    } else {
+      // eslint-disable-next-line no-param-reassign
+      experimentObject.inferredEndDate = null;
+    }
+  } else {
+    const inferredStartDateObj = new Date(experimentObject.inferredStartDate);
+    if (bundleTime < inferredStartDateObj) {
+      // eslint-disable-next-line no-param-reassign
+      experimentObject.inferredStartDate = time;
+    }
+    if (bundleDate < yesterday) {
+      if (!experimentObject.inferredEndDate
+        || (bundleTime > new Date(experimentObject.inferredEndDate))) {
+        // eslint-disable-next-line no-param-reassign
+        experimentObject.inferredEndDate = time;
+      }
+    }
+  }
+}
+
 function handler(bundles) {
   const experimentInsights = {};
   for (const bundle of bundles) {
-    const experimentEvent = bundle.events.find((e) => e.checkpoint === 'experiment');
+    const experimentEvent = bundle.events?.find((e) => e.checkpoint === 'experiment');
     if (experimentEvent) {
-      const { url, weight } = bundle;
+      const { url, weight, time } = bundle;
       if (!experimentInsights[url]) {
         experimentInsights[url] = [];
       }
@@ -60,6 +97,7 @@ function handler(bundles) {
       const variantName = experimentEvent.target;
       const experimentObject = getOrCreateExperimentObject(experimentInsights[url], experimentName);
       const variantObject = getOrCreateVariantObject(experimentObject.variants, variantName);
+      updateInferredStartAndEndDate(experimentObject, time);
       variantObject.views += weight;
 
       const metrics = {};

package/src/functions/traffic-acquisition.js

@@ -0,0 +1,73 @@
+/*
+ * Copyright 2024 Adobe. All rights reserved.
+ * This file is licensed to you under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License. You may obtain a copy
+ * of the License at http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under
+ * the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR REPRESENTATIONS
+ * OF ANY KIND, either express or implied. See the License for the specific language
+ * governing permissions and limitations under the License.
+ */
+
+import { classifyTrafficSource } from '../common/traffic.js';
+
+function extractHints(bundle) {
+  const findEvent = (checkpoint, source = '') => bundle.events.find((e) => e.checkpoint === checkpoint && (!source || e.source === source)) || {};
+
+  const referrer = findEvent('enter').source || '';
+  const utmSource = findEvent('utm', 'utm_source').target || '';
+  const utmMedium = findEvent('utm', 'utm_medium').target || '';
+  const tracking = findEvent('paid').checkpoint || findEvent('email').checkpoint || '';
+
+  return {
+    url: bundle.url,
+    weight: bundle.weight,
+    referrer,
+    utmSource,
+    utmMedium,
+    tracking,
+  };
+}
+
+function collectByUrlAndTrafficSource(acc, { url, weight, trafficSource }) {
+  acc[url] = acc[url] || { total: 0 };
+  acc[url][trafficSource] = (acc[url][trafficSource] || 0) + weight;
+  acc[url].total += weight;
+  return acc;
+}
+
+function transformFormat(trafficSources) {
+  return Object.entries(trafficSources).map(([url, value]) => ({
+    url,
+    total: value.total,
+    sources: Object.entries(value)
+      .filter(([source]) => source !== 'total')
+      .map(([source, views]) => ({ type: source, views })),
+  }));
+}
+
+function handler(bundles) {
+  const trafficSources = bundles
+    .map(extractHints)
+    .map((row) => {
+      const {
+        type,
+        category,
+      } = classifyTrafficSource(row.url, row.referrer, row.utmSource, row.utmMedium, row.tracking);
+      return {
+        url: row.url,
+        weight: row.weight,
+        trafficSource: `${type}:${category}`,
+      };
+    })
+    .reduce(collectByUrlAndTrafficSource, {});
+
+  return transformFormat(trafficSources)
+    .sort((a, b) => b.total - a.total); // sort desc by total views
+}
+
+export default {
+  handler,
+  checkpoints: ['email', 'enter', 'paid', 'utm'],
+};

package/src/index.js

@@ -13,12 +13,14 @@ import { fetchBundles } from './common/rum-bundler-client.js';
 import notfound from './functions/404.js';
 import cwv from './functions/cwv.js';
 import experiment from './functions/experiment.js';
+import trafficAcquisition from './functions/traffic-acquisition.js';
 import variant from './functions/variant.js';
 
 const HANDLERS = {
   404: notfound,
   cwv,
   experiment,
+  'traffic-acquisition': trafficAcquisition,
   variant,
 };