@adobe/spacecat-shared-rum-api-client 2.5.2 → 2.5.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/package.json +3 -2
- package/src/common/traffic.js +105 -51
- package/src/functions/traffic-acquisition.js +77 -3
package/CHANGELOG.md
CHANGED
|
@@ -1,3 +1,17 @@
|
|
|
1
|
+
# [@adobe/spacecat-shared-rum-api-client-v2.5.4](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.5.3...@adobe/spacecat-shared-rum-api-client-v2.5.4) (2024-08-01)
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
### Bug Fixes
|
|
5
|
+
|
|
6
|
+
* improve traffic acquisition detection ([#310](https://github.com/adobe/spacecat-shared/issues/310)) ([25d46ff](https://github.com/adobe/spacecat-shared/commit/25d46ffed66643e0581f23067b0f6922beff8e0e))
|
|
7
|
+
|
|
8
|
+
# [@adobe/spacecat-shared-rum-api-client-v2.5.3](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.5.2...@adobe/spacecat-shared-rum-api-client-v2.5.3) (2024-07-31)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
### Bug Fixes
|
|
12
|
+
|
|
13
|
+
* work-around for bamboos custom exp engine ([#308](https://github.com/adobe/spacecat-shared/issues/308)) ([56b3b62](https://github.com/adobe/spacecat-shared/commit/56b3b62ea4c330d3d1003ab82e8c90ae890914f8))
|
|
14
|
+
|
|
1
15
|
# [@adobe/spacecat-shared-rum-api-client-v2.5.2](https://github.com/adobe/spacecat-shared/compare/@adobe/spacecat-shared-rum-api-client-v2.5.1...@adobe/spacecat-shared-rum-api-client-v2.5.2) (2024-07-30)
|
|
2
16
|
|
|
3
17
|
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@adobe/spacecat-shared-rum-api-client",
|
|
3
|
-
"version": "2.5.
|
|
3
|
+
"version": "2.5.4",
|
|
4
4
|
"description": "Shared modules of the Spacecat Services - Rum API client",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.js",
|
|
@@ -36,7 +36,8 @@
|
|
|
36
36
|
"@adobe/helix-universal": "5.0.5",
|
|
37
37
|
"@adobe/spacecat-shared-utils": "1.4.0",
|
|
38
38
|
"aws4": "1.13.0",
|
|
39
|
-
"d3-array": "3.2.4"
|
|
39
|
+
"d3-array": "3.2.4",
|
|
40
|
+
"urijs": "^1.19.11"
|
|
40
41
|
},
|
|
41
42
|
"devDependencies": {
|
|
42
43
|
"chai": "4.5.0",
|
package/src/common/traffic.js
CHANGED
|
@@ -12,33 +12,61 @@
|
|
|
12
12
|
/* eslint-disable object-curly-newline */
|
|
13
13
|
|
|
14
14
|
import { hasText } from '@adobe/spacecat-shared-utils';
|
|
15
|
+
import URI from 'urijs';
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Extracts the second-level domain (SLD) from a given URL.
|
|
19
|
+
*
|
|
20
|
+
* For example, for the URL `https://subdomain.example.co.uk/path`, this
|
|
21
|
+
* function will return `example` (excluding the TLD `co.uk`).
|
|
22
|
+
*
|
|
23
|
+
* @param {string} url - The URL from which to extract the second-level domain.
|
|
24
|
+
* @returns {string} The second-level domain of the given URL, or the original
|
|
25
|
+
* URL if it does not contain any text.
|
|
26
|
+
*/
|
|
27
|
+
function getSecondLevelDomain(url) {
|
|
28
|
+
if (!hasText(url)) return url;
|
|
29
|
+
const uri = new URI(url);
|
|
30
|
+
const domain = uri.domain();
|
|
31
|
+
const tld = uri.tld();
|
|
32
|
+
return domain.split(`.${tld}`)[0];
|
|
33
|
+
}
|
|
15
34
|
|
|
16
35
|
/*
|
|
17
36
|
* --------- DEFINITIONS ----------------
|
|
18
37
|
*/
|
|
19
38
|
|
|
20
39
|
// Referrer related
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
40
|
+
const referrers = {
|
|
41
|
+
search: /google|yahoo|bing|yandex|baidu|duckduckgo|brave|ecosia|aol|startpage|ask/,
|
|
42
|
+
social: /^\b(x)\b|(.*(facebook|tiktok|snapchat|x|twitter|pinterest|reddit|linkedin|threads|quora|discord|tumblr|mastodon|bluesky|instagram).*)$/,
|
|
43
|
+
ad: /googlesyndication|2mdn|doubleclick|syndicatedsearch/,
|
|
44
|
+
video: /youtube|vimeo|twitch|dailymotion|wistia/,
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
const mediums = {
|
|
48
|
+
paidall: /^\bpp\b|(.*(cp[acmuv]|ppc|paid).*)$/, // matches 'pp', *cp[acmuv]*, *ppc*, *paid*
|
|
49
|
+
paidsearch: /google|paidsearch|sea|sem|maps/,
|
|
50
|
+
paidsocial: /paidsocial|socialpaid|fbig|facebook|gnews|instagram|line|linkedin|metasearch/,
|
|
51
|
+
organic: /organic/,
|
|
52
|
+
socialall: /^\b(soc)\b|(.*(social).*)$/,
|
|
53
|
+
display: /display|banner|poster|placement|image|dcm|businesslistings/,
|
|
54
|
+
video: /video/,
|
|
55
|
+
affiliate: /^aff|(.*(patrocinados|referral)).*$/,
|
|
56
|
+
email: ['em', 'email', 'mail', 'newsletter'],
|
|
57
|
+
sms: ['sms', 'mms'],
|
|
58
|
+
qr: ['qr', 'qrcode'],
|
|
59
|
+
push: ['push', 'pushnotification'],
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
const sources = {
|
|
63
|
+
social: /^\b(ig|fb|x|soc)\b|(.*(meta|tiktok|facebook|snapchat|twitter|igshopping|instagram|linkedin|reddit).*)$/,
|
|
64
|
+
search: /^\b(goo)\b|(.*(sea|google|yahoo|bing|yandex|baidu|duckduckgo|brave|ecosia|aol|startpage|ask).*)$/,
|
|
65
|
+
video: /youtube|vimeo|twitch|dailymotion|wistia/,
|
|
66
|
+
display: /optumib2b|jun|googleads|dv36|dv360|microsoft|flipboard|programmatic|yext|gdn|banner|newsshowcase/,
|
|
67
|
+
affiliate: /brandreward|yieldkit|fashionistatop|partner|linkbux|stylesblog|linkinbio|affiliate/,
|
|
68
|
+
email: /sfmc|email/,
|
|
69
|
+
};
|
|
42
70
|
|
|
43
71
|
// Tracking params - based on the checkpoints we have in rum-enhancer now
|
|
44
72
|
// const organicTrackingParams = ['srsltid']; WE DO NOT HAVE THIS AS OF NOW
|
|
@@ -73,51 +101,77 @@ const notEmpty = (text) => hasText(text);
|
|
|
73
101
|
*/
|
|
74
102
|
|
|
75
103
|
// ORDER IS IMPORTANT
|
|
76
|
-
const RULES = (
|
|
104
|
+
const RULES = (domain) => ([
|
|
77
105
|
// PAID
|
|
78
|
-
{ type: 'paid', category: 'search', referrer: anyOf(
|
|
79
|
-
{ type: 'paid', category: 'search', referrer: anyOf(
|
|
80
|
-
{ type: 'paid', category: '
|
|
81
|
-
{ type: 'paid', category: '
|
|
82
|
-
|
|
83
|
-
{ type: 'paid', category: '
|
|
84
|
-
{ type: 'paid', category: '
|
|
85
|
-
{ type: 'paid', category: '
|
|
86
|
-
{ type: 'paid', category: '
|
|
87
|
-
{ type: 'paid', category: '
|
|
88
|
-
{ type: 'paid', category: '
|
|
89
|
-
{ type: 'paid', category: '
|
|
106
|
+
{ type: 'paid', category: 'search', referrer: anyOf(referrers.search), utmSource: any, utmMedium: anyOf(mediums.paidsearch), tracking: none },
|
|
107
|
+
{ type: 'paid', category: 'search', referrer: anyOf(referrers.search), utmSource: any, utmMedium: any, tracking: anyOf(paidTrackingParams) },
|
|
108
|
+
{ type: 'paid', category: 'search', referrer: anyOf(referrers.ad), utmSource: any, utmMedium: anyOf(mediums.paidsearch), tracking: any },
|
|
109
|
+
{ type: 'paid', category: 'search', referrer: none, utmSource: anyOf(sources.search), utmMedium: anyOf(mediums.paidsearch), tracking: any },
|
|
110
|
+
|
|
111
|
+
{ type: 'paid', category: 'social', referrer: anyOf(referrers.social), utmSource: any, utmMedium: anyOf(mediums.paidsocial), tracking: none },
|
|
112
|
+
{ type: 'paid', category: 'social', referrer: anyOf(referrers.social), utmSource: any, utmMedium: any, tracking: anyOf(paidTrackingParams) },
|
|
113
|
+
{ type: 'paid', category: 'social', referrer: anyOf(referrers.social), utmSource: notEmpty, utmMedium: anyOf(mediums.socialall), tracking: any },
|
|
114
|
+
{ type: 'paid', category: 'social', referrer: none, utmSource: anyOf(sources.social), utmMedium: anyOf(mediums.paidsocial), tracking: any },
|
|
115
|
+
{ type: 'paid', category: 'social', referrer: none, utmSource: anyOf(sources.social), utmMedium: anyOf(mediums.paidall), tracking: any },
|
|
116
|
+
{ type: 'paid', category: 'social', referrer: anyOf(referrers.social), utmSource: notEmpty, utmMedium: notEmpty, tracking: any },
|
|
117
|
+
{ type: 'paid', category: 'social', referrer: none, utmSource: anyOf(sources.social), utmMedium: anyOf(mediums.socialall), tracking: any },
|
|
118
|
+
|
|
119
|
+
{ type: 'paid', category: 'video', referrer: anyOf(referrers.video), utmSource: any, utmMedium: anyOf(mediums.paidall), tracking: any },
|
|
120
|
+
{ type: 'paid', category: 'video', referrer: anyOf(referrers.video), utmSource: any, utmMedium: any, tracking: anyOf(paidTrackingParams) },
|
|
121
|
+
{ type: 'paid', category: 'video', referrer: none, utmSource: anyOf(sources.video), utmMedium: anyOf(mediums.video), tracking: any },
|
|
122
|
+
|
|
123
|
+
{ type: 'paid', category: 'display', referrer: notEmpty, utmSource: any, utmMedium: anyOf(mediums.paidall), tracking: any },
|
|
124
|
+
{ type: 'paid', category: 'display', referrer: notEmpty, utmSource: any, utmMedium: anyOf(mediums.display), tracking: any },
|
|
125
|
+
{ type: 'paid', category: 'display', referrer: anyOf(referrers.ad), utmSource: any, utmMedium: any, tracking: any },
|
|
126
|
+
{ type: 'paid', category: 'display', referrer: notEmpty, utmSource: anyOf(sources.display), utmMedium: any, tracking: any },
|
|
127
|
+
{ type: 'paid', category: 'display', referrer: none, utmSource: notEmpty, utmMedium: anyOf(mediums.display), tracking: any },
|
|
128
|
+
{ type: 'paid', category: 'display', referrer: none, utmSource: notEmpty, utmMedium: anyOf(mediums.paidall), tracking: any },
|
|
129
|
+
{ type: 'paid', category: 'display', referrer: none, utmSource: anyOf(sources.display), utmMedium: notEmpty, tracking: any },
|
|
130
|
+
{ type: 'paid', category: 'display', referrer: any, utmSource: any, utmMedium: any, tracking: anyOf(paidTrackingParams) },
|
|
131
|
+
{ type: 'paid', category: 'display', referrer: anyOf(referrers.ad), utmSource: any, utmMedium: any, tracking: any },
|
|
132
|
+
|
|
133
|
+
{ type: 'paid', category: 'affiliate', referrer: any, utmSource: any, utmMedium: anyOf(mediums.affiliate), tracking: any },
|
|
134
|
+
|
|
135
|
+
// low prio PAIDs
|
|
136
|
+
{ type: 'paid', category: 'search', referrer: none, utmSource: anyOf(sources.search), utmMedium: any, tracking: any },
|
|
137
|
+
{ type: 'paid', category: 'uncategorized', referrer: not(domain), utmSource: any, utmMedium: anyOf(mediums.paidall), tracking: any },
|
|
138
|
+
{ type: 'paid', category: 'uncategorized', referrer: not(domain), utmSource: any, utmMedium: any, tracking: anyOf(paidTrackingParams) },
|
|
90
139
|
|
|
91
140
|
// EARNED
|
|
92
|
-
{ type: 'earned', category: 'search', referrer: anyOf(
|
|
93
|
-
{ type: 'earned', category: 'search', referrer: anyOf(
|
|
94
|
-
{ type: 'earned', category: '
|
|
95
|
-
{ type: 'earned', category: 'social', referrer:
|
|
96
|
-
{ type: 'earned', category: '
|
|
97
|
-
{ type: 'earned', category: 'video', referrer: anyOf(
|
|
98
|
-
{ type: 'earned', category: '
|
|
141
|
+
{ type: 'earned', category: 'search', referrer: anyOf(referrers.search), utmSource: none, utmMedium: none, tracking: none },
|
|
142
|
+
{ type: 'earned', category: 'search', referrer: anyOf(referrers.search), utmSource: any, utmMedium: not(mediums.paidall), tracking: not(paidTrackingParams) },
|
|
143
|
+
{ type: 'earned', category: 'search', referrer: anyOf(referrers.search), utmSource: any, utmMedium: anyOf(mediums.organic), tracking: none },
|
|
144
|
+
{ type: 'earned', category: 'social', referrer: anyOf(referrers.social), utmSource: none, utmMedium: none, tracking: none },
|
|
145
|
+
{ type: 'earned', category: 'social', referrer: not(domain), utmSource: any, utmMedium: anyOf(mediums.organic), tracking: none },
|
|
146
|
+
{ type: 'earned', category: 'video', referrer: anyOf(referrers.video), utmSource: none, utmMedium: none, tracking: none },
|
|
147
|
+
{ type: 'earned', category: 'video', referrer: anyOf(referrers.video), utmSource: any, utmMedium: not(mediums.paidall), tracking: none },
|
|
148
|
+
{ type: 'earned', category: 'referral', referrer: not(domain), utmSource: none, utmMedium: none, tracking: none },
|
|
99
149
|
|
|
100
150
|
// OWNED
|
|
101
151
|
{ type: 'owned', category: 'direct', referrer: none, utmSource: none, utmMedium: none, tracking: none },
|
|
102
|
-
{ type: 'owned', category: 'internal', referrer: anyOf(
|
|
152
|
+
{ type: 'owned', category: 'internal', referrer: anyOf(domain), utmSource: none, utmMedium: none, tracking: none },
|
|
103
153
|
{ type: 'owned', category: 'email', referrer: any, utmSource: any, utmMedium: any, tracking: anyOf(emailTrackingParams) },
|
|
104
|
-
{ type: 'owned', category: 'email', referrer: any, utmSource: any, utmMedium: anyOf(
|
|
105
|
-
{ type: 'owned', category: 'sms', referrer: none, utmSource: any, utmMedium: anyOf(
|
|
106
|
-
{ type: 'owned', category: 'qr', referrer: none, utmSource: any, utmMedium: anyOf(
|
|
107
|
-
{ type: 'owned', category: 'push', referrer: none, utmSource: any, utmMedium: anyOf(
|
|
154
|
+
{ type: 'owned', category: 'email', referrer: any, utmSource: any, utmMedium: anyOf(mediums.email), tracking: any },
|
|
155
|
+
{ type: 'owned', category: 'sms', referrer: none, utmSource: any, utmMedium: anyOf(mediums.sms), tracking: none },
|
|
156
|
+
{ type: 'owned', category: 'qr', referrer: none, utmSource: any, utmMedium: anyOf(mediums.qr), tracking: none },
|
|
157
|
+
{ type: 'owned', category: 'push', referrer: none, utmSource: any, utmMedium: anyOf(mediums.push), tracking: none },
|
|
108
158
|
|
|
109
159
|
// FALLBACK
|
|
110
160
|
{ type: 'owned', category: 'uncategorized', referrer: any, utmSource: any, utmMedium: any, tracking: any },
|
|
111
161
|
]);
|
|
112
162
|
|
|
113
163
|
export function classifyTrafficSource(url, referrer, utmSource, utmMedium, trackingParams) {
|
|
114
|
-
const
|
|
115
|
-
const rules = RULES(
|
|
164
|
+
const secondLevelDomain = getSecondLevelDomain(url);
|
|
165
|
+
const rules = RULES(secondLevelDomain);
|
|
166
|
+
|
|
167
|
+
const referrerDomain = getSecondLevelDomain(referrer);
|
|
168
|
+
|
|
169
|
+
const sanitize = (str) => (str || '').toLowerCase().replace(/[^a-zA-Z0-9]/, '');
|
|
116
170
|
|
|
117
171
|
const { type, category } = rules.find((rule) => (
|
|
118
|
-
rule.referrer(
|
|
119
|
-
&& rule.utmSource(utmSource)
|
|
120
|
-
&& rule.utmMedium(utmMedium)
|
|
172
|
+
rule.referrer(referrerDomain)
|
|
173
|
+
&& rule.utmSource(sanitize(utmSource))
|
|
174
|
+
&& rule.utmMedium(sanitize(utmMedium))
|
|
121
175
|
&& rule.tracking(trackingParams)
|
|
122
176
|
));
|
|
123
177
|
|
|
@@ -11,6 +11,7 @@
|
|
|
11
11
|
*/
|
|
12
12
|
|
|
13
13
|
import { classifyTrafficSource } from '../common/traffic.js';
|
|
14
|
+
import { fetch } from '../utils.js';
|
|
14
15
|
|
|
15
16
|
function extractHints(bundle) {
|
|
16
17
|
const findEvent = (checkpoint, source = '') => bundle.events.find((e) => e.checkpoint === checkpoint && (!source || e.source === source)) || {};
|
|
@@ -47,8 +48,81 @@ function transformFormat(trafficSources) {
|
|
|
47
48
|
}));
|
|
48
49
|
}
|
|
49
50
|
|
|
50
|
-
|
|
51
|
-
|
|
51
|
+
/* c8 ignore start */
|
|
52
|
+
/*
|
|
53
|
+
* throw-away code for a single customer who customized the experimentation engine
|
|
54
|
+
* this code will be removed once they start using the default exp engine
|
|
55
|
+
*
|
|
56
|
+
* this function fetches experiment manifests, then merges variants data into controls data
|
|
57
|
+
*
|
|
58
|
+
* ie:
|
|
59
|
+
*
|
|
60
|
+
* if the customer runs for an experiment where variants are as following:
|
|
61
|
+
* control: /
|
|
62
|
+
* challenger-1: /a1/
|
|
63
|
+
* challenger-2: /a2/
|
|
64
|
+
*
|
|
65
|
+
* then data for the `/a1/` and `/a2` are counted towards `/`'s data
|
|
66
|
+
*/
|
|
67
|
+
async function mergeBundlesWithSameId(bundles) {
|
|
68
|
+
if (!bundles[0]?.url.includes('bamboohr.com')) return bundles;
|
|
69
|
+
const manifestUrls = [
|
|
70
|
+
...new Set(bundles.flatMap((bundle) => bundle.events
|
|
71
|
+
.filter((e) => e.checkpoint === 'experiment')
|
|
72
|
+
.map((e) => e.source))),
|
|
73
|
+
].map((experiment) => fetch(`https://www.bamboohr.com/experiments/${experiment}/manifest.json`));
|
|
74
|
+
|
|
75
|
+
const experiments = await Promise.all(manifestUrls);
|
|
76
|
+
const variants = (await Promise.all(experiments.map((e) => e.json().catch(() => {}))))
|
|
77
|
+
.filter((json) => json && Object.keys(json).length > 0)
|
|
78
|
+
.flatMap((json) => json.experiences?.data ?? [])
|
|
79
|
+
.filter((data) => data.Name === 'Pages');
|
|
80
|
+
|
|
81
|
+
const mapping = variants.reduce((acc, cur) => {
|
|
82
|
+
Object.entries(cur)
|
|
83
|
+
.filter(([k]) => !['Name', 'Control'].includes(k))
|
|
84
|
+
.forEach(([, v]) => {
|
|
85
|
+
acc[new URL(v).pathname] = new URL(cur.Control).pathname;
|
|
86
|
+
});
|
|
87
|
+
return acc;
|
|
88
|
+
}, {});
|
|
89
|
+
|
|
90
|
+
const variantPaths = Object.keys(mapping);
|
|
91
|
+
|
|
92
|
+
const getControlPath = (url) => {
|
|
93
|
+
const path = new URL(url).pathname;
|
|
94
|
+
if (variantPaths.includes(path)) return mapping[path];
|
|
95
|
+
return path;
|
|
96
|
+
};
|
|
97
|
+
|
|
98
|
+
const byIdAndPath = bundles.reduce((acc, cur) => {
|
|
99
|
+
const controlPath = getControlPath(cur.url);
|
|
100
|
+
const key = `${cur.id}-${controlPath}`;
|
|
101
|
+
if (!acc[key]) acc[key] = [];
|
|
102
|
+
if (variantPaths.includes(new URL(cur.url).pathname)) {
|
|
103
|
+
// eslint-disable-next-line no-param-reassign
|
|
104
|
+
cur.url = new URL(controlPath, cur.url).href;
|
|
105
|
+
}
|
|
106
|
+
acc[key].push(cur);
|
|
107
|
+
return acc;
|
|
108
|
+
}, {});
|
|
109
|
+
|
|
110
|
+
const merged = Object.entries(byIdAndPath).flatMap(([, v]) => {
|
|
111
|
+
let value = v;
|
|
112
|
+
if (v.length > 1) {
|
|
113
|
+
v[0].events.push(...v.slice(1).flatMap((bundle) => bundle.events));
|
|
114
|
+
value = [v[0]];
|
|
115
|
+
}
|
|
116
|
+
return value;
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
return Object.values(merged);
|
|
120
|
+
}
|
|
121
|
+
/* c8 ignore end */
|
|
122
|
+
|
|
123
|
+
async function handler(bundles) {
|
|
124
|
+
const merged = await mergeBundlesWithSameId(bundles);
|
|
125
|
+
const trafficSources = merged
|
|
52
126
|
.map(extractHints)
|
|
53
127
|
.map((row) => {
|
|
54
128
|
const {
|
|
@@ -69,5 +143,5 @@ function handler(bundles) {
|
|
|
69
143
|
|
|
70
144
|
export default {
|
|
71
145
|
handler,
|
|
72
|
-
checkpoints: ['email', 'enter', 'paid', 'utm'],
|
|
146
|
+
checkpoints: ['email', 'enter', 'paid', 'utm', 'experiment'],
|
|
73
147
|
};
|