@mparticle/web-sdk 3.22.0 → 3.24.0

package/src/pageUrl.ts

@@ -0,0 +1,191 @@
+import { isAllowedQueryParam } from './pageViewTracker';
+import { getHref, valueof } from './utils';
+
+// The one rule for a URL that leaves the SDK, shared with kits through
+// `mParticle._cleanUrl`. `path` keeps scheme, host and path. `full` adds the
+// query params an auto page view captures by default and the campaign
+// attribution params, then the fragment.
+export const UrlMode = {
+    Path: 'path',
+    Full: 'full',
+} as const;
+
+export type UrlMode = valueof<typeof UrlMode>;
+
+// Campaign attribution and ad-network click ids. URLs keep them; an auto page view
+// carries one as an attribute only when an input configures it.
+const URL_ATTRIBUTION_QUERY_PARAMS: string[] = [
+    'utm_source',
+    'utm_medium',
+    'utm_campaign',
+    'utm_term',
+    'utm_content',
+    'utm_id',
+    'utm_source_platform',
+    'utm_creative_format',
+    'utm_marketing_tactic',
+    'gad_source',
+    'gad_campaignid',
+    'mc_cid',
+    'gclid',
+    'gbraid',
+    'wbraid',
+    'fbclid',
+    'msclkid',
+    'ttclid',
+    'twclid',
+    'li_fat_id',
+    'dclid',
+    'srsltid',
+    'epik',
+    'ScCid',
+    'irclickid',
+    'rtid',
+    'rclid',
+];
+
+const URL_ATTRIBUTION_QUERY_PARAMS_LOWER: string[] = URL_ATTRIBUTION_QUERY_PARAMS.map(
+    name => name.toLowerCase()
+);
+
+const isUrlQueryParam = (name: string): boolean =>
+    isAllowedQueryParam(name) ||
+    URL_ATTRIBUTION_QUERY_PARAMS_LOWER.indexOf(name.toLowerCase()) !== -1;
+
+// OAuth 2.0 / OIDC authorization-flow params. They stay off URLs even where the
+// page view allowlist admits them, because a URL travels on every event.
+export const EXCLUDED_URL_QUERY_PARAMS: string[] = [
+    'client_id',
+    'redirect_uri',
+    'response_type',
+    'scope',
+    'state',
+    'code',
+    'nonce',
+];
+
+// An OAuth implicit or OIDC fragment response puts its tokens in `#k=v&...`.
+export const EXCLUDED_URL_FRAGMENT_PARAMS: string[] = EXCLUDED_URL_QUERY_PARAMS.concat(
+    [
+        'access_token',
+        'id_token',
+        'refresh_token',
+        'provider_token',
+        'provider_refresh_token',
+    ]
+);
+
+// Allowlisted params whose value is itself a URL, so it can carry a query.
+const URL_VALUED_QUERY_PARAMS: string[] = ['ref', 'referrer'];
+
+const USERINFO = /^((?:[a-z][a-z0-9+.-]*:)?\/\/)[^/]*@/i;
+
+const decodeQueryComponent = (component: string): string =>
+    decodeURIComponent(component.replace(/\+/g, ' '));
+
+const cleanQueryPair = (pair: string): string | null => {
+    const separator = pair.indexOf('=');
+    const rawName = separator === -1 ? pair : pair.slice(0, separator);
+    let name: string;
+    try {
+        name = decodeQueryComponent(rawName);
+    } catch (e) {
+        return null;
+    }
+
+    const lowerName = name.toLowerCase();
+    if (
+        !name ||
+        !isUrlQueryParam(name) ||
+        EXCLUDED_URL_QUERY_PARAMS.indexOf(lowerName) !== -1
+    ) {
+        return null;
+    }
+
+    if (separator === -1 || URL_VALUED_QUERY_PARAMS.indexOf(lowerName) === -1) {
+        return pair;
+    }
+
+    let value: string;
+    try {
+        value = decodeQueryComponent(pair.slice(separator + 1));
+    } catch (e) {
+        return null;
+    }
+    const cleanedValue = cleanUrl(value, UrlMode.Path);
+    return cleanedValue === value
+        ? pair
+        : `${rawName}=${encodeURIComponent(cleanedValue)}`;
+};
+
+const cleanQuery = (query: string): string =>
+    query
+        .split('&')
+        .map(cleanQueryPair)
+        .filter((pair): pair is string => pair !== null)
+        .join('&');
+
+const keepsFragmentPair = (pair: string): boolean => {
+    try {
+        const name = decodeQueryComponent(pair.split('=')[0]);
+        return EXCLUDED_URL_FRAGMENT_PARAMS.indexOf(name.toLowerCase()) === -1;
+    } catch (e) {
+        return false;
+    }
+};
+
+// A hash route (`#/...`, `#!...`, or a path with no `=` before its `?`) keeps
+// its path; a query inside it gets the same allowlist as the URL's own query.
+// A fragment of bare pairs loses the OAuth/OIDC ones.
+const cleanFragment = (hash: string): string => {
+    const queryStart = hash.indexOf('?');
+    const route = queryStart === -1 ? hash : hash.slice(0, queryStart);
+    const isRoute =
+        /^#[/!]/.test(hash) || (queryStart !== -1 && route.indexOf('=') === -1);
+
+    if (isRoute && queryStart !== -1) {
+        const query = cleanQuery(hash.slice(queryStart + 1));
+        return route + (query ? `?${query}` : '');
+    }
+
+    if (hash.indexOf('=') === -1) {
+        return hash;
+    }
+
+    const pairs = hash
+        .slice(1)
+        .split('&')
+        .filter(keepsFragmentPair)
+        .join('&');
+    return pairs ? `#${pairs}` : '';
+};
+
+// String-based rather than `new URL`, so kept pairs and the fragment go out
+// exactly as the browser reported them and no environment can parse differently.
+// An unknown mode is treated as `path`.
+export const cleanUrl = (href: string, mode: UrlMode): string => {
+    if (typeof href !== 'string' || !href) {
+        return '';
+    }
+
+    const hashStart = href.indexOf('#');
+    const hash = hashStart === -1 ? '' : href.slice(hashStart);
+    const beforeHash = hashStart === -1 ? href : href.slice(0, hashStart);
+    const queryStart = beforeHash.indexOf('?');
+    const path = (queryStart === -1
+        ? beforeHash
+        : beforeHash.slice(0, queryStart)
+    ).replace(USERINFO, '$1');
+
+    if (mode !== UrlMode.Full) {
+        return path;
+    }
+
+    const query =
+        queryStart === -1 ? '' : cleanQuery(beforeHash.slice(queryStart + 1));
+
+    return path + (query ? `?${query}` : '') + cleanFragment(hash);
+};
+
+// The current page as it may appear on an event.
+export const cleanPageHref = (): string => cleanUrl(getHref(), UrlMode.Full);

package/src/pageViewTracker.ts

@@ -37,45 +37,9 @@ type WindowWithApv = Window & {
     [WIN_APV_KEY]?: IApvState;
 };
 
-// The query params an auto page view may carry. An allowlist, not a denylist:
-// partner URLs routinely hold order ids, email addresses and session tokens, and
-// none of those should reach the event stream because someone forgot to exclude
-// them. Anything absent from this list is dropped.
-//
-// SECURITY: `code`, `state` and `nonce` are the OAuth 2.0 / OIDC authorization
-// code and the CSRF/replay tokens. They are credentials until redeemed, and
-// attaching them here persists them in the event store and forwards them to
-// every configured kit. They are on the list by explicit product decision —
-// deleting that line is the whole of the fix if that decision is revisited.
+// Parameters that identify which page a view is of. Authorization-flow and campaign
+// attribution parameters are deliberately absent; an input can add them explicitly.
 export const ALLOWED_QUERY_PARAMS: string[] = [
-    // Campaign attribution
-    'utm_source',
-    'utm_medium',
-    'utm_campaign',
-    'utm_term',
-    'utm_content',
-    'utm_id',
-
-    // Ad-network click ids
-    'gclid',
-    'gbraid',
-    'wbraid',
-    'fbclid',
-    'msclkid',
-    'ttclid',
-    'twclid',
-    'li_fat_id',
-    'dclid',
-
-    // OAuth / OIDC — see the SECURITY note above
-    'client_id',
-    'redirect_uri',
-    'response_type',
-    'scope',
-    'state',
-    'code',
-    'nonce',
-
     // Pagination
     'page',
     'limit',
@@ -120,9 +84,62 @@ export const ALLOWED_QUERY_PARAMS: string[] = [
     'referrer',
 ];
 
+export const AUTO_PAGE_VIEW_ATTRIBUTE = 'is_auto_page_view';
+
+const CORE_EVENT_FIELDS = ['hostname', 'title', 'path', AUTO_PAGE_VIEW_ATTRIBUTE];
+
+const PROTOTYPE_MEMBER_NAMES = ['constructor', '__proto__', 'prototype'];
+
+// Server flag values are the strings 'True' and 'False', so these get typed here by habit.
+const FLAG_VALUE_NAMES = ['true', 'false'];
+
+const RESERVED_QUERY_PARAMS = CORE_EVENT_FIELDS.concat(
+    PROTOTYPE_MEMBER_NAMES,
+    FLAG_VALUE_NAMES
+);
+
+const BUILT_IN_NAMES = ALLOWED_QUERY_PARAMS.map(name => name.toLowerCase());
+
+const isBuiltIn = (name: string): boolean =>
+    BUILT_IN_NAMES.indexOf(name.toLowerCase()) !== -1;
+
+// Excludes `&`, `=`, `%` and whitespace, which would distort the dedup key.
+const QUERY_PARAM_NAME = /^[a-z0-9_][a-z0-9_.-]{0,63}$/;
+
+export const MAX_CUSTOM_QUERY_PARAMS = 25;
+
+export interface IQueryParamAllowlist {
+    allowed: string[];
+
+    // Positions, never names: a rejected entry may be a secret pasted into the wrong field.
+    rejectedPositions: number[];
+
+    overLimit: number;
+}
+
+export const MAX_CUSTOM_QUERY_PARAM_VALUE_LENGTH = 512;
+
+// A list, not a dictionary: its order is the order the dedup key is built in.
+export interface ICapturedParam {
+    name: string;
+    value: string;
+}
+
+// Params matched by name prefix, for platforms that put a per-product id in the
+// name. Salesforce Commerce Cloud spells a variant choice
+// `dwvar_<productId>_<attribute>`, e.g. `dwvar_12345_color=navy`.
+export const ALLOWED_QUERY_PARAM_PREFIXES: string[] = ['dwvar_'];
+
+const hasAllowedPrefix = (name: string): boolean =>
+    ALLOWED_QUERY_PARAM_PREFIXES.some(prefix => name.indexOf(prefix) === 0);
+
+// Whether a query param name, in any case, is one an auto page view captures by default.
+export const isAllowedQueryParam = (name: string): boolean =>
+    isBuiltIn(name) || hasAllowedPrefix(name.toLowerCase());
+
 interface IPageSnapshot {
     path: string;
-    params: Dictionary<string>;
+    params: ICapturedParam[];
 }
 
 interface IPageViewData extends IPageSnapshot {
@@ -147,15 +164,114 @@ interface IPendingNavigation {
 // on their own, which is where the interesting rules live.
 // ---------------------------------------------------------------------------
 
-// Pulls the allowlisted query params off a URL. Delegates to the SDK's own
-// parser, which lowercases keys (so `?UTM_Source=` and `?utm_source=` land on one
-// attribute), drops empty values, and carries the fallback for browsers without
-// URLSearchParams. Tolerates an empty href, so SSR yields no params rather than
-// throwing.
-export const allowedQueryParams = (href: string): Dictionary<string> =>
-    queryStringParser(href, ALLOWED_QUERY_PARAMS);
+// Validated here as well as in the dashboard, because remote config is untrusted input.
+export const parseQueryParamAllowlist = (
+    configured: unknown
+): IQueryParamAllowlist => {
+    const allowed: string[] = [];
+    const rejectedPositions: number[] = [];
+    let overLimit = 0;
 
-export const AUTO_PAGE_VIEW_ATTRIBUTE = 'is_auto_page_view';
+    if (!configured) {
+        return { allowed, rejectedPositions, overLimit };
+    }
+
+    const entries: unknown[] = Array.isArray(configured)
+        ? configured
+        : String(configured).split(',');
+
+    entries.forEach((entry, index) => {
+        const position = index + 1;
+        const name = String(entry)
+            .trim()
+            .toLowerCase();
+
+        if (!name) {
+            return;
+        }
+
+        if (
+            !QUERY_PARAM_NAME.test(name) ||
+            RESERVED_QUERY_PARAMS.indexOf(name) !== -1
+        ) {
+            rejectedPositions.push(position);
+            return;
+        }
+
+        if (isBuiltIn(name) || allowed.indexOf(name) !== -1) {
+            return;
+        }
+
+        if (allowed.length >= MAX_CUSTOM_QUERY_PARAMS) {
+            overLimit++;
+            return;
+        }
+
+        allowed.push(name);
+    });
+
+    return { allowed, rejectedPositions, overLimit };
+};
+
+// Not localeCompare: this order feeds the dedup key, so it must be identical in every
+// browser, and localeCompare is locale-dependent.
+const byName = (a: string, b: string): number => (a < b ? -1 : a > b ? 1 : 0);
+
+// Built-ins keep their positions and additions sort after them, so a dedup key depends on
+// which params are configured, not the order they were typed.
+export const effectiveAllowlist = (extras: string[] = []): string[] =>
+    ALLOWED_QUERY_PARAMS.concat(
+        extras.filter(name => !isBuiltIn(name)).sort(byName)
+    );
+
+// Over-long custom values are dropped rather than truncated: two truncated values could
+// share a dedup key and swallow a real page view.
+const isWithinValueLimit = (name: string, value: string): boolean =>
+    isBuiltIn(name) || value.length <= MAX_CUSTOM_QUERY_PARAM_VALUE_LENGTH;
+
+// A prefix match is a platform name, not a configured extra, so the custom
+// value limit does not apply. A name already on the allowlist keeps that
+// position instead of being appended again.
+const prefixMatchedParams = (
+    href: string,
+    allowlist: string[]
+): ICapturedParam[] => {
+    const allParams = queryStringParser(href);
+
+    return Object.keys(allParams)
+        .filter(
+            name =>
+                hasAllowedPrefix(name) &&
+                allParams[name] &&
+                allowlist.indexOf(name) === -1
+        )
+        .sort(byName)
+        .map(name => ({ name, value: allParams[name] }));
+};
+
+export const allowedQueryParams = (
+    href: string,
+    extras: string[] = []
+): ICapturedParam[] => {
+    const allowlist = effectiveAllowlist(extras);
+    const found = queryStringParser(href, allowlist);
+    const listed = allowlist
+        .filter(name => hasOwnProp(found, name))
+        .filter(name => isWithinValueLimit(name, found[name]))
+        .map(name => ({ name, value: found[name] }));
+
+    return listed.concat(prefixMatchedParams(href, allowlist));
+};
+
+export const paramsToAttributes = (
+    params: ICapturedParam[]
+): Dictionary<string> => {
+    const attributes: Dictionary<string> = {};
+    params.forEach(({ name, value }) => {
+        attributes[name] = value;
+    });
+    return attributes;
+};
 
 // Absent rather than false on a manual page view, so the attribute is only ever
 // added by the automatic emitters. Shared by both of them so the key and that
@@ -165,32 +281,11 @@ export const autoPageViewAttribute = (
 ): Dictionary<boolean> =>
     isAutoPageView ? { [AUTO_PAGE_VIEW_ATTRIBUTE]: true } : {};
 
-// The captured params, in allowlist order. Ordering comes from the constant
-// rather than a sort: it is deterministic without needing a comparator, and it
-// does not depend on the object's insertion order, so reordering the query string
-// cannot produce a different key.
-//
-// Membership is an own-property check, not `in`. `in` walks the prototype
-// chain, so a name matching an Object.prototype member reports as present on
-// any plain object. ALLOWED_QUERY_PARAMS contains no such name, which was the
-// only thing making `in` safe here — and it stops being a safe assumption the
-// moment this list can be extended from configuration.
-const capturedNames = (params: Dictionary<string>): string[] =>
-    ALLOWED_QUERY_PARAMS.filter(name => hasOwnProp(params, name));
-
-// The dedup key: pathname plus the allowlisted params in a fixed order, so that
-// reordering the query string is not a new page. Params outside the allowlist
-// never make it into `page.params` and so cannot key a view — nor can the hash.
-//
-// Values are re-encoded because queryStringParser hands them back DECODED. A
-// value holding the pair delimiters would otherwise serialize exactly like two
-// separate params — `{q: 'a&search=b'}` and `{q: 'a', search: 'b'}` both becoming
-// `q=a&search=b` — and dedup would treat a real navigation between them as the
-// same page and drop the view. `q`, `search` and `redirect_uri` carry `&` and `=`
-// routinely, so this is reachable rather than theoretical.
+// Values are re-encoded because they arrive decoded: `{q: 'a&search=b'}` would otherwise
+// key the same as `{q: 'a', search: 'b'}` and dedup a real navigation away.
 export const pageKey = (page: IPageSnapshot): string => {
-    const query = capturedNames(page.params)
-        .map(name => `${name}=${encodeURIComponent(page.params[name])}`)
+    const query = page.params
+        .map(({ name, value }) => `${name}=${encodeURIComponent(value)}`)
         .join('&');
 
     return query ? `${page.path}?${query}` : page.path;
@@ -220,7 +315,7 @@ export const buildPageViewEvent = ({
     // wins. No allowlist entry collides with hostname/title/path today; naming
     // them here is what keeps a later addition from silently overwriting one.
     data: {
-        ...params,
+        ...paramsToAttributes(params),
         hostname,
         title,
         path,
@@ -310,9 +405,9 @@ const clearActiveTracker = (tracker: PageViewTracker): void => {
     }
 };
 
-const currentPage = (): IPageSnapshot => ({
+const currentPage = (extras: string[]): IPageSnapshot => ({
     path: window.location.pathname,
-    params: allowedQueryParams(getHref()),
+    params: allowedQueryParams(getHref(), extras),
 });
 
 // Log-safe description of a page: the path, plus the NAMES of the captured
@@ -326,7 +421,7 @@ const describePage = (page: IPageSnapshot | null): string => {
         return '';
     }
 
-    const names = capturedNames(page.params);
+    const names = page.params.map(({ name }) => name);
     return names.length ? `${page.path} (params: ${names.join()})` : page.path;
 };
 
@@ -344,6 +439,8 @@ export class PageViewTracker {
 
     private lastPage: IPageSnapshot | null = null;
     private active = false;
+
+    private customQueryParams: string[] = [];
     private pendingNavigations: IPendingNavigation[] = [];
 
     private undoHistoryPatch: (() => void) | null = null;
@@ -388,7 +485,8 @@ export class PageViewTracker {
         }
 
         this.active = true;
-        this.lastPage = currentPage();
+        this.customQueryParams = this.readCustomQueryParams();
+        this.lastPage = currentPage(this.customQueryParams);
         this.log(`[init] seeded lastPage: ${describePage(this.lastPage)}`);
 
         this.undoHistoryPatch = subscribeToRouteChange(
@@ -421,6 +519,36 @@ export class PageViewTracker {
         clearActiveTracker(this);
     }
 
+    private readCustomQueryParams(): string[] {
+        const {
+            allowed = [],
+            rejectedPositions = [],
+            overLimit = 0,
+        } = (this.mpInstance._Store.SDKConfig.autoLogPageViewQueryParams ||
+            {}) as IQueryParamAllowlist;
+
+        if (rejectedPositions.length) {
+            this.mpInstance.Logger.warning(
+                'mParticle APV: ignoring invalid additional page view query ' +
+                    `parameters at positions ${rejectedPositions.join(', ')}`
+            );
+        }
+
+        if (overLimit) {
+            this.mpInstance.Logger.warning(
+                `mParticle APV: ignoring ${overLimit} additional page view ` +
+                    `query parameters beyond the limit of ` +
+                    `${MAX_CUSTOM_QUERY_PARAMS}`
+            );
+        }
+
+        if (allowed.length) {
+            this.log(`[init] additional query params: ${allowed.join()}`);
+        }
+
+        return allowed;
+    }
+
     // Stops the outgoing tracker and returns the pages it had queued so this
     // tracker can re-schedule them once it is active. Private, but reachable
     // across instances: the handoff protocol stays internal to the class.
@@ -468,7 +596,7 @@ export class PageViewTracker {
     }
 
     private handleNavigation(source: NavigationSource): void {
-        const candidate = currentPage();
+        const candidate = currentPage(this.customQueryParams);
         const lastKey = this.lastPage ? pageKey(this.lastPage) : null;
 
         this.log(